@huaqiu/dsh-plugin-log 0.6.23 → 0.6.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -0
- package/lib/client.d.ts +9 -3
- package/lib/client.js +73 -17
- package/lib/index.d.mts +303 -19
- package/lib/index.mjs +2376 -350
- package/package.json +1 -1
- package/src/client.ts +39 -16
- package/src/fields.ts +31 -0
- package/src/fs-utils.ts +245 -0
- package/src/identity.ts +263 -0
- package/src/index.ts +408 -314
- package/src/ownership.ts +268 -0
- package/src/paths.ts +88 -41
- package/src/recovery.ts +433 -0
- package/src/redact.ts +9 -0
- package/src/rotation.ts +724 -0
- package/src/runtime.ts +633 -0
package/src/rotation.ts
ADDED
|
@@ -0,0 +1,724 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Asynchronous, fault-tolerant rotation + retention for ONE DSH process's log
|
|
3
|
+
* file inside the directory that process exclusively owns.
|
|
4
|
+
*
|
|
5
|
+
* The previous `@huaqiu/dsh-plugin-log` rotation was a synchronous
|
|
6
|
+
* `rename`-shift chain that (a) assumed the process could rename files another
|
|
7
|
+
* DSH process might hold open and (b) produced `.1 .2 .3` siblings with no
|
|
8
|
+
* ownership metadata. Both are wrong once several DSH processes share an
|
|
9
|
+
* installation, so this implementation follows the same rules as the HQ Edge
|
|
10
|
+
* logging subsystem (docs/tasks/dsh-plugin-log-isolation.md §5):
|
|
11
|
+
*
|
|
12
|
+
* - compression is streamed and asynchronous: no whole-file sync gzip;
|
|
13
|
+
* - the closing file is SEALED under a unique name first, so a crash leaves
|
|
14
|
+
* a recoverable artifact rather than a half-renamed member of a chain;
|
|
15
|
+
* - the archive is VERIFIED (gzip trailer included) before publication;
|
|
16
|
+
* - publication is atomic and NEVER overwrites an existing archive;
|
|
17
|
+
* - the sealed source is deleted only after successful publication;
|
|
18
|
+
* - unresolved artifacts are capped, so a filesystem that keeps rejecting
|
|
19
|
+
* publication cannot accumulate temp files without bound.
|
|
20
|
+
*
|
|
21
|
+
* Lifecycle:
|
|
22
|
+
* ACTIVE → ROTATION_REQUESTED → SEALING → ARCHIVING → VERIFYING
|
|
23
|
+
* → PUBLISHING → COMPLETED
|
|
24
|
+
* Every failure state preserves the original data for historical recovery.
|
|
25
|
+
*/
|
|
26
|
+
import {
|
|
27
|
+
createReadStream,
|
|
28
|
+
createWriteStream,
|
|
29
|
+
existsSync,
|
|
30
|
+
promises as fsPromises,
|
|
31
|
+
readdirSync,
|
|
32
|
+
renameSync,
|
|
33
|
+
unlinkSync,
|
|
34
|
+
} from 'node:fs'
|
|
35
|
+
import type { WriteStream } from 'node:fs'
|
|
36
|
+
import { createGunzip, createGzip } from 'node:zlib'
|
|
37
|
+
import { pipeline } from 'node:stream/promises'
|
|
38
|
+
import { Writable } from 'node:stream'
|
|
39
|
+
import { randomUUID } from 'node:crypto'
|
|
40
|
+
import { basename, join } from 'node:path'
|
|
41
|
+
|
|
42
|
+
import { appendDiagnostic, atomicPublish, errorCode, errorMessage, sizeOrZero, withFsRetries } from './fs-utils.js'
|
|
43
|
+
|
|
44
|
+
/** Rotation + retention policy for the DSH log stream. */
|
|
45
|
+
export interface DshRotationConfig {
|
|
46
|
+
/** Rotation frequency. Only "daily" is supported. */
|
|
47
|
+
frequency: 'daily'
|
|
48
|
+
/** Max size of a single log file before rotation, e.g. "10m". */
|
|
49
|
+
maxSize: string
|
|
50
|
+
/** Days to keep rotated (compressed) files. */
|
|
51
|
+
retentionDays: number
|
|
52
|
+
/** Maximum total bytes this DSH instance directory may consume. */
|
|
53
|
+
maxStorageBytes: number
|
|
54
|
+
/** Gzip rotated files. */
|
|
55
|
+
compress: boolean
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Defaults for DSH plugin logs: plugin diagnostics are low volume, so 10 MiB
|
|
60
|
+
* per file with 14 days of history is plenty, and 256 MiB caps a pathological
|
|
61
|
+
* plugin without competing with the parent Edge instance's own budget.
|
|
62
|
+
*/
|
|
63
|
+
export const DSH_ROTATION: DshRotationConfig = {
|
|
64
|
+
frequency: 'daily',
|
|
65
|
+
maxSize: '10m',
|
|
66
|
+
retentionDays: 14,
|
|
67
|
+
maxStorageBytes: 256 * 1024 * 1024,
|
|
68
|
+
compress: true,
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Parses a human size string ("10m", "1g", "500k", or "1024") into bytes. */
|
|
72
|
+
export function parseSize(input: string): number {
|
|
73
|
+
const match = /^(\d+(?:\.\d+)?)\s*([kmg]?b?)?$/i.exec(input.trim())
|
|
74
|
+
if (!match) return 10 * 1024 * 1024
|
|
75
|
+
const value = Number.parseFloat(match[1] ?? '')
|
|
76
|
+
if (!Number.isFinite(value)) return 10 * 1024 * 1024
|
|
77
|
+
const unit = (match[2] || 'm').toLowerCase()
|
|
78
|
+
switch (unit[0]) {
|
|
79
|
+
case 'k':
|
|
80
|
+
return Math.floor(value * 1024)
|
|
81
|
+
case 'g':
|
|
82
|
+
return Math.floor(value * 1024 * 1024 * 1024)
|
|
83
|
+
default:
|
|
84
|
+
return Math.floor(value * 1024 * 1024)
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Current UTC date as `yyyy-MM-dd`. UTC keeps multi-timezone installs stable. */
|
|
89
|
+
export function todayString(date: Date = new Date()): string {
|
|
90
|
+
return date.toISOString().slice(0, 10)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export const SEALED_MARKER = '.log.sealed-'
|
|
94
|
+
export const TEMP_MARKER = '.gz.tmp-'
|
|
95
|
+
export const QUARANTINE_MARKER = '.quarantine-'
|
|
96
|
+
export const ARCHIVE_SUFFIX = '.log.gz'
|
|
97
|
+
|
|
98
|
+
/** Canonical name for an in-progress gzip archive, e.g. `dsh.<date>.log.gz.tmp-<uuid>`. */
|
|
99
|
+
export function tempArchivePathFor(archivePath: string): string {
|
|
100
|
+
return `${archivePath.slice(0, -'.gz'.length)}${TEMP_MARKER}${randomUUID()}`
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Inverse of {@link tempArchivePathFor}, for a TEMP_MARKER-bearing name. */
|
|
104
|
+
export function archivePathFromTempName(dir: string, name: string): string {
|
|
105
|
+
return join(dir, `${name.slice(0, name.indexOf(TEMP_MARKER))}.gz`)
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** True for names that are published or legacy archives. */
|
|
109
|
+
export function isArchiveName(name: string): boolean {
|
|
110
|
+
return name.endsWith('.gz') || /\.gz\.\d+$/.test(name)
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** True for names that are incomplete rotation artifacts. */
|
|
114
|
+
export function isRecoveryArtifactName(name: string): boolean {
|
|
115
|
+
return name.includes(SEALED_MARKER) || name.includes(TEMP_MARKER)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export type LoggingHealthState = 'initializing' | 'healthy' | 'degraded' | 'unavailable'
|
|
119
|
+
|
|
120
|
+
export interface LoggingHealth {
|
|
121
|
+
state: LoggingHealthState
|
|
122
|
+
reason: string
|
|
123
|
+
droppedRecords: number
|
|
124
|
+
rotationCount: number
|
|
125
|
+
lastError: string | null
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Injected faults — production never sets these; tests do. */
|
|
129
|
+
export interface DshRotationFaults {
|
|
130
|
+
gzip?: boolean
|
|
131
|
+
publish?: boolean
|
|
132
|
+
seal?: boolean
|
|
133
|
+
unlink?: boolean
|
|
134
|
+
streamOpen?: boolean
|
|
135
|
+
/** Delay BEFORE compression — lets a test kill the process mid-compression. */
|
|
136
|
+
gzipDelayMs?: number
|
|
137
|
+
/** Delay BEFORE publication — lets a test kill the process mid-publication. */
|
|
138
|
+
publishDelayMs?: number
|
|
139
|
+
/** Simulate ENOSPC on the active write stream. */
|
|
140
|
+
diskFull?: boolean
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export interface DshLogWriterOptions {
|
|
144
|
+
/** The exclusively-owned instance directory. */
|
|
145
|
+
dir: string
|
|
146
|
+
/** Active file stem. Default `dsh`. */
|
|
147
|
+
baseName?: string
|
|
148
|
+
cfg?: DshRotationConfig
|
|
149
|
+
faults?: DshRotationFaults
|
|
150
|
+
now?: () => Date
|
|
151
|
+
/** Diagnostic sink; defaults to `logging-health.log` in the same directory. */
|
|
152
|
+
onDiagnostic?: (record: Record<string, unknown>) => void
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const MAX_PENDING_BYTES = 4 * 1024 * 1024
|
|
156
|
+
const MAX_PENDING_RECORDS = 20_000
|
|
157
|
+
const MAX_DIAGNOSTICS = 40
|
|
158
|
+
const MAX_DELETES_PER_SWEEP = 64
|
|
159
|
+
const MAX_OPEN_ATTEMPTS = 3
|
|
160
|
+
const MAX_UNRESOLVED_ARTIFACTS = 6
|
|
161
|
+
const UNRESOLVED_COOLDOWN_MS = 5_000
|
|
162
|
+
const STREAM_CLOSE_TIMEOUT_MS = 2_000
|
|
163
|
+
const DRAIN_TIMEOUT_MS = 2_000
|
|
164
|
+
const TEMP_MAX_AGE_MS = 24 * 60 * 60 * 1000
|
|
165
|
+
|
|
166
|
+
/** Streams `src` into a gzip temp file. Never loads the whole log into memory. */
|
|
167
|
+
export async function compressFileToArchive(
|
|
168
|
+
src: string,
|
|
169
|
+
tmp: string,
|
|
170
|
+
faults?: DshRotationFaults,
|
|
171
|
+
): Promise<void> {
|
|
172
|
+
if (faults?.gzipDelayMs) await new Promise((resolve) => setTimeout(resolve, faults.gzipDelayMs))
|
|
173
|
+
if (faults?.gzip) throw new Error('injected gzip failure')
|
|
174
|
+
await pipeline(createReadStream(src), createGzip({ level: 6 }), createWriteStream(tmp, { flags: 'wx' }))
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Verifies a gzip archive is complete and readable (trailer included). */
|
|
178
|
+
export async function verifyGzipArchive(path: string): Promise<number> {
|
|
179
|
+
let bytes = 0
|
|
180
|
+
const counter = new Writable({
|
|
181
|
+
write(chunk: Buffer, _encoding: string, callback: (err?: Error | null) => void): void {
|
|
182
|
+
bytes += chunk.length
|
|
183
|
+
callback()
|
|
184
|
+
},
|
|
185
|
+
})
|
|
186
|
+
await pipeline(createReadStream(path), createGunzip(), counter)
|
|
187
|
+
return bytes
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Serialized writer for one DSH log stream.
|
|
192
|
+
*
|
|
193
|
+
* Every mutation — append, seal, compress, publish, sweep — runs on ONE promise
|
|
194
|
+
* chain, so two plugins logging concurrently can never interleave a rotation
|
|
195
|
+
* with a write, and `flush()` has a single, exact meaning: "everything enqueued
|
|
196
|
+
* so far, including any rotation it triggered, has finished".
|
|
197
|
+
*/
|
|
198
|
+
export class DshLogWriter {
|
|
199
|
+
private readonly dir: string
|
|
200
|
+
private readonly baseName: string
|
|
201
|
+
private readonly cfg: DshRotationConfig
|
|
202
|
+
private readonly maxSizeBytes: number
|
|
203
|
+
private readonly now: () => Date
|
|
204
|
+
private readonly faults: DshRotationFaults
|
|
205
|
+
private readonly onDiagnostic?: (record: Record<string, unknown>) => void
|
|
206
|
+
|
|
207
|
+
private stream: WriteStream | null = null
|
|
208
|
+
private currentDate: string
|
|
209
|
+
private currentPath: string
|
|
210
|
+
private bytesWritten = 0
|
|
211
|
+
|
|
212
|
+
private queue: Promise<void> = Promise.resolve()
|
|
213
|
+
private pendingBytes = 0
|
|
214
|
+
private pendingRecords = 0
|
|
215
|
+
private droppedRecords = 0
|
|
216
|
+
private openAttempts = 0
|
|
217
|
+
private rotationCount = 0
|
|
218
|
+
private deletions = 0
|
|
219
|
+
private protectedFiles = new Set<string>()
|
|
220
|
+
private diagnosticsEmitted = 0
|
|
221
|
+
private closing = false
|
|
222
|
+
private opened = false
|
|
223
|
+
|
|
224
|
+
private health: LoggingHealthState = 'initializing'
|
|
225
|
+
private healthReason = 'awaiting instance ownership'
|
|
226
|
+
private lastError: string | null = null
|
|
227
|
+
|
|
228
|
+
constructor(options: DshLogWriterOptions) {
|
|
229
|
+
this.dir = options.dir
|
|
230
|
+
this.baseName = options.baseName ?? 'dsh'
|
|
231
|
+
this.cfg = options.cfg ?? DSH_ROTATION
|
|
232
|
+
this.maxSizeBytes = parseSize(this.cfg.maxSize)
|
|
233
|
+
this.now = options.now ?? (() => new Date())
|
|
234
|
+
this.faults = options.faults ?? {}
|
|
235
|
+
this.onDiagnostic = options.onDiagnostic
|
|
236
|
+
this.currentDate = todayString(this.now())
|
|
237
|
+
this.currentPath = this.pathForDate(this.currentDate)
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Absolute path of the active file (whether or not it is open yet). */
|
|
241
|
+
get path(): string {
|
|
242
|
+
return this.currentPath
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
getHealth(): LoggingHealth {
|
|
246
|
+
return {
|
|
247
|
+
state: this.health,
|
|
248
|
+
reason: this.healthReason,
|
|
249
|
+
droppedRecords: this.droppedRecords,
|
|
250
|
+
rotationCount: this.rotationCount,
|
|
251
|
+
lastError: this.lastError,
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** Opens the active file. Called only after the directory is owned. */
|
|
256
|
+
open(): void {
|
|
257
|
+
if (this.closing || this.opened) return
|
|
258
|
+
this.opened = true
|
|
259
|
+
this.openStream()
|
|
260
|
+
// Sweep on the same serialized chain as writes: retention must never race
|
|
261
|
+
// a rotation that is currently publishing an archive.
|
|
262
|
+
this.queue = this.queue.then(() => this.runSweepSafely())
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
private pathForDate(date: string): string {
|
|
266
|
+
return join(this.dir, `${this.baseName}.${date}.log`)
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
private diagnose(event: string, err?: unknown, extra: Record<string, unknown> = {}): void {
|
|
270
|
+
if (this.diagnosticsEmitted >= MAX_DIAGNOSTICS) return
|
|
271
|
+
this.diagnosticsEmitted += 1
|
|
272
|
+
const record: Record<string, unknown> = { event: `dsh-logging-${event}`, instanceDir: this.dir, ...extra }
|
|
273
|
+
if (err !== undefined) {
|
|
274
|
+
record.code = errorCode(err)
|
|
275
|
+
record.message = errorMessage(err).slice(0, 500)
|
|
276
|
+
this.lastError = errorMessage(err).slice(0, 500)
|
|
277
|
+
}
|
|
278
|
+
if (this.onDiagnostic) {
|
|
279
|
+
try {
|
|
280
|
+
this.onDiagnostic(record)
|
|
281
|
+
return
|
|
282
|
+
} catch {
|
|
283
|
+
/* a diagnostic sink must never re-enter the logging failure */
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
appendDiagnostic(join(this.dir, 'logging-health.log'), record)
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
private setHealth(state: LoggingHealthState, reason: string, err?: unknown): void {
|
|
290
|
+
this.health = state
|
|
291
|
+
this.healthReason = reason
|
|
292
|
+
if (err !== undefined) this.lastError = errorMessage(err).slice(0, 500)
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
private openStream(): void {
|
|
296
|
+
if (this.closing) return
|
|
297
|
+
if (this.faults.streamOpen) {
|
|
298
|
+
this.setHealth('unavailable', 'injected stream-open failure')
|
|
299
|
+
this.diagnose('open-failed', new Error('injected stream-open failure'))
|
|
300
|
+
return
|
|
301
|
+
}
|
|
302
|
+
try {
|
|
303
|
+
const stream = createWriteStream(this.currentPath, { flags: 'a' })
|
|
304
|
+
stream.on('error', (err: unknown) => {
|
|
305
|
+
this.diagnose('write-stream-error', err)
|
|
306
|
+
this.setHealth('degraded', 'active write stream error', err)
|
|
307
|
+
if (this.stream !== stream) return
|
|
308
|
+
this.stream = null
|
|
309
|
+
stream.destroy()
|
|
310
|
+
})
|
|
311
|
+
this.stream = stream
|
|
312
|
+
this.bytesWritten = sizeOrZero(this.currentPath)
|
|
313
|
+
this.setHealth('healthy', 'active log stream open')
|
|
314
|
+
if (this.faults.diskFull) {
|
|
315
|
+
const err = Object.assign(new Error('ENOSPC: no space left on device'), { code: 'ENOSPC' })
|
|
316
|
+
setImmediate(() => stream.emit('error', err))
|
|
317
|
+
}
|
|
318
|
+
} catch (err) {
|
|
319
|
+
this.setHealth('degraded', 'cannot open active log file', err)
|
|
320
|
+
this.diagnose('open-failed', err)
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** Total in-memory queue pressure, used to reject unbounded buffering. */
|
|
325
|
+
private canEnqueue(bytes: number): boolean {
|
|
326
|
+
if (this.pendingBytes + bytes <= MAX_PENDING_BYTES && this.pendingRecords < MAX_PENDING_RECORDS) return true
|
|
327
|
+
this.droppedRecords += 1
|
|
328
|
+
this.diagnose('pending-overflow', undefined, { droppedTotal: this.droppedRecords })
|
|
329
|
+
return false
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Enqueues one already-serialized JSONL record. Never throws and never
|
|
334
|
+
* blocks the caller: a plugin's business call returns immediately.
|
|
335
|
+
*/
|
|
336
|
+
write(line: string): void {
|
|
337
|
+
if (this.closing) {
|
|
338
|
+
this.droppedRecords += 1
|
|
339
|
+
return
|
|
340
|
+
}
|
|
341
|
+
const bytes = Buffer.byteLength(line, 'utf8')
|
|
342
|
+
if (!this.canEnqueue(bytes)) return
|
|
343
|
+
this.pendingBytes += bytes
|
|
344
|
+
this.pendingRecords += 1
|
|
345
|
+
this.queue = this.queue
|
|
346
|
+
.then(async () => {
|
|
347
|
+
this.pendingBytes -= bytes
|
|
348
|
+
this.pendingRecords -= 1
|
|
349
|
+
await this.append(line)
|
|
350
|
+
})
|
|
351
|
+
.catch((err: unknown) => {
|
|
352
|
+
// A logging failure must never surface as a plugin failure.
|
|
353
|
+
this.diagnose('append-failed', err)
|
|
354
|
+
this.setHealth('degraded', 'append failed', err)
|
|
355
|
+
})
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
private async append(line: string): Promise<void> {
|
|
359
|
+
if (this.closing || this.health === 'unavailable') {
|
|
360
|
+
this.droppedRecords += 1
|
|
361
|
+
return
|
|
362
|
+
}
|
|
363
|
+
const today = todayString(this.now())
|
|
364
|
+
if (today !== this.currentDate) {
|
|
365
|
+
await this.rotate('date')
|
|
366
|
+
}
|
|
367
|
+
if (!this.stream) {
|
|
368
|
+
if (this.openAttempts >= MAX_OPEN_ATTEMPTS) {
|
|
369
|
+
this.droppedRecords += 1
|
|
370
|
+
return
|
|
371
|
+
}
|
|
372
|
+
this.openAttempts += 1
|
|
373
|
+
this.openStream()
|
|
374
|
+
if (!this.stream) {
|
|
375
|
+
this.droppedRecords += 1
|
|
376
|
+
return
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
const stream = this.stream
|
|
380
|
+
try {
|
|
381
|
+
await this.writeChunk(stream, line)
|
|
382
|
+
this.bytesWritten += Buffer.byteLength(line, 'utf8')
|
|
383
|
+
// Only a WRITE that actually reached the file proves the destination is
|
|
384
|
+
// healthy again. Resetting on open would let a persistently broken
|
|
385
|
+
// stream reopen once per record forever.
|
|
386
|
+
this.openAttempts = 0
|
|
387
|
+
} catch (err) {
|
|
388
|
+
this.diagnose('write-failed', err)
|
|
389
|
+
this.setHealth('degraded', 'write failed', err)
|
|
390
|
+
this.droppedRecords += 1
|
|
391
|
+
return
|
|
392
|
+
}
|
|
393
|
+
if (this.bytesWritten >= this.maxSizeBytes) {
|
|
394
|
+
await this.rotate('size')
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Writes one chunk and resolves only once the stream has flushed it, so
|
|
400
|
+
* `flush()` is a real durability boundary (not just "handed to the stream").
|
|
401
|
+
* A bounded timeout keeps a wedged stream from stalling shutdown forever.
|
|
402
|
+
*/
|
|
403
|
+
private writeChunk(stream: WriteStream, chunk: string): Promise<void> {
|
|
404
|
+
return new Promise<void>((resolve, reject) => {
|
|
405
|
+
let settled = false
|
|
406
|
+
let timer: NodeJS.Timeout | null = null
|
|
407
|
+
const cleanup = () => {
|
|
408
|
+
if (timer) clearTimeout(timer)
|
|
409
|
+
stream.removeListener('error', onError)
|
|
410
|
+
}
|
|
411
|
+
const finish = (err?: Error) => {
|
|
412
|
+
if (settled) return
|
|
413
|
+
settled = true
|
|
414
|
+
cleanup()
|
|
415
|
+
if (err) reject(err)
|
|
416
|
+
else resolve()
|
|
417
|
+
}
|
|
418
|
+
const onError = (err: Error) => finish(err)
|
|
419
|
+
stream.once('error', onError)
|
|
420
|
+
try {
|
|
421
|
+
const accepted = stream.write(chunk, (err?: Error | null) => {
|
|
422
|
+
if (err) finish(err)
|
|
423
|
+
else finish()
|
|
424
|
+
})
|
|
425
|
+
if (!accepted) {
|
|
426
|
+
timer = setTimeout(() => finish(), DRAIN_TIMEOUT_MS)
|
|
427
|
+
timer.unref()
|
|
428
|
+
}
|
|
429
|
+
} catch (err) {
|
|
430
|
+
finish(err instanceof Error ? err : new Error(String(err)))
|
|
431
|
+
}
|
|
432
|
+
})
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
private sealedPathFor(date: string, reason: 'date' | 'size', sequence: number): string {
|
|
436
|
+
const stem = reason === 'date' ? `${this.baseName}.${date}` : `${this.baseName}.${date}.${sequence}`
|
|
437
|
+
// SEALED_MARKER already begins with ".log.", so no extra ".log" here.
|
|
438
|
+
return join(this.dir, `${stem}${SEALED_MARKER}${randomUUID()}`)
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
private archivePathFor(sealedPath: string): string {
|
|
442
|
+
const name = basename(sealedPath)
|
|
443
|
+
const marker = name.indexOf(SEALED_MARKER)
|
|
444
|
+
const stem = marker > 0 ? name.slice(0, marker) : name
|
|
445
|
+
return join(this.dir, `${stem}${ARCHIVE_SUFFIX}`)
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/** Never overwrite: pick the first free sequence, never append `.gz.N`. */
|
|
449
|
+
private chooseArchivePath(preferred: string, startSeq = 1): string {
|
|
450
|
+
if (startSeq === 1 && !existsSync(preferred)) return preferred
|
|
451
|
+
const bare = preferred.endsWith(ARCHIVE_SUFFIX) ? preferred.slice(0, -ARCHIVE_SUFFIX.length) : preferred
|
|
452
|
+
const stem = bare.replace(/\.\d+$/, '')
|
|
453
|
+
for (let seq = startSeq; seq < 10_000; seq += 1) {
|
|
454
|
+
const candidate = `${stem}.${seq}${ARCHIVE_SUFFIX}`
|
|
455
|
+
if (!existsSync(candidate)) return candidate
|
|
456
|
+
}
|
|
457
|
+
return `${stem}.${randomUUID()}${ARCHIVE_SUFFIX}`
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
private unresolvedArtifactCount(): number {
|
|
461
|
+
try {
|
|
462
|
+
// Synchronous by design: this runs on the serialized rotation path and
|
|
463
|
+
// must see the directory as it is at this exact point.
|
|
464
|
+
return readdirSync(this.dir).filter((name) => isRecoveryArtifactName(name)).length
|
|
465
|
+
} catch {
|
|
466
|
+
return 0
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
private closeStream(): Promise<void> {
|
|
471
|
+
const stream = this.stream
|
|
472
|
+
this.stream = null
|
|
473
|
+
if (!stream) return Promise.resolve()
|
|
474
|
+
return new Promise<void>((resolve) => {
|
|
475
|
+
let settled = false
|
|
476
|
+
const done = () => {
|
|
477
|
+
if (settled) return
|
|
478
|
+
settled = true
|
|
479
|
+
resolve()
|
|
480
|
+
}
|
|
481
|
+
stream.once('finish', done)
|
|
482
|
+
stream.once('error', done)
|
|
483
|
+
stream.once('close', done)
|
|
484
|
+
try {
|
|
485
|
+
stream.end()
|
|
486
|
+
} catch {
|
|
487
|
+
done()
|
|
488
|
+
}
|
|
489
|
+
const timer = setTimeout(done, STREAM_CLOSE_TIMEOUT_MS)
|
|
490
|
+
timer.unref()
|
|
491
|
+
})
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
private async rotate(reason: 'date' | 'size'): Promise<void> {
|
|
495
|
+
if (this.closing || this.health === 'unavailable') return
|
|
496
|
+
|
|
497
|
+
// A date rollover must still seal yesterday's file, but a size rotation
|
|
498
|
+
// must not add yet another artifact while recovery is already backlogged.
|
|
499
|
+
const unresolved = this.unresolvedArtifactCount()
|
|
500
|
+
if (reason === 'size' && unresolved >= MAX_UNRESOLVED_ARTIFACTS) {
|
|
501
|
+
this.diagnose('rotation-deferred', undefined, { unresolved, cap: MAX_UNRESOLVED_ARTIFACTS })
|
|
502
|
+
this.setHealth('degraded', 'unresolved rotation artifacts at cap; continuing on active file')
|
|
503
|
+
return
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
this.rotationCount += 1
|
|
507
|
+
const closingDate = this.currentDate
|
|
508
|
+
const closingPath = this.currentPath
|
|
509
|
+
|
|
510
|
+
// ---- SEALING ---------------------------------------------------------
|
|
511
|
+
await this.closeStream()
|
|
512
|
+
this.currentDate = todayString(this.now())
|
|
513
|
+
this.currentPath = this.pathForDate(this.currentDate)
|
|
514
|
+
|
|
515
|
+
if (!existsSync(closingPath) || sizeOrZero(closingPath) === 0) {
|
|
516
|
+
try {
|
|
517
|
+
if (existsSync(closingPath)) unlinkSync(closingPath)
|
|
518
|
+
} catch {
|
|
519
|
+
/* an empty file is harmless */
|
|
520
|
+
}
|
|
521
|
+
this.openStream()
|
|
522
|
+
return
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
const sequence = reason === 'size' ? this.rotationCount : 0
|
|
526
|
+
const sealedPath = this.sealedPathFor(closingDate, reason, sequence)
|
|
527
|
+
try {
|
|
528
|
+
if (this.faults.seal) throw new Error('injected seal failure')
|
|
529
|
+
await withFsRetries(() => renameSync(closingPath, sealedPath), {
|
|
530
|
+
attempts: 5,
|
|
531
|
+
baseDelayMs: 25,
|
|
532
|
+
totalTimeoutMs: 2_000,
|
|
533
|
+
})
|
|
534
|
+
} catch (err) {
|
|
535
|
+
// Preserve the original: continue appending to the SAME active file.
|
|
536
|
+
this.diagnose('seal-failed', err, { closingPath })
|
|
537
|
+
this.setHealth('degraded', 'seal failed; continuing on active file', err)
|
|
538
|
+
this.currentDate = closingDate
|
|
539
|
+
this.currentPath = closingPath
|
|
540
|
+
this.openStream()
|
|
541
|
+
return
|
|
542
|
+
}
|
|
543
|
+
this.protectedFiles.add(sealedPath)
|
|
544
|
+
|
|
545
|
+
// Writes resume on the fresh active file BEFORE compression starts, so a
|
|
546
|
+
// slow gzip never holds up plugin logging.
|
|
547
|
+
this.openStream()
|
|
548
|
+
|
|
549
|
+
if (!this.cfg.compress) {
|
|
550
|
+
this.protectedFiles.delete(sealedPath)
|
|
551
|
+
return
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
// ---- ARCHIVING / VERIFYING / PUBLISHING ------------------------------
|
|
555
|
+
const preferredArchive = this.chooseArchivePath(this.archivePathFor(sealedPath))
|
|
556
|
+
const tmpPath = tempArchivePathFor(preferredArchive)
|
|
557
|
+
this.protectedFiles.add(tmpPath)
|
|
558
|
+
let archiveVerified = false
|
|
559
|
+
try {
|
|
560
|
+
await compressFileToArchive(sealedPath, tmpPath, this.faults)
|
|
561
|
+
await verifyGzipArchive(tmpPath)
|
|
562
|
+
archiveVerified = true
|
|
563
|
+
if (this.faults.publishDelayMs) await new Promise((resolve) => setTimeout(resolve, this.faults.publishDelayMs))
|
|
564
|
+
|
|
565
|
+
if (this.faults.publish) throw new Error('injected publish failure')
|
|
566
|
+
let published = false
|
|
567
|
+
let archivePath = preferredArchive
|
|
568
|
+
for (let attempt = 0; attempt < 32 && !published; attempt += 1) {
|
|
569
|
+
try {
|
|
570
|
+
published = await withFsRetries(() => atomicPublish(tmpPath, archivePath), {
|
|
571
|
+
attempts: 4,
|
|
572
|
+
baseDelayMs: 25,
|
|
573
|
+
totalTimeoutMs: 2_000,
|
|
574
|
+
isRetryable: (err) => errorCode(err) !== 'EEXIST' && errorCode(err) !== 'ENOENT',
|
|
575
|
+
})
|
|
576
|
+
} catch (err) {
|
|
577
|
+
this.diagnose('publish-failed', err, { archivePath })
|
|
578
|
+
break
|
|
579
|
+
}
|
|
580
|
+
if (!published) archivePath = this.chooseArchivePath(preferredArchive, attempt + 2)
|
|
581
|
+
}
|
|
582
|
+
if (!published) {
|
|
583
|
+
// Sealed source + verified temp archive stay for historical recovery.
|
|
584
|
+
this.diagnose('publish-unresolved', undefined, { sealedPath, tmpPath, archivePath })
|
|
585
|
+
this.setHealth('degraded', 'archive publication deferred to recovery')
|
|
586
|
+
return
|
|
587
|
+
}
|
|
588
|
+
this.protectedFiles.delete(tmpPath)
|
|
589
|
+
this.diagnose('archive-published', undefined, { archivePath })
|
|
590
|
+
|
|
591
|
+
// ---- COMPLETED: delete the source only AFTER publication -----------
|
|
592
|
+
try {
|
|
593
|
+
if (this.faults.unlink) throw new Error('injected unlink failure')
|
|
594
|
+
await withFsRetries(() => unlinkSync(sealedPath), {
|
|
595
|
+
attempts: 4,
|
|
596
|
+
baseDelayMs: 25,
|
|
597
|
+
totalTimeoutMs: 2_000,
|
|
598
|
+
})
|
|
599
|
+
} catch (err) {
|
|
600
|
+
// A leftover sealed log is harmless and recoverable; never lose data.
|
|
601
|
+
this.diagnose('source-unlink-failed', err, { sealedPath })
|
|
602
|
+
}
|
|
603
|
+
this.setHealth('healthy', 'rotation completed')
|
|
604
|
+
} catch (err) {
|
|
605
|
+
if (archiveVerified) {
|
|
606
|
+
// Publication failed AFTER verification: KEEP the complete temp
|
|
607
|
+
// archive so recovery can publish it without recompressing.
|
|
608
|
+
this.diagnose('publish-failed-after-verify', err, { sealedPath, tmpPath })
|
|
609
|
+
this.setHealth('degraded', 'archive publication deferred to recovery')
|
|
610
|
+
} else {
|
|
611
|
+
// Compression / verification failure: discard the INCOMPLETE temp
|
|
612
|
+
// archive, but preserve the sealed source.
|
|
613
|
+
this.diagnose('compression-failed', err, { sealedPath, tmpPath })
|
|
614
|
+
this.setHealth('degraded', 'rotation failed; sealed source preserved', err)
|
|
615
|
+
try {
|
|
616
|
+
if (existsSync(tmpPath)) unlinkSync(tmpPath)
|
|
617
|
+
} catch {
|
|
618
|
+
/* recovery will quarantine an incomplete temp archive */
|
|
619
|
+
}
|
|
620
|
+
this.protectedFiles.delete(tmpPath)
|
|
621
|
+
}
|
|
622
|
+
} finally {
|
|
623
|
+
this.protectedFiles.delete(sealedPath)
|
|
624
|
+
await this.runSweepSafely()
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
private async runSweepSafely(): Promise<void> {
|
|
629
|
+
try {
|
|
630
|
+
await this.runSweep()
|
|
631
|
+
} catch (err) {
|
|
632
|
+
this.diagnose('sweep-failed', err)
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/**
|
|
637
|
+
* Capacity + retention. Counts ACTUAL disk consumption of the instance
|
|
638
|
+
* directory. The active file and any file involved in the current rotation
|
|
639
|
+
* are never deleted.
|
|
640
|
+
*/
|
|
641
|
+
private async runSweep(): Promise<void> {
|
|
642
|
+
this.deletions = 0
|
|
643
|
+
let entries: string[]
|
|
644
|
+
try {
|
|
645
|
+
entries = await fsPromises.readdir(this.dir)
|
|
646
|
+
} catch {
|
|
647
|
+
return
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
const activeName = `${this.baseName}.${this.currentDate}.log`
|
|
651
|
+
const now = Date.now()
|
|
652
|
+
const retentionMs = this.cfg.retentionDays * 24 * 60 * 60 * 1000
|
|
653
|
+
const candidates: Array<{ name: string; path: string; size: number; mtime: number }> = []
|
|
654
|
+
let totalSize = 0
|
|
655
|
+
|
|
656
|
+
for (const name of entries.slice(0, 20_000)) {
|
|
657
|
+
const path = join(this.dir, name)
|
|
658
|
+
try {
|
|
659
|
+
const stat = await fsPromises.stat(path)
|
|
660
|
+
if (!stat.isFile()) continue
|
|
661
|
+
totalSize += stat.size
|
|
662
|
+
if (name === activeName || this.protectedFiles.has(path)) continue
|
|
663
|
+
if (isArchiveName(name)) {
|
|
664
|
+
candidates.push({ name, path, size: stat.size, mtime: stat.mtimeMs })
|
|
665
|
+
} else if (name.includes(TEMP_MARKER) && now - stat.mtimeMs > TEMP_MAX_AGE_MS) {
|
|
666
|
+
candidates.push({ name, path, size: stat.size, mtime: stat.mtimeMs })
|
|
667
|
+
}
|
|
668
|
+
} catch {
|
|
669
|
+
/* an unreadable entry is preserved, never deleted */
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// 1. Retention by the date in the name (mtime is the fallback).
|
|
674
|
+
for (const file of candidates) {
|
|
675
|
+
const match = /\.(\d{4}-\d{2}-\d{2})\./.exec(file.name)
|
|
676
|
+
const stamp = match?.[1] ? Date.parse(match[1]) : Number.NaN
|
|
677
|
+
const age = Number.isNaN(stamp) ? now - file.mtime : now - stamp
|
|
678
|
+
if (age <= retentionMs) continue
|
|
679
|
+
if (await this.deleteEligible(file)) totalSize -= file.size
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// 2. Capacity: delete the oldest eligible archive first, bounded per sweep.
|
|
683
|
+
if (totalSize > this.cfg.maxStorageBytes) {
|
|
684
|
+
const sorted = candidates.filter((file) => existsSync(file.path)).sort((a, b) => a.mtime - b.mtime)
|
|
685
|
+
for (const file of sorted) {
|
|
686
|
+
if (totalSize <= this.cfg.maxStorageBytes) break
|
|
687
|
+
if (await this.deleteEligible(file)) totalSize -= file.size
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
private async deleteEligible(file: { name: string; path: string }): Promise<boolean> {
|
|
693
|
+
if (this.deletions >= MAX_DELETES_PER_SWEEP) return false
|
|
694
|
+
if (!existsSync(file.path)) return false
|
|
695
|
+
try {
|
|
696
|
+
await withFsRetries(() => unlinkSync(file.path), { attempts: 3, baseDelayMs: 25, totalTimeoutMs: 1_000 })
|
|
697
|
+
this.deletions += 1
|
|
698
|
+
return true
|
|
699
|
+
} catch (err) {
|
|
700
|
+
this.diagnose('cleanup-failed', err, { file: file.name })
|
|
701
|
+
return false
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
/**
|
|
706
|
+
* Waits until everything enqueued so far — including a rotation it
|
|
707
|
+
* triggered — has completed. Never called from plugin code; only from DSH
|
|
708
|
+
* shutdown and tests.
|
|
709
|
+
*/
|
|
710
|
+
async flush(): Promise<void> {
|
|
711
|
+
await this.queue
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
/** Bounded, final close. Deletes nothing and finishes the active file. */
|
|
715
|
+
async close(): Promise<void> {
|
|
716
|
+
if (this.closing) {
|
|
717
|
+
await this.queue
|
|
718
|
+
return
|
|
719
|
+
}
|
|
720
|
+
this.closing = true
|
|
721
|
+
await this.queue
|
|
722
|
+
await this.closeStream()
|
|
723
|
+
}
|
|
724
|
+
}
|