dsh-runtime-log 0.0.0-stage → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/LICENSE +21 -0
- package/README.md +142 -2
- package/cordis.patch.yml +7 -0
- package/package.json +40 -4
- package/src/index.js +284 -0
- package/src/log-sink.js +489 -0
- package/src/retention.js +348 -0
package/src/retention.js
ADDED
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-runtime-log - bounded retention for per-process run files.
|
|
3
|
+
*
|
|
4
|
+
* pruneRuns({
|
|
5
|
+
* directory,
|
|
6
|
+
* currentFilePath,
|
|
7
|
+
* activeFilePaths = [],
|
|
8
|
+
* maxTotalFiles = 16,
|
|
9
|
+
* reserveCurrentRun = false,
|
|
10
|
+
* currentProcessPid = process.pid,
|
|
11
|
+
* pruneCurrentProcessRuns = false,
|
|
12
|
+
* fs?,
|
|
13
|
+
* isAlive?,
|
|
14
|
+
* })
|
|
15
|
+
* returns { deletedFiles, retainedFiles, errors } and never rejects.
|
|
16
|
+
*
|
|
17
|
+
* A run file is recognized only when its basename is exactly
|
|
18
|
+
* host-<positive decimal pid>-<valid UUID>.jsonl with an optional .1 ... .31
|
|
19
|
+
* rotation suffix. Recognized files are grouped by their run basename (the
|
|
20
|
+
* rotation suffixes belong to the same run), and only regular files are
|
|
21
|
+
* considered: symbolic links, junctions, directories and unrelated names are
|
|
22
|
+
* ignored and never followed.
|
|
23
|
+
*
|
|
24
|
+
* Every protected run group is always retained: the current run, identified
|
|
25
|
+
* by currentFilePath (including any rotation), and each group named by an
|
|
26
|
+
* activeFilePaths entry (again including any rotation). Unrecognized or
|
|
27
|
+
* malformed entries are ignored rather than treated as errors.
|
|
28
|
+
*
|
|
29
|
+
* When the number of recognized files exceeds maxTotalFiles, whole run groups
|
|
30
|
+
* whose process is definitely dead are removed oldest first, where a group's
|
|
31
|
+
* age is the newest mtime among its files. Groups with a live or unknown
|
|
32
|
+
* process are never removed. Pruning stops as soon as the cap is met or no
|
|
33
|
+
* dead groups remain, so live/unknown/protected runs can legitimately leave
|
|
34
|
+
* the directory above the cap.
|
|
35
|
+
*
|
|
36
|
+
* maxTotalFiles is a target based on files that actually exist, not a hard
|
|
37
|
+
* quota. With reserveCurrentRun = true the caller declares that the current
|
|
38
|
+
* run may not have written its first segment yet (the sink creates it lazily),
|
|
39
|
+
* so exactly one slot is reserved - and only when the current run's file is not
|
|
40
|
+
* part of this scan. Once the current segment is on disk it is counted like any
|
|
41
|
+
* other retained file; future rotations are never reserved up front. Live
|
|
42
|
+
* runs and later rotations may legitimately exceed the target.
|
|
43
|
+
*
|
|
44
|
+
* process.pid is the current process, so its own older run groups are not
|
|
45
|
+
* "definitely dead": by default (pruneCurrentProcessRuns = false) any group
|
|
46
|
+
* whose PID equals currentProcessPid is retained without probing, exactly as a
|
|
47
|
+
* live run would be. A caller that tracks every active run of this process can
|
|
48
|
+
* opt in with pruneCurrentProcessRuns = true; only then is an unprotected
|
|
49
|
+
* same-PID group treated as definitely dead without consulting isAlive, while
|
|
50
|
+
* every other PID keeps the isAlive contract and still needs an ESRCH-backed
|
|
51
|
+
* "dead" result.
|
|
52
|
+
*
|
|
53
|
+
* isAlive(pid) is synchronous and returns "live", "dead" or "unknown". The
|
|
54
|
+
* default probes with process.kill(pid, 0): only ESRCH means "dead"; success
|
|
55
|
+
* means "live"; every other failure (EPERM, EACCES, ...) means "unknown".
|
|
56
|
+
* A throwing predicate, an unexpected return value, or any unexpected
|
|
57
|
+
* filesystem failure is reported as a sanitized error code and treated as
|
|
58
|
+
* "retain" - this module never deletes on uncertainty.
|
|
59
|
+
*
|
|
60
|
+
* Concurrent pruners are tolerated: an ENOENT from unlink means the file is
|
|
61
|
+
* already gone and is counted as removed. Other errors are returned as short
|
|
62
|
+
* codes (for example "EACCES") without raw messages, paths or secrets. The
|
|
63
|
+
* function only reads the configured directory (no recursion, no traversal),
|
|
64
|
+
* only unlinks recognized files, and never renames anything.
|
|
65
|
+
*
|
|
66
|
+
* This is best-effort retention, not a hard disk quota: a directory containing
|
|
67
|
+
* only current/live/unknown runs can stay above maxTotalFiles.
|
|
68
|
+
*/
|
|
69
|
+
import defaultFs from 'node:fs/promises'
|
|
70
|
+
import nodePath from 'node:path'
|
|
71
|
+
|
|
72
|
+
const DEFAULT_MAX_TOTAL_FILES = 16
|
|
73
|
+
|
|
74
|
+
// host-<pid>-<uuid>.jsonl[.1..31]
|
|
75
|
+
// PID: positive decimal without a leading zero.
|
|
76
|
+
// UUID: canonical 8-4-4-4-12, RFC 4122 version 1-8 and variant 8/9/a/b.
|
|
77
|
+
const RUN_FILE_RE = /^host-([1-9]\d*)-([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})\.jsonl(?:\.([1-9]|[12]\d|3[01]))?$/
|
|
78
|
+
|
|
79
|
+
function defaultIsAlive(pid) {
|
|
80
|
+
try {
|
|
81
|
+
process.kill(pid, 0)
|
|
82
|
+
return 'live'
|
|
83
|
+
} catch (error) {
|
|
84
|
+
try {
|
|
85
|
+
return error && error.code === 'ESRCH' ? 'dead' : 'unknown'
|
|
86
|
+
} catch {
|
|
87
|
+
return 'unknown'
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function resolveFs(input) {
|
|
93
|
+
if (!input || typeof input !== 'object') return defaultFs
|
|
94
|
+
return {
|
|
95
|
+
readdir: typeof input.readdir === 'function' ? input.readdir.bind(input) : defaultFs.readdir,
|
|
96
|
+
lstat: typeof input.lstat === 'function' ? input.lstat.bind(input) : defaultFs.lstat,
|
|
97
|
+
unlink: typeof input.unlink === 'function' ? input.unlink.bind(input) : defaultFs.unlink,
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function resolveMaxTotalFiles(value) {
|
|
102
|
+
if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) return value
|
|
103
|
+
return DEFAULT_MAX_TOTAL_FILES
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function parseRunFile(name) {
|
|
107
|
+
if (typeof name !== 'string') return null
|
|
108
|
+
const match = RUN_FILE_RE.exec(name)
|
|
109
|
+
if (match === null) return null
|
|
110
|
+
const pid = Number(match[1])
|
|
111
|
+
if (!Number.isSafeInteger(pid) || pid <= 0) return null
|
|
112
|
+
const runId = match[2].toLowerCase()
|
|
113
|
+
return {
|
|
114
|
+
pid,
|
|
115
|
+
rotation: match[3] === undefined ? 0 : Number(match[3]),
|
|
116
|
+
base: 'host-' + pid + '-' + runId + '.jsonl',
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function currentGroupBase(currentFilePath) {
|
|
121
|
+
if (typeof currentFilePath !== 'string' || currentFilePath.length === 0) return null
|
|
122
|
+
let name
|
|
123
|
+
try {
|
|
124
|
+
name = nodePath.basename(currentFilePath)
|
|
125
|
+
} catch {
|
|
126
|
+
return null
|
|
127
|
+
}
|
|
128
|
+
const parsed = parseRunFile(name)
|
|
129
|
+
return parsed === null ? null : parsed.base
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Run-group bases that must never be pruned: the current file plus every
|
|
133
|
+
// caller-declared active file. Rotations resolve to their base, so declaring
|
|
134
|
+
// any segment protects the whole group. Malformed entries are ignored.
|
|
135
|
+
function protectedGroupBases(settings, currentBase) {
|
|
136
|
+
const bases = new Set()
|
|
137
|
+
if (currentBase !== null) bases.add(currentBase)
|
|
138
|
+
const activeFilePaths = settings.activeFilePaths
|
|
139
|
+
if (Array.isArray(activeFilePaths)) {
|
|
140
|
+
for (const value of activeFilePaths) {
|
|
141
|
+
try {
|
|
142
|
+
const base = currentGroupBase(value)
|
|
143
|
+
if (base !== null) bases.add(base)
|
|
144
|
+
} catch {
|
|
145
|
+
// An exotic value (throwing toString/getter) is simply not protected.
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return bases
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function resolveCurrentProcessPid(value) {
|
|
153
|
+
if (typeof value === 'number' && Number.isSafeInteger(value) && value > 0) return value
|
|
154
|
+
return process.pid
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Strict opt-in: any non-boolean truthy value stays with the conservative path.
|
|
158
|
+
function resolvePruneCurrentProcessRuns(value) {
|
|
159
|
+
return value === true
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function isRegularFile(stats) {
|
|
163
|
+
try {
|
|
164
|
+
if (!stats || typeof stats.isFile !== 'function') return false
|
|
165
|
+
if (typeof stats.isSymbolicLink === 'function' && stats.isSymbolicLink()) return false
|
|
166
|
+
return stats.isFile()
|
|
167
|
+
} catch {
|
|
168
|
+
return false
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function mtimeOf(stats) {
|
|
173
|
+
try {
|
|
174
|
+
const value = Number(stats.mtimeMs)
|
|
175
|
+
return Number.isFinite(value) ? value : 0
|
|
176
|
+
} catch {
|
|
177
|
+
return 0
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function sanitizedErrorCode(error) {
|
|
182
|
+
try {
|
|
183
|
+
const code = error && typeof error.code === 'string' ? error.code : ''
|
|
184
|
+
return /^[A-Z][A-Z0-9_]{0,31}$/.test(code) ? code : 'EUNKNOWN'
|
|
185
|
+
} catch {
|
|
186
|
+
return 'EUNKNOWN'
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export async function pruneRuns(options = {}) {
|
|
191
|
+
const result = { deletedFiles: 0, retainedFiles: 0, errors: [] }
|
|
192
|
+
const errorCodes = new Set()
|
|
193
|
+
let deletedFiles = 0
|
|
194
|
+
let retainedFiles = 0
|
|
195
|
+
|
|
196
|
+
const noteError = (error) => {
|
|
197
|
+
errorCodes.add(sanitizedErrorCode(error))
|
|
198
|
+
}
|
|
199
|
+
const finalize = () => {
|
|
200
|
+
result.deletedFiles = deletedFiles
|
|
201
|
+
result.retainedFiles = retainedFiles
|
|
202
|
+
result.errors = Array.from(errorCodes)
|
|
203
|
+
return result
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
try {
|
|
207
|
+
const settings = options !== null && typeof options === 'object' ? options : null
|
|
208
|
+
if (settings === null) {
|
|
209
|
+
errorCodes.add('EINVAL')
|
|
210
|
+
return finalize()
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const directory = settings.directory
|
|
214
|
+
if (typeof directory !== 'string' || directory.trim().length === 0) {
|
|
215
|
+
errorCodes.add('EINVAL')
|
|
216
|
+
return finalize()
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const fsp = resolveFs(settings.fs)
|
|
220
|
+
const isAlive = typeof settings.isAlive === 'function' ? settings.isAlive : defaultIsAlive
|
|
221
|
+
const maxTotalFiles = resolveMaxTotalFiles(settings.maxTotalFiles)
|
|
222
|
+
const currentBase = currentGroupBase(settings.currentFilePath)
|
|
223
|
+
const protectedBases = protectedGroupBases(settings, currentBase)
|
|
224
|
+
const reserveCurrentRun = settings.reserveCurrentRun === true
|
|
225
|
+
const currentProcessPid = resolveCurrentProcessPid(settings.currentProcessPid)
|
|
226
|
+
const pruneCurrentProcessRuns = resolvePruneCurrentProcessRuns(settings.pruneCurrentProcessRuns)
|
|
227
|
+
|
|
228
|
+
let names
|
|
229
|
+
try {
|
|
230
|
+
names = await fsp.readdir(directory)
|
|
231
|
+
} catch (error) {
|
|
232
|
+
// A missing directory simply means there is nothing to prune yet.
|
|
233
|
+
if (!(error && error.code === 'ENOENT')) noteError(error)
|
|
234
|
+
return finalize()
|
|
235
|
+
}
|
|
236
|
+
if (!Array.isArray(names)) return finalize()
|
|
237
|
+
|
|
238
|
+
const files = []
|
|
239
|
+
for (const name of names) {
|
|
240
|
+
const parsed = parseRunFile(name)
|
|
241
|
+
if (parsed === null) continue
|
|
242
|
+
const filePath = nodePath.join(directory, name)
|
|
243
|
+
let stats
|
|
244
|
+
try {
|
|
245
|
+
stats = await fsp.lstat(filePath)
|
|
246
|
+
} catch (error) {
|
|
247
|
+
if (error && error.code === 'ENOENT') continue
|
|
248
|
+
noteError(error)
|
|
249
|
+
continue
|
|
250
|
+
}
|
|
251
|
+
if (!isRegularFile(stats)) continue
|
|
252
|
+
files.push({
|
|
253
|
+
path: filePath,
|
|
254
|
+
base: parsed.base,
|
|
255
|
+
pid: parsed.pid,
|
|
256
|
+
rotation: parsed.rotation,
|
|
257
|
+
mtimeMs: mtimeOf(stats),
|
|
258
|
+
})
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
retainedFiles = files.length
|
|
262
|
+
// Budget from the files that exist. The cap only gives up a slot when the
|
|
263
|
+
// current run's first segment is not on disk yet; it never reserves every
|
|
264
|
+
// future rotation of the current run.
|
|
265
|
+
let budget = maxTotalFiles
|
|
266
|
+
if (reserveCurrentRun && !(currentBase !== null && files.some((file) => file.base === currentBase))) {
|
|
267
|
+
budget = Math.max(0, budget - 1)
|
|
268
|
+
}
|
|
269
|
+
if (retainedFiles <= budget) return finalize()
|
|
270
|
+
|
|
271
|
+
const groups = new Map()
|
|
272
|
+
for (const file of files) {
|
|
273
|
+
let group = groups.get(file.base)
|
|
274
|
+
if (group === undefined) {
|
|
275
|
+
group = { base: file.base, pid: file.pid, files: [], latestMtime: 0 }
|
|
276
|
+
groups.set(file.base, group)
|
|
277
|
+
}
|
|
278
|
+
group.files.push(file)
|
|
279
|
+
if (file.mtimeMs > group.latestMtime) group.latestMtime = file.mtimeMs
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Oldest group first, using the newest file in each group. The basename
|
|
283
|
+
// tie-break keeps the order deterministic when mtimes are equal.
|
|
284
|
+
const ordered = Array.from(groups.values()).sort((left, right) => {
|
|
285
|
+
if (left.latestMtime !== right.latestMtime) return left.latestMtime - right.latestMtime
|
|
286
|
+
if (left.base === right.base) return 0
|
|
287
|
+
return left.base < right.base ? -1 : 1
|
|
288
|
+
})
|
|
289
|
+
|
|
290
|
+
const isCurrentlyActive = (group) => {
|
|
291
|
+
if (typeof settings.isRunActive !== 'function') return false
|
|
292
|
+
try {
|
|
293
|
+
return settings.isRunActive(group.pid, group.base) === true
|
|
294
|
+
} catch (error) {
|
|
295
|
+
noteError(error)
|
|
296
|
+
return true
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
for (const group of ordered) {
|
|
301
|
+
if (retainedFiles <= budget) break
|
|
302
|
+
// The directory listing is asynchronous. Re-check the caller's live registry
|
|
303
|
+
// after that await so new same-PID runs are not mistaken for closed runs.
|
|
304
|
+
if (protectedBases.has(group.base) || isCurrentlyActive(group)) continue
|
|
305
|
+
|
|
306
|
+
let status
|
|
307
|
+
if (group.pid === currentProcessPid) {
|
|
308
|
+
// This process is still running, so its own runs are never "definitely
|
|
309
|
+
// dead" unless the caller explicitly asserts it tracks every active
|
|
310
|
+
// run of this process.
|
|
311
|
+
if (!pruneCurrentProcessRuns) continue
|
|
312
|
+
status = 'dead'
|
|
313
|
+
} else {
|
|
314
|
+
try {
|
|
315
|
+
status = isAlive(group.pid)
|
|
316
|
+
} catch (error) {
|
|
317
|
+
noteError(error)
|
|
318
|
+
status = 'unknown'
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
if (status !== 'dead') continue
|
|
322
|
+
|
|
323
|
+
// Remove the oldest segments first and leave the base file for last, so
|
|
324
|
+
// an interrupted prune never discards the newest part of a run first.
|
|
325
|
+
const orderedFiles = group.files.slice().sort((left, right) => right.rotation - left.rotation)
|
|
326
|
+
for (const file of orderedFiles) {
|
|
327
|
+
if (isCurrentlyActive(group)) break
|
|
328
|
+
try {
|
|
329
|
+
await fsp.unlink(file.path)
|
|
330
|
+
deletedFiles += 1
|
|
331
|
+
retainedFiles -= 1
|
|
332
|
+
} catch (error) {
|
|
333
|
+
if (error && error.code === 'ENOENT') {
|
|
334
|
+
// Another pruner already removed it; the group still shrank.
|
|
335
|
+
deletedFiles += 1
|
|
336
|
+
retainedFiles -= 1
|
|
337
|
+
} else {
|
|
338
|
+
noteError(error)
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
} catch (error) {
|
|
344
|
+
noteError(error)
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
return finalize()
|
|
348
|
+
}
|