dsh-aiterm 0.5.1 → 0.5.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRACTS.md +104 -22
- package/README.md +202 -26
- package/client/HistoryPanel.jsx +342 -29
- package/client/api.js +44 -3
- package/client/locale.js +40 -2
- package/cordis.patch.yml +1 -0
- package/lib/client.js +30 -30
- package/lib/host/autolog.js +107 -4
- package/lib/host/config.js +14 -0
- package/lib/host/disclosure.js +357 -0
- package/lib/host/exporter.js +447 -159
- package/lib/host/history-store.js +1735 -295
- package/lib/host/permission.js +13 -0
- package/lib/host/prompt.js +24 -4
- package/lib/host/registry.js +858 -76
- package/lib/host/routes.js +45 -2
- package/lib/host/tools.js +280 -22
- package/lib/index.js +2 -2
- package/package.json +2 -2
- package/scripts/smoke-client.mjs +193 -6
package/lib/host/exporter.js
CHANGED
|
@@ -1,25 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Human-readable export of stored terminal history.
|
|
3
3
|
*
|
|
4
|
-
* The durable journal
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* search machinery.
|
|
4
|
+
* The durable journal is the source of truth: exporting reads it directly and
|
|
5
|
+
* turns every output line into `<ISO timestamp> <line>`, prefixed by a small
|
|
6
|
+
* `#` header block. Journals are streamed in bounded chunks, so a 512 MiB
|
|
7
|
+
* `historyMaxBytesPerSession` journal is exported without materializing it in
|
|
8
|
+
* memory and without an artificial size ceiling that would silently reject a
|
|
9
|
+
* configured maximum.
|
|
11
10
|
*
|
|
12
|
-
* Selection
|
|
13
|
-
* - `all: true` — every
|
|
14
|
-
*
|
|
15
|
-
* -
|
|
11
|
+
* Selection is by **incarnation address**, not by logical session id alone:
|
|
12
|
+
* - `all: true` — every readable catalog entry / live incarnation (duplicates
|
|
13
|
+
* with the same logical id are all included, each with its own file).
|
|
14
|
+
* - `journalKey` — exactly one address (a `historyId` or a live/legacy id).
|
|
15
|
+
* - `sessionId` — compatibility fallback: the first/newest matching address.
|
|
16
|
+
* - neither — the only available incarnation; two or more is `BAD_TARGET`.
|
|
16
17
|
*
|
|
17
|
-
* Writes are atomic (temp file + rename)
|
|
18
|
-
*
|
|
18
|
+
* Writes are atomic (unique temp file + rename) and final file names carry an
|
|
19
|
+
* incarnation token plus a random suffix, so two concurrent exports — even of
|
|
20
|
+
* the same incarnation — cannot overwrite each other.
|
|
19
21
|
*/
|
|
20
22
|
|
|
21
|
-
import {
|
|
23
|
+
import { randomUUID } from 'node:crypto'
|
|
24
|
+
import { open, rename, rm, stat } from 'node:fs/promises'
|
|
22
25
|
import { join, resolve } from 'node:path'
|
|
26
|
+
import { StringDecoder } from 'node:string_decoder'
|
|
23
27
|
|
|
24
28
|
import { coded, ensureDir } from './util.js'
|
|
25
29
|
|
|
@@ -27,30 +31,49 @@ import { coded, ensureDir } from './util.js'
|
|
|
27
31
|
const MAX_TITLE_SIZE = 40
|
|
28
32
|
/** Longest exported session id component in a file name. */
|
|
29
33
|
const MAX_ID_SIZE = 64
|
|
30
|
-
/** Hard cap on exported
|
|
31
|
-
const MAX_EXPORT_SESSIONS =
|
|
34
|
+
/** Hard cap on exported addresses per call. */
|
|
35
|
+
const MAX_EXPORT_SESSIONS = 5000
|
|
36
|
+
/** Chunk size for reading a journal and buffering the output. */
|
|
37
|
+
const STREAM_CHUNK_BYTES = 64 * 1024
|
|
38
|
+
/** Flush buffered output to disk at this size. */
|
|
39
|
+
const WRITE_CHUNK_BYTES = 64 * 1024
|
|
40
|
+
/** Conservative total output budget for one export call. */
|
|
41
|
+
const MAX_EXPORT_TOTAL_BYTES = 512 * 1024 * 1024
|
|
32
42
|
|
|
33
43
|
/**
|
|
34
44
|
* @typedef {object} ExportFile
|
|
35
45
|
* @property {string} sessionId
|
|
46
|
+
* @property {string} [historyId]
|
|
36
47
|
* @property {string} path absolute path of the written `.log` file.
|
|
37
48
|
* @property {number} bytes file size in bytes.
|
|
38
49
|
* @property {number} lines number of written lines (header included).
|
|
39
50
|
*/
|
|
40
51
|
|
|
52
|
+
/**
|
|
53
|
+
* @typedef {object} ExportSkipped
|
|
54
|
+
* @property {string} sessionId
|
|
55
|
+
* @property {string} [historyId]
|
|
56
|
+
* @property {string} code stable reason code.
|
|
57
|
+
* @property {string} message explanation.
|
|
58
|
+
*/
|
|
59
|
+
|
|
41
60
|
/**
|
|
42
61
|
* @typedef {object} ExportResult
|
|
43
62
|
* @property {string} dir absolute export directory.
|
|
44
63
|
* @property {ExportFile[]} files written files.
|
|
64
|
+
* @property {ExportSkipped[]} skipped up to 100 detailed entries intentionally not written.
|
|
65
|
+
* @property {number} skippedTotal exact number of skipped entries.
|
|
66
|
+
* @property {{ count: number, limit: number, reason: string }} [omitted] aggregate count of entries not attempted.
|
|
45
67
|
*/
|
|
46
68
|
|
|
47
69
|
/**
|
|
48
70
|
* @typedef {object} ExportOptions
|
|
49
|
-
* @property {import('./history-store.js').HistoryStore} historyStore
|
|
50
|
-
* @property {Array<Record<string, any>>} [sessions] live
|
|
71
|
+
* @property {import('./history-store.js').HistoryStore} historyStore journal facade.
|
|
72
|
+
* @property {Array<Record<string, any>>} [sessions] live session metadata.
|
|
51
73
|
* @property {string} dir output directory.
|
|
52
|
-
* @property {string} [sessionId]
|
|
53
|
-
* @property {
|
|
74
|
+
* @property {string} [sessionId] compatibility logical session id.
|
|
75
|
+
* @property {string} [journalKey] exact incarnation address.
|
|
76
|
+
* @property {boolean} [all] export every address.
|
|
54
77
|
* @property {number | string} [since] lower time bound.
|
|
55
78
|
* @property {number | string} [until] upper time bound.
|
|
56
79
|
*/
|
|
@@ -58,7 +81,7 @@ const MAX_EXPORT_SESSIONS = 500
|
|
|
58
81
|
/**
|
|
59
82
|
* Export stored output history as text log files.
|
|
60
83
|
* @param {ExportOptions} options export options.
|
|
61
|
-
* @returns {Promise<ExportResult>} written files.
|
|
84
|
+
* @returns {Promise<ExportResult>} written files and skipped entries.
|
|
62
85
|
* @throws {import('./util.js').CodedError} `HISTORY_UNAVAILABLE`, `SESSION_NOT_FOUND`
|
|
63
86
|
* or `BAD_TARGET`.
|
|
64
87
|
*/
|
|
@@ -81,11 +104,21 @@ export async function exportSessions(options = {}) {
|
|
|
81
104
|
/** @type {Array<Record<string, any>>} */
|
|
82
105
|
let catalog = []
|
|
83
106
|
try {
|
|
84
|
-
catalog = await historyStore.listSessions({ limit:
|
|
107
|
+
catalog = await historyStore.listSessions({ limit: 20_000 })
|
|
85
108
|
} catch {
|
|
86
109
|
catalog = []
|
|
87
110
|
}
|
|
88
|
-
const
|
|
111
|
+
const maxFiles = typeof opts.maxFiles === 'number' && Number.isFinite(opts.maxFiles)
|
|
112
|
+
? Math.max(1, Math.min(MAX_EXPORT_SESSIONS, Math.floor(opts.maxFiles)))
|
|
113
|
+
: MAX_EXPORT_SESSIONS
|
|
114
|
+
const selection = resolveSelection({
|
|
115
|
+
live,
|
|
116
|
+
catalog,
|
|
117
|
+
sessionId: opts.sessionId,
|
|
118
|
+
journalKey: opts.journalKey,
|
|
119
|
+
all: opts.all === true,
|
|
120
|
+
maxFiles,
|
|
121
|
+
})
|
|
89
122
|
// A live journal may still hold buffered chunks; flush once before reading.
|
|
90
123
|
try {
|
|
91
124
|
if (typeof historyStore.flush === 'function') await historyStore.flush()
|
|
@@ -94,66 +127,201 @@ export async function exportSessions(options = {}) {
|
|
|
94
127
|
}
|
|
95
128
|
const exportedAt = Date.now()
|
|
96
129
|
const stamp = fileStamp(new Date(exportedAt))
|
|
130
|
+
const maxTotalBytes = typeof opts.maxTotalBytes === 'number' && Number.isFinite(opts.maxTotalBytes)
|
|
131
|
+
? Math.max(1, Math.floor(opts.maxTotalBytes))
|
|
132
|
+
: MAX_EXPORT_TOTAL_BYTES
|
|
133
|
+
let producedBytes = 0
|
|
97
134
|
/** @type {ExportFile[]} */
|
|
98
135
|
const files = []
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if (
|
|
105
|
-
const records = journal.records.filter(
|
|
106
|
-
(record) => (since === undefined || record.t >= since) && (until === undefined || record.t <= until),
|
|
107
|
-
)
|
|
108
|
-
const content = buildContent({ id, meta, records, since, until, exportedAt })
|
|
109
|
-
const path = join(dir, `${safeId(id)}-${safeTitle(meta.title, id)}-${stamp}.log`)
|
|
110
|
-
await writeAtomic(path, content)
|
|
111
|
-
files.push({ sessionId: id, path, bytes: Buffer.byteLength(content, 'utf8'), lines: countLines(content) })
|
|
136
|
+
/** @type {ExportSkipped[]} */
|
|
137
|
+
const skipped = []
|
|
138
|
+
let skippedTotal = 0
|
|
139
|
+
const addSkipped = (entry) => {
|
|
140
|
+
skippedTotal += 1
|
|
141
|
+
if (skipped.length < 100) skipped.push(entry)
|
|
112
142
|
}
|
|
113
|
-
|
|
143
|
+
for (const key of selection.keys) {
|
|
144
|
+
const meta = selection.meta.get(key) ?? { id: key }
|
|
145
|
+
const id = typeof meta.id === 'string' && meta.id.length > 0 ? meta.id : key
|
|
146
|
+
const historyId = typeof meta.historyId === 'string' && meta.historyId.length > 0 ? meta.historyId : undefined
|
|
147
|
+
const skip = (code, detail) => {
|
|
148
|
+
addSkipped({ sessionId: id, ...(historyId === undefined ? {} : { historyId }), code, message: detail })
|
|
149
|
+
}
|
|
150
|
+
let file
|
|
151
|
+
try {
|
|
152
|
+
file = historyStore.fileFor(key)
|
|
153
|
+
} catch (error) {
|
|
154
|
+
const failure = fixedExportFailure(error)
|
|
155
|
+
if (selection.all === true) {
|
|
156
|
+
addSkipped({
|
|
157
|
+
sessionId: id,
|
|
158
|
+
...(historyId === undefined ? {} : { historyId }),
|
|
159
|
+
code: failure.code,
|
|
160
|
+
message: failure.message,
|
|
161
|
+
})
|
|
162
|
+
continue
|
|
163
|
+
}
|
|
164
|
+
throw coded(failure.code, failure.message, failure.status)
|
|
165
|
+
}
|
|
166
|
+
if (typeof file !== 'string' || file.length === 0) {
|
|
167
|
+
if (selection.all === true) {
|
|
168
|
+
skip('SESSION_NOT_FOUND', 'no journal file exists for this incarnation')
|
|
169
|
+
continue
|
|
170
|
+
}
|
|
171
|
+
throw coded('SESSION_NOT_FOUND', `no history journal exists for session ${id}`, 404)
|
|
172
|
+
}
|
|
173
|
+
let info
|
|
174
|
+
try {
|
|
175
|
+
info = await stat(file)
|
|
176
|
+
} catch (error) {
|
|
177
|
+
const missing = error?.code === 'ENOENT'
|
|
178
|
+
if (selection.all === true) {
|
|
179
|
+
skip(missing ? 'SESSION_NOT_FOUND' : 'HISTORY_READ_FAILED', safeErrorMessage(error))
|
|
180
|
+
continue
|
|
181
|
+
}
|
|
182
|
+
throw missing
|
|
183
|
+
? coded('SESSION_NOT_FOUND', `no history journal exists for session ${id}`, 404)
|
|
184
|
+
: coded('HISTORY_READ_FAILED', 'the history source could not be read', 500)
|
|
185
|
+
}
|
|
186
|
+
if (!info.isFile()) {
|
|
187
|
+
if (selection.all === true) {
|
|
188
|
+
skip('HISTORY_READ_FAILED', 'the journal source is not a regular file')
|
|
189
|
+
continue
|
|
190
|
+
}
|
|
191
|
+
throw coded('HISTORY_READ_FAILED', `the journal for session ${id} is not a regular file`, 500)
|
|
192
|
+
}
|
|
193
|
+
// Freeze the source boundary at the size observed before any read. This
|
|
194
|
+
// prevents a continuously growing live journal from streaming to EOF and
|
|
195
|
+
// excludes output produced after this boundary (including after a
|
|
196
|
+
// permission change).
|
|
197
|
+
const snapshotBytes = info.size
|
|
198
|
+
// Re-validate owner/actor immediately before the first read, while the
|
|
199
|
+
// boundary above is already fixed. The registry callback uses current live
|
|
200
|
+
// permission and the durable recording after a concurrent close.
|
|
201
|
+
try {
|
|
202
|
+
if (typeof historyStore.beforeRead === 'function') await historyStore.beforeRead(key)
|
|
203
|
+
} catch (error) {
|
|
204
|
+
const failure = fixedExportFailure(error)
|
|
205
|
+
if (selection.all === true) {
|
|
206
|
+
skip(failure.code, failure.message)
|
|
207
|
+
continue
|
|
208
|
+
}
|
|
209
|
+
throw coded(failure.code, failure.message, failure.status)
|
|
210
|
+
}
|
|
211
|
+
const remaining = maxTotalBytes - producedBytes
|
|
212
|
+
const path = uniqueExportPath(dir, id, historyId, meta, stamp)
|
|
213
|
+
let written
|
|
214
|
+
try {
|
|
215
|
+
written = await streamJournalExport({
|
|
216
|
+
file,
|
|
217
|
+
path,
|
|
218
|
+
id,
|
|
219
|
+
meta,
|
|
220
|
+
since,
|
|
221
|
+
until,
|
|
222
|
+
exportedAt,
|
|
223
|
+
snapshotBytes,
|
|
224
|
+
maxOutputBytes: remaining,
|
|
225
|
+
writeAll: opts.writeAll,
|
|
226
|
+
})
|
|
227
|
+
} catch (error) {
|
|
228
|
+
const failure = fixedExportFailure(error)
|
|
229
|
+
if (selection.all === true) {
|
|
230
|
+
skip(failure.code, failure.message)
|
|
231
|
+
continue
|
|
232
|
+
}
|
|
233
|
+
throw coded(failure.code, failure.message, failure.status)
|
|
234
|
+
}
|
|
235
|
+
producedBytes += written.bytes
|
|
236
|
+
files.push({
|
|
237
|
+
sessionId: id,
|
|
238
|
+
...(historyId === undefined ? {} : { historyId }),
|
|
239
|
+
path,
|
|
240
|
+
bytes: written.bytes,
|
|
241
|
+
lines: written.lines,
|
|
242
|
+
})
|
|
243
|
+
}
|
|
244
|
+
const omittedCount = selection.omittedCount ?? 0
|
|
245
|
+
const result = { dir, files, skipped, skippedTotal }
|
|
246
|
+
if (omittedCount > 0) {
|
|
247
|
+
result.omitted = {
|
|
248
|
+
count: omittedCount,
|
|
249
|
+
limit: maxFiles,
|
|
250
|
+
reason: 'EXPORT_LIMIT',
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
return result
|
|
114
254
|
}
|
|
115
255
|
|
|
116
256
|
/**
|
|
117
257
|
* @typedef {object} Selection
|
|
118
|
-
* @property {string[]}
|
|
119
|
-
* @property {Map<string, Record<string, any>>} meta metadata by
|
|
258
|
+
* @property {string[]} keys incarnation addresses to export.
|
|
259
|
+
* @property {Map<string, Record<string, any>>} meta metadata by address.
|
|
120
260
|
* @property {boolean} all whether the caller asked for everything.
|
|
261
|
+
* @property {number} [omittedCount] addresses beyond the per-call export limit.
|
|
262
|
+
*/
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Stable address for one catalog/live entry.
|
|
266
|
+
* @param {Record<string, any>} entry entry.
|
|
267
|
+
* @returns {string} address.
|
|
121
268
|
*/
|
|
269
|
+
function addressOf(entry) {
|
|
270
|
+
if (typeof entry.address === 'string' && entry.address.length > 0) return entry.address
|
|
271
|
+
return typeof entry.historyId === 'string' && entry.historyId.length > 0 ? entry.historyId : entry.id
|
|
272
|
+
}
|
|
122
273
|
|
|
123
274
|
/**
|
|
124
|
-
* Decide which
|
|
125
|
-
* @param {{ live: Array<Record<string, any>>, catalog: Array<Record<string, any>>, sessionId?: unknown, all: boolean }} input candidates.
|
|
275
|
+
* Decide which incarnations an export covers.
|
|
276
|
+
* @param {{ live: Array<Record<string, any>>, catalog: Array<Record<string, any>>, sessionId?: unknown, journalKey?: unknown, all: boolean, maxFiles?: number }} input candidates.
|
|
126
277
|
* @returns {Selection} selection.
|
|
127
278
|
*/
|
|
128
279
|
function resolveSelection(input) {
|
|
129
280
|
/** @type {Map<string, Record<string, any>>} */
|
|
130
281
|
const meta = new Map()
|
|
131
|
-
for (const entry of input.catalog) {
|
|
132
|
-
if (entry !== null && typeof entry === 'object' && typeof entry.id === 'string' && entry.id.length > 0) meta.set(entry.id, entry)
|
|
133
|
-
}
|
|
134
|
-
for (const session of input.live) {
|
|
135
|
-
if (typeof session.id !== 'string' || session.id.length === 0) continue
|
|
136
|
-
// A live session's metadata is fresher (title can be renamed mid-session).
|
|
137
|
-
meta.set(session.id, { ...(meta.get(session.id) ?? {}), ...session })
|
|
138
|
-
}
|
|
139
282
|
/** @type {string[]} */
|
|
140
283
|
const ordered = []
|
|
141
284
|
const seen = new Set()
|
|
142
|
-
|
|
143
|
-
if (
|
|
144
|
-
|
|
145
|
-
|
|
285
|
+
const add = (entry, override) => {
|
|
286
|
+
if (entry === null || typeof entry !== 'object') return
|
|
287
|
+
if (typeof entry.id !== 'string' || entry.id.length === 0) return
|
|
288
|
+
const key = addressOf(entry)
|
|
289
|
+
if (override === true || !meta.has(key)) meta.set(key, { ...(meta.get(key) ?? {}), ...entry })
|
|
290
|
+
if (!seen.has(key)) {
|
|
291
|
+
seen.add(key)
|
|
292
|
+
ordered.push(key)
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
for (const session of input.live) add(session, true)
|
|
296
|
+
for (const entry of input.catalog) add(entry, false)
|
|
297
|
+
// Catalog rows with no journal are still listed by the store; the exporter
|
|
298
|
+
// decides skip/throw per the caller's scope below.
|
|
299
|
+
|
|
300
|
+
if (typeof input.journalKey === 'string' && input.journalKey.length > 0) {
|
|
301
|
+
if (!meta.has(input.journalKey)) throw coded('SESSION_NOT_FOUND', 'the requested history incarnation no longer exists', 404)
|
|
302
|
+
return { keys: [input.journalKey], meta, all: false }
|
|
303
|
+
}
|
|
304
|
+
if (input.all === true) {
|
|
305
|
+
return {
|
|
306
|
+
keys: ordered.slice(0, input.maxFiles ?? MAX_EXPORT_SESSIONS),
|
|
307
|
+
meta,
|
|
308
|
+
all: true,
|
|
309
|
+
omittedCount: ordered.length - (input.maxFiles ?? MAX_EXPORT_SESSIONS),
|
|
310
|
+
}
|
|
146
311
|
}
|
|
147
|
-
if (input.all === true) return { ids: ordered.slice(0, MAX_EXPORT_SESSIONS), meta, all: true }
|
|
148
312
|
if (typeof input.sessionId === 'string' && input.sessionId.length > 0) {
|
|
149
|
-
const
|
|
150
|
-
if (
|
|
151
|
-
|
|
313
|
+
const matches = ordered.filter((key) => (meta.get(key)?.id ?? key) === input.sessionId)
|
|
314
|
+
if (matches.length === 0) throw coded('SESSION_NOT_FOUND', `no session ${input.sessionId} to export`, 404)
|
|
315
|
+
// Compatibility: a caller that only knows the logical id exports the first
|
|
316
|
+
// (live, then newest catalog) incarnation. Exact callers pass journalKey.
|
|
317
|
+
return { keys: [matches[0]], meta, all: false }
|
|
152
318
|
}
|
|
153
|
-
const
|
|
154
|
-
|
|
319
|
+
const liveKeys = input.live
|
|
320
|
+
.filter((session) => typeof session.id === 'string' && session.id.length > 0)
|
|
321
|
+
.map((session) => addressOf(session))
|
|
322
|
+
const candidates = liveKeys.length > 0 ? liveKeys : ordered
|
|
155
323
|
if (candidates.length === 0) throw coded('BAD_TARGET', 'no session to export; connect one or pass all: true')
|
|
156
|
-
if (candidates.length === 1) return {
|
|
324
|
+
if (candidates.length === 1) return { keys: [candidates[0]], meta, all: false }
|
|
157
325
|
throw coded(
|
|
158
326
|
'BAD_TARGET',
|
|
159
327
|
`several sessions are available (${candidates.slice(0, 8).join(', ')}); pass sessionId or all: true`,
|
|
@@ -161,97 +329,185 @@ function resolveSelection(input) {
|
|
|
161
329
|
}
|
|
162
330
|
|
|
163
331
|
/**
|
|
164
|
-
*
|
|
165
|
-
* @param {
|
|
166
|
-
* @param {string} id session id.
|
|
167
|
-
* @
|
|
332
|
+
* Build a final path that cannot collide with another export.
|
|
333
|
+
* @param {string} dir output directory.
|
|
334
|
+
* @param {string} id logical session id.
|
|
335
|
+
* @param {string | undefined} historyId incarnation address.
|
|
336
|
+
* @param {Record<string, any>} meta metadata.
|
|
337
|
+
* @param {string} stamp timestamp stem.
|
|
338
|
+
* @returns {string} absolute output path.
|
|
168
339
|
*/
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
}
|
|
176
|
-
/** @type {Array<{ t: number, text: string }>} */
|
|
177
|
-
const records = []
|
|
178
|
-
let lineStart = 0
|
|
179
|
-
for (let index = 0; index <= buffer.length; index += 1) {
|
|
180
|
-
if (index !== buffer.length && buffer[index] !== 0x0a) continue
|
|
181
|
-
if (index > lineStart) {
|
|
182
|
-
const line = buffer.subarray(lineStart, index).toString('utf8').trim()
|
|
183
|
-
if (line.length > 0) {
|
|
184
|
-
const record = parseRecord(line)
|
|
185
|
-
if (record !== undefined) records.push(record)
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
lineStart = index + 1
|
|
189
|
-
}
|
|
190
|
-
return { exists: true, records }
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
/**
|
|
194
|
-
* Decode one NDJSON journal line.
|
|
195
|
-
* @param {string} line raw line.
|
|
196
|
-
* @returns {{ t: number, text: string } | undefined} decoded record.
|
|
197
|
-
*/
|
|
198
|
-
function parseRecord(line) {
|
|
199
|
-
let parsed
|
|
200
|
-
try {
|
|
201
|
-
parsed = JSON.parse(line)
|
|
202
|
-
} catch {
|
|
203
|
-
// A partially written line from a tail-truncated journal is skipped.
|
|
204
|
-
return undefined
|
|
205
|
-
}
|
|
206
|
-
if (parsed === null || typeof parsed !== 'object') return undefined
|
|
207
|
-
if (typeof parsed.t !== 'number' || !Number.isFinite(parsed.t) || typeof parsed.d !== 'string') return undefined
|
|
208
|
-
let text
|
|
209
|
-
try {
|
|
210
|
-
text = Buffer.from(parsed.d, 'base64').toString('utf8')
|
|
211
|
-
} catch {
|
|
212
|
-
return undefined
|
|
213
|
-
}
|
|
214
|
-
return text.length > 0 ? { t: Math.floor(parsed.t), text } : undefined
|
|
340
|
+
function uniqueExportPath(dir, id, historyId, meta, stamp) {
|
|
341
|
+
const incarnation = historyId === undefined ? '' : `-${shortToken(historyId)}`
|
|
342
|
+
// Random suffix: two concurrent exports of the same incarnation must not
|
|
343
|
+
// overwrite each other or share a temp file.
|
|
344
|
+
const nonce = randomUUID().slice(0, 8)
|
|
345
|
+
return join(dir, `${safeId(id)}${incarnation}-${safeTitle(meta.title, id)}-${stamp}-${nonce}.log`)
|
|
215
346
|
}
|
|
216
347
|
|
|
217
348
|
/**
|
|
218
|
-
*
|
|
349
|
+
* Stream one journal into a human-readable log file.
|
|
219
350
|
*
|
|
220
351
|
* Output chunks are reassembled into lines before timestamping, so a line split
|
|
221
352
|
* across two journal records still appears once, carrying the timestamp of the
|
|
222
|
-
* record that started it.
|
|
223
|
-
*
|
|
224
|
-
* @param {string} input
|
|
225
|
-
* @
|
|
226
|
-
* @param {Array<{ t: number, text: string }>} input.records time-filtered records.
|
|
227
|
-
* @param {number | undefined} input.since window start.
|
|
228
|
-
* @param {number | undefined} input.until window end.
|
|
229
|
-
* @param {number} input.exportedAt export time.
|
|
230
|
-
* @returns {string} file content.
|
|
353
|
+
* record that started it. Memory is bounded by the chunk buffer plus the
|
|
354
|
+
* largest single output line.
|
|
355
|
+
* @param {{ file: string, path: string, id: string, meta: Record<string, any>, since?: number, until?: number, exportedAt: number }} input export input.
|
|
356
|
+
* @returns {Promise<{ bytes: number, lines: number }>} written size/line count.
|
|
231
357
|
*/
|
|
232
|
-
function
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
358
|
+
async function streamJournalExport({
|
|
359
|
+
file,
|
|
360
|
+
path,
|
|
361
|
+
id,
|
|
362
|
+
meta,
|
|
363
|
+
since,
|
|
364
|
+
until,
|
|
365
|
+
exportedAt,
|
|
366
|
+
snapshotBytes,
|
|
367
|
+
maxOutputBytes,
|
|
368
|
+
writeAll = writeAllBytes,
|
|
369
|
+
}) {
|
|
370
|
+
const temp = `${path}.${process.pid}.${randomUUID()}.tmp`
|
|
371
|
+
let handle
|
|
372
|
+
let reader
|
|
373
|
+
let bytes = 0
|
|
374
|
+
let lines = 0
|
|
239
375
|
/** @type {string[]} */
|
|
240
|
-
const
|
|
241
|
-
let
|
|
242
|
-
let
|
|
243
|
-
|
|
244
|
-
if (
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
376
|
+
const buffered = []
|
|
377
|
+
let bufferedBytes = 0
|
|
378
|
+
let projectedBytes = 0
|
|
379
|
+
const write = async (text) => {
|
|
380
|
+
if (text.length === 0) return
|
|
381
|
+
const textBytes = Buffer.byteLength(text, 'utf8')
|
|
382
|
+
if (typeof maxOutputBytes === 'number' && projectedBytes + textBytes > maxOutputBytes) {
|
|
383
|
+
throw coded('EXPORT_BUDGET', 'the per-call export byte budget would be exceeded', 413)
|
|
384
|
+
}
|
|
385
|
+
projectedBytes += textBytes
|
|
386
|
+
buffered.push(text)
|
|
387
|
+
bufferedBytes += textBytes
|
|
388
|
+
if (bufferedBytes >= WRITE_CHUNK_BYTES) {
|
|
389
|
+
const chunk = buffered.join('')
|
|
390
|
+
buffered.length = 0
|
|
391
|
+
bufferedBytes = 0
|
|
392
|
+
await writeAll(handle, chunk)
|
|
393
|
+
bytes += Buffer.byteLength(chunk, 'utf8')
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
const emitted = async (line) => {
|
|
397
|
+
await write(`${line}\n`)
|
|
398
|
+
lines += 1
|
|
399
|
+
}
|
|
400
|
+
try {
|
|
401
|
+
handle = await open(temp, 'w', 0o600)
|
|
402
|
+
const header = [headerLine(id, meta), `# exported ${new Date(exportedAt).toISOString()}`]
|
|
403
|
+
if (since !== undefined || until !== undefined) {
|
|
404
|
+
header.push(
|
|
405
|
+
`# window since ${since === undefined ? '-' : new Date(since).toISOString()} until ${until === undefined ? '-' : new Date(until).toISOString()}`,
|
|
406
|
+
)
|
|
407
|
+
}
|
|
408
|
+
for (const line of header) await emitted(line)
|
|
409
|
+
|
|
410
|
+
let pending = ''
|
|
411
|
+
let pendingAt = 0
|
|
412
|
+
const consume = async (text, at) => {
|
|
413
|
+
if (pending.length === 0) pendingAt = at
|
|
414
|
+
pending += text
|
|
415
|
+
let index
|
|
416
|
+
while ((index = pending.indexOf('\n')) !== -1) {
|
|
417
|
+
await emitted(`${new Date(pendingAt).toISOString()} ${trimCarriageReturn(pending.slice(0, index))}`)
|
|
418
|
+
pending = pending.slice(index + 1)
|
|
419
|
+
pendingAt = at
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
const processLine = async (line) => {
|
|
423
|
+
const trimmed = line.trim()
|
|
424
|
+
if (trimmed.length === 0) return
|
|
425
|
+
let parsed
|
|
426
|
+
try {
|
|
427
|
+
parsed = JSON.parse(trimmed)
|
|
428
|
+
} catch {
|
|
429
|
+
return
|
|
430
|
+
}
|
|
431
|
+
if (parsed === null || typeof parsed !== 'object') return
|
|
432
|
+
if (typeof parsed.t !== 'number' || !Number.isFinite(parsed.t) || typeof parsed.d !== 'string') return
|
|
433
|
+
const at = Math.floor(parsed.t)
|
|
434
|
+
if (since !== undefined && at < since) return
|
|
435
|
+
if (until !== undefined && at > until) return
|
|
436
|
+
let text
|
|
437
|
+
try {
|
|
438
|
+
text = Buffer.from(parsed.d, 'base64').toString('utf8')
|
|
439
|
+
} catch {
|
|
440
|
+
return
|
|
441
|
+
}
|
|
442
|
+
if (text.length === 0) return
|
|
443
|
+
await consume(text, at)
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
reader = await open(file, 'r')
|
|
447
|
+
const decoder = new StringDecoder('utf8')
|
|
448
|
+
const raw = Buffer.allocUnsafe(STREAM_CHUNK_BYTES)
|
|
449
|
+
let carry = ''
|
|
450
|
+
let remainingSource = typeof snapshotBytes === 'number' ? Math.max(0, Math.floor(snapshotBytes)) : Infinity
|
|
451
|
+
while (remainingSource > 0) {
|
|
452
|
+
const want = remainingSource === Infinity ? raw.length : Math.min(raw.length, remainingSource)
|
|
453
|
+
const { bytesRead } = await reader.read(raw, 0, want, null)
|
|
454
|
+
if (bytesRead === 0) break
|
|
455
|
+
remainingSource -= bytesRead
|
|
456
|
+
carry += decoder.write(raw.subarray(0, bytesRead))
|
|
457
|
+
let index
|
|
458
|
+
while ((index = carry.indexOf('\n')) !== -1) {
|
|
459
|
+
const line = carry.slice(0, index)
|
|
460
|
+
carry = carry.slice(index + 1)
|
|
461
|
+
await processLine(line)
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
carry += decoder.end()
|
|
465
|
+
if (carry.trim().length > 0) await processLine(carry)
|
|
466
|
+
if (pending.length > 0) {
|
|
467
|
+
await emitted(`${new Date(pendingAt).toISOString()} ${trimCarriageReturn(pending)}`)
|
|
468
|
+
}
|
|
469
|
+
if (bufferedBytes > 0) {
|
|
470
|
+
const chunk = buffered.join('')
|
|
471
|
+
buffered.length = 0
|
|
472
|
+
bufferedBytes = 0
|
|
473
|
+
await writeAll(handle, chunk)
|
|
474
|
+
bytes += Buffer.byteLength(chunk, 'utf8')
|
|
475
|
+
}
|
|
476
|
+
await handle.sync()
|
|
477
|
+
await handle.close()
|
|
478
|
+
handle = undefined
|
|
479
|
+
try {
|
|
480
|
+
await reader.close()
|
|
481
|
+
} catch {
|
|
482
|
+
// already closed
|
|
483
|
+
}
|
|
484
|
+
reader = undefined
|
|
485
|
+
await rename(temp, path)
|
|
486
|
+
return { bytes, lines }
|
|
487
|
+
} catch (error) {
|
|
488
|
+
try {
|
|
489
|
+
if (reader !== undefined) await reader.close()
|
|
490
|
+
} catch {
|
|
491
|
+
// best effort
|
|
492
|
+
}
|
|
493
|
+
try {
|
|
494
|
+
if (handle !== undefined) await handle.close()
|
|
495
|
+
} catch {
|
|
496
|
+
// best effort
|
|
497
|
+
}
|
|
498
|
+
try {
|
|
499
|
+
await rm(temp, { force: true })
|
|
500
|
+
} catch {
|
|
501
|
+
// best effort
|
|
502
|
+
}
|
|
503
|
+
throw error
|
|
504
|
+
} finally {
|
|
505
|
+
try {
|
|
506
|
+
if (reader !== undefined) await reader.close()
|
|
507
|
+
} catch {
|
|
508
|
+
// already closed
|
|
251
509
|
}
|
|
252
510
|
}
|
|
253
|
-
if (pending.length > 0) lines.push(`${new Date(pendingAt).toISOString()} ${trimCarriageReturn(pending)}`)
|
|
254
|
-
return `${lines.join('\n')}\n`
|
|
255
511
|
}
|
|
256
512
|
|
|
257
513
|
/**
|
|
@@ -276,15 +532,46 @@ function headerLine(id, meta) {
|
|
|
276
532
|
}
|
|
277
533
|
|
|
278
534
|
/**
|
|
279
|
-
* Write a
|
|
280
|
-
* @param {
|
|
281
|
-
* @param {string}
|
|
535
|
+
* Write a whole string, handling partial `FileHandle.write` results.
|
|
536
|
+
* @param {import('node:fs/promises').FileHandle} handle open file.
|
|
537
|
+
* @param {string} text text to write.
|
|
282
538
|
* @returns {Promise<void>} completion.
|
|
283
539
|
*/
|
|
284
|
-
async function
|
|
285
|
-
const
|
|
286
|
-
|
|
287
|
-
|
|
540
|
+
async function writeAllBytes(handle, text) {
|
|
541
|
+
const buffer = Buffer.from(text, 'utf8')
|
|
542
|
+
let offset = 0
|
|
543
|
+
while (offset < buffer.length) {
|
|
544
|
+
const { bytesWritten } = await handle.write(buffer, offset, buffer.length - offset)
|
|
545
|
+
if (!Number.isInteger(bytesWritten) || bytesWritten <= 0) {
|
|
546
|
+
throw new Error('write made no progress')
|
|
547
|
+
}
|
|
548
|
+
offset += bytesWritten
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
/**
|
|
553
|
+
* Map an internal failure to a stable public code/message. Never includes a
|
|
554
|
+
* private journal path or an OS error string.
|
|
555
|
+
* @param {unknown} error thrown value.
|
|
556
|
+
* @returns {{ code: string, message: string, status: number }} fixed failure.
|
|
557
|
+
*/
|
|
558
|
+
function fixedExportFailure(error) {
|
|
559
|
+
const code = typeof error?.code === 'string' ? error.code : 'HISTORY_READ_FAILED'
|
|
560
|
+
if (code === 'AI_READ_DENIED') return { code, message: 'AI read is not permitted for this incarnation', status: 403 }
|
|
561
|
+
if (code === 'SESSION_NOT_FOUND') return { code, message: 'the history incarnation is not available', status: 404 }
|
|
562
|
+
if (code === 'EXPORT_BUDGET') return { code, message: 'the per-call export byte budget was exceeded', status: 413 }
|
|
563
|
+
if (code === 'HISTORY_TOO_LARGE') return { code, message: 'the history source is too large to export', status: 413 }
|
|
564
|
+
return { code: 'HISTORY_READ_FAILED', message: 'the history source could not be read or exported', status: 500 }
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* Error text safe to expose/log without a private journal path.
|
|
569
|
+
* @param {unknown} error thrown value.
|
|
570
|
+
* @returns {string} fixed message or a path-free code.
|
|
571
|
+
*/
|
|
572
|
+
function safeErrorMessage(error) {
|
|
573
|
+
const code = typeof error?.code === 'string' && /^[A-Z0-9_]+$/.test(error.code) ? error.code : ''
|
|
574
|
+
return code.length > 0 ? `history source error (${code})` : 'history source error'
|
|
288
575
|
}
|
|
289
576
|
|
|
290
577
|
/**
|
|
@@ -310,6 +597,19 @@ function timeValue(value) {
|
|
|
310
597
|
return Number.isFinite(parsed) ? parsed : undefined
|
|
311
598
|
}
|
|
312
599
|
|
|
600
|
+
/**
|
|
601
|
+
* Short stable token for one incarnation, used to keep export filenames unique.
|
|
602
|
+
* @param {string} historyId opaque address.
|
|
603
|
+
* @returns {string} short token.
|
|
604
|
+
*/
|
|
605
|
+
function shortToken(historyId) {
|
|
606
|
+
const parts = String(historyId).split('~')
|
|
607
|
+
const raw = parts.length > 1 ? parts[parts.length - 1] : String(historyId)
|
|
608
|
+
const safe = raw.replace(/[^A-Za-z0-9]/g, '')
|
|
609
|
+
if (safe.length === 0) return safeId(historyId)
|
|
610
|
+
return safe.slice(0, 8)
|
|
611
|
+
}
|
|
612
|
+
|
|
313
613
|
/**
|
|
314
614
|
* `yyyyMMdd-HHmmss` in local time.
|
|
315
615
|
* @param {Date} date date.
|
|
@@ -365,18 +665,6 @@ function trimCarriageReturn(line) {
|
|
|
365
665
|
return line.endsWith('\r') ? line.slice(0, -1) : line
|
|
366
666
|
}
|
|
367
667
|
|
|
368
|
-
/**
|
|
369
|
-
* @param {string} content file content.
|
|
370
|
-
* @returns {number} number of lines (content always ends with a newline).
|
|
371
|
-
*/
|
|
372
|
-
function countLines(content) {
|
|
373
|
-
let count = 0
|
|
374
|
-
for (let index = 0; index < content.length; index += 1) {
|
|
375
|
-
if (content.charCodeAt(index) === 0x0a) count += 1
|
|
376
|
-
}
|
|
377
|
-
return count
|
|
378
|
-
}
|
|
379
|
-
|
|
380
668
|
/**
|
|
381
669
|
* @param {unknown} error thrown value.
|
|
382
670
|
* @returns {string} message.
|