@anhlp/alp-daemon 0.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/store.ts ADDED
@@ -0,0 +1,336 @@
1
+ import { appendFile, mkdir, open, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
2
+ import { replaceFile } from './lock.js';
3
+ import { randomBytes } from 'node:crypto';
4
+ import path from 'node:path';
5
+ import { createLogger } from '@anhlp/alp-core/log';
6
+ import type { Envelope, SessionSnapshot, SessionSpec } from '@anhlp/alp-runtime/index';
7
+ import type { SessionError, SessionStatus } from '@anhlp/alp-protocol/session';
8
+
9
+ /**
10
+ * alpd's durable state (plans/reference/ALPD.md §5, §66): one JSON record per session,
11
+ * written atomically, one append-only JSONL timeline per tree, and one receipt per
12
+ * delivered prompt. Writes survive a crash of the alpd process once their promise
13
+ * resolves; nothing is fsynced, so a power loss or a failing disk may lose recent writes.
14
+ */
15
+
16
+ // The session lifecycle and the error shape are neutral (ALPD §18): the protocol owns them so
17
+ // viewers and integrations speak one language. Re-exported here to keep the daemon's API stable.
18
+ export type { SessionError, SessionStatus } from '@anhlp/alp-protocol/session';
19
+
20
+ export type SessionRecord = {
21
+ version: 1;
22
+ id: string;
23
+ rootId: string;
24
+ /** The client's request, kept for roots so they can be resumed. */
25
+ spec?: SessionSpec;
26
+ delegation?: boolean;
27
+ session?: SessionSnapshot;
28
+ status: SessionStatus;
29
+ lastError?: SessionError;
30
+ title?: string;
31
+ /** Set aside by the user: closed, and listed only when asked (ALPD §61). */
32
+ archived?: boolean;
33
+ createdAt: string;
34
+ updatedAt: string;
35
+ };
36
+
37
+ /**
38
+ * Delivery of one clientMessageId; `pending` means alpd may have stopped while sending it.
39
+ * A completed receipt says what the runtime answered: `unknown` when the provider may or
40
+ * may not have the message (its process died or timed out mid-request).
41
+ */
42
+ export type Receipt = {
43
+ version: 1;
44
+ sessionId: string;
45
+ clientMessageId: string;
46
+ fingerprint: string;
47
+ state: 'pending' | 'completed';
48
+ outcome?: 'accepted' | 'failed' | 'unknown';
49
+ error?: { message: string; code?: string };
50
+ /** Completed at the next start from the tree's timeline, not by the delivery itself. */
51
+ reconciled?: boolean;
52
+ updatedAt: string;
53
+ };
54
+
55
+ /** A receipt file that exists but cannot be read as one: its delivery's outcome is unknown. */
56
+ export type UnreadableReceipt = { version: 1; state: 'unreadable' };
57
+
58
+ /** Failed writes since alpd started; `ok` until the first one (ALPD §66). */
59
+ export type StoreHealth = { ok: boolean; failures: number; lastError?: { at: string; operation: string; message: string } };
60
+
61
+ const log = createLogger({ component: 'store' });
62
+
63
+ const SAFE = /[^\w.:-]/g;
64
+ const fileName = (id: string) => id.replace(SAFE, '_');
65
+ const TAIL_CHUNK = 64 * 1024;
66
+
67
+ export type Store = ReturnType<typeof createStore>;
68
+
69
+ export function createStore(root: string) {
70
+ const sessionsDir = path.join(root, 'sessions');
71
+ const timelineDir = path.join(root, 'timeline');
72
+ const receiptsDir = path.join(root, 'receipts');
73
+ const writes = new Map<string, Promise<void>>();
74
+ const appends = new Map<string, Promise<void>>();
75
+ /** Keys whose last write failed (records, receipts) or that lost any append (timelines). */
76
+ const failed = new Map<string, Error>();
77
+ /** Timelines whose tail this process has checked for a line a crash cut short. */
78
+ const repaired = new Set<string>();
79
+ const health: StoreHealth = { ok: true, failures: 0 };
80
+ let ready: Promise<void> | undefined;
81
+ const prepare = () => ready ??= Promise.all([
82
+ mkdir(sessionsDir, { recursive: true, mode: 0o700 }),
83
+ mkdir(timelineDir, { recursive: true, mode: 0o700 }),
84
+ mkdir(receiptsDir, { recursive: true, mode: 0o700 }),
85
+ ]).then(() => {}, error => { ready = undefined; throw error; });
86
+
87
+ async function writeAtomic(file: string, body: string) {
88
+ const temporary = `${file}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
89
+ try {
90
+ await writeFile(temporary, body, { mode: 0o600 });
91
+ await replaceFile(temporary, file);
92
+ } catch (error) {
93
+ await rm(temporary, { force: true }).catch(() => {});
94
+ throw error;
95
+ }
96
+ }
97
+
98
+ /**
99
+ * Serializes work per key so a later write never lands before an earlier one. The
100
+ * returned promise rejects when the write fails; the queue itself goes on.
101
+ */
102
+ function queue(map: Map<string, Promise<void>>, key: string, operation: string, work: () => Promise<void>, sticky = false, ids: { sessionId?: string; rootId?: string; clientMessageId?: string } = {}) {
103
+ const next = (map.get(key) ?? Promise.resolve()).then(prepare).then(work);
104
+ const failure = map === appends ? `timeline:${key}` : key;
105
+ const settled = next.then(
106
+ () => { if (!sticky) failed.delete(failure); },
107
+ error => {
108
+ failed.set(failure, error);
109
+ health.ok = false;
110
+ health.failures++;
111
+ health.lastError = { at: new Date().toISOString(), operation, message: error?.message ?? String(error) };
112
+ log.error(`${operation} failed`, { code: 'ALP_PERSISTENCE_WRITE_FAILED', operation, error, ...ids });
113
+ });
114
+ map.set(key, settled);
115
+ void settled.then(() => { if (map.get(key) === settled) map.delete(key); });
116
+ return next;
117
+ }
118
+
119
+ /**
120
+ * A crash during an append can leave a timeline's last line cut short. Before this
121
+ * process appends to it, a cut line is removed, or completed when it parses whole,
122
+ * so the next line does not run into it.
123
+ */
124
+ async function repairTail(file: string) {
125
+ const handle = await open(file, 'r+').catch((error: any) => { if (error?.code === 'ENOENT') return undefined; throw error; });
126
+ if (!handle) return;
127
+ try {
128
+ const { size } = await handle.stat();
129
+ if (!size) return;
130
+ const last = Buffer.alloc(1);
131
+ await handle.read(last, 0, 1, size - 1);
132
+ if (last[0] === 0x0a) return;
133
+ let start = 0;
134
+ for (let end = size; end > 0 && !start; end -= TAIL_CHUNK) {
135
+ const from = Math.max(0, end - TAIL_CHUNK);
136
+ const chunk = Buffer.alloc(end - from);
137
+ await handle.read(chunk, 0, chunk.length, from);
138
+ const newline = chunk.lastIndexOf(0x0a);
139
+ if (newline >= 0) start = from + newline + 1;
140
+ }
141
+ const tail = Buffer.alloc(size - start);
142
+ await handle.read(tail, 0, tail.length, start);
143
+ let whole = false;
144
+ try { JSON.parse(tail.toString('utf8')); whole = true; } catch {}
145
+ if (whole) await handle.write('\n', size);
146
+ else await handle.truncate(start);
147
+ log.warn(`${whole ? 'completed' : 'removed'} a timeline line cut short in ${path.basename(file)} (${tail.length} bytes)`, { code: 'ALP_TIMELINE_REPAIRED', file: path.basename(file), bytes: tail.length });
148
+ } finally {
149
+ await handle.close();
150
+ }
151
+ }
152
+
153
+ /** Envelopes of a timeline text; lines without a cursor (written before cursors) count from the previous one. */
154
+ function parseTimeline(text: string, name: string): Envelope[] {
155
+ const envelopes: Envelope[] = [];
156
+ let cursor = 0;
157
+ let skipped = 0;
158
+ for (const line of text.split('\n')) {
159
+ if (!line) continue;
160
+ let envelope: Envelope;
161
+ try { envelope = JSON.parse(line); } catch { skipped++; continue; }
162
+ cursor = typeof envelope.cursor === 'number' && envelope.cursor > cursor ? envelope.cursor : cursor + 1;
163
+ envelopes.push({ ...envelope, cursor });
164
+ }
165
+ if (skipped) log.warn(`skipped ${skipped} unreadable timeline lines in ${name}`, { code: 'ALP_TIMELINE_UNREADABLE', file: name, skipped });
166
+ return envelopes;
167
+ }
168
+
169
+ const receiptFile = (key: string) => path.join(receiptsDir, `${fileName(key)}.json`);
170
+
171
+ return {
172
+ put(record: SessionRecord) {
173
+ const file = path.join(sessionsDir, `${fileName(record.id)}.json`);
174
+ const body = JSON.stringify(record);
175
+ return queue(writes, record.id, `record ${record.id}`, () => writeAtomic(file, body), false, { sessionId: record.id, rootId: record.rootId });
176
+ },
177
+
178
+ async receipt(key: string): Promise<Receipt | UnreadableReceipt | undefined> {
179
+ await writes.get(`receipt:${key}`);
180
+ let text: string;
181
+ try {
182
+ text = await readFile(receiptFile(key), 'utf8');
183
+ } catch (error: any) {
184
+ if (error?.code === 'ENOENT') return undefined;
185
+ throw error;
186
+ }
187
+ try {
188
+ const receipt = JSON.parse(text);
189
+ if (receipt?.version === 1 && typeof receipt.fingerprint === 'string' && (receipt.state === 'pending' || receipt.state === 'completed')) return receipt;
190
+ } catch {}
191
+ return { version: 1, state: 'unreadable' };
192
+ },
193
+
194
+ putReceipt(key: string, receipt: Receipt) {
195
+ const body = JSON.stringify(receipt);
196
+ return queue(writes, `receipt:${key}`, `receipt ${receipt.clientMessageId}`, () => writeAtomic(receiptFile(key), body), false, { sessionId: receipt.sessionId, clientMessageId: receipt.clientMessageId });
197
+ },
198
+
199
+ removeReceipt(key: string) {
200
+ return queue(writes, `receipt:${key}`, `receipt removal ${key}`, () => rm(receiptFile(key), { force: true }));
201
+ },
202
+
203
+ /** Every readable receipt with its key and when its file was last written. */
204
+ async receipts(): Promise<Array<{ key: string; receipt: Receipt; writtenAt: number }>> {
205
+ await prepare();
206
+ const found: Array<{ key: string; receipt: Receipt; writtenAt: number }> = [];
207
+ for (const name of await readdir(receiptsDir)) {
208
+ if (!name.endsWith('.json')) continue;
209
+ const key = name.slice(0, -'.json'.length);
210
+ try {
211
+ const file = path.join(receiptsDir, name);
212
+ const receipt = JSON.parse(await readFile(file, 'utf8'));
213
+ if (receipt?.version === 1 && typeof receipt.sessionId === 'string') found.push({ key, receipt, writtenAt: (await stat(file)).mtimeMs });
214
+ } catch {}
215
+ }
216
+ return found;
217
+ },
218
+
219
+ /** When the newest of these records' files was written (a restore writes them anew); 0 when none is there. */
220
+ async writtenAt(ids: string[]) {
221
+ let newest = 0;
222
+ for (const id of ids) newest = Math.max(newest, await stat(path.join(sessionsDir, `${fileName(id)}.json`)).then(info => info.mtimeMs, () => 0));
223
+ return newest;
224
+ },
225
+
226
+ /**
227
+ * Deletes receipts not written for `maxAgeMs` and, with `temporaries` (only at start, when no
228
+ * write can be under way), temporary files a crash left behind.
229
+ */
230
+ async pruneReceipts(maxAgeMs: number, { temporaries = true } = {}) {
231
+ await prepare();
232
+ const now = Date.now();
233
+ for (const directory of [sessionsDir, receiptsDir]) {
234
+ for (const name of await readdir(directory)) {
235
+ const file = path.join(directory, name);
236
+ if (name.endsWith('.tmp')) { if (temporaries) await rm(file, { force: true }); continue; }
237
+ if (directory !== receiptsDir) continue;
238
+ const info = await stat(file).catch(() => undefined);
239
+ if (info && now - info.mtimeMs > maxAgeMs) await rm(file, { force: true });
240
+ }
241
+ }
242
+ },
243
+
244
+ async list(): Promise<SessionRecord[]> {
245
+ await prepare();
246
+ const records: SessionRecord[] = [];
247
+ for (const name of await readdir(sessionsDir)) {
248
+ if (!name.endsWith('.json')) continue;
249
+ try {
250
+ const record = JSON.parse(await readFile(path.join(sessionsDir, name), 'utf8'));
251
+ if (record?.version === 1 && typeof record.id === 'string') records.push(record);
252
+ else log.warn(`skipped invalid record ${name}`, { code: 'ALP_RECORD_UNREADABLE', file: name });
253
+ } catch {
254
+ log.warn(`skipped unreadable record ${name}`, { code: 'ALP_RECORD_UNREADABLE', file: name });
255
+ }
256
+ }
257
+ return records;
258
+ },
259
+
260
+ append(rootId: string, envelope: Envelope) {
261
+ const file = path.join(timelineDir, `${fileName(rootId)}.jsonl`);
262
+ const line = `${JSON.stringify(envelope)}\n`;
263
+ // A lost line leaves a gap in the timeline for good, so a timeline stays failed once it lost one.
264
+ return queue(appends, rootId, `timeline ${rootId}`, async () => {
265
+ if (!repaired.has(rootId)) { await repairTail(file); repaired.add(rootId); }
266
+ await appendFile(file, line, { mode: 0o600 });
267
+ }, true, { rootId });
268
+ },
269
+
270
+ async timeline(rootId: string): Promise<Envelope[]> {
271
+ await appends.get(rootId);
272
+ const name = `${fileName(rootId)}.jsonl`;
273
+ try {
274
+ return parseTimeline(await readFile(path.join(timelineDir, name), 'utf8'), name);
275
+ } catch (error: any) {
276
+ if (error?.code === 'ENOENT') return [];
277
+ throw error;
278
+ }
279
+ },
280
+
281
+ /**
282
+ * Waits for the queued writes of these records and timelines, and names those that
283
+ * failed: a record whose last write failed, or a timeline that lost a line.
284
+ */
285
+ async drain({ records = [] as string[], timelines = [] as string[] }) {
286
+ await Promise.all([...records.map(id => writes.get(id)), ...timelines.map(id => appends.get(id))]);
287
+ return [...records.filter(id => failed.has(id)), ...timelines.filter(id => failed.has(`timeline:${id}`))];
288
+ },
289
+
290
+ /**
291
+ * Whether the state directory can be read and written right now: a small file is written,
292
+ * read back and removed. `StoreHealth` only says what failed since alpd started.
293
+ */
294
+ async probe(): Promise<{ ok: boolean; code?: string; message?: string }> {
295
+ const file = path.join(root, `.probe-${process.pid}-${randomBytes(3).toString('hex')}.tmp`);
296
+ const token = randomBytes(8).toString('hex');
297
+ let step: 'write' | 'read' = 'write';
298
+ try {
299
+ await prepare();
300
+ await writeFile(file, token, { mode: 0o600 });
301
+ step = 'read';
302
+ if (await readFile(file, 'utf8') !== token) throw new Error('what was written is not what was read back');
303
+ return { ok: true };
304
+ } catch (error: any) {
305
+ return { ok: false, code: step === 'write' ? 'ALP_STATE_NOT_WRITABLE' : 'ALP_STATE_NOT_READABLE', message: `${error?.code ? `${error.code}: ` : ''}${error?.message ?? error}` };
306
+ } finally {
307
+ await rm(file, { force: true }).catch(() => {});
308
+ }
309
+ },
310
+
311
+ /** How many records, receipts and timelines are failed now: a later successful write clears a record, a lost timeline line never clears. */
312
+ failing() {
313
+ return failed.size;
314
+ },
315
+
316
+ health(): StoreHealth {
317
+ return { ...health, ...(health.lastError ? { lastError: { ...health.lastError } } : {}) };
318
+ },
319
+
320
+ /** Removes a whole tree: its records and its timeline. */
321
+ async remove(rootId: string, ids: string[]) {
322
+ await Promise.all([...ids.map(id => writes.get(id)), appends.get(rootId)]);
323
+ await Promise.all([
324
+ ...ids.map(id => rm(path.join(sessionsDir, `${fileName(id)}.json`), { force: true })),
325
+ rm(path.join(timelineDir, `${fileName(rootId)}.jsonl`), { force: true }),
326
+ ]);
327
+ for (const id of ids) failed.delete(id);
328
+ failed.delete(`timeline:${rootId}`);
329
+ repaired.delete(rootId);
330
+ },
331
+
332
+ async flush() {
333
+ await Promise.all([...writes.values(), ...appends.values()]);
334
+ },
335
+ };
336
+ }