@playola-radio/slipstream 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +132 -0
- package/SKILL.md +63 -0
- package/THIRD-PARTY-NOTICES.md +36 -0
- package/contracts/interface/v2/schema.json +480 -0
- package/dist/answer-forwarder.js +58 -0
- package/dist/attribution-engine.js +164 -0
- package/dist/attribution-producer.js +153 -0
- package/dist/attribution-scoring.js +86 -0
- package/dist/attribution.js +120 -0
- package/dist/blob-source.js +100 -0
- package/dist/boundary-registry.js +47 -0
- package/dist/cas.js +99 -0
- package/dist/change-view.js +201 -0
- package/dist/claude-root-transcript.js +110 -0
- package/dist/cli.js +629 -0
- package/dist/clip-blob-reader.js +104 -0
- package/dist/clip-function-parser.js +83 -0
- package/dist/clip-language.js +15 -0
- package/dist/clip-projection-service.js +240 -0
- package/dist/clip-projection-worker.js +18 -0
- package/dist/clip-projection.js +585 -0
- package/dist/clip-worker-pool.js +118 -0
- package/dist/config.js +154 -0
- package/dist/control-client.js +98 -0
- package/dist/control-protocol.js +72 -0
- package/dist/daemon-location.js +33 -0
- package/dist/daemon.js +951 -0
- package/dist/display-fold.js +332 -0
- package/dist/engine.js +112 -0
- package/dist/entrypoint.js +17 -0
- package/dist/event.js +79 -0
- package/dist/evidence-ingest.js +60 -0
- package/dist/harness-context/claude.js +42 -0
- package/dist/harness-context/codex.js +49 -0
- package/dist/harness-context.js +49 -0
- package/dist/health.js +65 -0
- package/dist/http-reader.js +655 -0
- package/dist/http-security.js +50 -0
- package/dist/interface-projection.js +243 -0
- package/dist/interface-range-resolver.js +230 -0
- package/dist/interface-service.js +509 -0
- package/dist/interface-ts-pool.js +111 -0
- package/dist/interface-ts-worker.js +35 -0
- package/dist/interface-v2-comparison.js +135 -0
- package/dist/interface-v2-core.js +123 -0
- package/dist/interface-v2-typescript.js +448 -0
- package/dist/lock.js +197 -0
- package/dist/log-reader.js +148 -0
- package/dist/log.js +59 -0
- package/dist/maintenance.js +255 -0
- package/dist/mcp-forwarder.js +161 -0
- package/dist/mcp-protocol.js +92 -0
- package/dist/platform.js +21 -0
- package/dist/projection-admission.js +296 -0
- package/dist/projection-trace.js +17 -0
- package/dist/public-events.js +27 -0
- package/dist/question-hook.js +110 -0
- package/dist/questions.js +167 -0
- package/dist/reader-runtime.js +38 -0
- package/dist/reader.js +93 -0
- package/dist/recovery.js +303 -0
- package/dist/schema.js +108 -0
- package/dist/session.js +866 -0
- package/dist/sibling-module.js +10 -0
- package/dist/snapshot.js +15 -0
- package/dist/storage.js +111 -0
- package/dist/store-reader.js +185 -0
- package/dist/swift-grammar.js +268 -0
- package/dist/swift-interface-extract.js +344 -0
- package/dist/swift-interface.js +62 -0
- package/dist/swift-parse-host.js +174 -0
- package/dist/swift-parse-worker.js +34 -0
- package/dist/swift-parse.js +134 -0
- package/dist/swift-spans.js +62 -0
- package/dist/task-forwarder.js +152 -0
- package/dist/transcript/claude.js +199 -0
- package/dist/transcript/codex.js +132 -0
- package/dist/transcript/discovery.js +365 -0
- package/dist/transcript/file-reader.js +128 -0
- package/dist/transcript/fs-io.js +287 -0
- package/dist/transcript/runner.js +62 -0
- package/dist/transcript/steppers.js +30 -0
- package/dist/transcript/types.js +30 -0
- package/dist/transcript/watcher.js +133 -0
- package/dist/tui.js +168 -0
- package/package.json +60 -0
- package/schemas/projections/clip.v1.json +195 -0
- package/schemas/projections/clip.v2.json +212 -0
- package/schemas/projections/clip.v3.json +212 -0
- package/schemas/slipstream.capture.baseline.completed.v1.json +62 -0
- package/schemas/slipstream.capture.gap.v1.json +112 -0
- package/schemas/slipstream.change.attribution.v1.json +115 -0
- package/schemas/slipstream.enrichment.configured.v1.json +77 -0
- package/schemas/slipstream.enrichment.coverage.v1.json +95 -0
- package/schemas/slipstream.file.baselined.v1.json +122 -0
- package/schemas/slipstream.file.changed.v1.json +179 -0
- package/schemas/slipstream.harness.evidence.v1.json +145 -0
- package/schemas/slipstream.question.answered.v1.json +29 -0
- package/schemas/slipstream.question.dispatch_attempted.v1.json +28 -0
- package/schemas/slipstream.question.queued.v1.json +55 -0
- package/schemas/slipstream.session.resumed.v1.json +70 -0
- package/schemas/slipstream.session.started.v1.json +69 -0
- package/schemas/slipstream.task.started.v1.json +72 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// src/log-reader.ts
|
|
2
|
+
import { open } from 'node:fs/promises';
|
|
3
|
+
export class LogCorruptError extends Error {
|
|
4
|
+
}
|
|
5
|
+
/** Optional bounded scans can stop before materializing an oversized record. */
|
|
6
|
+
export class LogReadLimitError extends Error {
|
|
7
|
+
}
|
|
8
|
+
export class LogReadAbortedError extends Error {
|
|
9
|
+
}
|
|
10
|
+
const CURSOR_RE = /^(0|[1-9][0-9]*)$/;
|
|
11
|
+
const SEQ_RE = /^[1-9][0-9]*$/;
|
|
12
|
+
export function parseCursor(raw) {
|
|
13
|
+
if (raw === undefined)
|
|
14
|
+
return 0n;
|
|
15
|
+
if (!CURSOR_RE.test(raw))
|
|
16
|
+
return null;
|
|
17
|
+
return BigInt(raw);
|
|
18
|
+
}
|
|
19
|
+
export function parseLine(line) {
|
|
20
|
+
let obj;
|
|
21
|
+
try {
|
|
22
|
+
obj = JSON.parse(line);
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
throw new LogCorruptError(`invalid JSON: ${line.slice(0, 80)}`);
|
|
26
|
+
}
|
|
27
|
+
if (typeof obj !== 'object' || obj === null)
|
|
28
|
+
throw new LogCorruptError('line is not an object');
|
|
29
|
+
const rec = obj;
|
|
30
|
+
const seq = rec.seq;
|
|
31
|
+
const id = rec.id;
|
|
32
|
+
const source = rec.source;
|
|
33
|
+
const type = rec.type;
|
|
34
|
+
const data = rec.data;
|
|
35
|
+
if (typeof seq !== 'string' || !SEQ_RE.test(seq))
|
|
36
|
+
throw new LogCorruptError('bad seq');
|
|
37
|
+
if (id !== undefined && typeof id !== 'string')
|
|
38
|
+
throw new LogCorruptError('bad id');
|
|
39
|
+
if (source !== undefined && typeof source !== 'string')
|
|
40
|
+
throw new LogCorruptError('bad source');
|
|
41
|
+
if (typeof type !== 'string')
|
|
42
|
+
throw new LogCorruptError('bad type');
|
|
43
|
+
if (typeof data !== 'object' || data === null)
|
|
44
|
+
throw new LogCorruptError('bad data');
|
|
45
|
+
// Callers that read from disk (openLogCursor) know the true consumed byte
|
|
46
|
+
// count — including a stripped BOM — and overwrite this default afterward.
|
|
47
|
+
return { seq: BigInt(seq), id, source, type, raw: line, bytesRead: Buffer.byteLength(line, 'utf8') + 1, data: data };
|
|
48
|
+
}
|
|
49
|
+
// Each batch retains at most 256 records / 256 KiB, plus one oversized record.
|
|
50
|
+
// A single record cannot be split on the wire; memory also depends on its size.
|
|
51
|
+
export const LOG_BATCH_RECORDS = 256;
|
|
52
|
+
const BATCH_BYTES = 256 * 1024;
|
|
53
|
+
const READ_BYTES = 64 * 1024;
|
|
54
|
+
export async function openLogCursor(logPath, after) {
|
|
55
|
+
const handle = await open(logPath, 'r');
|
|
56
|
+
let offset = 0; // sole committed byte position, advanced only over complete lines
|
|
57
|
+
let lastSeq = after;
|
|
58
|
+
let positioned = after === 0n;
|
|
59
|
+
let window = Buffer.alloc(0);
|
|
60
|
+
let windowStart = 0;
|
|
61
|
+
// Recovery decodes the whole file with one non-streaming call, so the
|
|
62
|
+
// WHATWG decoder strips a BOM only if it is the file's first three bytes.
|
|
63
|
+
// Match that exactly: strip a BOM only for the record at byte offset 0, and
|
|
64
|
+
// preserve (so parseLine rejects) a BOM anywhere else, mid-log.
|
|
65
|
+
const firstLineDecoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: false });
|
|
66
|
+
const laterLineDecoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
|
|
67
|
+
async function nextLine(maxBytes, signal) {
|
|
68
|
+
let pos = offset;
|
|
69
|
+
const pieces = [];
|
|
70
|
+
let lineBytes = 0;
|
|
71
|
+
for (;;) {
|
|
72
|
+
if (signal?.aborted)
|
|
73
|
+
throw new LogReadAbortedError('log read aborted');
|
|
74
|
+
if (lineBytes >= maxBytes)
|
|
75
|
+
throw new LogReadLimitError('log record exceeds scan byte budget');
|
|
76
|
+
if (pos < windowStart || pos >= windowStart + window.length) {
|
|
77
|
+
const buffer = Buffer.allocUnsafe(Math.min(READ_BYTES, maxBytes - lineBytes + 1));
|
|
78
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, pos);
|
|
79
|
+
window = buffer.subarray(0, bytesRead);
|
|
80
|
+
windowStart = pos;
|
|
81
|
+
if (!bytesRead)
|
|
82
|
+
return null; // torn tail: offset stays at the line start
|
|
83
|
+
}
|
|
84
|
+
const start = pos - windowStart;
|
|
85
|
+
const nl = window.indexOf(0x0a, start);
|
|
86
|
+
if (nl >= 0) {
|
|
87
|
+
if (lineBytes + nl - start + 1 > maxBytes)
|
|
88
|
+
throw new LogReadLimitError('log record exceeds scan byte budget');
|
|
89
|
+
pieces.push(window.subarray(start, nl));
|
|
90
|
+
return Buffer.concat(pieces);
|
|
91
|
+
}
|
|
92
|
+
lineBytes += window.length - start;
|
|
93
|
+
if (lineBytes >= maxBytes)
|
|
94
|
+
throw new LogReadLimitError('log record exceeds scan byte budget');
|
|
95
|
+
pieces.push(window.subarray(start));
|
|
96
|
+
pos = windowStart + window.length;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return {
|
|
100
|
+
async readThrough(boundary, limits = {}) {
|
|
101
|
+
const out = [];
|
|
102
|
+
let bytes = 0;
|
|
103
|
+
while (lastSeq < boundary && out.length < Math.min(LOG_BATCH_RECORDS, limits.maxRecords ?? LOG_BATCH_RECORDS)
|
|
104
|
+
&& bytes < BATCH_BYTES && bytes < (limits.maxBytes ?? Infinity)) {
|
|
105
|
+
const lineStart = offset;
|
|
106
|
+
let slice;
|
|
107
|
+
try {
|
|
108
|
+
slice = await nextLine((limits.maxBytes ?? Infinity) - bytes, limits.signal);
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
// Offset already advanced for records in this batch. Return them before
|
|
112
|
+
// reporting the limit on the next call, so a retry cannot skip them.
|
|
113
|
+
if (out.length > 0 && (error instanceof LogReadLimitError || error instanceof LogReadAbortedError))
|
|
114
|
+
return out;
|
|
115
|
+
throw error;
|
|
116
|
+
}
|
|
117
|
+
if (slice === null)
|
|
118
|
+
break;
|
|
119
|
+
let line;
|
|
120
|
+
try {
|
|
121
|
+
line = (lineStart === 0 ? firstLineDecoder : laterLineDecoder).decode(slice);
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
throw new LogCorruptError('invalid UTF-8 in log record');
|
|
125
|
+
}
|
|
126
|
+
const ev = parseLine(line);
|
|
127
|
+
const lineBytes = slice.length + 1;
|
|
128
|
+
ev.bytesRead = lineBytes;
|
|
129
|
+
if (!positioned) {
|
|
130
|
+
if (ev.seq <= after) {
|
|
131
|
+
offset += lineBytes;
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
positioned = true;
|
|
135
|
+
}
|
|
136
|
+
if (ev.seq !== lastSeq + 1n) {
|
|
137
|
+
throw new LogCorruptError(`non-contiguous seq: expected ${lastSeq + 1n} got ${ev.seq}`);
|
|
138
|
+
}
|
|
139
|
+
out.push(ev);
|
|
140
|
+
lastSeq = ev.seq;
|
|
141
|
+
offset += lineBytes;
|
|
142
|
+
bytes += lineBytes;
|
|
143
|
+
}
|
|
144
|
+
return out;
|
|
145
|
+
},
|
|
146
|
+
async close() { await handle.close(); },
|
|
147
|
+
};
|
|
148
|
+
}
|
package/dist/log.js
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { open, stat } from 'node:fs/promises';
|
|
2
|
+
import { dirname } from 'node:path';
|
|
3
|
+
import { buildPublicEnvelope as buildEnvelope } from "./public-events.js";
|
|
4
|
+
import { FILE_MODE, StorageError, fsyncDir, writeAll } from "./storage.js";
|
|
5
|
+
/**
|
|
6
|
+
* The single writer for a session's `events.jsonl`. Appends are serialized onto
|
|
7
|
+
* a tail promise so seq assignment and byte order never interleave. Each append
|
|
8
|
+
* is durable before it is acknowledged: writeAll then fsync, then the high-water
|
|
9
|
+
* seq advances. A write failure poisons the log — every later append rejects
|
|
10
|
+
* rather than concatenate onto a possibly half-written line (the log is the
|
|
11
|
+
* source of truth). Recovery repairs and reopens; it does not append through a
|
|
12
|
+
* poisoned handle.
|
|
13
|
+
*/
|
|
14
|
+
export async function createLog(opts) {
|
|
15
|
+
const { filePath, sessionId, sequencer } = opts;
|
|
16
|
+
const isNew = !(await stat(filePath).then(() => true).catch(() => false));
|
|
17
|
+
const handle = await open(filePath, 'a', FILE_MODE);
|
|
18
|
+
if (isNew) {
|
|
19
|
+
try {
|
|
20
|
+
await handle.sync();
|
|
21
|
+
await fsyncDir(dirname(filePath));
|
|
22
|
+
}
|
|
23
|
+
catch (err) {
|
|
24
|
+
await handle.close();
|
|
25
|
+
throw err instanceof StorageError ? err : new StorageError('create-log', err);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
let durableSeq = opts.startSeq ?? 0n;
|
|
29
|
+
let tail = Promise.resolve();
|
|
30
|
+
let poison = null;
|
|
31
|
+
const append = (input) => {
|
|
32
|
+
const result = tail.then(async () => {
|
|
33
|
+
if (poison)
|
|
34
|
+
throw poison;
|
|
35
|
+
const seq = durableSeq + 1n;
|
|
36
|
+
const event = buildEnvelope(sequencer ? sequencer.enrich(input) : input, seq, sessionId);
|
|
37
|
+
try {
|
|
38
|
+
await writeAll(handle, Buffer.from(`${JSON.stringify(event)}\n`));
|
|
39
|
+
await handle.sync();
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
poison = err instanceof StorageError ? err : new StorageError('append', err);
|
|
43
|
+
throw poison;
|
|
44
|
+
}
|
|
45
|
+
durableSeq = seq;
|
|
46
|
+
// Ordered and durable: advance any sequencer state before the next append
|
|
47
|
+
// is processed, so a later change sees this commit and no earlier one does.
|
|
48
|
+
sequencer?.onCommitted(event);
|
|
49
|
+
return event;
|
|
50
|
+
});
|
|
51
|
+
tail = result.catch(() => undefined);
|
|
52
|
+
return result;
|
|
53
|
+
};
|
|
54
|
+
const close = async () => {
|
|
55
|
+
await tail;
|
|
56
|
+
await handle.close();
|
|
57
|
+
};
|
|
58
|
+
return { append, durableSeq: () => durableSeq, close };
|
|
59
|
+
}
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { access, open, readdir, rename, unlink } from 'node:fs/promises';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { dirname, join } from 'node:path';
|
|
4
|
+
import { FILE_MODE, StorageError, fsyncDir, writeAll } from "./storage.js";
|
|
5
|
+
import { blobPath, isValidHex, listSessions, onDiskHighWater, sessionLogPath, tombstonePath, } from "./store-reader.js";
|
|
6
|
+
import { openLogCursor, LogCorruptError } from "./log-reader.js";
|
|
7
|
+
import { PUBLIC_EVENT_TYPES as EVENT_TYPES } from "./public-events.js";
|
|
8
|
+
/**
|
|
9
|
+
* Store maintenance for detached-only session deletion + GC (Stage 3, P5).
|
|
10
|
+
*
|
|
11
|
+
* Deletion is durable-tombstone-first: {@link publishTombstone} makes
|
|
12
|
+
* `removed.json` durable BEFORE {@link removeSessionHistory} removes the log, so
|
|
13
|
+
* a crash between the two leaves a durable tombstone the reader already 410s and
|
|
14
|
+
* a residual log GC finishes later. Nothing here touches the ACTIVE session or
|
|
15
|
+
* decides retention policy — the daemon gates admission (detached-only) and the
|
|
16
|
+
* caller supplies the session ids.
|
|
17
|
+
*
|
|
18
|
+
* Blob reclamation ({@link reclaimUnreferencedBlobs}) is a conservative,
|
|
19
|
+
* whole-store mark-and-sweep over the GLOBAL content-addressed store. Blobs are
|
|
20
|
+
* shared across sessions, so the live set is every hash referenced by every
|
|
21
|
+
* NON-removed session's durable log; anything else is swept. The mark phase is
|
|
22
|
+
* all-or-nothing: a missing, corrupt, or uninterpretable retained log aborts the
|
|
23
|
+
* whole sweep (deletes nothing) rather than risk deleting a live blob.
|
|
24
|
+
*/
|
|
25
|
+
const TOMBSTONE_BODY = Buffer.from('{"version":1}\n', 'utf8');
|
|
26
|
+
const HISTORY_FILES = ['events.jsonl'];
|
|
27
|
+
const SHARD_RE = /^[0-9a-f]{2}$/;
|
|
28
|
+
const KNOWN_EVENT_TYPES = new Set(EVENT_TYPES);
|
|
29
|
+
function sessionDir(storeDir, id) {
|
|
30
|
+
return dirname(tombstonePath(storeDir, id));
|
|
31
|
+
}
|
|
32
|
+
/** blobs/sha256 — the CAS root, derived from store-reader's blobPath so the
|
|
33
|
+
* layout lives in one place (blobPath gives `.../sha256/<ab>/<hex>`). */
|
|
34
|
+
function blobBaseDir(storeDir) {
|
|
35
|
+
return dirname(dirname(blobPath(storeDir, '0'.repeat(64))));
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Durably publish `sessions/<id>/removed.json = {"version":1}`: unique temp,
|
|
39
|
+
* `wx`/0600, write, fsync file, atomic rename over any prior marker, fsync the
|
|
40
|
+
* session dir. Idempotent — republishing is safe and required, because an
|
|
41
|
+
* existing marker does not prove the earlier dir-fsync completed.
|
|
42
|
+
*/
|
|
43
|
+
export async function publishTombstone(storeDir, id) {
|
|
44
|
+
const dest = tombstonePath(storeDir, id);
|
|
45
|
+
const dir = dirname(dest);
|
|
46
|
+
const tmp = `${dest}.${randomUUID()}.tmp`;
|
|
47
|
+
let handle;
|
|
48
|
+
try {
|
|
49
|
+
try {
|
|
50
|
+
handle = await open(tmp, 'wx', FILE_MODE);
|
|
51
|
+
await writeAll(handle, TOMBSTONE_BODY);
|
|
52
|
+
await handle.sync();
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
throw new StorageError('write-tombstone', err);
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
await handle?.close();
|
|
59
|
+
}
|
|
60
|
+
try {
|
|
61
|
+
await rename(tmp, dest);
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
throw new StorageError('rename-tombstone', err);
|
|
65
|
+
}
|
|
66
|
+
await fsyncDir(dir);
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
await unlink(tmp).catch(() => { });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Remove a removed session's history by an EXPLICIT inventory (never a recursive
|
|
74
|
+
* dir removal), preserving `removed.json` and the session dir so the reader still
|
|
75
|
+
* lists `{removed:true}` and 410s. fsync the dir so the removal is durable.
|
|
76
|
+
*/
|
|
77
|
+
export async function removeSessionHistory(storeDir, id) {
|
|
78
|
+
const dir = sessionDir(storeDir, id);
|
|
79
|
+
let removedAny = false;
|
|
80
|
+
for (const name of HISTORY_FILES) {
|
|
81
|
+
try {
|
|
82
|
+
await unlink(join(dir, name));
|
|
83
|
+
removedAny = true;
|
|
84
|
+
}
|
|
85
|
+
catch (err) {
|
|
86
|
+
if (err.code !== 'ENOENT')
|
|
87
|
+
throw new StorageError('remove-history', err);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
if (removedAny)
|
|
91
|
+
await fsyncDir(dir);
|
|
92
|
+
}
|
|
93
|
+
/** Whether `sessions/<id>/` exists. A well-formed id naming no dir is a deletion
|
|
94
|
+
* target the caller reports as not-found rather than manufacturing a tombstone. */
|
|
95
|
+
export async function sessionExists(storeDir, id) {
|
|
96
|
+
try {
|
|
97
|
+
await access(sessionDir(storeDir, id));
|
|
98
|
+
return true;
|
|
99
|
+
}
|
|
100
|
+
catch (err) {
|
|
101
|
+
if (err.code === 'ENOENT')
|
|
102
|
+
return false;
|
|
103
|
+
throw new StorageError('stat-session', err);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Mark-and-sweep the global CAS: delete every blob NOT referenced by any
|
|
108
|
+
* non-removed session's durable log. Returns the number of blobs unlinked.
|
|
109
|
+
* Throws (deleting nothing further) if the mark phase cannot be completed
|
|
110
|
+
* safely, or if the {@link SweepAbort} predicate trips before an unlink.
|
|
111
|
+
*/
|
|
112
|
+
export async function reclaimUnreferencedBlobs(storeDir, abort) {
|
|
113
|
+
const live = new Set();
|
|
114
|
+
for (const s of await listSessions(storeDir)) {
|
|
115
|
+
if (s.removed)
|
|
116
|
+
continue;
|
|
117
|
+
await collectBlobRefs(storeDir, s.id, live); // throws on any unsafe condition
|
|
118
|
+
}
|
|
119
|
+
return sweepBlobs(storeDir, live, abort);
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Add every blob hash one non-removed session's durable log references to `live`.
|
|
123
|
+
* Reads the whole log up to its durable high-water. Throws — never returns a
|
|
124
|
+
* partial set — if the log is missing, corrupt, carries an event type this
|
|
125
|
+
* version cannot interpret, or holds a content snapshot without a valid sha256.
|
|
126
|
+
*/
|
|
127
|
+
async function collectBlobRefs(storeDir, id, live) {
|
|
128
|
+
const logPath = sessionLogPath(storeDir, id);
|
|
129
|
+
const boundary = await onDiskHighWater(logPath); // throws LogCorruptError on a bad tail
|
|
130
|
+
let cursor;
|
|
131
|
+
try {
|
|
132
|
+
cursor = await openLogCursor(logPath, 0n);
|
|
133
|
+
}
|
|
134
|
+
catch (err) {
|
|
135
|
+
// A non-removed session must have a readable log; a missing one is unsafe to
|
|
136
|
+
// treat as an empty reference set (onDiskHighWater returns 0n for BOTH a
|
|
137
|
+
// missing and an empty log, so distinguish them here).
|
|
138
|
+
throw new StorageError('gc-mark-open', err);
|
|
139
|
+
}
|
|
140
|
+
try {
|
|
141
|
+
// Validate every complete record through the log's PHYSICAL end — never stop at
|
|
142
|
+
// the final record's declared seq. `readThrough(boundary + 1n)` keeps consuming
|
|
143
|
+
// past the high-water, so a log whose tail rewinds below an interior seq (e.g.
|
|
144
|
+
// 1, 2, 1) is caught by the cursor's contiguity check or the reached-seq
|
|
145
|
+
// assertion below, rather than silently skipping the records after the boundary.
|
|
146
|
+
let seq = 0n;
|
|
147
|
+
for (;;) {
|
|
148
|
+
const batch = await cursor.readThrough(boundary + 1n); // throws on non-contiguous seq
|
|
149
|
+
if (!batch.length)
|
|
150
|
+
break; // physical EOF (or a torn, uncommitted tail)
|
|
151
|
+
for (const ev of batch) {
|
|
152
|
+
addRefsFromEvent(ev.type, ev.data, live);
|
|
153
|
+
seq = ev.seq;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
if (seq !== boundary) {
|
|
157
|
+
// The last complete record read is not the durable high-water: the log is
|
|
158
|
+
// short of, or runs past, its own final seq — corruption. Delete nothing.
|
|
159
|
+
throw new LogCorruptError(`gc-mark: read to seq ${seq}, not durable high-water ${boundary}`);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
finally {
|
|
163
|
+
await cursor.close();
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
function addRefsFromEvent(type, data, live) {
|
|
167
|
+
if (!KNOWN_EVENT_TYPES.has(type)) {
|
|
168
|
+
// A newer/unknown type may reference blobs through a mechanism this version
|
|
169
|
+
// does not know. Refuse to sweep rather than risk deleting a live blob.
|
|
170
|
+
throw new LogCorruptError(`gc-mark: unknown event type ${type}; refusing to sweep`);
|
|
171
|
+
}
|
|
172
|
+
// A blob reference is exactly a canonical content snapshot ({kind:'content',
|
|
173
|
+
// sha256}). Collect them wherever they appear in the event's data — not just the
|
|
174
|
+
// fields this version happens to hard-code — so an additive blob-bearing field on
|
|
175
|
+
// an existing type is still marked, never orphaned and swept. Any other shape
|
|
176
|
+
// carries no blob and is ignored.
|
|
177
|
+
addRefsFromValue(data, live);
|
|
178
|
+
}
|
|
179
|
+
function addRefsFromValue(value, live) {
|
|
180
|
+
if (Array.isArray(value)) {
|
|
181
|
+
for (const item of value)
|
|
182
|
+
addRefsFromValue(item, live);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
if (typeof value !== 'object' || value === null)
|
|
186
|
+
return;
|
|
187
|
+
const record = value;
|
|
188
|
+
if (record.kind === 'content' || 'sha256' in record) {
|
|
189
|
+
// Anything shaped like a blob reference — a content snapshot, or any object
|
|
190
|
+
// carrying a sha256 — must be a fully valid content snapshot. A partial or
|
|
191
|
+
// malformed one (a sha256 without kind:'content', or a bad hex) is corruption
|
|
192
|
+
// this version cannot interpret: fail closed rather than silently drop the
|
|
193
|
+
// reference and sweep a blob it may protect.
|
|
194
|
+
const sha = record.sha256;
|
|
195
|
+
if (record.kind !== 'content' || typeof sha !== 'string' || !isValidHex(sha)) {
|
|
196
|
+
throw new LogCorruptError('gc-mark: malformed content snapshot');
|
|
197
|
+
}
|
|
198
|
+
live.add(sha);
|
|
199
|
+
}
|
|
200
|
+
for (const child of Object.values(record))
|
|
201
|
+
addRefsFromValue(child, live);
|
|
202
|
+
}
|
|
203
|
+
/** Delete validated CAS blobs not in `live`, fsyncing each changed shard. Only
|
|
204
|
+
* well-formed, correctly-sharded hash files are ever unlinked — a stray or
|
|
205
|
+
* misplaced file is left untouched. */
|
|
206
|
+
async function sweepBlobs(storeDir, live, abort) {
|
|
207
|
+
const base = blobBaseDir(storeDir);
|
|
208
|
+
let shards;
|
|
209
|
+
try {
|
|
210
|
+
shards = await readdir(base);
|
|
211
|
+
}
|
|
212
|
+
catch (err) {
|
|
213
|
+
if (err.code === 'ENOENT')
|
|
214
|
+
return 0;
|
|
215
|
+
throw new StorageError('gc-sweep-readdir', err);
|
|
216
|
+
}
|
|
217
|
+
let removed = 0;
|
|
218
|
+
for (const shard of shards) {
|
|
219
|
+
if (!SHARD_RE.test(shard))
|
|
220
|
+
continue; // not a CAS shard directory
|
|
221
|
+
const shardDir = join(base, shard);
|
|
222
|
+
let names;
|
|
223
|
+
try {
|
|
224
|
+
names = await readdir(shardDir);
|
|
225
|
+
}
|
|
226
|
+
catch (err) {
|
|
227
|
+
if (err.code === 'ENOENT')
|
|
228
|
+
continue;
|
|
229
|
+
throw new StorageError('gc-sweep-readdir', err);
|
|
230
|
+
}
|
|
231
|
+
let removedHere = 0;
|
|
232
|
+
for (const name of names) {
|
|
233
|
+
if (!isValidHex(name) || name.slice(0, 2) !== shard)
|
|
234
|
+
continue; // temp/stray/misplaced
|
|
235
|
+
if (live.has(name))
|
|
236
|
+
continue;
|
|
237
|
+
if (abort()) {
|
|
238
|
+
throw new StorageError('gc-sweep-aborted', new Error('store ownership lost mid-sweep'));
|
|
239
|
+
}
|
|
240
|
+
try {
|
|
241
|
+
await unlink(join(shardDir, name));
|
|
242
|
+
removed++;
|
|
243
|
+
removedHere++;
|
|
244
|
+
}
|
|
245
|
+
catch (err) {
|
|
246
|
+
if (err.code === 'ENOENT')
|
|
247
|
+
continue;
|
|
248
|
+
throw new StorageError('gc-sweep-unlink', err);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (removedHere > 0)
|
|
252
|
+
await fsyncDir(shardDir);
|
|
253
|
+
}
|
|
254
|
+
return removed;
|
|
255
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* The Slipstream MCP forwarder — the whole subprocess a harness spawns.
|
|
4
|
+
*
|
|
5
|
+
* It wires these pieces together and owns nothing else:
|
|
6
|
+
* - {@link ./mcp-protocol.ts} — hand-rolled MCP-over-stdio JSON-RPC transport.
|
|
7
|
+
* - {@link ./harness-context.ts} — fail-closed harness-identity resolver (SC3).
|
|
8
|
+
* - {@link ./task-forwarder.ts} — begin_task orchestration with honest retry.
|
|
9
|
+
* - {@link ./answer-forwarder.ts} — answer_question orchestration with honest retry.
|
|
10
|
+
* - {@link ./control-client.ts} — the unix-socket control channel to the daemon.
|
|
11
|
+
*
|
|
12
|
+
* It exposes two tools, `slipstream_begin_task(title)` and
|
|
13
|
+
* `slipstream_answer_question(question_id, text)`. The forwarder is a control
|
|
14
|
+
* client: it never attaches, never captures, and never touches the store — it resolves the daemon's control socket from `--store` (shared with the CLI)
|
|
15
|
+
* and forwards a verified identity triple. A missing daemon fails fast as
|
|
16
|
+
* DAEMON_UNAVAILABLE rather than hanging, because the tool must return promptly.
|
|
17
|
+
*/
|
|
18
|
+
import { isMainModule } from "./entrypoint.js";
|
|
19
|
+
import { dispatch, parseMessage } from "./mcp-protocol.js";
|
|
20
|
+
import { createHarnessContext } from "./harness-context.js";
|
|
21
|
+
import { forwardBeginTask } from "./task-forwarder.js";
|
|
22
|
+
import { forwardAnswer } from "./answer-forwarder.js";
|
|
23
|
+
import { sendControlRequest } from "./control-client.js";
|
|
24
|
+
import { controlSocketPath, resolveStoreDir } from "./daemon-location.js";
|
|
25
|
+
export const BEGIN_TASK_TOOL = {
|
|
26
|
+
name: 'slipstream_begin_task',
|
|
27
|
+
description: 'Declare the start of a task so the Slipstream feed groups the file changes you are about ' +
|
|
28
|
+
'to make under it. Call it right before you start working on a distinct piece of work.',
|
|
29
|
+
inputSchema: {
|
|
30
|
+
type: 'object',
|
|
31
|
+
properties: {
|
|
32
|
+
title: { type: 'string', description: 'A short human-readable title for the task.' },
|
|
33
|
+
},
|
|
34
|
+
required: ['title'],
|
|
35
|
+
additionalProperties: false,
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
export const ANSWER_QUESTION_TOOL = {
|
|
39
|
+
name: 'slipstream_answer_question',
|
|
40
|
+
description: 'Return your answer to a Slipstream question. Call it once per question with the exact ' +
|
|
41
|
+
'question_id from the Slipstream question context and your complete answer as text.',
|
|
42
|
+
inputSchema: {
|
|
43
|
+
type: 'object',
|
|
44
|
+
properties: {
|
|
45
|
+
question_id: { type: 'string', description: 'The exact Slipstream question id you are answering.' },
|
|
46
|
+
text: { type: 'string', description: 'Your complete answer for the user.' },
|
|
47
|
+
},
|
|
48
|
+
required: ['question_id', 'text'],
|
|
49
|
+
additionalProperties: false,
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
function isObject(v) {
|
|
53
|
+
return typeof v === 'object' && v !== null;
|
|
54
|
+
}
|
|
55
|
+
function stringArg(params, key) {
|
|
56
|
+
// Pass the raw value through; the daemon is the authority on validity (an
|
|
57
|
+
// empty/absent title or answer becomes its INVALID_TITLE / INVALID_ANSWER,
|
|
58
|
+
// not a second local rule).
|
|
59
|
+
const value = isObject(params) && isObject(params.arguments) ? params.arguments[key] : undefined;
|
|
60
|
+
return typeof value === 'string' ? value : '';
|
|
61
|
+
}
|
|
62
|
+
export function createForwarderHandlers(opts) {
|
|
63
|
+
const ctx = createHarnessContext();
|
|
64
|
+
return {
|
|
65
|
+
serverInfo: { name: 'slipstream-forwarder', version: '0.0.0' },
|
|
66
|
+
protocolVersion: '2025-06-18',
|
|
67
|
+
tools: [BEGIN_TASK_TOOL, ANSWER_QUESTION_TOOL],
|
|
68
|
+
onInitialize: (params) => {
|
|
69
|
+
const clientInfo = isObject(params) ? params.clientInfo : undefined;
|
|
70
|
+
ctx.initialize(clientInfo, opts.env);
|
|
71
|
+
},
|
|
72
|
+
callTool: (name, params) => {
|
|
73
|
+
const identity = ctx.identityForCall(params);
|
|
74
|
+
const send = (request) => sendControlRequest({ socketPath: opts.socketPath, request });
|
|
75
|
+
return name === ANSWER_QUESTION_TOOL.name
|
|
76
|
+
? forwardAnswer({ identity, questionId: stringArg(params, 'question_id'), text: stringArg(params, 'text'), send })
|
|
77
|
+
: forwardBeginTask({ identity, title: stringArg(params, 'title'), send });
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
/** The largest single JSON-RPC line the forwarder will buffer from stdin. A well-
|
|
82
|
+
* formed request is a few hundred bytes; this cap is generous for a title yet far
|
|
83
|
+
* short of exhausting memory on an unterminated line. */
|
|
84
|
+
export const MAX_STDIN_LINE_BYTES = 1024 * 1024;
|
|
85
|
+
const NEWLINE = 0x0a;
|
|
86
|
+
/**
|
|
87
|
+
* Split raw stdin bytes into newline-delimited lines with a hard byte cap. A line
|
|
88
|
+
* (terminated or not) that exceeds `maxBytes` is dropped and reported as
|
|
89
|
+
* `{ overflow: true }`; the bytes up to its terminating newline are then discarded
|
|
90
|
+
* rather than accumulated, so a peer streaming an endless unterminated line can
|
|
91
|
+
* never grow the buffer without bound. Blank lines are skipped.
|
|
92
|
+
*/
|
|
93
|
+
export function createStdinFramer(maxBytes = MAX_STDIN_LINE_BYTES) {
|
|
94
|
+
let buf = Buffer.alloc(0);
|
|
95
|
+
let discarding = false; // dropping the tail of an over-cap line until its newline
|
|
96
|
+
return {
|
|
97
|
+
push(chunk) {
|
|
98
|
+
buf = buf.length === 0 ? chunk : Buffer.concat([buf, chunk]);
|
|
99
|
+
const out = [];
|
|
100
|
+
for (;;) {
|
|
101
|
+
const nl = buf.indexOf(NEWLINE);
|
|
102
|
+
if (nl === -1)
|
|
103
|
+
break;
|
|
104
|
+
const lineBuf = buf.subarray(0, nl);
|
|
105
|
+
buf = buf.subarray(nl + 1);
|
|
106
|
+
if (discarding) {
|
|
107
|
+
discarding = false;
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (lineBuf.length > maxBytes) {
|
|
111
|
+
out.push({ overflow: true });
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
const line = lineBuf.toString('utf8');
|
|
115
|
+
if (line.trim().length === 0)
|
|
116
|
+
continue;
|
|
117
|
+
out.push({ line });
|
|
118
|
+
}
|
|
119
|
+
if (buf.length > maxBytes) {
|
|
120
|
+
if (!discarding)
|
|
121
|
+
out.push({ overflow: true });
|
|
122
|
+
buf = Buffer.alloc(0);
|
|
123
|
+
discarding = true;
|
|
124
|
+
}
|
|
125
|
+
return out;
|
|
126
|
+
},
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
export async function runForwarder() {
|
|
130
|
+
let socketPath;
|
|
131
|
+
try {
|
|
132
|
+
socketPath = controlSocketPath(resolveStoreDir(process.argv.slice(2)));
|
|
133
|
+
}
|
|
134
|
+
catch (err) {
|
|
135
|
+
process.stderr.write(`slipstream-forwarder: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
136
|
+
process.exitCode = 2;
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
const handlers = createForwarderHandlers({ socketPath, env: process.env });
|
|
140
|
+
const writeError = (code, message) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: null, error: { code, message } }) + '\n');
|
|
141
|
+
const framer = createStdinFramer();
|
|
142
|
+
for await (const chunk of process.stdin) {
|
|
143
|
+
for (const frame of framer.push(chunk)) {
|
|
144
|
+
if ('overflow' in frame) {
|
|
145
|
+
writeError(-32700, 'message exceeds byte cap');
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
const parsed = parseMessage(frame.line);
|
|
149
|
+
if (!parsed.ok) {
|
|
150
|
+
writeError(parsed.code, parsed.code === -32700 ? 'parse error' : 'invalid request');
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
const res = await dispatch(parsed.value, handlers);
|
|
154
|
+
if (res)
|
|
155
|
+
process.stdout.write(JSON.stringify(res) + '\n');
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (process.argv[1] && isMainModule(import.meta.url, process.argv[1])) {
|
|
160
|
+
await runForwarder();
|
|
161
|
+
}
|