mcp-for-i 0.1.13 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/dist/config/store.js +229 -109
- package/dist/controlplane/auth.js +128 -0
- package/dist/controlplane/clientIntegrationService.js +83 -4
- package/dist/controlplane/commands.js +14 -9
- package/dist/controlplane/connectionService.js +110 -5
- package/dist/controlplane/connectionTest.js +329 -0
- package/dist/controlplane/errors.js +12 -0
- package/dist/controlplane/runtimeService.js +98 -26
- package/dist/controlplane/server.js +122 -11
- package/dist/controlplane/serverShared.js +25 -6
- package/dist/controlplane/serviceManager.js +113 -100
- package/dist/controlplane/ui.js +1742 -920
- package/dist/ibmi/CompileTools.js +29 -1
- package/dist/ibmi/IBMiContent.js +435 -63
- package/dist/ibmi/IbmDocs.js +685 -0
- package/dist/ibmi/LocalLanguageActions.js +129 -3
- package/dist/ibmi/Search.js +158 -52
- package/dist/ibmi/Tools.js +7 -10
- package/dist/ibmi/client.js +220 -31
- package/dist/ibmi/evfevent.js +256 -0
- package/dist/ibmi/knownHosts.js +190 -0
- package/dist/ibmi/libraryList.js +93 -0
- package/dist/ibmi/sourceInspect.js +815 -0
- package/dist/ibmi/sqlJob.js +888 -0
- package/dist/ibmi/sqlParse.js +317 -0
- package/dist/ibmi/tn5250/certTrust.js +59 -0
- package/dist/ibmi/tn5250/client.js +310 -0
- package/dist/ibmi/tn5250/datastream.js +109 -0
- package/dist/ibmi/tn5250/ebcdic.js +74 -0
- package/dist/ibmi/tn5250/index.js +6 -0
- package/dist/ibmi/tn5250/screen.js +827 -0
- package/dist/ibmi/tn5250/telnet.js +186 -0
- package/dist/ibmi/variables.js +18 -7
- package/dist/index.js +43 -22
- package/dist/mcp/annotations.js +155 -0
- package/dist/mcp/audit.js +490 -133
- package/dist/mcp/confirm.js +48 -0
- package/dist/mcp/context.js +43 -5
- package/dist/mcp/deployMethods.js +192 -0
- package/dist/mcp/logger.js +5 -10
- package/dist/mcp/modules/bulkMembers.js +476 -0
- package/dist/mcp/modules/changedLines.js +189 -0
- package/dist/mcp/modules/compare.js +198 -0
- package/dist/mcp/modules/compile.js +515 -0
- package/dist/mcp/modules/ifsOps.js +123 -0
- package/dist/mcp/modules/index.js +38 -0
- package/dist/mcp/modules/inspect.js +811 -0
- package/dist/mcp/modules/libl.js +246 -0
- package/dist/mcp/modules/members.js +171 -0
- package/dist/mcp/modules/objects.js +339 -0
- package/dist/mcp/modules/system.js +266 -0
- package/dist/mcp/modules/tn5250.js +515 -0
- package/dist/mcp/policy.js +412 -0
- package/dist/mcp/tools.js +1700 -698
- package/dist/security/credentialStore.js +62 -13
- package/dist/security/localPaths.js +138 -0
- package/dist/security/redact.js +79 -0
- package/dist/util/atomicWrite.js +101 -0
- package/dist/util/fileLock.js +206 -0
- package/docs/capabilities.md +50 -7
- package/docs/configuration.md +189 -60
- package/docs/examples.md +109 -73
- package/docs/overview.md +15 -15
- package/docs/security.md +101 -16
- package/docs/tools.md +363 -188
- package/package.json +56 -53
package/dist/mcp/audit.js
CHANGED
|
@@ -2,20 +2,38 @@ import crypto from "crypto";
|
|
|
2
2
|
import fs from "fs/promises";
|
|
3
3
|
import path from "path";
|
|
4
4
|
import { getConfigDir } from "../config/store.js";
|
|
5
|
+
import { isTransientFsError, readFileWithRetry, sleep, writeFileAtomic } from "../util/atomicWrite.js";
|
|
6
|
+
import { redactSecrets } from "../security/redact.js";
|
|
7
|
+
import { withFileLock } from "../util/fileLock.js";
|
|
8
|
+
const LOCK_OPTIONS = { staleMs: 30_000, timeoutMs: 20_000 };
|
|
9
|
+
const MAX_ANCHOR_RESETS = 20;
|
|
10
|
+
// In-process chain of audit writes (appends and purges). Each link swallows its
|
|
11
|
+
// own failure, so one failed write never stops later ones; the caller still
|
|
12
|
+
// receives the error.
|
|
5
13
|
let appendQueue = Promise.resolve();
|
|
14
|
+
function enqueue(work) {
|
|
15
|
+
const run = appendQueue.then(work);
|
|
16
|
+
appendQueue = run.then(() => undefined, () => undefined);
|
|
17
|
+
return run;
|
|
18
|
+
}
|
|
6
19
|
export async function appendAuditRecord(input) {
|
|
7
20
|
const paths = resolveAuditPaths();
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
const
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
|
|
21
|
+
const redactedArgs = redactSecrets(input.args);
|
|
22
|
+
const argsHash = sha256Hex(stableStringify(redactedArgs));
|
|
23
|
+
return enqueue(() => withAuditLock(paths, async () => {
|
|
24
|
+
// Everything below reads the current state from disk under the cross-process
|
|
25
|
+
// lock, so several MCP processes extend one chain instead of forking it.
|
|
26
|
+
const key = await loadOrCreateKey(paths);
|
|
27
|
+
const anchor = await ensureAnchor(paths, key);
|
|
28
|
+
const tail = await readTail(paths.logPath);
|
|
29
|
+
const prevHash = tail.last ? tail.last.hash : anchor.records === 0 ? anchor.lastHash : "";
|
|
30
|
+
const seq = Math.max(anchor.lastSeq, tail.last?.seq ?? 0) + 1;
|
|
31
|
+
const base = toJsonValue({
|
|
32
|
+
version: 2,
|
|
33
|
+
seq,
|
|
34
|
+
keyId: key.id,
|
|
17
35
|
id: crypto.randomUUID(),
|
|
18
|
-
at:
|
|
36
|
+
at: new Date().toISOString(),
|
|
19
37
|
tool: input.tool,
|
|
20
38
|
status: input.status,
|
|
21
39
|
connectionName: input.connectionName,
|
|
@@ -27,23 +45,34 @@ export async function appendAuditRecord(input) {
|
|
|
27
45
|
error: input.error,
|
|
28
46
|
resultSummary: input.resultSummary,
|
|
29
47
|
prevHash
|
|
48
|
+
});
|
|
49
|
+
const record = { ...base, hash: keyedHash(key.secret, prevHash, base) };
|
|
50
|
+
const prefix = tail.size > 0 && !tail.endsWithNewline ? "\n" : "";
|
|
51
|
+
await appendWithRetry(paths.logPath, `${prefix}${JSON.stringify(record)}\n`);
|
|
52
|
+
const next = {
|
|
53
|
+
...stripMac(anchor),
|
|
54
|
+
records: anchor.records + 1,
|
|
55
|
+
lastSeq: seq,
|
|
56
|
+
lastHash: record.hash,
|
|
57
|
+
updatedAt: record.at
|
|
30
58
|
};
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
59
|
+
if (anchor.records === 0) {
|
|
60
|
+
next.firstHash = record.hash;
|
|
61
|
+
next.firstPrevHash = prevHash;
|
|
62
|
+
next.firstSeq = seq;
|
|
63
|
+
}
|
|
64
|
+
await writeAnchor(paths, key, next);
|
|
65
|
+
await writeHead(paths, record.hash);
|
|
66
|
+
return record;
|
|
67
|
+
}));
|
|
39
68
|
}
|
|
40
69
|
export async function listAuditRecords(filter = {}) {
|
|
41
70
|
const paths = resolveAuditPaths();
|
|
42
71
|
const limit = clampNumber(filter.limit, 200, 1, 5000);
|
|
43
|
-
const
|
|
72
|
+
const parsed = await readAuditLog(paths.logPath);
|
|
44
73
|
const sinceMs = filter.since ? Date.parse(filter.since) : Number.NaN;
|
|
45
74
|
const hasSince = Number.isFinite(sinceMs);
|
|
46
|
-
const filtered =
|
|
75
|
+
const filtered = parsed.records.filter(entry => {
|
|
47
76
|
if (filter.tool && entry.tool !== filter.tool)
|
|
48
77
|
return false;
|
|
49
78
|
if (filter.status && entry.status !== filter.status)
|
|
@@ -60,58 +89,139 @@ export async function listAuditRecords(filter = {}) {
|
|
|
60
89
|
return {
|
|
61
90
|
records: newestFirst,
|
|
62
91
|
total: filtered.length,
|
|
63
|
-
limit
|
|
92
|
+
limit,
|
|
93
|
+
skippedLines: parsed.unparseableLines.length
|
|
64
94
|
};
|
|
65
95
|
}
|
|
66
96
|
export async function verifyAuditChain() {
|
|
67
97
|
const paths = resolveAuditPaths();
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
98
|
+
// Taken under the lock so the log and the anchor are read as one consistent snapshot.
|
|
99
|
+
return withAuditLock(paths, async () => {
|
|
100
|
+
const parsed = await readAuditLog(paths.logPath);
|
|
101
|
+
const records = parsed.records;
|
|
102
|
+
const key = await readKey(paths);
|
|
103
|
+
const anchorRead = await readAnchor(paths, key);
|
|
104
|
+
const anchor = anchorRead.anchor;
|
|
105
|
+
const failureDetails = [];
|
|
106
|
+
const fail = (id, reason) => {
|
|
107
|
+
failureDetails.push({ id, reason });
|
|
108
|
+
};
|
|
109
|
+
for (const lineNumber of parsed.unparseableLines) {
|
|
110
|
+
fail(`line ${lineNumber}`, "unparseable line");
|
|
111
|
+
}
|
|
112
|
+
if (anchorRead.problem)
|
|
113
|
+
fail("ANCHOR", anchorRead.problem);
|
|
114
|
+
for (const reset of anchor?.resets || []) {
|
|
115
|
+
fail("ANCHOR", `anchor was rebuilt at ${reset.at}: ${reset.reason}`);
|
|
116
|
+
}
|
|
117
|
+
let previous = anchor ? anchor.firstPrevHash : "";
|
|
118
|
+
let lastSeq;
|
|
119
|
+
let seenKeyed = false;
|
|
120
|
+
let legacyCount = 0;
|
|
121
|
+
let keyedCount = 0;
|
|
122
|
+
const segments = [];
|
|
123
|
+
records.forEach((record, index) => {
|
|
124
|
+
const before = failureDetails.length;
|
|
125
|
+
if (index === 0 && anchor && record.hash !== anchor.firstHash) {
|
|
126
|
+
fail(record.id, "first record does not match the anchor (records were removed from the start of the log)");
|
|
127
|
+
}
|
|
128
|
+
if (record.prevHash !== previous) {
|
|
129
|
+
fail(record.id, "prevHash mismatch");
|
|
130
|
+
}
|
|
131
|
+
let type;
|
|
132
|
+
if (record.version === 2) {
|
|
133
|
+
type = "keyed";
|
|
134
|
+
seenKeyed = true;
|
|
135
|
+
keyedCount += 1;
|
|
136
|
+
if (!key) {
|
|
137
|
+
fail(record.id, "audit key file is missing; keyed record cannot be verified");
|
|
138
|
+
}
|
|
139
|
+
else if (record.keyId !== key.id) {
|
|
140
|
+
fail(record.id, "record was signed with a different audit key");
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
const { hash: _hash, ...base } = record;
|
|
144
|
+
if (!timingSafeEqualHex(keyedHash(key.secret, record.prevHash, base), record.hash)) {
|
|
145
|
+
fail(record.id, "hash mismatch");
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
if (typeof record.seq !== "number") {
|
|
149
|
+
fail(record.id, "missing sequence number");
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
if (lastSeq !== undefined && record.seq !== lastSeq + 1) {
|
|
153
|
+
fail(record.id, `sequence gap (expected ${lastSeq + 1}, found ${record.seq})`);
|
|
154
|
+
}
|
|
155
|
+
lastSeq = record.seq;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
else {
|
|
159
|
+
type = "legacy";
|
|
160
|
+
legacyCount += 1;
|
|
161
|
+
if (seenKeyed) {
|
|
162
|
+
fail(record.id, "legacy (unkeyed) record after keyed records");
|
|
163
|
+
}
|
|
164
|
+
if (legacyHash(record) !== record.hash) {
|
|
165
|
+
fail(record.id, "hash mismatch");
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
const failed = failureDetails.length - before;
|
|
169
|
+
const current = segments[segments.length - 1];
|
|
170
|
+
if (current && current.type === type) {
|
|
171
|
+
current.lastId = record.id;
|
|
172
|
+
current.records += 1;
|
|
173
|
+
current.failures += failed;
|
|
174
|
+
}
|
|
175
|
+
else {
|
|
176
|
+
segments.push({ type, firstId: record.id, lastId: record.id, records: 1, failures: failed });
|
|
177
|
+
}
|
|
76
178
|
previous = record.hash;
|
|
77
|
-
|
|
179
|
+
});
|
|
180
|
+
const headHash = await readHeadHash(paths.headPath);
|
|
181
|
+
const last = records[records.length - 1];
|
|
182
|
+
if (anchor) {
|
|
183
|
+
if (anchor.records !== records.length) {
|
|
184
|
+
fail("ANCHOR", `record count does not match the anchor (expected ${anchor.records}, found ${records.length})`);
|
|
185
|
+
}
|
|
186
|
+
if (last && last.hash !== anchor.lastHash) {
|
|
187
|
+
fail("ANCHOR", "last record does not match the anchor (records were removed from or added to the end of the log)");
|
|
188
|
+
}
|
|
189
|
+
if (last && last.version === 2 && last.seq !== anchor.lastSeq) {
|
|
190
|
+
fail("ANCHOR", `last sequence number does not match the anchor (expected ${anchor.lastSeq}, found ${last.seq})`);
|
|
191
|
+
}
|
|
78
192
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
correlationId: record.correlationId,
|
|
89
|
-
argsHash: record.argsHash,
|
|
90
|
-
args: record.args,
|
|
91
|
-
error: record.error,
|
|
92
|
-
resultSummary: record.resultSummary,
|
|
93
|
-
prevHash: record.prevHash
|
|
94
|
-
};
|
|
95
|
-
const expected = sha256Hex(`${record.prevHash}|${stableStringify(base)}`);
|
|
96
|
-
if (expected !== record.hash) {
|
|
97
|
-
failures += 1;
|
|
98
|
-
failureDetails.push({ id: record.id, reason: "hash mismatch" });
|
|
193
|
+
else if (!anchorRead.problem) {
|
|
194
|
+
if (keyedCount > 0) {
|
|
195
|
+
fail("ANCHOR", "anchor file is missing");
|
|
196
|
+
}
|
|
197
|
+
// Log written only by older versions: fall back to the legacy head check.
|
|
198
|
+
const headMatches = records.length === 0 ? headHash === "" : headHash === last.hash;
|
|
199
|
+
if (!headMatches) {
|
|
200
|
+
fail("HEAD", "head hash mismatch");
|
|
201
|
+
}
|
|
99
202
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
203
|
+
return {
|
|
204
|
+
ok: failureDetails.length === 0,
|
|
205
|
+
records: records.length,
|
|
206
|
+
failures: failureDetails.length,
|
|
207
|
+
details: failureDetails.slice(0, 50),
|
|
208
|
+
headHash,
|
|
209
|
+
keyedRecords: keyedCount,
|
|
210
|
+
legacyRecords: legacyCount,
|
|
211
|
+
unparseableLines: parsed.unparseableLines.length,
|
|
212
|
+
segments,
|
|
213
|
+
anchor: anchor
|
|
214
|
+
? {
|
|
215
|
+
records: anchor.records,
|
|
216
|
+
firstSeq: anchor.firstSeq,
|
|
217
|
+
lastSeq: anchor.lastSeq,
|
|
218
|
+
lastHash: anchor.lastHash,
|
|
219
|
+
updatedAt: anchor.updatedAt,
|
|
220
|
+
lastPurge: anchor.lastPurge
|
|
221
|
+
}
|
|
222
|
+
: null
|
|
223
|
+
};
|
|
224
|
+
});
|
|
115
225
|
}
|
|
116
226
|
export async function exportAuditRecords(options) {
|
|
117
227
|
const paths = resolveAuditPaths();
|
|
@@ -158,7 +268,7 @@ export async function exportAuditRecords(options) {
|
|
|
158
268
|
row.hash,
|
|
159
269
|
row.error || ""
|
|
160
270
|
];
|
|
161
|
-
lines.push(values.map(csvEscape).join(","));
|
|
271
|
+
lines.push(values.map(value => csvEscape(String(value ?? ""))).join(","));
|
|
162
272
|
}
|
|
163
273
|
await fs.writeFile(outputPath, `${lines.join("\n")}\n`, "utf8");
|
|
164
274
|
}
|
|
@@ -171,6 +281,7 @@ export async function exportAuditRecords(options) {
|
|
|
171
281
|
outputPath,
|
|
172
282
|
format,
|
|
173
283
|
exported: records.length,
|
|
284
|
+
skippedLines: listing.skippedLines,
|
|
174
285
|
sourcePath: paths.logPath
|
|
175
286
|
};
|
|
176
287
|
}
|
|
@@ -180,37 +291,48 @@ export async function purgeAuditRecords(options) {
|
|
|
180
291
|
if (!Number.isFinite(cutoff)) {
|
|
181
292
|
throw new Error(`Invalid before timestamp: ${options.before}`);
|
|
182
293
|
}
|
|
183
|
-
const paths = resolveAuditPaths();
|
|
184
|
-
await fs.mkdir(paths.dir, { recursive: true });
|
|
185
|
-
const records = await readAuditRecords(paths.logPath);
|
|
186
|
-
const keep = records.filter(record => Date.parse(record.at) >= cutoff);
|
|
187
|
-
const purgeCount = records.length - keep.length;
|
|
188
294
|
const dryRun = Boolean(options.dryRun);
|
|
189
|
-
|
|
190
|
-
|
|
295
|
+
const paths = resolveAuditPaths();
|
|
296
|
+
// Read and rewrite inside the write queue and the cross-process lock, so records
|
|
297
|
+
// appended while the purge runs are never dropped.
|
|
298
|
+
return enqueue(() => withAuditLock(paths, async () => {
|
|
299
|
+
const parsed = await readAuditLog(paths.logPath);
|
|
300
|
+
// Unparseable lines are kept: they are evidence for verify, not ours to discard.
|
|
301
|
+
const keepLines = parsed.lines.filter(line => !line.record || !(Date.parse(line.record.at) < cutoff));
|
|
302
|
+
const keepRecords = keepLines.map(line => line.record).filter((r) => Boolean(r));
|
|
303
|
+
const total = parsed.records.length;
|
|
304
|
+
const purgeCount = total - keepRecords.length;
|
|
305
|
+
const summary = {
|
|
191
306
|
ok: true,
|
|
192
307
|
dryRun,
|
|
193
308
|
before: new Date(cutoff).toISOString(),
|
|
194
|
-
total
|
|
309
|
+
total,
|
|
195
310
|
purged: purgeCount,
|
|
196
|
-
remaining:
|
|
311
|
+
remaining: keepRecords.length,
|
|
312
|
+
skippedLines: parsed.unparseableLines.length
|
|
197
313
|
};
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
const
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
const
|
|
204
|
-
await
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
314
|
+
if (dryRun || purgeCount === 0)
|
|
315
|
+
return summary;
|
|
316
|
+
const key = await loadOrCreateKey(paths);
|
|
317
|
+
const anchor = await ensureAnchor(paths, key);
|
|
318
|
+
// Kept records are not re-hashed: the anchor records where the chain now starts.
|
|
319
|
+
const payload = keepLines.map(line => line.raw).join("\n");
|
|
320
|
+
await writeFileAtomic(paths.logPath, payload.length > 0 ? `${payload}\n` : "");
|
|
321
|
+
const first = keepRecords[0];
|
|
322
|
+
const lastKept = keepRecords[keepRecords.length - 1];
|
|
323
|
+
const now = new Date().toISOString();
|
|
324
|
+
await writeAnchor(paths, key, {
|
|
325
|
+
...stripMac(anchor),
|
|
326
|
+
records: keepRecords.length,
|
|
327
|
+
firstHash: first ? first.hash : "",
|
|
328
|
+
firstPrevHash: first ? first.prevHash : anchor.lastHash,
|
|
329
|
+
firstSeq: first && typeof first.seq === "number" ? first.seq : null,
|
|
330
|
+
updatedAt: now,
|
|
331
|
+
lastPurge: { at: now, before: summary.before, purged: purgeCount }
|
|
332
|
+
});
|
|
333
|
+
await writeHead(paths, lastKept ? lastKept.hash : "");
|
|
334
|
+
return summary;
|
|
335
|
+
}));
|
|
214
336
|
}
|
|
215
337
|
function resolveAuditPaths() {
|
|
216
338
|
const explicit = process.env.MCP_FOR_I_AUDIT_LOG_PATH;
|
|
@@ -218,23 +340,120 @@ function resolveAuditPaths() {
|
|
|
218
340
|
? path.resolve(explicit)
|
|
219
341
|
: path.join(getConfigDir(), "audit", "tool-audit.jsonl");
|
|
220
342
|
const dir = path.dirname(logPath);
|
|
343
|
+
const explicitKey = process.env.MCP_FOR_I_AUDIT_KEY_PATH;
|
|
221
344
|
return {
|
|
222
345
|
dir,
|
|
223
346
|
logPath,
|
|
224
|
-
headPath: path.join(dir, "tool-audit.head")
|
|
347
|
+
headPath: path.join(dir, "tool-audit.head"),
|
|
348
|
+
anchorPath: path.join(dir, "tool-audit.anchor.json"),
|
|
349
|
+
keyPath: explicitKey && explicitKey.trim().length > 0 ? path.resolve(explicitKey) : path.join(dir, "tool-audit.key"),
|
|
350
|
+
lockPath: path.join(dir, "tool-audit.lock")
|
|
225
351
|
};
|
|
226
352
|
}
|
|
227
|
-
async function
|
|
353
|
+
async function withAuditLock(paths, work) {
|
|
354
|
+
await fs.mkdir(paths.dir, { recursive: true });
|
|
355
|
+
return withFileLock(paths.lockPath, work, LOCK_OPTIONS);
|
|
356
|
+
}
|
|
357
|
+
// ---------------------------------------------------------------------------
|
|
358
|
+
// Reading the log
|
|
359
|
+
async function readAuditLog(logPath) {
|
|
360
|
+
let raw = "";
|
|
361
|
+
try {
|
|
362
|
+
raw = await readFileWithRetry(logPath);
|
|
363
|
+
}
|
|
364
|
+
catch (err) {
|
|
365
|
+
if (err?.code !== "ENOENT")
|
|
366
|
+
throw err;
|
|
367
|
+
}
|
|
368
|
+
const result = { records: [], unparseableLines: [], lines: [] };
|
|
369
|
+
raw.split(/\r?\n/).forEach((text, index) => {
|
|
370
|
+
const line = text.trim();
|
|
371
|
+
if (!line)
|
|
372
|
+
return;
|
|
373
|
+
const record = parseRecordLine(line);
|
|
374
|
+
result.lines.push({ raw: line, lineNumber: index + 1, record });
|
|
375
|
+
if (record)
|
|
376
|
+
result.records.push(record);
|
|
377
|
+
else
|
|
378
|
+
result.unparseableLines.push(index + 1);
|
|
379
|
+
});
|
|
380
|
+
return result;
|
|
381
|
+
}
|
|
382
|
+
function parseRecordLine(line) {
|
|
383
|
+
if (!line)
|
|
384
|
+
return undefined;
|
|
228
385
|
try {
|
|
229
|
-
const
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
.
|
|
233
|
-
.
|
|
234
|
-
.
|
|
386
|
+
const value = JSON.parse(line);
|
|
387
|
+
if (value &&
|
|
388
|
+
typeof value === "object" &&
|
|
389
|
+
typeof value.hash === "string" &&
|
|
390
|
+
typeof value.prevHash === "string" &&
|
|
391
|
+
typeof value.at === "string") {
|
|
392
|
+
return value;
|
|
393
|
+
}
|
|
235
394
|
}
|
|
236
395
|
catch {
|
|
237
|
-
|
|
396
|
+
// fall through
|
|
397
|
+
}
|
|
398
|
+
return undefined;
|
|
399
|
+
}
|
|
400
|
+
/** Reads backwards from the end of the log to find the last valid record without loading the whole file. */
|
|
401
|
+
async function readTail(logPath) {
|
|
402
|
+
let handle;
|
|
403
|
+
try {
|
|
404
|
+
handle = await fs.open(logPath, "r");
|
|
405
|
+
}
|
|
406
|
+
catch (err) {
|
|
407
|
+
if (err?.code === "ENOENT")
|
|
408
|
+
return { size: 0, endsWithNewline: true };
|
|
409
|
+
throw err;
|
|
410
|
+
}
|
|
411
|
+
try {
|
|
412
|
+
const { size } = await handle.stat();
|
|
413
|
+
if (size === 0)
|
|
414
|
+
return { size, endsWithNewline: true };
|
|
415
|
+
const lastByte = Buffer.alloc(1);
|
|
416
|
+
await handle.read(lastByte, 0, 1, size - 1);
|
|
417
|
+
const endsWithNewline = lastByte[0] === 0x0a;
|
|
418
|
+
const chunkSize = 64 * 1024;
|
|
419
|
+
let position = size;
|
|
420
|
+
let carry = Buffer.alloc(0);
|
|
421
|
+
while (position > 0) {
|
|
422
|
+
const start = Math.max(0, position - chunkSize);
|
|
423
|
+
const chunk = Buffer.alloc(position - start);
|
|
424
|
+
await handle.read(chunk, 0, chunk.length, start);
|
|
425
|
+
position = start;
|
|
426
|
+
const buffer = Buffer.concat([chunk, carry]);
|
|
427
|
+
// Every byte range after a newline in the buffer is a complete line.
|
|
428
|
+
let end = buffer.length;
|
|
429
|
+
for (let i = buffer.length - 1; i >= 0; i--) {
|
|
430
|
+
if (buffer[i] !== 0x0a)
|
|
431
|
+
continue;
|
|
432
|
+
const record = parseRecordLine(buffer.subarray(i + 1, end).toString("utf8").trim());
|
|
433
|
+
if (record)
|
|
434
|
+
return { last: record, size, endsWithNewline };
|
|
435
|
+
end = i;
|
|
436
|
+
}
|
|
437
|
+
carry = buffer.subarray(0, end);
|
|
438
|
+
}
|
|
439
|
+
const record = parseRecordLine(carry.toString("utf8").trim());
|
|
440
|
+
return { last: record, size, endsWithNewline };
|
|
441
|
+
}
|
|
442
|
+
finally {
|
|
443
|
+
await handle.close();
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
async function appendWithRetry(filePath, content, retries = 8) {
|
|
447
|
+
for (let attempt = 0;; attempt++) {
|
|
448
|
+
try {
|
|
449
|
+
await fs.appendFile(filePath, content, "utf8");
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
catch (err) {
|
|
453
|
+
if (!isTransientFsError(err) || attempt >= retries)
|
|
454
|
+
throw err;
|
|
455
|
+
await sleep(20 * (attempt + 1));
|
|
456
|
+
}
|
|
238
457
|
}
|
|
239
458
|
}
|
|
240
459
|
async function readHeadHash(headPath) {
|
|
@@ -246,38 +465,160 @@ async function readHeadHash(headPath) {
|
|
|
246
465
|
return "";
|
|
247
466
|
}
|
|
248
467
|
}
|
|
249
|
-
async function
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
rebuilt.push(next);
|
|
278
|
-
previous = hash;
|
|
468
|
+
async function writeHead(paths, hash) {
|
|
469
|
+
await writeFileAtomic(paths.headPath, hash, { fsync: false });
|
|
470
|
+
}
|
|
471
|
+
async function readKey(paths) {
|
|
472
|
+
try {
|
|
473
|
+
const raw = (await readFileWithRetry(paths.keyPath)).trim();
|
|
474
|
+
if (!/^[0-9a-f]{64}$/i.test(raw))
|
|
475
|
+
return undefined;
|
|
476
|
+
const secret = Buffer.from(raw, "hex");
|
|
477
|
+
return { secret, id: sha256Hex(secret).slice(0, 16) };
|
|
478
|
+
}
|
|
479
|
+
catch {
|
|
480
|
+
return undefined;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
/** Must be called while holding the audit lock, so only one process creates the key. */
|
|
484
|
+
async function loadOrCreateKey(paths) {
|
|
485
|
+
const existing = await readKey(paths);
|
|
486
|
+
if (existing)
|
|
487
|
+
return existing;
|
|
488
|
+
let exists = true;
|
|
489
|
+
try {
|
|
490
|
+
await fs.access(paths.keyPath);
|
|
491
|
+
}
|
|
492
|
+
catch (err) {
|
|
493
|
+
if (err?.code !== "ENOENT")
|
|
494
|
+
throw err;
|
|
495
|
+
exists = false;
|
|
279
496
|
}
|
|
280
|
-
|
|
497
|
+
if (exists) {
|
|
498
|
+
throw new Error(`Audit key file ${paths.keyPath} does not contain a valid key; refusing to replace it.`);
|
|
499
|
+
}
|
|
500
|
+
const secret = crypto.randomBytes(32);
|
|
501
|
+
await fs.mkdir(path.dirname(paths.keyPath), { recursive: true });
|
|
502
|
+
await writeFileAtomic(paths.keyPath, secret.toString("hex"), { mode: 0o600 });
|
|
503
|
+
return { secret, id: sha256Hex(secret).slice(0, 16) };
|
|
504
|
+
}
|
|
505
|
+
// ---------------------------------------------------------------------------
|
|
506
|
+
// Anchor
|
|
507
|
+
async function readAnchor(paths, key) {
|
|
508
|
+
let raw;
|
|
509
|
+
try {
|
|
510
|
+
raw = await readFileWithRetry(paths.anchorPath);
|
|
511
|
+
}
|
|
512
|
+
catch (err) {
|
|
513
|
+
if (err?.code === "ENOENT")
|
|
514
|
+
return { exists: false };
|
|
515
|
+
return { exists: true, problem: `anchor file could not be read (${err?.code || err?.message})` };
|
|
516
|
+
}
|
|
517
|
+
let anchor;
|
|
518
|
+
try {
|
|
519
|
+
anchor = JSON.parse(raw);
|
|
520
|
+
}
|
|
521
|
+
catch {
|
|
522
|
+
return { exists: true, problem: "anchor file is not valid JSON" };
|
|
523
|
+
}
|
|
524
|
+
if (!anchor || typeof anchor !== "object" || typeof anchor.mac !== "string") {
|
|
525
|
+
return { exists: true, problem: "anchor file has an unexpected format" };
|
|
526
|
+
}
|
|
527
|
+
if (!Array.isArray(anchor.resets))
|
|
528
|
+
anchor.resets = [];
|
|
529
|
+
if (!key)
|
|
530
|
+
return { exists: true, anchor, problem: "audit key file is missing; the anchor cannot be verified" };
|
|
531
|
+
if (anchor.keyId !== key.id)
|
|
532
|
+
return { exists: true, anchor, problem: "anchor was signed with a different audit key" };
|
|
533
|
+
if (!timingSafeEqualHex(anchorMac(key.secret, stripMac(anchor)), anchor.mac)) {
|
|
534
|
+
return { exists: true, anchor, problem: "anchor signature mismatch (the anchor file was modified)" };
|
|
535
|
+
}
|
|
536
|
+
return { exists: true, anchor };
|
|
537
|
+
}
|
|
538
|
+
/**
|
|
539
|
+
* Returns a trusted anchor, creating it the first time (from the existing log) or
|
|
540
|
+
* rebuilding it when it is missing or invalid. A rebuild after keyed records were
|
|
541
|
+
* written is recorded in `resets`, which verify keeps reporting as a failure.
|
|
542
|
+
*/
|
|
543
|
+
async function ensureAnchor(paths, key) {
|
|
544
|
+
const current = await readAnchor(paths, key);
|
|
545
|
+
if (current.anchor && !current.problem)
|
|
546
|
+
return current.anchor;
|
|
547
|
+
const parsed = await readAuditLog(paths.logPath);
|
|
548
|
+
const records = parsed.records;
|
|
549
|
+
const first = records[0];
|
|
550
|
+
const last = records[records.length - 1];
|
|
551
|
+
const lastSeq = records.reduce((max, r) => (typeof r.seq === "number" && r.seq > max ? r.seq : max), 0);
|
|
552
|
+
const hasKeyed = records.some(r => r.version === 2);
|
|
553
|
+
const resets = current.anchor?.resets ? [...current.anchor.resets] : [];
|
|
554
|
+
let reason;
|
|
555
|
+
if (current.problem)
|
|
556
|
+
reason = current.problem;
|
|
557
|
+
else if (!current.exists && hasKeyed)
|
|
558
|
+
reason = "anchor file was missing";
|
|
559
|
+
if (reason) {
|
|
560
|
+
resets.push({ at: new Date().toISOString(), reason });
|
|
561
|
+
while (resets.length > MAX_ANCHOR_RESETS)
|
|
562
|
+
resets.shift();
|
|
563
|
+
}
|
|
564
|
+
return writeAnchor(paths, key, {
|
|
565
|
+
version: 1,
|
|
566
|
+
keyId: key.id,
|
|
567
|
+
records: records.length,
|
|
568
|
+
firstHash: first ? first.hash : "",
|
|
569
|
+
firstPrevHash: first ? first.prevHash : "",
|
|
570
|
+
firstSeq: first && typeof first.seq === "number" ? first.seq : null,
|
|
571
|
+
lastSeq,
|
|
572
|
+
lastHash: last ? last.hash : "",
|
|
573
|
+
updatedAt: new Date().toISOString(),
|
|
574
|
+
resets
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
async function writeAnchor(paths, key, anchor) {
|
|
578
|
+
const unsigned = toJsonValue({ ...anchor, keyId: key.id });
|
|
579
|
+
const signed = { ...unsigned, mac: anchorMac(key.secret, unsigned) };
|
|
580
|
+
await writeFileAtomic(paths.anchorPath, `${JSON.stringify(signed, null, 2)}\n`, { mode: 0o600 });
|
|
581
|
+
return signed;
|
|
582
|
+
}
|
|
583
|
+
function stripMac(anchor) {
|
|
584
|
+
const { mac: _mac, ...rest } = anchor;
|
|
585
|
+
return rest;
|
|
586
|
+
}
|
|
587
|
+
function anchorMac(secret, anchor) {
|
|
588
|
+
return crypto.createHmac("sha256", secret).update(`anchor|${canonicalJson(anchor)}`).digest("hex");
|
|
589
|
+
}
|
|
590
|
+
// ---------------------------------------------------------------------------
|
|
591
|
+
// Hashing
|
|
592
|
+
function keyedHash(secret, prevHash, base) {
|
|
593
|
+
return crypto.createHmac("sha256", secret).update(`${prevHash}|${canonicalJson(base)}`).digest("hex");
|
|
594
|
+
}
|
|
595
|
+
function legacyHash(record) {
|
|
596
|
+
const base = {
|
|
597
|
+
version: record.version,
|
|
598
|
+
id: record.id,
|
|
599
|
+
at: record.at,
|
|
600
|
+
tool: record.tool,
|
|
601
|
+
status: record.status,
|
|
602
|
+
connectionName: record.connectionName,
|
|
603
|
+
approve: record.approve,
|
|
604
|
+
durationMs: record.durationMs,
|
|
605
|
+
correlationId: record.correlationId,
|
|
606
|
+
argsHash: record.argsHash,
|
|
607
|
+
args: record.args,
|
|
608
|
+
error: record.error,
|
|
609
|
+
resultSummary: record.resultSummary,
|
|
610
|
+
prevHash: record.prevHash
|
|
611
|
+
};
|
|
612
|
+
return sha256Hex(`${record.prevHash}|${stableStringify(base)}`);
|
|
613
|
+
}
|
|
614
|
+
function timingSafeEqualHex(a, b) {
|
|
615
|
+
const left = Buffer.from(String(a), "utf8");
|
|
616
|
+
const right = Buffer.from(String(b), "utf8");
|
|
617
|
+
return left.length === right.length && crypto.timingSafeEqual(left, right);
|
|
618
|
+
}
|
|
619
|
+
/** Round-trips through JSON so the hashed value is exactly what is stored and read back. */
|
|
620
|
+
function toJsonValue(value) {
|
|
621
|
+
return JSON.parse(JSON.stringify(value));
|
|
281
622
|
}
|
|
282
623
|
function normalizeExportFormat(format) {
|
|
283
624
|
const normalized = String(format || "jsonl").trim().toLowerCase();
|
|
@@ -286,8 +627,12 @@ function normalizeExportFormat(format) {
|
|
|
286
627
|
}
|
|
287
628
|
throw new Error(`Unsupported export format: ${format}`);
|
|
288
629
|
}
|
|
289
|
-
function csvEscape(
|
|
290
|
-
|
|
630
|
+
export function csvEscape(input) {
|
|
631
|
+
let value = String(input ?? "");
|
|
632
|
+
// Spreadsheet apps run cells that start with these characters as formulas.
|
|
633
|
+
if (/^[=+\-@\t\r]/.test(value))
|
|
634
|
+
value = `'${value}`;
|
|
635
|
+
if (value.includes(",") || value.includes("\"") || value.includes("\n") || value.includes("\r")) {
|
|
291
636
|
return `"${value.replace(/"/g, "\"\"")}"`;
|
|
292
637
|
}
|
|
293
638
|
return value;
|
|
@@ -305,6 +650,7 @@ function clampNumber(value, fallback, min, max) {
|
|
|
305
650
|
function sha256Hex(value) {
|
|
306
651
|
return crypto.createHash("sha256").update(value).digest("hex");
|
|
307
652
|
}
|
|
653
|
+
/** Legacy canonical form (version 1 records and argsHash). Do not change: stored hashes depend on it. */
|
|
308
654
|
function stableStringify(value) {
|
|
309
655
|
if (value === null || typeof value !== "object")
|
|
310
656
|
return JSON.stringify(value);
|
|
@@ -314,6 +660,17 @@ function stableStringify(value) {
|
|
|
314
660
|
const props = entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`);
|
|
315
661
|
return `{${props.join(",")}}`;
|
|
316
662
|
}
|
|
663
|
+
/** Canonical form for keyed records: keys sorted by code unit, undefined members omitted. */
|
|
664
|
+
function canonicalJson(value) {
|
|
665
|
+
if (value === null || typeof value !== "object")
|
|
666
|
+
return JSON.stringify(value) ?? "null";
|
|
667
|
+
if (Array.isArray(value))
|
|
668
|
+
return `[${value.map(v => canonicalJson(v === undefined ? null : v)).join(",")}]`;
|
|
669
|
+
const entries = Object.entries(value)
|
|
670
|
+
.filter(([, v]) => v !== undefined)
|
|
671
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
|
672
|
+
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`;
|
|
673
|
+
}
|
|
317
674
|
function redactSensitive(input) {
|
|
318
675
|
const secrets = ["password", "passphrase", "secret", "token", "apikey", "apiKey", "authorization"];
|
|
319
676
|
if (Array.isArray(input)) {
|