@omnicross/daemon 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +1354 -363
- package/dist/cli.js +1336 -340
- package/dist/index.cjs +897 -107
- package/dist/index.d.cts +160 -43
- package/dist/index.d.ts +160 -43
- package/dist/index.js +895 -100
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/bootstrap.ts
|
|
2
|
-
import { accessSync, constants as fsConstants, existsSync as
|
|
2
|
+
import { accessSync, constants as fsConstants, existsSync as existsSync21 } from "fs";
|
|
3
3
|
import { DEFAULT_AUDIT_CONFIG } from "@omnicross/contracts/audit-types";
|
|
4
4
|
import { DEFAULT_BILLING_CONFIG } from "@omnicross/contracts/billing-types";
|
|
5
5
|
import { getGeminiCodeAssistProjectResolver } from "@omnicross/core/auth/GeminiCodeAssistProjectResolver";
|
|
@@ -33,7 +33,12 @@ import {
|
|
|
33
33
|
} from "@omnicross/core/provider-proxy";
|
|
34
34
|
import { routeLeaseDescriptorPort } from "@omnicross/cli-launcher";
|
|
35
35
|
import { KeySpendTracker } from "@omnicross/core/outbound-api";
|
|
36
|
-
import {
|
|
36
|
+
import {
|
|
37
|
+
__resetSharedUsageThroughputTrackerForTests,
|
|
38
|
+
getSharedUsageThroughputTracker as getSharedUsageThroughputTracker2,
|
|
39
|
+
PricingEngine,
|
|
40
|
+
UsageRecorder
|
|
41
|
+
} from "@omnicross/core/usage";
|
|
37
42
|
import {
|
|
38
43
|
setSubscriptionAccountService,
|
|
39
44
|
setSubscriptionProviderRegistry,
|
|
@@ -637,6 +642,37 @@ function handleAuditQuery(req, res, reader) {
|
|
|
637
642
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
638
643
|
res.end(JSON.stringify({ records }));
|
|
639
644
|
}
|
|
645
|
+
function handleAuditBodyQuery(req, res, reader) {
|
|
646
|
+
const url = new URL(req.url ?? "/", "http://localhost");
|
|
647
|
+
const id = url.searchParams.get("id")?.trim();
|
|
648
|
+
const sessionKey = url.searchParams.get("session")?.trim();
|
|
649
|
+
if (!id || !sessionKey) {
|
|
650
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
651
|
+
res.end(JSON.stringify({ error: "id and session are required" }));
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
const query2 = { id, sessionKey };
|
|
655
|
+
const ts = intParam(url.searchParams.get("ts"));
|
|
656
|
+
if (ts !== void 0) query2.ts = ts;
|
|
657
|
+
const body = reader ? reader(query2) : {};
|
|
658
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
659
|
+
res.end(JSON.stringify(body));
|
|
660
|
+
}
|
|
661
|
+
function handleAuditCompact(res, compact) {
|
|
662
|
+
if (!compact) {
|
|
663
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
664
|
+
res.end(JSON.stringify({ days: 0, shards: 0, savedBytes: 0 }));
|
|
665
|
+
return;
|
|
666
|
+
}
|
|
667
|
+
try {
|
|
668
|
+
const result = compact();
|
|
669
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
670
|
+
res.end(JSON.stringify(result));
|
|
671
|
+
} catch (error) {
|
|
672
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
673
|
+
res.end(JSON.stringify({ error: error instanceof Error ? error.message : "compaction failed" }));
|
|
674
|
+
}
|
|
675
|
+
}
|
|
640
676
|
async function handleAuditStatsQuery(req, res, reader) {
|
|
641
677
|
const url = new URL(req.url ?? "/", "http://localhost");
|
|
642
678
|
const query2 = {};
|
|
@@ -1264,6 +1300,35 @@ function validateApiKeys(raw) {
|
|
|
1264
1300
|
}
|
|
1265
1301
|
return out.length > 0 ? out : void 0;
|
|
1266
1302
|
}
|
|
1303
|
+
var THINK_LEVELS = /* @__PURE__ */ new Set([
|
|
1304
|
+
"none",
|
|
1305
|
+
"minimal",
|
|
1306
|
+
"low",
|
|
1307
|
+
"medium",
|
|
1308
|
+
"high",
|
|
1309
|
+
"xhigh",
|
|
1310
|
+
"max"
|
|
1311
|
+
]);
|
|
1312
|
+
function validateThinkingLevels(raw) {
|
|
1313
|
+
if (!Array.isArray(raw)) return void 0;
|
|
1314
|
+
if (!raw.every((level) => typeof level === "string" && THINK_LEVELS.has(level))) {
|
|
1315
|
+
return void 0;
|
|
1316
|
+
}
|
|
1317
|
+
return [...raw];
|
|
1318
|
+
}
|
|
1319
|
+
function validateThinkingTokenLimit(raw) {
|
|
1320
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return void 0;
|
|
1321
|
+
const bounds = raw;
|
|
1322
|
+
const min = bounds["min"];
|
|
1323
|
+
const max = bounds["max"];
|
|
1324
|
+
if (typeof min !== "number" || !Number.isFinite(min) || !Number.isInteger(min) || min < 0) {
|
|
1325
|
+
return void 0;
|
|
1326
|
+
}
|
|
1327
|
+
if (typeof max !== "number" || !Number.isFinite(max) || !Number.isInteger(max) || max < min) {
|
|
1328
|
+
return void 0;
|
|
1329
|
+
}
|
|
1330
|
+
return { min, max };
|
|
1331
|
+
}
|
|
1267
1332
|
function validateModelConfigs(raw) {
|
|
1268
1333
|
if (!Array.isArray(raw)) return void 0;
|
|
1269
1334
|
const out = [];
|
|
@@ -1278,6 +1343,10 @@ function validateModelConfigs(raw) {
|
|
|
1278
1343
|
if (typeof m["enabled"] === "boolean") entry.enabled = m["enabled"];
|
|
1279
1344
|
if (typeof m["vision"] === "boolean") entry.vision = m["vision"];
|
|
1280
1345
|
if (typeof m["reasoning"] === "boolean") entry.reasoning = m["reasoning"];
|
|
1346
|
+
const thinkingLevels = validateThinkingLevels(m["thinkingLevels"]);
|
|
1347
|
+
if (thinkingLevels) entry.thinkingLevels = thinkingLevels;
|
|
1348
|
+
const thinkingTokenLimit = validateThinkingTokenLimit(m["thinkingTokenLimit"]);
|
|
1349
|
+
if (thinkingTokenLimit) entry.thinkingTokenLimit = thinkingTokenLimit;
|
|
1281
1350
|
out.push(entry);
|
|
1282
1351
|
}
|
|
1283
1352
|
return out.length > 0 ? out : void 0;
|
|
@@ -3613,6 +3682,7 @@ function applyAuditConfig(config) {
|
|
|
3613
3682
|
} else {
|
|
3614
3683
|
setAuditCaptureConfig(null);
|
|
3615
3684
|
setAuditSink(null);
|
|
3685
|
+
writer?.reset();
|
|
3616
3686
|
if (sweeper) {
|
|
3617
3687
|
if (config) sweeper.configure(config);
|
|
3618
3688
|
sweeper.dispose();
|
|
@@ -3623,6 +3693,7 @@ function resetAuditRuntimeForTests() {
|
|
|
3623
3693
|
setAuditCaptureConfig(null);
|
|
3624
3694
|
setAuditSink(null);
|
|
3625
3695
|
setUpstreamTracePath(null);
|
|
3696
|
+
writer?.reset();
|
|
3626
3697
|
if (sweeper) sweeper.dispose();
|
|
3627
3698
|
writer = null;
|
|
3628
3699
|
sweeper = null;
|
|
@@ -4214,6 +4285,7 @@ async function handleImport(body, deps) {
|
|
|
4214
4285
|
}
|
|
4215
4286
|
|
|
4216
4287
|
// src/admin/usagePricing.ts
|
|
4288
|
+
import { getSharedUsageThroughputTracker } from "@omnicross/core/usage";
|
|
4217
4289
|
var err4 = (status, message) => ({
|
|
4218
4290
|
status,
|
|
4219
4291
|
body: { error: { type: "admin_api_error", message } }
|
|
@@ -4239,6 +4311,9 @@ var BUCKET_SPAN_MS = {
|
|
|
4239
4311
|
};
|
|
4240
4312
|
var MAX_TIMESERIES_BUCKETS = 2e3;
|
|
4241
4313
|
async function handleUsageGet(view, query2, deps) {
|
|
4314
|
+
if (view === "throughput") {
|
|
4315
|
+
return { status: 200, body: getSharedUsageThroughputTracker().snapshot() };
|
|
4316
|
+
}
|
|
4242
4317
|
const range = parseRange(query2);
|
|
4243
4318
|
if (!isRange(range)) return range;
|
|
4244
4319
|
switch (view) {
|
|
@@ -5044,6 +5119,12 @@ function parseModelConfigsInput(raw, existing) {
|
|
|
5044
5119
|
else if (typeof prior?.vision === "boolean") entry.vision = prior.vision;
|
|
5045
5120
|
if (typeof m["reasoning"] === "boolean") entry.reasoning = m["reasoning"];
|
|
5046
5121
|
else if (typeof prior?.reasoning === "boolean") entry.reasoning = prior.reasoning;
|
|
5122
|
+
const thinkingLevels = validateThinkingLevels(m["thinkingLevels"]);
|
|
5123
|
+
if (thinkingLevels) entry.thinkingLevels = thinkingLevels;
|
|
5124
|
+
else if (prior?.thinkingLevels) entry.thinkingLevels = prior.thinkingLevels;
|
|
5125
|
+
const thinkingTokenLimit = validateThinkingTokenLimit(m["thinkingTokenLimit"]);
|
|
5126
|
+
if (thinkingTokenLimit) entry.thinkingTokenLimit = thinkingTokenLimit;
|
|
5127
|
+
else if (prior?.thinkingTokenLimit) entry.thinkingTokenLimit = prior.thinkingTokenLimit;
|
|
5047
5128
|
out.push(entry);
|
|
5048
5129
|
}
|
|
5049
5130
|
return out.length > 0 ? out : void 0;
|
|
@@ -5964,7 +6045,7 @@ async function handleUiStatic(req, res, urlPath, uiDist) {
|
|
|
5964
6045
|
}
|
|
5965
6046
|
|
|
5966
6047
|
// src/admin/version.ts
|
|
5967
|
-
var DAEMON_VERSION = true ? "0.1.
|
|
6048
|
+
var DAEMON_VERSION = true ? "0.1.10" : "0.0.0-dev";
|
|
5968
6049
|
|
|
5969
6050
|
// src/admin/AdminServer.ts
|
|
5970
6051
|
var LOOPBACK_ADDR = "127.0.0.1";
|
|
@@ -6076,6 +6157,14 @@ var AdminServer = class {
|
|
|
6076
6157
|
await handleAuditStatsQuery(req, res, this.deps.auditStatsReader);
|
|
6077
6158
|
return;
|
|
6078
6159
|
}
|
|
6160
|
+
if (path2 === "/admin/api/audit/body" && (req.method === "GET" || req.method === "HEAD")) {
|
|
6161
|
+
handleAuditBodyQuery(req, res, this.deps.auditBodyReader);
|
|
6162
|
+
return;
|
|
6163
|
+
}
|
|
6164
|
+
if (path2 === "/admin/api/audit/compact" && req.method === "POST") {
|
|
6165
|
+
handleAuditCompact(res, this.deps.auditCompactor);
|
|
6166
|
+
return;
|
|
6167
|
+
}
|
|
6079
6168
|
if (path2 === "/admin/api/billing-status" && (req.method === "GET" || req.method === "HEAD")) {
|
|
6080
6169
|
handleBillingStatus(res, this.deps.billingStatusReader);
|
|
6081
6170
|
return;
|
|
@@ -6535,6 +6624,15 @@ function toLLMProvider(row) {
|
|
|
6535
6624
|
api_base_url: row.baseUrl,
|
|
6536
6625
|
api_key: resolvePreferredApiKey(row),
|
|
6537
6626
|
models,
|
|
6627
|
+
modelConfigs: row.modelConfigs?.map((config) => ({
|
|
6628
|
+
id: config.id,
|
|
6629
|
+
name: config.name ?? config.id,
|
|
6630
|
+
enabled: config.enabled ?? true,
|
|
6631
|
+
vision: config.vision,
|
|
6632
|
+
reasoning: config.reasoning,
|
|
6633
|
+
thinkingLevels: config.thinkingLevels,
|
|
6634
|
+
thinkingTokenLimit: config.thinkingTokenLimit
|
|
6635
|
+
})),
|
|
6538
6636
|
enabled: true,
|
|
6539
6637
|
transformer,
|
|
6540
6638
|
// app-parity-2 child 3: POPULATE the coding-plan endpoint onto the core
|
|
@@ -8818,18 +8916,214 @@ var AccountHealthSweeper = class {
|
|
|
8818
8916
|
};
|
|
8819
8917
|
|
|
8820
8918
|
// src/audit/AuditPruneSweeper.ts
|
|
8821
|
-
import { existsSync as
|
|
8822
|
-
import { join as
|
|
8919
|
+
import { createReadStream as createReadStream2, createWriteStream as createWriteStream2, existsSync as existsSync16, readdirSync as readdirSync3, rmSync as rmSync4, unlinkSync as unlinkSync4 } from "fs";
|
|
8920
|
+
import { join as join8 } from "path";
|
|
8921
|
+
import { pipeline } from "stream/promises";
|
|
8922
|
+
import { createGzip } from "zlib";
|
|
8923
|
+
|
|
8924
|
+
// src/audit/auditDictionary.ts
|
|
8925
|
+
import { existsSync as existsSync14, readdirSync, readFileSync as readFileSync14, renameSync as renameSync5, unlinkSync as unlinkSync3, writeFileSync as writeFileSync12 } from "fs";
|
|
8926
|
+
import { join as join6 } from "path";
|
|
8927
|
+
|
|
8928
|
+
// src/audit/auditBodyStore.ts
|
|
8929
|
+
var ANCHOR_EVERY = 64;
|
|
8930
|
+
var ANCHOR_DELTA_RATIO = 0.75;
|
|
8931
|
+
var DIVERGED_PREFIX_RATIO = 0.25;
|
|
8932
|
+
var MAX_BASE_CHARS = 4e6;
|
|
8933
|
+
var CACHE_BUDGET_CHARS = 16e6;
|
|
8934
|
+
var CACHE_MAX_SESSIONS = 32;
|
|
8935
|
+
var MAX_BASES_PER_SESSION = 4;
|
|
8936
|
+
var isHighSurrogate = (code) => code >= 55296 && code <= 56319;
|
|
8937
|
+
var isLowSurrogate = (code) => code >= 56320 && code <= 57343;
|
|
8938
|
+
function computeBodyDelta(prev, next) {
|
|
8939
|
+
const shortest = Math.min(prev.length, next.length);
|
|
8940
|
+
let pre = 0;
|
|
8941
|
+
while (pre < shortest && prev.charCodeAt(pre) === next.charCodeAt(pre)) pre += 1;
|
|
8942
|
+
if (pre > 0 && isHighSurrogate(prev.charCodeAt(pre - 1))) pre -= 1;
|
|
8943
|
+
const maxSuf = shortest - pre;
|
|
8944
|
+
let suf = 0;
|
|
8945
|
+
while (suf < maxSuf && prev.charCodeAt(prev.length - 1 - suf) === next.charCodeAt(next.length - 1 - suf)) {
|
|
8946
|
+
suf += 1;
|
|
8947
|
+
}
|
|
8948
|
+
if (suf > 0 && isLowSurrogate(prev.charCodeAt(prev.length - suf))) suf -= 1;
|
|
8949
|
+
return { pre, suf, ins: next.slice(pre, next.length - suf) };
|
|
8950
|
+
}
|
|
8951
|
+
function applyBodyDelta(prev, delta) {
|
|
8952
|
+
const head = delta.pre > 0 ? prev.slice(0, delta.pre) : "";
|
|
8953
|
+
const tail = delta.suf > 0 ? prev.slice(prev.length - delta.suf) : "";
|
|
8954
|
+
return head + delta.ins + tail;
|
|
8955
|
+
}
|
|
8956
|
+
var SessionBaseCache = class {
|
|
8957
|
+
constructor(maxSessions = CACHE_MAX_SESSIONS, budgetChars = CACHE_BUDGET_CHARS, maxBaseChars = MAX_BASE_CHARS, maxHeads = MAX_BASES_PER_SESSION) {
|
|
8958
|
+
this.maxSessions = maxSessions;
|
|
8959
|
+
this.budgetChars = budgetChars;
|
|
8960
|
+
this.maxBaseChars = maxBaseChars;
|
|
8961
|
+
this.maxHeads = maxHeads;
|
|
8962
|
+
}
|
|
8963
|
+
maxSessions;
|
|
8964
|
+
budgetChars;
|
|
8965
|
+
maxBaseChars;
|
|
8966
|
+
maxHeads;
|
|
8967
|
+
/** Session key to its retained heads, most-recent first. */
|
|
8968
|
+
entries = /* @__PURE__ */ new Map();
|
|
8969
|
+
chars = 0;
|
|
8970
|
+
/** Retained sessions (tests + diagnostics). */
|
|
8971
|
+
get size() {
|
|
8972
|
+
return this.entries.size;
|
|
8973
|
+
}
|
|
8974
|
+
/** A session's retained heads, most-recent first. Refreshes LRU recency. */
|
|
8975
|
+
get(sessionKey) {
|
|
8976
|
+
const found = this.entries.get(sessionKey);
|
|
8977
|
+
if (!found) return [];
|
|
8978
|
+
this.entries.delete(sessionKey);
|
|
8979
|
+
this.entries.set(sessionKey, found);
|
|
8980
|
+
return found;
|
|
8981
|
+
}
|
|
8982
|
+
/**
|
|
8983
|
+
* Retain `base` as a head of `sessionKey`.
|
|
8984
|
+
*
|
|
8985
|
+
* `replacesId` is the head this turn CONTINUES (its body was preserved whole
|
|
8986
|
+
* inside the new one), which is swapped out so a linear conversation keeps
|
|
8987
|
+
* exactly one head. Omit it when the turn started a distinct stream %s that
|
|
8988
|
+
* head is added alongside, which is what keeps a fork's branches apart.
|
|
8989
|
+
*
|
|
8990
|
+
* A body larger than `maxBaseChars` is not retained: the next turn anchors
|
|
8991
|
+
* rather than letting one oversized session monopolize the budget.
|
|
8992
|
+
*/
|
|
8993
|
+
remember(sessionKey, base, replacesId) {
|
|
8994
|
+
const heads = this.entries.get(sessionKey) ?? [];
|
|
8995
|
+
if (replacesId !== void 0) {
|
|
8996
|
+
const at = heads.findIndex((head) => head.lastId === replacesId);
|
|
8997
|
+
if (at >= 0) {
|
|
8998
|
+
this.chars -= heads[at].text.length;
|
|
8999
|
+
heads.splice(at, 1);
|
|
9000
|
+
}
|
|
9001
|
+
}
|
|
9002
|
+
if (base.text.length <= this.maxBaseChars) {
|
|
9003
|
+
heads.unshift(base);
|
|
9004
|
+
this.chars += base.text.length;
|
|
9005
|
+
}
|
|
9006
|
+
while (heads.length > this.maxHeads) {
|
|
9007
|
+
const dropped = heads.pop();
|
|
9008
|
+
if (dropped) this.chars -= dropped.text.length;
|
|
9009
|
+
}
|
|
9010
|
+
this.entries.delete(sessionKey);
|
|
9011
|
+
if (heads.length > 0) this.entries.set(sessionKey, heads);
|
|
9012
|
+
this.evict();
|
|
9013
|
+
}
|
|
9014
|
+
/** Drop a session's heads (eviction, or a write failure invalidating them). */
|
|
9015
|
+
forget(sessionKey) {
|
|
9016
|
+
const heads = this.entries.get(sessionKey);
|
|
9017
|
+
if (!heads) return;
|
|
9018
|
+
for (const head of heads) this.chars -= head.text.length;
|
|
9019
|
+
this.entries.delete(sessionKey);
|
|
9020
|
+
}
|
|
9021
|
+
/** Drop everything (writer disposal / test teardown). */
|
|
9022
|
+
clear() {
|
|
9023
|
+
this.entries.clear();
|
|
9024
|
+
this.chars = 0;
|
|
9025
|
+
}
|
|
9026
|
+
/** Evict least-recently-used sessions until both bounds hold. */
|
|
9027
|
+
evict() {
|
|
9028
|
+
while (this.entries.size > this.maxSessions || this.chars > this.budgetChars && this.entries.size > 1) {
|
|
9029
|
+
const oldest = this.entries.keys().next();
|
|
9030
|
+
if (oldest.done) break;
|
|
9031
|
+
this.forget(oldest.value);
|
|
9032
|
+
}
|
|
9033
|
+
}
|
|
9034
|
+
};
|
|
9035
|
+
function anchorReason(base, dayDir, delta, nextLength) {
|
|
9036
|
+
if (!base || !delta) return "new";
|
|
9037
|
+
if (base.dayDir !== dayDir) return "day";
|
|
9038
|
+
if (base.chainLen >= ANCHOR_EVERY) return "chain";
|
|
9039
|
+
if (delta.pre < base.text.length * DIVERGED_PREFIX_RATIO) return "diverged";
|
|
9040
|
+
if (delta.ins.length > nextLength * ANCHOR_DELTA_RATIO) return "costly";
|
|
9041
|
+
return null;
|
|
9042
|
+
}
|
|
9043
|
+
function pickBase(heads, next) {
|
|
9044
|
+
let best = null;
|
|
9045
|
+
for (const base of heads) {
|
|
9046
|
+
const delta = computeBodyDelta(base.text, next);
|
|
9047
|
+
if (best !== null && delta.ins.length >= best.delta.ins.length) continue;
|
|
9048
|
+
best = { base, delta, continues: delta.pre + delta.suf >= base.text.length };
|
|
9049
|
+
}
|
|
9050
|
+
return best;
|
|
9051
|
+
}
|
|
9052
|
+
function encodeBodyEntry(record, sessionKey, dayDir, cache) {
|
|
9053
|
+
const requestBody = record.requestBody;
|
|
9054
|
+
const responseBody = record.responseBody;
|
|
9055
|
+
if (requestBody === void 0 && responseBody === void 0) return null;
|
|
9056
|
+
const entry = { id: record.id, ts: record.ts };
|
|
9057
|
+
if (requestBody !== void 0) {
|
|
9058
|
+
const heads = cache.get(sessionKey);
|
|
9059
|
+
const sameDay = heads.filter((head) => head.dayDir === dayDir);
|
|
9060
|
+
const chosen = pickBase(sameDay, requestBody);
|
|
9061
|
+
const reason = heads.length > 0 && sameDay.length === 0 ? "day" : anchorReason(chosen?.base, dayDir, chosen?.delta ?? null, requestBody.length);
|
|
9062
|
+
if (reason !== null) {
|
|
9063
|
+
entry.req = { base: null, anchor: reason, pre: 0, suf: 0, ins: requestBody };
|
|
9064
|
+
cache.remember(
|
|
9065
|
+
sessionKey,
|
|
9066
|
+
{ dayDir, lastId: record.id, text: requestBody, chainLen: 0 },
|
|
9067
|
+
chosen?.continues === true ? chosen.base.lastId : void 0
|
|
9068
|
+
);
|
|
9069
|
+
} else {
|
|
9070
|
+
const picked = chosen;
|
|
9071
|
+
entry.req = {
|
|
9072
|
+
base: picked.base.lastId,
|
|
9073
|
+
...picked.continues ? { cont: true } : {},
|
|
9074
|
+
pre: picked.delta.pre,
|
|
9075
|
+
suf: picked.delta.suf,
|
|
9076
|
+
ins: picked.delta.ins
|
|
9077
|
+
};
|
|
9078
|
+
cache.remember(
|
|
9079
|
+
sessionKey,
|
|
9080
|
+
{ dayDir, lastId: record.id, text: requestBody, chainLen: picked.base.chainLen + 1 },
|
|
9081
|
+
picked.continues ? picked.base.lastId : void 0
|
|
9082
|
+
);
|
|
9083
|
+
}
|
|
9084
|
+
}
|
|
9085
|
+
if (responseBody !== void 0) entry.res = responseBody;
|
|
9086
|
+
return JSON.stringify(entry);
|
|
9087
|
+
}
|
|
9088
|
+
function isAuditBodyEntry(value) {
|
|
9089
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
9090
|
+
const entry = value;
|
|
9091
|
+
if (typeof entry["id"] !== "string" || typeof entry["ts"] !== "number") return false;
|
|
9092
|
+
if (entry["res"] !== void 0 && typeof entry["res"] !== "string") return false;
|
|
9093
|
+
const req = entry["req"];
|
|
9094
|
+
if (req === void 0) return true;
|
|
9095
|
+
if (!req || typeof req !== "object" || Array.isArray(req)) return false;
|
|
9096
|
+
const delta = req;
|
|
9097
|
+
if (delta["anchor"] !== void 0 && typeof delta["anchor"] !== "string") return false;
|
|
9098
|
+
if (delta["cont"] !== void 0 && typeof delta["cont"] !== "boolean") return false;
|
|
9099
|
+
return (delta["base"] === null || typeof delta["base"] === "string") && Number.isSafeInteger(delta["pre"]) && delta["pre"] >= 0 && Number.isSafeInteger(delta["suf"]) && delta["suf"] >= 0 && typeof delta["ins"] === "string";
|
|
9100
|
+
}
|
|
8823
9101
|
|
|
8824
9102
|
// src/audit/auditFiles.ts
|
|
8825
9103
|
var AUDIT_FILE_RE = /^audit-(\d{4})-(\d{2})-(\d{2})\.jsonl$/;
|
|
9104
|
+
var AUDIT_DAY_DIR_RE = /^audit-(\d{4})-(\d{2})-(\d{2})$/;
|
|
9105
|
+
var AUDIT_META_FILE = "meta.jsonl";
|
|
9106
|
+
var AUDIT_BODIES_DIR = "bodies";
|
|
9107
|
+
var AUDIT_SESSION_KEY_RE = /^[0-9a-f]{8,64}$/;
|
|
8826
9108
|
var pad22 = (n) => String(n).padStart(2, "0");
|
|
8827
|
-
|
|
9109
|
+
var localDateStamp = (ts) => {
|
|
8828
9110
|
const d = new Date(ts);
|
|
8829
|
-
return
|
|
9111
|
+
return `${d.getFullYear()}-${pad22(d.getMonth() + 1)}-${pad22(d.getDate())}`;
|
|
9112
|
+
};
|
|
9113
|
+
function auditDayDirName(ts) {
|
|
9114
|
+
return `audit-${localDateStamp(ts)}`;
|
|
9115
|
+
}
|
|
9116
|
+
function isAuditDayDir(name) {
|
|
9117
|
+
return AUDIT_DAY_DIR_RE.test(name);
|
|
9118
|
+
}
|
|
9119
|
+
function isSafeSessionKey(key) {
|
|
9120
|
+
return typeof key === "string" && AUDIT_SESSION_KEY_RE.test(key);
|
|
8830
9121
|
}
|
|
8831
|
-
function
|
|
8832
|
-
|
|
9122
|
+
function auditBodyFileName(sessionKey) {
|
|
9123
|
+
return `${sessionKey}.jsonl`;
|
|
9124
|
+
}
|
|
9125
|
+
function auditFileDateMs(name) {
|
|
9126
|
+
const m = AUDIT_FILE_RE.exec(name) ?? AUDIT_DAY_DIR_RE.exec(name);
|
|
8833
9127
|
if (!m) return null;
|
|
8834
9128
|
const year = Number(m[1]);
|
|
8835
9129
|
const month = Number(m[2]);
|
|
@@ -8841,16 +9135,153 @@ function auditFileDateMs(fileName) {
|
|
|
8841
9135
|
return d.getTime();
|
|
8842
9136
|
}
|
|
8843
9137
|
|
|
9138
|
+
// src/audit/auditDictionary.ts
|
|
9139
|
+
var AUDIT_DICT_FILE = "_dict.jsonl";
|
|
9140
|
+
var DICT_BASE_PREFIX = "dict:";
|
|
9141
|
+
var DICT_CANDIDATES = 3;
|
|
9142
|
+
var MIN_SAVING_RATIO = 0.2;
|
|
9143
|
+
function parseEntries(raw) {
|
|
9144
|
+
const entries = [];
|
|
9145
|
+
for (const line of raw.split("\n")) {
|
|
9146
|
+
const trimmed = line.trim();
|
|
9147
|
+
if (!trimmed) continue;
|
|
9148
|
+
try {
|
|
9149
|
+
const parsed = JSON.parse(trimmed);
|
|
9150
|
+
if (isAuditBodyEntry(parsed)) entries.push(parsed);
|
|
9151
|
+
} catch {
|
|
9152
|
+
}
|
|
9153
|
+
}
|
|
9154
|
+
return entries;
|
|
9155
|
+
}
|
|
9156
|
+
function plainShards(bodiesPath) {
|
|
9157
|
+
try {
|
|
9158
|
+
return readdirSync(bodiesPath).filter(
|
|
9159
|
+
(file) => file.endsWith(".jsonl") && isSafeSessionKey(file.slice(0, -".jsonl".length))
|
|
9160
|
+
);
|
|
9161
|
+
} catch {
|
|
9162
|
+
return [];
|
|
9163
|
+
}
|
|
9164
|
+
}
|
|
9165
|
+
function chooseDictionary(anchors) {
|
|
9166
|
+
if (anchors.length < 2) return null;
|
|
9167
|
+
const total = anchors.reduce((sum, body) => sum + body.length, 0);
|
|
9168
|
+
const candidates = [...anchors].sort((a, b) => b.length - a.length).slice(0, DICT_CANDIDATES);
|
|
9169
|
+
let best = null;
|
|
9170
|
+
for (const candidate of candidates) {
|
|
9171
|
+
let cost = candidate.length;
|
|
9172
|
+
for (const body of anchors) {
|
|
9173
|
+
cost += body === candidate ? 0 : computeBodyDelta(candidate, body).ins.length;
|
|
9174
|
+
}
|
|
9175
|
+
if (best === null || cost < best.cost) best = { body: candidate, cost };
|
|
9176
|
+
}
|
|
9177
|
+
if (best === null) return null;
|
|
9178
|
+
return total - best.cost >= total * MIN_SAVING_RATIO ? best.body : null;
|
|
9179
|
+
}
|
|
9180
|
+
var EMPTY = { shards: 0, anchors: 0, savedBytes: 0 };
|
|
9181
|
+
function compactAuditDay(dayPath) {
|
|
9182
|
+
const bodiesPath = join6(dayPath, AUDIT_BODIES_DIR);
|
|
9183
|
+
if (!existsSync14(bodiesPath)) return EMPTY;
|
|
9184
|
+
const dictPath = join6(bodiesPath, AUDIT_DICT_FILE);
|
|
9185
|
+
if (existsSync14(dictPath) || existsSync14(`${dictPath}.gz`)) return EMPTY;
|
|
9186
|
+
const shardFiles = plainShards(bodiesPath);
|
|
9187
|
+
if (shardFiles.length < 2) return EMPTY;
|
|
9188
|
+
const loaded = /* @__PURE__ */ new Map();
|
|
9189
|
+
const anchors = [];
|
|
9190
|
+
for (const file of shardFiles) {
|
|
9191
|
+
let entries;
|
|
9192
|
+
try {
|
|
9193
|
+
entries = parseEntries(readFileSync14(join6(bodiesPath, file), "utf8"));
|
|
9194
|
+
} catch {
|
|
9195
|
+
continue;
|
|
9196
|
+
}
|
|
9197
|
+
loaded.set(file, entries);
|
|
9198
|
+
for (const entry of entries) {
|
|
9199
|
+
if (entry.req && entry.req.base === null) anchors.push(entry.req.ins);
|
|
9200
|
+
}
|
|
9201
|
+
}
|
|
9202
|
+
if (anchors.length < 2) return EMPTY;
|
|
9203
|
+
const dictionary = chooseDictionary(anchors);
|
|
9204
|
+
if (dictionary === null) return EMPTY;
|
|
9205
|
+
const dictEntry = {
|
|
9206
|
+
id: `${DICT_BASE_PREFIX}0`,
|
|
9207
|
+
ts: 0,
|
|
9208
|
+
req: { base: null, anchor: "dict", pre: 0, suf: 0, ins: dictionary }
|
|
9209
|
+
};
|
|
9210
|
+
writeFileSync12(dictPath, JSON.stringify(dictEntry) + "\n", "utf8");
|
|
9211
|
+
const result = { shards: 0, anchors: 0, savedBytes: 0 };
|
|
9212
|
+
for (const [file, entries] of loaded) {
|
|
9213
|
+
let changed = false;
|
|
9214
|
+
let saved = 0;
|
|
9215
|
+
const rewritten = entries.map((entry) => {
|
|
9216
|
+
if (!entry.req || entry.req.base !== null || entry.req.ins === dictionary) return entry;
|
|
9217
|
+
const delta = computeBodyDelta(dictionary, entry.req.ins);
|
|
9218
|
+
if (delta.ins.length >= entry.req.ins.length) return entry;
|
|
9219
|
+
changed = true;
|
|
9220
|
+
saved += entry.req.ins.length - delta.ins.length;
|
|
9221
|
+
return {
|
|
9222
|
+
...entry,
|
|
9223
|
+
req: { base: dictEntry.id, pre: delta.pre, suf: delta.suf, ins: delta.ins }
|
|
9224
|
+
};
|
|
9225
|
+
});
|
|
9226
|
+
if (!changed) continue;
|
|
9227
|
+
const target = join6(bodiesPath, file);
|
|
9228
|
+
const temp = `${target}.compacting`;
|
|
9229
|
+
try {
|
|
9230
|
+
writeFileSync12(temp, rewritten.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8");
|
|
9231
|
+
renameSync5(temp, target);
|
|
9232
|
+
} catch {
|
|
9233
|
+
try {
|
|
9234
|
+
if (existsSync14(temp)) unlinkSync3(temp);
|
|
9235
|
+
} catch {
|
|
9236
|
+
}
|
|
9237
|
+
continue;
|
|
9238
|
+
}
|
|
9239
|
+
result.shards += 1;
|
|
9240
|
+
result.anchors += rewritten.filter((e) => e.req?.base === dictEntry.id).length;
|
|
9241
|
+
result.savedBytes += saved;
|
|
9242
|
+
}
|
|
9243
|
+
if (result.shards === 0) {
|
|
9244
|
+
try {
|
|
9245
|
+
unlinkSync3(dictPath);
|
|
9246
|
+
} catch {
|
|
9247
|
+
}
|
|
9248
|
+
}
|
|
9249
|
+
return result;
|
|
9250
|
+
}
|
|
9251
|
+
function compactAllClosedAuditDays(auditDir, now = Date.now) {
|
|
9252
|
+
const run = { days: 0, shards: 0, savedBytes: 0 };
|
|
9253
|
+
if (!existsSync14(auditDir)) return run;
|
|
9254
|
+
const today = auditDayDirName(now());
|
|
9255
|
+
let names;
|
|
9256
|
+
try {
|
|
9257
|
+
names = readdirSync(auditDir).filter(isAuditDayDir).sort();
|
|
9258
|
+
} catch {
|
|
9259
|
+
return run;
|
|
9260
|
+
}
|
|
9261
|
+
for (const name of names) {
|
|
9262
|
+
if (name === today) continue;
|
|
9263
|
+
try {
|
|
9264
|
+
const result = compactAuditDay(join6(auditDir, name));
|
|
9265
|
+
if (result.shards === 0) continue;
|
|
9266
|
+
run.days += 1;
|
|
9267
|
+
run.shards += result.shards;
|
|
9268
|
+
run.savedBytes += result.savedBytes;
|
|
9269
|
+
} catch {
|
|
9270
|
+
}
|
|
9271
|
+
}
|
|
9272
|
+
return run;
|
|
9273
|
+
}
|
|
9274
|
+
|
|
8844
9275
|
// src/audit/auditStats.ts
|
|
8845
9276
|
import {
|
|
8846
9277
|
createReadStream,
|
|
8847
|
-
existsSync as
|
|
8848
|
-
readFileSync as
|
|
8849
|
-
readdirSync,
|
|
9278
|
+
existsSync as existsSync15,
|
|
9279
|
+
readFileSync as readFileSync15,
|
|
9280
|
+
readdirSync as readdirSync2,
|
|
8850
9281
|
statSync as statSync3,
|
|
8851
|
-
writeFileSync as
|
|
9282
|
+
writeFileSync as writeFileSync13
|
|
8852
9283
|
} from "fs";
|
|
8853
|
-
import { basename, dirname as dirname7, join as
|
|
9284
|
+
import { basename, dirname as dirname7, join as join7 } from "path";
|
|
8854
9285
|
var SIDECAR_VERSION = 1;
|
|
8855
9286
|
var META_PREFIX_BYTES = 64 * 1024;
|
|
8856
9287
|
var READ_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
@@ -8858,9 +9289,9 @@ function auditStatsFileName(auditFile) {
|
|
|
8858
9289
|
return auditFile.replace(/\.jsonl$/, ".stats.json");
|
|
8859
9290
|
}
|
|
8860
9291
|
function readPersisted(path2) {
|
|
8861
|
-
if (!
|
|
9292
|
+
if (!existsSync15(path2)) return null;
|
|
8862
9293
|
try {
|
|
8863
|
-
const value = JSON.parse(
|
|
9294
|
+
const value = JSON.parse(readFileSync15(path2, "utf8"));
|
|
8864
9295
|
if (value.version !== SIDECAR_VERSION || !Number.isSafeInteger(value.auditBytes) || (value.auditBytes ?? -1) < 0 || !Number.isSafeInteger(value.requestCount) || (value.requestCount ?? -1) < 0 || !Number.isSafeInteger(value.errorCount) || (value.errorCount ?? -1) < 0 || (value.errorCount ?? 0) > (value.requestCount ?? -1) || typeof value.complete !== "boolean" || value.minTs !== null && !Number.isFinite(value.minTs) || value.maxTs !== null && !Number.isFinite(value.maxTs)) {
|
|
8865
9296
|
return null;
|
|
8866
9297
|
}
|
|
@@ -8870,7 +9301,7 @@ function readPersisted(path2) {
|
|
|
8870
9301
|
}
|
|
8871
9302
|
}
|
|
8872
9303
|
function updateAuditStatsAfterAppend(auditPath, auditBytesBefore, auditBytesAfter, record) {
|
|
8873
|
-
const statsPath =
|
|
9304
|
+
const statsPath = join7(dirname7(auditPath), auditStatsFileName(basename(auditPath)));
|
|
8874
9305
|
const previous = auditBytesBefore === 0 ? {
|
|
8875
9306
|
version: SIDECAR_VERSION,
|
|
8876
9307
|
auditBytes: 0,
|
|
@@ -8890,13 +9321,13 @@ function updateAuditStatsAfterAppend(auditPath, auditBytesBefore, auditBytesAfte
|
|
|
8890
9321
|
minTs: previous.minTs === null ? record.ts : Math.min(previous.minTs, record.ts),
|
|
8891
9322
|
maxTs: previous.maxTs === null ? record.ts : Math.max(previous.maxTs, record.ts)
|
|
8892
9323
|
};
|
|
8893
|
-
|
|
9324
|
+
writeFileSync13(statsPath, JSON.stringify(next), "utf8");
|
|
8894
9325
|
}
|
|
8895
9326
|
function queryCovers(stats, from, to) {
|
|
8896
9327
|
return stats.requestCount === 0 || stats.minTs !== null && stats.maxTs !== null && from <= stats.minTs && to >= stats.maxTs;
|
|
8897
9328
|
}
|
|
8898
|
-
function fileOverlaps(
|
|
8899
|
-
const start = auditFileDateMs(
|
|
9329
|
+
function fileOverlaps(name, from, to) {
|
|
9330
|
+
const start = auditFileDateMs(name);
|
|
8900
9331
|
if (start === null) return false;
|
|
8901
9332
|
const date = new Date(start);
|
|
8902
9333
|
const end = new Date(date.getFullYear(), date.getMonth(), date.getDate() + 1).getTime();
|
|
@@ -9001,21 +9432,27 @@ function mergePersistedStats(previous, appended) {
|
|
|
9001
9432
|
};
|
|
9002
9433
|
}
|
|
9003
9434
|
async function readAuditStats(auditDir, query2 = {}) {
|
|
9004
|
-
if (!
|
|
9435
|
+
if (!existsSync15(auditDir)) return { requestCount: 0, errorCount: 0, complete: true };
|
|
9005
9436
|
const from = typeof query2.from === "number" ? query2.from : -Infinity;
|
|
9006
9437
|
const to = typeof query2.to === "number" ? query2.to : Infinity;
|
|
9007
|
-
let
|
|
9438
|
+
let sources;
|
|
9008
9439
|
try {
|
|
9009
|
-
|
|
9440
|
+
sources = readdirSync2(auditDir).filter((name) => fileOverlaps(name, from, to)).sort().map(
|
|
9441
|
+
(name) => AUDIT_DAY_DIR_RE.test(name) ? {
|
|
9442
|
+
auditPath: join7(auditDir, name, AUDIT_META_FILE),
|
|
9443
|
+
statsPath: join7(auditDir, name, auditStatsFileName(AUDIT_META_FILE))
|
|
9444
|
+
} : {
|
|
9445
|
+
auditPath: join7(auditDir, name),
|
|
9446
|
+
statsPath: join7(auditDir, auditStatsFileName(name))
|
|
9447
|
+
}
|
|
9448
|
+
).filter((source) => existsSync15(source.auditPath));
|
|
9010
9449
|
} catch {
|
|
9011
9450
|
return { requestCount: 0, errorCount: 0, complete: false };
|
|
9012
9451
|
}
|
|
9013
9452
|
const total = { requestCount: 0, errorCount: 0, complete: true };
|
|
9014
|
-
for (const
|
|
9015
|
-
const auditPath = join6(auditDir, file);
|
|
9453
|
+
for (const { auditPath, statsPath } of sources) {
|
|
9016
9454
|
try {
|
|
9017
9455
|
const auditBytes = statSync3(auditPath).size;
|
|
9018
|
-
const statsPath = join6(auditDir, auditStatsFileName(file));
|
|
9019
9456
|
const persisted = readPersisted(statsPath);
|
|
9020
9457
|
if (persisted && persisted.complete && persisted.auditBytes === auditBytes && queryCovers(persisted, from, to)) {
|
|
9021
9458
|
total.requestCount += persisted.requestCount;
|
|
@@ -9034,7 +9471,7 @@ async function readAuditStats(auditDir, query2 = {}) {
|
|
|
9034
9471
|
total.errorCount += scanned.filtered.errorCount + (resumable?.errorCount ?? 0);
|
|
9035
9472
|
total.complete = total.complete && scanned.filtered.complete;
|
|
9036
9473
|
const current = resumable ? mergePersistedStats(resumable, scanned.all) : scanned.all;
|
|
9037
|
-
if (current.complete)
|
|
9474
|
+
if (current.complete) writeFileSync13(statsPath, JSON.stringify(current), "utf8");
|
|
9038
9475
|
} catch {
|
|
9039
9476
|
total.complete = false;
|
|
9040
9477
|
}
|
|
@@ -9045,6 +9482,7 @@ async function readAuditStats(auditDir, query2 = {}) {
|
|
|
9045
9482
|
// src/audit/AuditPruneSweeper.ts
|
|
9046
9483
|
var DAY_MS = 24 * 60 * 6e4;
|
|
9047
9484
|
var SWEEP_INTERVAL_MS2 = 60 * 6e4;
|
|
9485
|
+
var ARCHIVE_BATCH = 64;
|
|
9048
9486
|
var AuditPruneSweeper = class {
|
|
9049
9487
|
constructor(auditDir, logger, config, intervalMs = SWEEP_INTERVAL_MS2, now = Date.now) {
|
|
9050
9488
|
this.auditDir = auditDir;
|
|
@@ -9060,6 +9498,7 @@ var AuditPruneSweeper = class {
|
|
|
9060
9498
|
now;
|
|
9061
9499
|
timer = null;
|
|
9062
9500
|
sweeping = false;
|
|
9501
|
+
archiving = false;
|
|
9063
9502
|
/** Whether pruning is active (audit enabled). */
|
|
9064
9503
|
get enabled() {
|
|
9065
9504
|
return this.config.enabled;
|
|
@@ -9069,13 +9508,13 @@ var AuditPruneSweeper = class {
|
|
|
9069
9508
|
this.config = config;
|
|
9070
9509
|
}
|
|
9071
9510
|
/**
|
|
9072
|
-
* Arm the
|
|
9073
|
-
*
|
|
9511
|
+
* Arm the interval AND run one pass immediately (boot cleanup). No-op when
|
|
9512
|
+
* audit is disabled (zero regression). Idempotent.
|
|
9074
9513
|
*/
|
|
9075
9514
|
start() {
|
|
9076
9515
|
if (this.timer || !this.config.enabled) return;
|
|
9077
|
-
void this.
|
|
9078
|
-
this.timer = setInterval(() => void this.
|
|
9516
|
+
void this.runOnce();
|
|
9517
|
+
this.timer = setInterval(() => void this.runOnce(), this.intervalMs);
|
|
9079
9518
|
this.timer.unref?.();
|
|
9080
9519
|
}
|
|
9081
9520
|
/** Clear the interval (daemon shutdown / test teardown). Idempotent. */
|
|
@@ -9085,31 +9524,43 @@ var AuditPruneSweeper = class {
|
|
|
9085
9524
|
this.timer = null;
|
|
9086
9525
|
}
|
|
9087
9526
|
}
|
|
9527
|
+
/** Prune first, then archive — never spend CPU compressing a day about to go. */
|
|
9528
|
+
async runOnce() {
|
|
9529
|
+
await this.sweep();
|
|
9530
|
+
await this.archive();
|
|
9531
|
+
}
|
|
9532
|
+
/** The LOCAL-midnight epoch ms of the current day. */
|
|
9533
|
+
todayMidnight() {
|
|
9534
|
+
const today = new Date(this.now());
|
|
9535
|
+
return new Date(today.getFullYear(), today.getMonth(), today.getDate()).getTime();
|
|
9536
|
+
}
|
|
9088
9537
|
/**
|
|
9089
|
-
* One prune:
|
|
9090
|
-
*
|
|
9091
|
-
*
|
|
9538
|
+
* One prune: remove every audit day strictly OLDER than the retention cutoff
|
|
9539
|
+
* (`now - retentionDays` days, at local-midnight granularity). Exposed for
|
|
9540
|
+
* tests; never throws. Returns the number of days removed.
|
|
9092
9541
|
*/
|
|
9093
9542
|
async sweep() {
|
|
9094
9543
|
if (!this.config.enabled || this.sweeping) return 0;
|
|
9095
9544
|
this.sweeping = true;
|
|
9096
9545
|
try {
|
|
9097
|
-
if (!
|
|
9098
|
-
const
|
|
9099
|
-
const todayMidnight = new Date(today.getFullYear(), today.getMonth(), today.getDate()).getTime();
|
|
9100
|
-
const cutoff = todayMidnight - (this.config.retentionDays - 1) * DAY_MS;
|
|
9546
|
+
if (!existsSync16(this.auditDir)) return 0;
|
|
9547
|
+
const cutoff = this.todayMidnight() - (this.config.retentionDays - 1) * DAY_MS;
|
|
9101
9548
|
let removed = 0;
|
|
9102
|
-
for (const
|
|
9103
|
-
const dateMs = auditFileDateMs(
|
|
9549
|
+
for (const name of readdirSync3(this.auditDir)) {
|
|
9550
|
+
const dateMs = auditFileDateMs(name);
|
|
9104
9551
|
if (dateMs === null || dateMs >= cutoff) continue;
|
|
9105
9552
|
try {
|
|
9106
|
-
|
|
9553
|
+
if (isAuditDayDir(name)) {
|
|
9554
|
+
rmSync4(join8(this.auditDir, name), { recursive: true, force: true });
|
|
9555
|
+
} else {
|
|
9556
|
+
unlinkSync4(join8(this.auditDir, name));
|
|
9557
|
+
const statsPath = join8(this.auditDir, auditStatsFileName(name));
|
|
9558
|
+
if (existsSync16(statsPath)) unlinkSync4(statsPath);
|
|
9559
|
+
}
|
|
9107
9560
|
removed += 1;
|
|
9108
|
-
const statsPath = join7(this.auditDir, auditStatsFileName(file));
|
|
9109
|
-
if (existsSync15(statsPath)) unlinkSync3(statsPath);
|
|
9110
9561
|
} catch (error) {
|
|
9111
|
-
this.logger.warn("[AuditPruneSweeper] failed to
|
|
9112
|
-
|
|
9562
|
+
this.logger.warn("[AuditPruneSweeper] failed to remove expired audit day", {
|
|
9563
|
+
name,
|
|
9113
9564
|
error: error instanceof Error ? error.message : String(error)
|
|
9114
9565
|
});
|
|
9115
9566
|
}
|
|
@@ -9125,59 +9576,349 @@ var AuditPruneSweeper = class {
|
|
|
9125
9576
|
this.sweeping = false;
|
|
9126
9577
|
}
|
|
9127
9578
|
}
|
|
9579
|
+
/**
|
|
9580
|
+
* Gzip the body shards of every CLOSED day (anything before today). Today is
|
|
9581
|
+
* deliberately left as plain text so it stays greppable while it is the day you
|
|
9582
|
+
* are debugging. Exposed for tests; never throws. Returns shards compressed.
|
|
9583
|
+
*/
|
|
9584
|
+
async archive() {
|
|
9585
|
+
if (!this.config.enabled || this.archiving) return 0;
|
|
9586
|
+
this.archiving = true;
|
|
9587
|
+
try {
|
|
9588
|
+
if (!existsSync16(this.auditDir)) return 0;
|
|
9589
|
+
const today = this.todayMidnight();
|
|
9590
|
+
let compressed = 0;
|
|
9591
|
+
for (const name of readdirSync3(this.auditDir)) {
|
|
9592
|
+
if (compressed >= ARCHIVE_BATCH) break;
|
|
9593
|
+
const dateMs = auditFileDateMs(name);
|
|
9594
|
+
if (dateMs === null || dateMs >= today || !isAuditDayDir(name)) continue;
|
|
9595
|
+
const dayPath = join8(this.auditDir, name);
|
|
9596
|
+
try {
|
|
9597
|
+
const compaction = compactAuditDay(dayPath);
|
|
9598
|
+
if (compaction.shards > 0) {
|
|
9599
|
+
this.logger.debug("audit cross-session compaction complete", {
|
|
9600
|
+
day: name,
|
|
9601
|
+
shards: compaction.shards,
|
|
9602
|
+
anchors: compaction.anchors,
|
|
9603
|
+
savedBytes: compaction.savedBytes
|
|
9604
|
+
});
|
|
9605
|
+
}
|
|
9606
|
+
} catch (error) {
|
|
9607
|
+
this.logger.warn("[AuditPruneSweeper] cross-session compaction failed", {
|
|
9608
|
+
day: name,
|
|
9609
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9610
|
+
});
|
|
9611
|
+
}
|
|
9612
|
+
compressed += await this.archiveDay(
|
|
9613
|
+
join8(dayPath, AUDIT_BODIES_DIR),
|
|
9614
|
+
ARCHIVE_BATCH - compressed
|
|
9615
|
+
);
|
|
9616
|
+
}
|
|
9617
|
+
if (compressed > 0) this.logger.debug("audit archive complete", { compressed });
|
|
9618
|
+
return compressed;
|
|
9619
|
+
} catch (error) {
|
|
9620
|
+
this.logger.warn("audit archive pass failed", {
|
|
9621
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9622
|
+
});
|
|
9623
|
+
return 0;
|
|
9624
|
+
} finally {
|
|
9625
|
+
this.archiving = false;
|
|
9626
|
+
}
|
|
9627
|
+
}
|
|
9628
|
+
/** Gzip up to `budget` plain shards in one day's `bodies/` directory. */
|
|
9629
|
+
async archiveDay(bodiesPath, budget) {
|
|
9630
|
+
let shards;
|
|
9631
|
+
try {
|
|
9632
|
+
shards = readdirSync3(bodiesPath).filter((file) => file.endsWith(".jsonl"));
|
|
9633
|
+
} catch {
|
|
9634
|
+
return 0;
|
|
9635
|
+
}
|
|
9636
|
+
let compressed = 0;
|
|
9637
|
+
for (const shard of shards) {
|
|
9638
|
+
if (compressed >= budget) break;
|
|
9639
|
+
const source = join8(bodiesPath, shard);
|
|
9640
|
+
const target = `${source}.gz`;
|
|
9641
|
+
try {
|
|
9642
|
+
if (existsSync16(target)) {
|
|
9643
|
+
unlinkSync4(source);
|
|
9644
|
+
continue;
|
|
9645
|
+
}
|
|
9646
|
+
await pipeline(createReadStream2(source), createGzip(), createWriteStream2(target));
|
|
9647
|
+
unlinkSync4(source);
|
|
9648
|
+
compressed += 1;
|
|
9649
|
+
} catch (error) {
|
|
9650
|
+
try {
|
|
9651
|
+
if (existsSync16(target)) unlinkSync4(target);
|
|
9652
|
+
} catch {
|
|
9653
|
+
}
|
|
9654
|
+
this.logger.warn("[AuditPruneSweeper] failed to archive audit body shard", {
|
|
9655
|
+
shard,
|
|
9656
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9657
|
+
});
|
|
9658
|
+
}
|
|
9659
|
+
}
|
|
9660
|
+
return compressed;
|
|
9661
|
+
}
|
|
9128
9662
|
};
|
|
9129
9663
|
|
|
9130
|
-
// src/audit/
|
|
9131
|
-
import { existsSync as
|
|
9132
|
-
import { join as
|
|
9133
|
-
|
|
9134
|
-
|
|
9135
|
-
|
|
9136
|
-
|
|
9137
|
-
|
|
9664
|
+
// src/audit/auditBodyReader.ts
|
|
9665
|
+
import { existsSync as existsSync17, readdirSync as readdirSync4, readFileSync as readFileSync16, statSync as statSync5 } from "fs";
|
|
9666
|
+
import { join as join9 } from "path";
|
|
9667
|
+
import { gunzipSync } from "zlib";
|
|
9668
|
+
|
|
9669
|
+
// src/audit/auditJsonl.ts
|
|
9670
|
+
import { closeSync, openSync, readSync, statSync as statSync4 } from "fs";
|
|
9671
|
+
var WINDOW_BYTES = 1 << 20;
|
|
9672
|
+
var MAX_LINE_BYTES = 32 * 1024 * 1024;
|
|
9673
|
+
var NEWLINE = 10;
|
|
9674
|
+
function forEachLineFromTail(path2, onLine) {
|
|
9675
|
+
let fd;
|
|
9676
|
+
let end;
|
|
9677
|
+
try {
|
|
9678
|
+
end = statSync4(path2).size;
|
|
9679
|
+
if (end === 0) return;
|
|
9680
|
+
fd = openSync(path2, "r");
|
|
9681
|
+
} catch {
|
|
9682
|
+
return;
|
|
9683
|
+
}
|
|
9684
|
+
try {
|
|
9685
|
+
let carry = Buffer.alloc(0);
|
|
9686
|
+
while (end > 0) {
|
|
9687
|
+
const start = Math.max(0, end - WINDOW_BYTES);
|
|
9688
|
+
const window = Buffer.allocUnsafe(end - start);
|
|
9689
|
+
let read;
|
|
9690
|
+
try {
|
|
9691
|
+
read = readSync(fd, window, 0, end - start, start);
|
|
9692
|
+
} catch {
|
|
9693
|
+
return;
|
|
9694
|
+
}
|
|
9695
|
+
const chunk = carry.length > 0 ? Buffer.concat([window.subarray(0, read), carry]) : window.subarray(0, read);
|
|
9696
|
+
let lineEnd = chunk.length;
|
|
9697
|
+
let nl = lineEnd > 0 ? chunk.lastIndexOf(NEWLINE, lineEnd - 1) : -1;
|
|
9698
|
+
while (nl >= 0) {
|
|
9699
|
+
if (nl + 1 < lineEnd) {
|
|
9700
|
+
const line = chunk.subarray(nl + 1, lineEnd).toString("utf8").trim();
|
|
9701
|
+
if (line && onLine(line)) return;
|
|
9702
|
+
}
|
|
9703
|
+
lineEnd = nl;
|
|
9704
|
+
nl = lineEnd > 0 ? chunk.lastIndexOf(NEWLINE, lineEnd - 1) : -1;
|
|
9705
|
+
}
|
|
9706
|
+
if (start === 0) {
|
|
9707
|
+
if (lineEnd > 0) {
|
|
9708
|
+
const line = chunk.subarray(0, lineEnd).toString("utf8").trim();
|
|
9709
|
+
if (line) onLine(line);
|
|
9710
|
+
}
|
|
9711
|
+
return;
|
|
9712
|
+
}
|
|
9713
|
+
if (lineEnd > MAX_LINE_BYTES) return;
|
|
9714
|
+
carry = Buffer.from(chunk.subarray(0, lineEnd));
|
|
9715
|
+
end = start;
|
|
9716
|
+
}
|
|
9717
|
+
} finally {
|
|
9718
|
+
try {
|
|
9719
|
+
closeSync(fd);
|
|
9720
|
+
} catch {
|
|
9721
|
+
}
|
|
9722
|
+
}
|
|
9723
|
+
}
|
|
9724
|
+
|
|
9725
|
+
// src/audit/auditBodyReader.ts
|
|
9726
|
+
function candidateDays(auditDir, ts) {
|
|
9727
|
+
if (typeof ts === "number" && Number.isFinite(ts)) {
|
|
9728
|
+
const named = auditDayDirName(ts);
|
|
9729
|
+
if (existsSync17(join9(auditDir, named))) return [named];
|
|
9730
|
+
}
|
|
9138
9731
|
try {
|
|
9139
|
-
|
|
9732
|
+
return readdirSync4(auditDir).filter(isAuditDayDir).sort().reverse();
|
|
9140
9733
|
} catch {
|
|
9141
9734
|
return [];
|
|
9142
9735
|
}
|
|
9143
|
-
|
|
9144
|
-
|
|
9145
|
-
const
|
|
9146
|
-
|
|
9147
|
-
|
|
9148
|
-
|
|
9736
|
+
}
|
|
9737
|
+
function readShard(auditDir, day, sessionKey) {
|
|
9738
|
+
const base = join9(auditDir, day, AUDIT_BODIES_DIR, auditBodyFileName(sessionKey));
|
|
9739
|
+
try {
|
|
9740
|
+
if (existsSync17(base)) return readFileSync16(base, "utf8");
|
|
9741
|
+
const gz = `${base}.gz`;
|
|
9742
|
+
if (existsSync17(gz)) return gunzipSync(readFileSync16(gz)).toString("utf8");
|
|
9743
|
+
} catch {
|
|
9744
|
+
return null;
|
|
9745
|
+
}
|
|
9746
|
+
return null;
|
|
9747
|
+
}
|
|
9748
|
+
function parseShard(raw) {
|
|
9749
|
+
const entries = /* @__PURE__ */ new Map();
|
|
9750
|
+
for (const line of raw.split("\n")) {
|
|
9751
|
+
const trimmed = line.trim();
|
|
9752
|
+
if (!trimmed) continue;
|
|
9753
|
+
let parsed;
|
|
9149
9754
|
try {
|
|
9150
|
-
|
|
9755
|
+
parsed = JSON.parse(trimmed);
|
|
9151
9756
|
} catch {
|
|
9152
9757
|
continue;
|
|
9153
9758
|
}
|
|
9154
|
-
|
|
9155
|
-
|
|
9156
|
-
|
|
9157
|
-
|
|
9759
|
+
if (isAuditBodyEntry(parsed)) entries.set(parsed.id, parsed);
|
|
9760
|
+
}
|
|
9761
|
+
return entries;
|
|
9762
|
+
}
|
|
9763
|
+
function withDictionary(auditDir, day, entries) {
|
|
9764
|
+
let needed = false;
|
|
9765
|
+
for (const entry of entries.values()) {
|
|
9766
|
+
if (entry.req?.base?.startsWith(DICT_BASE_PREFIX)) {
|
|
9767
|
+
needed = true;
|
|
9768
|
+
break;
|
|
9769
|
+
}
|
|
9770
|
+
}
|
|
9771
|
+
if (!needed) return entries;
|
|
9772
|
+
const base = join9(auditDir, day, AUDIT_BODIES_DIR, AUDIT_DICT_FILE);
|
|
9773
|
+
let raw = null;
|
|
9774
|
+
try {
|
|
9775
|
+
if (existsSync17(base)) raw = readFileSync16(base, "utf8");
|
|
9776
|
+
else if (existsSync17(`${base}.gz`)) raw = gunzipSync(readFileSync16(`${base}.gz`)).toString("utf8");
|
|
9777
|
+
} catch {
|
|
9778
|
+
return entries;
|
|
9779
|
+
}
|
|
9780
|
+
if (raw === null) return entries;
|
|
9781
|
+
for (const [id, entry] of parseShard(raw)) entries.set(id, entry);
|
|
9782
|
+
return entries;
|
|
9783
|
+
}
|
|
9784
|
+
function reconstructRequest(entries, entry) {
|
|
9785
|
+
if (!entry.req) return void 0;
|
|
9786
|
+
const chain = [];
|
|
9787
|
+
const visited = /* @__PURE__ */ new Set();
|
|
9788
|
+
let cursor = entry;
|
|
9789
|
+
while (cursor?.req) {
|
|
9790
|
+
if (visited.has(cursor.id)) return void 0;
|
|
9791
|
+
visited.add(cursor.id);
|
|
9792
|
+
chain.push(cursor);
|
|
9793
|
+
if (cursor.req.base === null) break;
|
|
9794
|
+
cursor = entries.get(cursor.req.base);
|
|
9795
|
+
}
|
|
9796
|
+
const anchor = chain[chain.length - 1];
|
|
9797
|
+
if (!anchor?.req || anchor.req.base !== null) return void 0;
|
|
9798
|
+
let text = anchor.req.ins;
|
|
9799
|
+
for (let i = chain.length - 2; i >= 0; i -= 1) {
|
|
9800
|
+
const delta = chain[i]?.req;
|
|
9801
|
+
if (!delta) return void 0;
|
|
9802
|
+
if (delta.pre > text.length || delta.suf > text.length - delta.pre) return void 0;
|
|
9803
|
+
text = applyBodyDelta(text, delta);
|
|
9804
|
+
}
|
|
9805
|
+
return text;
|
|
9806
|
+
}
|
|
9807
|
+
function readAuditBody(auditDir, query2) {
|
|
9808
|
+
if (!isSafeSessionKey(query2.sessionKey) || !query2.id) return {};
|
|
9809
|
+
if (!existsSync17(auditDir)) return {};
|
|
9810
|
+
for (const day of candidateDays(auditDir, query2.ts)) {
|
|
9811
|
+
const raw = readShard(auditDir, day, query2.sessionKey);
|
|
9812
|
+
if (raw === null) continue;
|
|
9813
|
+
const entries = withDictionary(auditDir, day, parseShard(raw));
|
|
9814
|
+
const entry = entries.get(query2.id);
|
|
9815
|
+
if (!entry) continue;
|
|
9816
|
+
const result = {};
|
|
9817
|
+
const requestBody = reconstructRequest(entries, entry);
|
|
9818
|
+
if (requestBody !== void 0) result.requestBody = requestBody;
|
|
9819
|
+
if (entry.res !== void 0) result.responseBody = entry.res;
|
|
9820
|
+
return result;
|
|
9821
|
+
}
|
|
9822
|
+
return readLegacyInlineBody(auditDir, query2.id);
|
|
9823
|
+
}
|
|
9824
|
+
function readLegacyInlineBody(auditDir, id) {
|
|
9825
|
+
let names;
|
|
9826
|
+
try {
|
|
9827
|
+
names = readdirSync4(auditDir).filter((name) => AUDIT_FILE_RE.test(name)).sort().reverse();
|
|
9828
|
+
} catch {
|
|
9829
|
+
return {};
|
|
9830
|
+
}
|
|
9831
|
+
const needle = JSON.stringify(id);
|
|
9832
|
+
let found = {};
|
|
9833
|
+
for (const name of names) {
|
|
9834
|
+
forEachLineFromTail(join9(auditDir, name), (line) => {
|
|
9835
|
+
if (!line.includes(needle)) return false;
|
|
9836
|
+
let parsed;
|
|
9158
9837
|
try {
|
|
9159
|
-
|
|
9838
|
+
parsed = JSON.parse(line);
|
|
9160
9839
|
} catch {
|
|
9161
|
-
|
|
9840
|
+
return false;
|
|
9162
9841
|
}
|
|
9163
|
-
|
|
9164
|
-
if (
|
|
9165
|
-
|
|
9166
|
-
|
|
9842
|
+
const record = parsed;
|
|
9843
|
+
if (record.id !== id) return false;
|
|
9844
|
+
const result = {};
|
|
9845
|
+
if (typeof record.requestBody === "string") result.requestBody = record.requestBody;
|
|
9846
|
+
if (typeof record.responseBody === "string") result.responseBody = record.responseBody;
|
|
9847
|
+
found = result;
|
|
9848
|
+
return true;
|
|
9849
|
+
});
|
|
9850
|
+
if (found.requestBody !== void 0 || found.responseBody !== void 0) break;
|
|
9851
|
+
}
|
|
9852
|
+
return found;
|
|
9853
|
+
}
|
|
9854
|
+
|
|
9855
|
+
// src/audit/auditReader.ts
|
|
9856
|
+
import { existsSync as existsSync18, readdirSync as readdirSync5 } from "fs";
|
|
9857
|
+
import { join as join10 } from "path";
|
|
9858
|
+
var DEFAULT_LIMIT = 200;
|
|
9859
|
+
var MAX_LIMIT = 2e3;
|
|
9860
|
+
var OVERSCAN = 256;
|
|
9861
|
+
function daySources(auditDir) {
|
|
9862
|
+
let names;
|
|
9863
|
+
try {
|
|
9864
|
+
names = readdirSync5(auditDir);
|
|
9865
|
+
} catch {
|
|
9866
|
+
return [];
|
|
9867
|
+
}
|
|
9868
|
+
const sources = [];
|
|
9869
|
+
for (const name of names) {
|
|
9870
|
+
const dateMs = auditFileDateMs(name);
|
|
9871
|
+
if (dateMs === null) continue;
|
|
9872
|
+
if (AUDIT_DAY_DIR_RE.test(name)) {
|
|
9873
|
+
const path2 = join10(auditDir, name, AUDIT_META_FILE);
|
|
9874
|
+
if (existsSync18(path2)) sources.push({ path: path2, dateMs });
|
|
9875
|
+
} else if (AUDIT_FILE_RE.test(name)) {
|
|
9876
|
+
sources.push({ path: join10(auditDir, name), dateMs });
|
|
9167
9877
|
}
|
|
9168
9878
|
}
|
|
9169
|
-
|
|
9170
|
-
return matched.slice(0, limit);
|
|
9879
|
+
return sources.sort((a, b) => b.dateMs - a.dateMs);
|
|
9171
9880
|
}
|
|
9172
9881
|
function isAuditRecord(value) {
|
|
9173
9882
|
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
9174
9883
|
const r = value;
|
|
9175
9884
|
return typeof r["id"] === "string" && typeof r["ts"] === "number" && typeof r["method"] === "string" && typeof r["path"] === "string" && typeof r["status"] === "number";
|
|
9176
9885
|
}
|
|
9886
|
+
function toMetaRecord(record) {
|
|
9887
|
+
if (record.requestBody === void 0 && record.responseBody === void 0) return record;
|
|
9888
|
+
const { requestBody: _req, responseBody: _res, ...meta } = record;
|
|
9889
|
+
return { ...meta, hasBody: true };
|
|
9890
|
+
}
|
|
9891
|
+
function readAuditRecords(auditDir, query2 = {}) {
|
|
9892
|
+
if (!existsSync18(auditDir)) return [];
|
|
9893
|
+
const from = typeof query2.from === "number" ? query2.from : -Infinity;
|
|
9894
|
+
const to = typeof query2.to === "number" ? query2.to : Infinity;
|
|
9895
|
+
const limit = Math.min(MAX_LIMIT, Math.max(1, Math.trunc(query2.limit ?? DEFAULT_LIMIT)));
|
|
9896
|
+
const matched = [];
|
|
9897
|
+
for (const source of daySources(auditDir)) {
|
|
9898
|
+
const before = matched.length;
|
|
9899
|
+
forEachLineFromTail(source.path, (line) => {
|
|
9900
|
+
let parsed;
|
|
9901
|
+
try {
|
|
9902
|
+
parsed = JSON.parse(line);
|
|
9903
|
+
} catch {
|
|
9904
|
+
return false;
|
|
9905
|
+
}
|
|
9906
|
+
if (!isAuditRecord(parsed)) return false;
|
|
9907
|
+
if (query2.keyId !== void 0 && parsed.keyId !== query2.keyId) return false;
|
|
9908
|
+
if (query2.sessionKey !== void 0 && parsed.sessionKey !== query2.sessionKey) return false;
|
|
9909
|
+
if (parsed.ts < from || parsed.ts > to) return false;
|
|
9910
|
+
matched.push(toMetaRecord(parsed));
|
|
9911
|
+
return matched.length - before >= limit + OVERSCAN;
|
|
9912
|
+
});
|
|
9913
|
+
if (matched.length >= limit) break;
|
|
9914
|
+
}
|
|
9915
|
+
matched.sort((a, b) => b.ts - a.ts);
|
|
9916
|
+
return matched.slice(0, limit);
|
|
9917
|
+
}
|
|
9177
9918
|
|
|
9178
9919
|
// src/audit/AuditWriter.ts
|
|
9179
|
-
import { appendFileSync as appendFileSync2, existsSync as
|
|
9180
|
-
import { join as
|
|
9920
|
+
import { appendFileSync as appendFileSync2, existsSync as existsSync19, mkdirSync as mkdirSync5, statSync as statSync6 } from "fs";
|
|
9921
|
+
import { join as join11 } from "path";
|
|
9181
9922
|
var AuditWriter = class {
|
|
9182
9923
|
constructor(auditDir, logger, defer = (fn) => setTimeout(fn, 0)) {
|
|
9183
9924
|
this.auditDir = auditDir;
|
|
@@ -9187,10 +9928,13 @@ var AuditWriter = class {
|
|
|
9187
9928
|
auditDir;
|
|
9188
9929
|
logger;
|
|
9189
9930
|
defer;
|
|
9190
|
-
|
|
9931
|
+
/** Day directories already created this process (avoids an mkdir per record). */
|
|
9932
|
+
ensuredDirs = /* @__PURE__ */ new Set();
|
|
9933
|
+
/** Per-session encoding bases. Memory-only; a miss simply writes a full snapshot. */
|
|
9934
|
+
bases = new SessionBaseCache();
|
|
9191
9935
|
/**
|
|
9192
|
-
* Enqueue one record
|
|
9193
|
-
*
|
|
9936
|
+
* Enqueue one record. Returns IMMEDIATELY (fire-and-forget); every fs write and
|
|
9937
|
+
* the delta encoding happen on the deferred tick. A failure is logged, never thrown.
|
|
9194
9938
|
*/
|
|
9195
9939
|
record(record) {
|
|
9196
9940
|
this.defer(() => {
|
|
@@ -9203,25 +9947,41 @@ var AuditWriter = class {
|
|
|
9203
9947
|
}
|
|
9204
9948
|
});
|
|
9205
9949
|
}
|
|
9950
|
+
/** Drop all retained encoding bases (config reload / shutdown / test teardown). */
|
|
9951
|
+
reset() {
|
|
9952
|
+
this.bases.clear();
|
|
9953
|
+
this.ensuredDirs.clear();
|
|
9954
|
+
}
|
|
9206
9955
|
/**
|
|
9207
|
-
* Append synchronously — the awaitable form tests use to assert
|
|
9208
|
-
*
|
|
9209
|
-
* store's lazy file creation).
|
|
9956
|
+
* Append synchronously — the awaitable form tests use to assert a line landed.
|
|
9957
|
+
* Writes the metadata line first (canonical), then the body shard.
|
|
9210
9958
|
*/
|
|
9211
9959
|
appendNow(record) {
|
|
9212
|
-
|
|
9213
|
-
|
|
9214
|
-
|
|
9215
|
-
|
|
9216
|
-
|
|
9217
|
-
|
|
9218
|
-
|
|
9960
|
+
const dayDir = auditDayDirName(record.ts);
|
|
9961
|
+
const dayPath = this.ensureDir(join11(this.auditDir, dayDir));
|
|
9962
|
+
this.appendMeta(dayPath, record);
|
|
9963
|
+
this.appendBody(dayPath, dayDir, record);
|
|
9964
|
+
}
|
|
9965
|
+
/** Create a directory once per process and remember it. */
|
|
9966
|
+
ensureDir(path2) {
|
|
9967
|
+
if (!this.ensuredDirs.has(path2)) {
|
|
9968
|
+
mkdirSync5(path2, { recursive: true });
|
|
9969
|
+
this.ensuredDirs.add(path2);
|
|
9970
|
+
}
|
|
9971
|
+
return path2;
|
|
9972
|
+
}
|
|
9973
|
+
/** Write the body-free metadata line + refresh the exact-count sidecar. */
|
|
9974
|
+
appendMeta(dayPath, record) {
|
|
9975
|
+
const { requestBody: _req, responseBody: _res, ...meta } = record;
|
|
9976
|
+
const file = join11(dayPath, AUDIT_META_FILE);
|
|
9977
|
+
const line = JSON.stringify(meta) + "\n";
|
|
9978
|
+
const bytesBefore = existsSync19(file) ? statSync6(file).size : 0;
|
|
9219
9979
|
appendFileSync2(file, line, "utf8");
|
|
9220
9980
|
try {
|
|
9221
9981
|
updateAuditStatsAfterAppend(
|
|
9222
9982
|
file,
|
|
9223
|
-
|
|
9224
|
-
|
|
9983
|
+
bytesBefore,
|
|
9984
|
+
bytesBefore + Buffer.byteLength(line, "utf8"),
|
|
9225
9985
|
record
|
|
9226
9986
|
);
|
|
9227
9987
|
} catch (error) {
|
|
@@ -9230,12 +9990,39 @@ var AuditWriter = class {
|
|
|
9230
9990
|
});
|
|
9231
9991
|
}
|
|
9232
9992
|
}
|
|
9993
|
+
/**
|
|
9994
|
+
* Write the delta-encoded body shard for one record. A no-op when nothing was
|
|
9995
|
+
* captured or when the session key is missing/unsafe — in which case the body
|
|
9996
|
+
* is dropped rather than written to an unvalidated path.
|
|
9997
|
+
*/
|
|
9998
|
+
appendBody(dayPath, dayDir, record) {
|
|
9999
|
+
if (record.requestBody === void 0 && record.responseBody === void 0) return;
|
|
10000
|
+
const sessionKey = record.sessionKey;
|
|
10001
|
+
if (!isSafeSessionKey(sessionKey)) {
|
|
10002
|
+
this.logger.warn("[AuditWriter] dropping audit body with no usable session key", {
|
|
10003
|
+
id: record.id
|
|
10004
|
+
});
|
|
10005
|
+
return;
|
|
10006
|
+
}
|
|
10007
|
+
try {
|
|
10008
|
+
const line = encodeBodyEntry(record, sessionKey, dayDir, this.bases);
|
|
10009
|
+
if (line === null) return;
|
|
10010
|
+
const bodiesPath = this.ensureDir(join11(dayPath, AUDIT_BODIES_DIR));
|
|
10011
|
+
appendFileSync2(join11(bodiesPath, auditBodyFileName(sessionKey)), line + "\n", "utf8");
|
|
10012
|
+
} catch (error) {
|
|
10013
|
+
this.bases.forget(sessionKey);
|
|
10014
|
+
this.logger.warn("[AuditWriter] failed to append audit body shard", {
|
|
10015
|
+
id: record.id,
|
|
10016
|
+
error: error instanceof Error ? error.message : String(error)
|
|
10017
|
+
});
|
|
10018
|
+
}
|
|
10019
|
+
}
|
|
9233
10020
|
};
|
|
9234
10021
|
|
|
9235
10022
|
// src/billing/BillingPublisher.ts
|
|
9236
10023
|
import { appendFileSync as appendFileSync3, mkdirSync as mkdirSync6 } from "fs";
|
|
9237
10024
|
import { createHmac } from "crypto";
|
|
9238
|
-
import { join as
|
|
10025
|
+
import { join as join12 } from "path";
|
|
9239
10026
|
import { fetchUpstream as fetchUpstream5 } from "@omnicross/core/pipeline/upstreamFetch";
|
|
9240
10027
|
|
|
9241
10028
|
// src/billing/billingFiles.ts
|
|
@@ -9306,7 +10093,7 @@ var BillingPublisher = class {
|
|
|
9306
10093
|
*/
|
|
9307
10094
|
appendNow(event) {
|
|
9308
10095
|
this.ensureDir();
|
|
9309
|
-
const file =
|
|
10096
|
+
const file = join12(this.billingDir, billingFileName(event.ts));
|
|
9310
10097
|
appendFileSync3(file, JSON.stringify(event) + "\n", "utf8");
|
|
9311
10098
|
}
|
|
9312
10099
|
/**
|
|
@@ -9356,7 +10143,7 @@ var BillingPublisher = class {
|
|
|
9356
10143
|
markDelivered(event) {
|
|
9357
10144
|
try {
|
|
9358
10145
|
this.ensureDir();
|
|
9359
|
-
const file =
|
|
10146
|
+
const file = join12(this.billingDir, deliveredFileName(event.ts));
|
|
9360
10147
|
appendFileSync3(file, JSON.stringify({ id: event.id, deliveredAt: this.now() }) + "\n", "utf8");
|
|
9361
10148
|
} catch (error) {
|
|
9362
10149
|
this.logger.warn("[BillingPublisher] failed to append delivery marker", {
|
|
@@ -9372,14 +10159,14 @@ var BillingPublisher = class {
|
|
|
9372
10159
|
};
|
|
9373
10160
|
|
|
9374
10161
|
// src/billing/billingReader.ts
|
|
9375
|
-
import { existsSync as
|
|
9376
|
-
import { join as
|
|
10162
|
+
import { existsSync as existsSync20, readdirSync as readdirSync6, readFileSync as readFileSync17 } from "fs";
|
|
10163
|
+
import { join as join13 } from "path";
|
|
9377
10164
|
function readBillingLedger(billingDir) {
|
|
9378
10165
|
const view = { events: [], deliveredIds: /* @__PURE__ */ new Set() };
|
|
9379
|
-
if (!
|
|
10166
|
+
if (!existsSync20(billingDir)) return view;
|
|
9380
10167
|
let files;
|
|
9381
10168
|
try {
|
|
9382
|
-
files =
|
|
10169
|
+
files = readdirSync6(billingDir);
|
|
9383
10170
|
} catch {
|
|
9384
10171
|
return view;
|
|
9385
10172
|
}
|
|
@@ -9410,7 +10197,7 @@ function readBillingStatus(billingDir) {
|
|
|
9410
10197
|
function parseLines(dir, file) {
|
|
9411
10198
|
let raw;
|
|
9412
10199
|
try {
|
|
9413
|
-
raw =
|
|
10200
|
+
raw = readFileSync17(join13(dir, file), "utf8");
|
|
9414
10201
|
} catch {
|
|
9415
10202
|
return [];
|
|
9416
10203
|
}
|
|
@@ -9914,8 +10701,10 @@ function buildDaemon(config, paths) {
|
|
|
9914
10701
|
async (providerId, model) => await pricingEngine.getEntry(providerId, model) !== null
|
|
9915
10702
|
);
|
|
9916
10703
|
const keySpendTracker = new KeySpendTracker(usageEventStore);
|
|
10704
|
+
const usageThroughput = getSharedUsageThroughputTracker2();
|
|
9917
10705
|
const usageRecorder = new UsageRecorder(usageEventStore, pricingEngine, logger, {
|
|
9918
|
-
onRecord: (apiKeyId, costUsd, at) => keySpendTracker.add(apiKeyId, costUsd, at)
|
|
10706
|
+
onRecord: (apiKeyId, costUsd, at) => keySpendTracker.add(apiKeyId, costUsd, at),
|
|
10707
|
+
onEvent: (row, at) => usageThroughput.record(row, at)
|
|
9919
10708
|
});
|
|
9920
10709
|
const providerProxy = getProviderProxy({ llmConfig, apiKeyPool, usageRecorder });
|
|
9921
10710
|
const routeLeaseManager = new RouteLeaseManager(
|
|
@@ -10064,6 +10853,11 @@ function buildDaemon(config, paths) {
|
|
|
10064
10853
|
// NEVER unauth, NEVER on `/health`. Routed in `AdminServer` (not `adminApi.ts`).
|
|
10065
10854
|
auditReader: (query2) => readAuditRecords(auditDir, query2),
|
|
10066
10855
|
auditStatsReader: (query2) => readAuditStats(auditDir, query2),
|
|
10856
|
+
// audit-store-sharding: bodies live in per-session shards, so opening ONE
|
|
10857
|
+
// record's payload is a separate authed call that replays its delta chain.
|
|
10858
|
+
auditBodyReader: (query2) => readAuditBody(auditDir, query2),
|
|
10859
|
+
// audit-store-sharding D8: the manual counterpart to the daily pass.
|
|
10860
|
+
auditCompactor: () => compactAllClosedAuditDays(auditDir),
|
|
10067
10861
|
// billing-event-stream: the AUTHED `GET /admin/api/billing-status` returns the
|
|
10068
10862
|
// secret-free total/delivered/pending counts of the durable ledger.
|
|
10069
10863
|
billingStatusReader: () => readBillingStatus(billingDir)
|
|
@@ -10137,10 +10931,11 @@ function resetDaemonSingletonsForTests() {
|
|
|
10137
10931
|
__resetSharedIdentityStoreForTests();
|
|
10138
10932
|
__resetSharedAccountAllowanceStoreForTests();
|
|
10139
10933
|
__resetSharedAccountAllowanceSchedulingForTests();
|
|
10934
|
+
__resetSharedUsageThroughputTrackerForTests();
|
|
10140
10935
|
}
|
|
10141
10936
|
function isTokensStoreReadable(tokensPath) {
|
|
10142
10937
|
try {
|
|
10143
|
-
if (!
|
|
10938
|
+
if (!existsSync21(tokensPath)) return true;
|
|
10144
10939
|
accessSync(tokensPath, fsConstants.R_OK);
|
|
10145
10940
|
return true;
|
|
10146
10941
|
} catch {
|