@omnicross/daemon 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +1312 -369
- package/dist/cli.js +1309 -361
- package/dist/index.cjs +849 -107
- package/dist/index.d.cts +147 -38
- package/dist/index.d.ts +147 -38
- package/dist/index.js +847 -100
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -53,7 +53,7 @@ __export(src_exports, {
|
|
|
53
53
|
module.exports = __toCommonJS(src_exports);
|
|
54
54
|
|
|
55
55
|
// src/bootstrap.ts
|
|
56
|
-
var
|
|
56
|
+
var import_node_fs26 = require("fs");
|
|
57
57
|
var import_audit_types = require("@omnicross/contracts/audit-types");
|
|
58
58
|
var import_billing_types = require("@omnicross/contracts/billing-types");
|
|
59
59
|
var import_GeminiCodeAssistProjectResolver = require("@omnicross/core/auth/GeminiCodeAssistProjectResolver");
|
|
@@ -69,7 +69,7 @@ var import_gemini_code_assist_resolver = require("@omnicross/core/ports/gemini-c
|
|
|
69
69
|
var import_provider_proxy4 = require("@omnicross/core/provider-proxy");
|
|
70
70
|
var import_cli_launcher2 = require("@omnicross/cli-launcher");
|
|
71
71
|
var import_outbound_api5 = require("@omnicross/core/outbound-api");
|
|
72
|
-
var
|
|
72
|
+
var import_usage2 = require("@omnicross/core/usage");
|
|
73
73
|
var import_subscriptions4 = require("@omnicross/subscriptions");
|
|
74
74
|
|
|
75
75
|
// src/admin/accountsCodexOAuth.ts
|
|
@@ -650,6 +650,37 @@ function handleAuditQuery(req, res, reader) {
|
|
|
650
650
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
651
651
|
res.end(JSON.stringify({ records }));
|
|
652
652
|
}
|
|
653
|
+
function handleAuditBodyQuery(req, res, reader) {
|
|
654
|
+
const url = new URL(req.url ?? "/", "http://localhost");
|
|
655
|
+
const id = url.searchParams.get("id")?.trim();
|
|
656
|
+
const sessionKey = url.searchParams.get("session")?.trim();
|
|
657
|
+
if (!id || !sessionKey) {
|
|
658
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
659
|
+
res.end(JSON.stringify({ error: "id and session are required" }));
|
|
660
|
+
return;
|
|
661
|
+
}
|
|
662
|
+
const query2 = { id, sessionKey };
|
|
663
|
+
const ts = intParam(url.searchParams.get("ts"));
|
|
664
|
+
if (ts !== void 0) query2.ts = ts;
|
|
665
|
+
const body = reader ? reader(query2) : {};
|
|
666
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
667
|
+
res.end(JSON.stringify(body));
|
|
668
|
+
}
|
|
669
|
+
function handleAuditCompact(res, compact) {
|
|
670
|
+
if (!compact) {
|
|
671
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
672
|
+
res.end(JSON.stringify({ days: 0, shards: 0, savedBytes: 0 }));
|
|
673
|
+
return;
|
|
674
|
+
}
|
|
675
|
+
try {
|
|
676
|
+
const result = compact();
|
|
677
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
678
|
+
res.end(JSON.stringify(result));
|
|
679
|
+
} catch (error) {
|
|
680
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
681
|
+
res.end(JSON.stringify({ error: error instanceof Error ? error.message : "compaction failed" }));
|
|
682
|
+
}
|
|
683
|
+
}
|
|
653
684
|
async function handleAuditStatsQuery(req, res, reader) {
|
|
654
685
|
const url = new URL(req.url ?? "/", "http://localhost");
|
|
655
686
|
const query2 = {};
|
|
@@ -3618,6 +3649,7 @@ function applyAuditConfig(config) {
|
|
|
3618
3649
|
} else {
|
|
3619
3650
|
(0, import_auditSink.setAuditCaptureConfig)(null);
|
|
3620
3651
|
(0, import_auditSink.setAuditSink)(null);
|
|
3652
|
+
writer?.reset();
|
|
3621
3653
|
if (sweeper) {
|
|
3622
3654
|
if (config) sweeper.configure(config);
|
|
3623
3655
|
sweeper.dispose();
|
|
@@ -3628,6 +3660,7 @@ function resetAuditRuntimeForTests() {
|
|
|
3628
3660
|
(0, import_auditSink.setAuditCaptureConfig)(null);
|
|
3629
3661
|
(0, import_auditSink.setAuditSink)(null);
|
|
3630
3662
|
(0, import_upstreamTrace.setUpstreamTracePath)(null);
|
|
3663
|
+
writer?.reset();
|
|
3631
3664
|
if (sweeper) sweeper.dispose();
|
|
3632
3665
|
writer = null;
|
|
3633
3666
|
sweeper = null;
|
|
@@ -4219,6 +4252,7 @@ async function handleImport(body, deps) {
|
|
|
4219
4252
|
}
|
|
4220
4253
|
|
|
4221
4254
|
// src/admin/usagePricing.ts
|
|
4255
|
+
var import_usage = require("@omnicross/core/usage");
|
|
4222
4256
|
var err4 = (status, message) => ({
|
|
4223
4257
|
status,
|
|
4224
4258
|
body: { error: { type: "admin_api_error", message } }
|
|
@@ -4244,6 +4278,9 @@ var BUCKET_SPAN_MS = {
|
|
|
4244
4278
|
};
|
|
4245
4279
|
var MAX_TIMESERIES_BUCKETS = 2e3;
|
|
4246
4280
|
async function handleUsageGet(view, query2, deps) {
|
|
4281
|
+
if (view === "throughput") {
|
|
4282
|
+
return { status: 200, body: (0, import_usage.getSharedUsageThroughputTracker)().snapshot() };
|
|
4283
|
+
}
|
|
4247
4284
|
const range = parseRange(query2);
|
|
4248
4285
|
if (!isRange(range)) return range;
|
|
4249
4286
|
switch (view) {
|
|
@@ -5973,7 +6010,7 @@ async function handleUiStatic(req, res, urlPath, uiDist) {
|
|
|
5973
6010
|
}
|
|
5974
6011
|
|
|
5975
6012
|
// src/admin/version.ts
|
|
5976
|
-
var DAEMON_VERSION = true ? "0.1.
|
|
6013
|
+
var DAEMON_VERSION = true ? "0.1.10" : "0.0.0-dev";
|
|
5977
6014
|
|
|
5978
6015
|
// src/admin/AdminServer.ts
|
|
5979
6016
|
var LOOPBACK_ADDR = "127.0.0.1";
|
|
@@ -6085,6 +6122,14 @@ var AdminServer = class {
|
|
|
6085
6122
|
await handleAuditStatsQuery(req, res, this.deps.auditStatsReader);
|
|
6086
6123
|
return;
|
|
6087
6124
|
}
|
|
6125
|
+
if (path2 === "/admin/api/audit/body" && (req.method === "GET" || req.method === "HEAD")) {
|
|
6126
|
+
handleAuditBodyQuery(req, res, this.deps.auditBodyReader);
|
|
6127
|
+
return;
|
|
6128
|
+
}
|
|
6129
|
+
if (path2 === "/admin/api/audit/compact" && req.method === "POST") {
|
|
6130
|
+
handleAuditCompact(res, this.deps.auditCompactor);
|
|
6131
|
+
return;
|
|
6132
|
+
}
|
|
6088
6133
|
if (path2 === "/admin/api/billing-status" && (req.method === "GET" || req.method === "HEAD")) {
|
|
6089
6134
|
handleBillingStatus(res, this.deps.billingStatusReader);
|
|
6090
6135
|
return;
|
|
@@ -8826,18 +8871,214 @@ var AccountHealthSweeper = class {
|
|
|
8826
8871
|
};
|
|
8827
8872
|
|
|
8828
8873
|
// src/audit/AuditPruneSweeper.ts
|
|
8829
|
-
var
|
|
8830
|
-
var
|
|
8874
|
+
var import_node_fs19 = require("fs");
|
|
8875
|
+
var import_node_path12 = require("path");
|
|
8876
|
+
var import_promises2 = require("stream/promises");
|
|
8877
|
+
var import_node_zlib = require("zlib");
|
|
8878
|
+
|
|
8879
|
+
// src/audit/auditDictionary.ts
|
|
8880
|
+
var import_node_fs17 = require("fs");
|
|
8881
|
+
var import_node_path10 = require("path");
|
|
8882
|
+
|
|
8883
|
+
// src/audit/auditBodyStore.ts
|
|
8884
|
+
var ANCHOR_EVERY = 64;
|
|
8885
|
+
var ANCHOR_DELTA_RATIO = 0.75;
|
|
8886
|
+
var DIVERGED_PREFIX_RATIO = 0.25;
|
|
8887
|
+
var MAX_BASE_CHARS = 4e6;
|
|
8888
|
+
var CACHE_BUDGET_CHARS = 16e6;
|
|
8889
|
+
var CACHE_MAX_SESSIONS = 32;
|
|
8890
|
+
var MAX_BASES_PER_SESSION = 4;
|
|
8891
|
+
var isHighSurrogate = (code) => code >= 55296 && code <= 56319;
|
|
8892
|
+
var isLowSurrogate = (code) => code >= 56320 && code <= 57343;
|
|
8893
|
+
function computeBodyDelta(prev, next) {
|
|
8894
|
+
const shortest = Math.min(prev.length, next.length);
|
|
8895
|
+
let pre = 0;
|
|
8896
|
+
while (pre < shortest && prev.charCodeAt(pre) === next.charCodeAt(pre)) pre += 1;
|
|
8897
|
+
if (pre > 0 && isHighSurrogate(prev.charCodeAt(pre - 1))) pre -= 1;
|
|
8898
|
+
const maxSuf = shortest - pre;
|
|
8899
|
+
let suf = 0;
|
|
8900
|
+
while (suf < maxSuf && prev.charCodeAt(prev.length - 1 - suf) === next.charCodeAt(next.length - 1 - suf)) {
|
|
8901
|
+
suf += 1;
|
|
8902
|
+
}
|
|
8903
|
+
if (suf > 0 && isLowSurrogate(prev.charCodeAt(prev.length - suf))) suf -= 1;
|
|
8904
|
+
return { pre, suf, ins: next.slice(pre, next.length - suf) };
|
|
8905
|
+
}
|
|
8906
|
+
function applyBodyDelta(prev, delta) {
|
|
8907
|
+
const head = delta.pre > 0 ? prev.slice(0, delta.pre) : "";
|
|
8908
|
+
const tail = delta.suf > 0 ? prev.slice(prev.length - delta.suf) : "";
|
|
8909
|
+
return head + delta.ins + tail;
|
|
8910
|
+
}
|
|
8911
|
+
var SessionBaseCache = class {
|
|
8912
|
+
constructor(maxSessions = CACHE_MAX_SESSIONS, budgetChars = CACHE_BUDGET_CHARS, maxBaseChars = MAX_BASE_CHARS, maxHeads = MAX_BASES_PER_SESSION) {
|
|
8913
|
+
this.maxSessions = maxSessions;
|
|
8914
|
+
this.budgetChars = budgetChars;
|
|
8915
|
+
this.maxBaseChars = maxBaseChars;
|
|
8916
|
+
this.maxHeads = maxHeads;
|
|
8917
|
+
}
|
|
8918
|
+
maxSessions;
|
|
8919
|
+
budgetChars;
|
|
8920
|
+
maxBaseChars;
|
|
8921
|
+
maxHeads;
|
|
8922
|
+
/** Session key to its retained heads, most-recent first. */
|
|
8923
|
+
entries = /* @__PURE__ */ new Map();
|
|
8924
|
+
chars = 0;
|
|
8925
|
+
/** Retained sessions (tests + diagnostics). */
|
|
8926
|
+
get size() {
|
|
8927
|
+
return this.entries.size;
|
|
8928
|
+
}
|
|
8929
|
+
/** A session's retained heads, most-recent first. Refreshes LRU recency. */
|
|
8930
|
+
get(sessionKey) {
|
|
8931
|
+
const found = this.entries.get(sessionKey);
|
|
8932
|
+
if (!found) return [];
|
|
8933
|
+
this.entries.delete(sessionKey);
|
|
8934
|
+
this.entries.set(sessionKey, found);
|
|
8935
|
+
return found;
|
|
8936
|
+
}
|
|
8937
|
+
/**
|
|
8938
|
+
* Retain `base` as a head of `sessionKey`.
|
|
8939
|
+
*
|
|
8940
|
+
* `replacesId` is the head this turn CONTINUES (its body was preserved whole
|
|
8941
|
+
* inside the new one), which is swapped out so a linear conversation keeps
|
|
8942
|
+
* exactly one head. Omit it when the turn started a distinct stream %s that
|
|
8943
|
+
* head is added alongside, which is what keeps a fork's branches apart.
|
|
8944
|
+
*
|
|
8945
|
+
* A body larger than `maxBaseChars` is not retained: the next turn anchors
|
|
8946
|
+
* rather than letting one oversized session monopolize the budget.
|
|
8947
|
+
*/
|
|
8948
|
+
remember(sessionKey, base, replacesId) {
|
|
8949
|
+
const heads = this.entries.get(sessionKey) ?? [];
|
|
8950
|
+
if (replacesId !== void 0) {
|
|
8951
|
+
const at = heads.findIndex((head) => head.lastId === replacesId);
|
|
8952
|
+
if (at >= 0) {
|
|
8953
|
+
this.chars -= heads[at].text.length;
|
|
8954
|
+
heads.splice(at, 1);
|
|
8955
|
+
}
|
|
8956
|
+
}
|
|
8957
|
+
if (base.text.length <= this.maxBaseChars) {
|
|
8958
|
+
heads.unshift(base);
|
|
8959
|
+
this.chars += base.text.length;
|
|
8960
|
+
}
|
|
8961
|
+
while (heads.length > this.maxHeads) {
|
|
8962
|
+
const dropped = heads.pop();
|
|
8963
|
+
if (dropped) this.chars -= dropped.text.length;
|
|
8964
|
+
}
|
|
8965
|
+
this.entries.delete(sessionKey);
|
|
8966
|
+
if (heads.length > 0) this.entries.set(sessionKey, heads);
|
|
8967
|
+
this.evict();
|
|
8968
|
+
}
|
|
8969
|
+
/** Drop a session's heads (eviction, or a write failure invalidating them). */
|
|
8970
|
+
forget(sessionKey) {
|
|
8971
|
+
const heads = this.entries.get(sessionKey);
|
|
8972
|
+
if (!heads) return;
|
|
8973
|
+
for (const head of heads) this.chars -= head.text.length;
|
|
8974
|
+
this.entries.delete(sessionKey);
|
|
8975
|
+
}
|
|
8976
|
+
/** Drop everything (writer disposal / test teardown). */
|
|
8977
|
+
clear() {
|
|
8978
|
+
this.entries.clear();
|
|
8979
|
+
this.chars = 0;
|
|
8980
|
+
}
|
|
8981
|
+
/** Evict least-recently-used sessions until both bounds hold. */
|
|
8982
|
+
evict() {
|
|
8983
|
+
while (this.entries.size > this.maxSessions || this.chars > this.budgetChars && this.entries.size > 1) {
|
|
8984
|
+
const oldest = this.entries.keys().next();
|
|
8985
|
+
if (oldest.done) break;
|
|
8986
|
+
this.forget(oldest.value);
|
|
8987
|
+
}
|
|
8988
|
+
}
|
|
8989
|
+
};
|
|
8990
|
+
function anchorReason(base, dayDir, delta, nextLength) {
|
|
8991
|
+
if (!base || !delta) return "new";
|
|
8992
|
+
if (base.dayDir !== dayDir) return "day";
|
|
8993
|
+
if (base.chainLen >= ANCHOR_EVERY) return "chain";
|
|
8994
|
+
if (delta.pre < base.text.length * DIVERGED_PREFIX_RATIO) return "diverged";
|
|
8995
|
+
if (delta.ins.length > nextLength * ANCHOR_DELTA_RATIO) return "costly";
|
|
8996
|
+
return null;
|
|
8997
|
+
}
|
|
8998
|
+
function pickBase(heads, next) {
|
|
8999
|
+
let best = null;
|
|
9000
|
+
for (const base of heads) {
|
|
9001
|
+
const delta = computeBodyDelta(base.text, next);
|
|
9002
|
+
if (best !== null && delta.ins.length >= best.delta.ins.length) continue;
|
|
9003
|
+
best = { base, delta, continues: delta.pre + delta.suf >= base.text.length };
|
|
9004
|
+
}
|
|
9005
|
+
return best;
|
|
9006
|
+
}
|
|
9007
|
+
function encodeBodyEntry(record, sessionKey, dayDir, cache) {
|
|
9008
|
+
const requestBody = record.requestBody;
|
|
9009
|
+
const responseBody = record.responseBody;
|
|
9010
|
+
if (requestBody === void 0 && responseBody === void 0) return null;
|
|
9011
|
+
const entry = { id: record.id, ts: record.ts };
|
|
9012
|
+
if (requestBody !== void 0) {
|
|
9013
|
+
const heads = cache.get(sessionKey);
|
|
9014
|
+
const sameDay = heads.filter((head) => head.dayDir === dayDir);
|
|
9015
|
+
const chosen = pickBase(sameDay, requestBody);
|
|
9016
|
+
const reason = heads.length > 0 && sameDay.length === 0 ? "day" : anchorReason(chosen?.base, dayDir, chosen?.delta ?? null, requestBody.length);
|
|
9017
|
+
if (reason !== null) {
|
|
9018
|
+
entry.req = { base: null, anchor: reason, pre: 0, suf: 0, ins: requestBody };
|
|
9019
|
+
cache.remember(
|
|
9020
|
+
sessionKey,
|
|
9021
|
+
{ dayDir, lastId: record.id, text: requestBody, chainLen: 0 },
|
|
9022
|
+
chosen?.continues === true ? chosen.base.lastId : void 0
|
|
9023
|
+
);
|
|
9024
|
+
} else {
|
|
9025
|
+
const picked = chosen;
|
|
9026
|
+
entry.req = {
|
|
9027
|
+
base: picked.base.lastId,
|
|
9028
|
+
...picked.continues ? { cont: true } : {},
|
|
9029
|
+
pre: picked.delta.pre,
|
|
9030
|
+
suf: picked.delta.suf,
|
|
9031
|
+
ins: picked.delta.ins
|
|
9032
|
+
};
|
|
9033
|
+
cache.remember(
|
|
9034
|
+
sessionKey,
|
|
9035
|
+
{ dayDir, lastId: record.id, text: requestBody, chainLen: picked.base.chainLen + 1 },
|
|
9036
|
+
picked.continues ? picked.base.lastId : void 0
|
|
9037
|
+
);
|
|
9038
|
+
}
|
|
9039
|
+
}
|
|
9040
|
+
if (responseBody !== void 0) entry.res = responseBody;
|
|
9041
|
+
return JSON.stringify(entry);
|
|
9042
|
+
}
|
|
9043
|
+
function isAuditBodyEntry(value) {
|
|
9044
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
9045
|
+
const entry = value;
|
|
9046
|
+
if (typeof entry["id"] !== "string" || typeof entry["ts"] !== "number") return false;
|
|
9047
|
+
if (entry["res"] !== void 0 && typeof entry["res"] !== "string") return false;
|
|
9048
|
+
const req = entry["req"];
|
|
9049
|
+
if (req === void 0) return true;
|
|
9050
|
+
if (!req || typeof req !== "object" || Array.isArray(req)) return false;
|
|
9051
|
+
const delta = req;
|
|
9052
|
+
if (delta["anchor"] !== void 0 && typeof delta["anchor"] !== "string") return false;
|
|
9053
|
+
if (delta["cont"] !== void 0 && typeof delta["cont"] !== "boolean") return false;
|
|
9054
|
+
return (delta["base"] === null || typeof delta["base"] === "string") && Number.isSafeInteger(delta["pre"]) && delta["pre"] >= 0 && Number.isSafeInteger(delta["suf"]) && delta["suf"] >= 0 && typeof delta["ins"] === "string";
|
|
9055
|
+
}
|
|
8831
9056
|
|
|
8832
9057
|
// src/audit/auditFiles.ts
|
|
8833
9058
|
var AUDIT_FILE_RE = /^audit-(\d{4})-(\d{2})-(\d{2})\.jsonl$/;
|
|
9059
|
+
var AUDIT_DAY_DIR_RE = /^audit-(\d{4})-(\d{2})-(\d{2})$/;
|
|
9060
|
+
var AUDIT_META_FILE = "meta.jsonl";
|
|
9061
|
+
var AUDIT_BODIES_DIR = "bodies";
|
|
9062
|
+
var AUDIT_SESSION_KEY_RE = /^[0-9a-f]{8,64}$/;
|
|
8834
9063
|
var pad22 = (n) => String(n).padStart(2, "0");
|
|
8835
|
-
|
|
9064
|
+
var localDateStamp = (ts) => {
|
|
8836
9065
|
const d = new Date(ts);
|
|
8837
|
-
return
|
|
9066
|
+
return `${d.getFullYear()}-${pad22(d.getMonth() + 1)}-${pad22(d.getDate())}`;
|
|
9067
|
+
};
|
|
9068
|
+
function auditDayDirName(ts) {
|
|
9069
|
+
return `audit-${localDateStamp(ts)}`;
|
|
9070
|
+
}
|
|
9071
|
+
function isAuditDayDir(name) {
|
|
9072
|
+
return AUDIT_DAY_DIR_RE.test(name);
|
|
9073
|
+
}
|
|
9074
|
+
function isSafeSessionKey(key) {
|
|
9075
|
+
return typeof key === "string" && AUDIT_SESSION_KEY_RE.test(key);
|
|
9076
|
+
}
|
|
9077
|
+
function auditBodyFileName(sessionKey) {
|
|
9078
|
+
return `${sessionKey}.jsonl`;
|
|
8838
9079
|
}
|
|
8839
|
-
function auditFileDateMs(
|
|
8840
|
-
const m = AUDIT_FILE_RE.exec(
|
|
9080
|
+
function auditFileDateMs(name) {
|
|
9081
|
+
const m = AUDIT_FILE_RE.exec(name) ?? AUDIT_DAY_DIR_RE.exec(name);
|
|
8841
9082
|
if (!m) return null;
|
|
8842
9083
|
const year = Number(m[1]);
|
|
8843
9084
|
const month = Number(m[2]);
|
|
@@ -8849,9 +9090,146 @@ function auditFileDateMs(fileName) {
|
|
|
8849
9090
|
return d.getTime();
|
|
8850
9091
|
}
|
|
8851
9092
|
|
|
9093
|
+
// src/audit/auditDictionary.ts
|
|
9094
|
+
var AUDIT_DICT_FILE = "_dict.jsonl";
|
|
9095
|
+
var DICT_BASE_PREFIX = "dict:";
|
|
9096
|
+
var DICT_CANDIDATES = 3;
|
|
9097
|
+
var MIN_SAVING_RATIO = 0.2;
|
|
9098
|
+
function parseEntries(raw) {
|
|
9099
|
+
const entries = [];
|
|
9100
|
+
for (const line of raw.split("\n")) {
|
|
9101
|
+
const trimmed = line.trim();
|
|
9102
|
+
if (!trimmed) continue;
|
|
9103
|
+
try {
|
|
9104
|
+
const parsed = JSON.parse(trimmed);
|
|
9105
|
+
if (isAuditBodyEntry(parsed)) entries.push(parsed);
|
|
9106
|
+
} catch {
|
|
9107
|
+
}
|
|
9108
|
+
}
|
|
9109
|
+
return entries;
|
|
9110
|
+
}
|
|
9111
|
+
function plainShards(bodiesPath) {
|
|
9112
|
+
try {
|
|
9113
|
+
return (0, import_node_fs17.readdirSync)(bodiesPath).filter(
|
|
9114
|
+
(file) => file.endsWith(".jsonl") && isSafeSessionKey(file.slice(0, -".jsonl".length))
|
|
9115
|
+
);
|
|
9116
|
+
} catch {
|
|
9117
|
+
return [];
|
|
9118
|
+
}
|
|
9119
|
+
}
|
|
9120
|
+
function chooseDictionary(anchors) {
|
|
9121
|
+
if (anchors.length < 2) return null;
|
|
9122
|
+
const total = anchors.reduce((sum, body) => sum + body.length, 0);
|
|
9123
|
+
const candidates = [...anchors].sort((a, b) => b.length - a.length).slice(0, DICT_CANDIDATES);
|
|
9124
|
+
let best = null;
|
|
9125
|
+
for (const candidate of candidates) {
|
|
9126
|
+
let cost = candidate.length;
|
|
9127
|
+
for (const body of anchors) {
|
|
9128
|
+
cost += body === candidate ? 0 : computeBodyDelta(candidate, body).ins.length;
|
|
9129
|
+
}
|
|
9130
|
+
if (best === null || cost < best.cost) best = { body: candidate, cost };
|
|
9131
|
+
}
|
|
9132
|
+
if (best === null) return null;
|
|
9133
|
+
return total - best.cost >= total * MIN_SAVING_RATIO ? best.body : null;
|
|
9134
|
+
}
|
|
9135
|
+
var EMPTY = { shards: 0, anchors: 0, savedBytes: 0 };
|
|
9136
|
+
function compactAuditDay(dayPath) {
|
|
9137
|
+
const bodiesPath = (0, import_node_path10.join)(dayPath, AUDIT_BODIES_DIR);
|
|
9138
|
+
if (!(0, import_node_fs17.existsSync)(bodiesPath)) return EMPTY;
|
|
9139
|
+
const dictPath = (0, import_node_path10.join)(bodiesPath, AUDIT_DICT_FILE);
|
|
9140
|
+
if ((0, import_node_fs17.existsSync)(dictPath) || (0, import_node_fs17.existsSync)(`${dictPath}.gz`)) return EMPTY;
|
|
9141
|
+
const shardFiles = plainShards(bodiesPath);
|
|
9142
|
+
if (shardFiles.length < 2) return EMPTY;
|
|
9143
|
+
const loaded = /* @__PURE__ */ new Map();
|
|
9144
|
+
const anchors = [];
|
|
9145
|
+
for (const file of shardFiles) {
|
|
9146
|
+
let entries;
|
|
9147
|
+
try {
|
|
9148
|
+
entries = parseEntries((0, import_node_fs17.readFileSync)((0, import_node_path10.join)(bodiesPath, file), "utf8"));
|
|
9149
|
+
} catch {
|
|
9150
|
+
continue;
|
|
9151
|
+
}
|
|
9152
|
+
loaded.set(file, entries);
|
|
9153
|
+
for (const entry of entries) {
|
|
9154
|
+
if (entry.req && entry.req.base === null) anchors.push(entry.req.ins);
|
|
9155
|
+
}
|
|
9156
|
+
}
|
|
9157
|
+
if (anchors.length < 2) return EMPTY;
|
|
9158
|
+
const dictionary = chooseDictionary(anchors);
|
|
9159
|
+
if (dictionary === null) return EMPTY;
|
|
9160
|
+
const dictEntry = {
|
|
9161
|
+
id: `${DICT_BASE_PREFIX}0`,
|
|
9162
|
+
ts: 0,
|
|
9163
|
+
req: { base: null, anchor: "dict", pre: 0, suf: 0, ins: dictionary }
|
|
9164
|
+
};
|
|
9165
|
+
(0, import_node_fs17.writeFileSync)(dictPath, JSON.stringify(dictEntry) + "\n", "utf8");
|
|
9166
|
+
const result = { shards: 0, anchors: 0, savedBytes: 0 };
|
|
9167
|
+
for (const [file, entries] of loaded) {
|
|
9168
|
+
let changed = false;
|
|
9169
|
+
let saved = 0;
|
|
9170
|
+
const rewritten = entries.map((entry) => {
|
|
9171
|
+
if (!entry.req || entry.req.base !== null || entry.req.ins === dictionary) return entry;
|
|
9172
|
+
const delta = computeBodyDelta(dictionary, entry.req.ins);
|
|
9173
|
+
if (delta.ins.length >= entry.req.ins.length) return entry;
|
|
9174
|
+
changed = true;
|
|
9175
|
+
saved += entry.req.ins.length - delta.ins.length;
|
|
9176
|
+
return {
|
|
9177
|
+
...entry,
|
|
9178
|
+
req: { base: dictEntry.id, pre: delta.pre, suf: delta.suf, ins: delta.ins }
|
|
9179
|
+
};
|
|
9180
|
+
});
|
|
9181
|
+
if (!changed) continue;
|
|
9182
|
+
const target = (0, import_node_path10.join)(bodiesPath, file);
|
|
9183
|
+
const temp = `${target}.compacting`;
|
|
9184
|
+
try {
|
|
9185
|
+
(0, import_node_fs17.writeFileSync)(temp, rewritten.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8");
|
|
9186
|
+
(0, import_node_fs17.renameSync)(temp, target);
|
|
9187
|
+
} catch {
|
|
9188
|
+
try {
|
|
9189
|
+
if ((0, import_node_fs17.existsSync)(temp)) (0, import_node_fs17.unlinkSync)(temp);
|
|
9190
|
+
} catch {
|
|
9191
|
+
}
|
|
9192
|
+
continue;
|
|
9193
|
+
}
|
|
9194
|
+
result.shards += 1;
|
|
9195
|
+
result.anchors += rewritten.filter((e) => e.req?.base === dictEntry.id).length;
|
|
9196
|
+
result.savedBytes += saved;
|
|
9197
|
+
}
|
|
9198
|
+
if (result.shards === 0) {
|
|
9199
|
+
try {
|
|
9200
|
+
(0, import_node_fs17.unlinkSync)(dictPath);
|
|
9201
|
+
} catch {
|
|
9202
|
+
}
|
|
9203
|
+
}
|
|
9204
|
+
return result;
|
|
9205
|
+
}
|
|
9206
|
+
function compactAllClosedAuditDays(auditDir, now = Date.now) {
|
|
9207
|
+
const run = { days: 0, shards: 0, savedBytes: 0 };
|
|
9208
|
+
if (!(0, import_node_fs17.existsSync)(auditDir)) return run;
|
|
9209
|
+
const today = auditDayDirName(now());
|
|
9210
|
+
let names;
|
|
9211
|
+
try {
|
|
9212
|
+
names = (0, import_node_fs17.readdirSync)(auditDir).filter(isAuditDayDir).sort();
|
|
9213
|
+
} catch {
|
|
9214
|
+
return run;
|
|
9215
|
+
}
|
|
9216
|
+
for (const name of names) {
|
|
9217
|
+
if (name === today) continue;
|
|
9218
|
+
try {
|
|
9219
|
+
const result = compactAuditDay((0, import_node_path10.join)(auditDir, name));
|
|
9220
|
+
if (result.shards === 0) continue;
|
|
9221
|
+
run.days += 1;
|
|
9222
|
+
run.shards += result.shards;
|
|
9223
|
+
run.savedBytes += result.savedBytes;
|
|
9224
|
+
} catch {
|
|
9225
|
+
}
|
|
9226
|
+
}
|
|
9227
|
+
return run;
|
|
9228
|
+
}
|
|
9229
|
+
|
|
8852
9230
|
// src/audit/auditStats.ts
|
|
8853
|
-
var
|
|
8854
|
-
var
|
|
9231
|
+
var import_node_fs18 = require("fs");
|
|
9232
|
+
var import_node_path11 = require("path");
|
|
8855
9233
|
var SIDECAR_VERSION = 1;
|
|
8856
9234
|
var META_PREFIX_BYTES = 64 * 1024;
|
|
8857
9235
|
var READ_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
@@ -8859,9 +9237,9 @@ function auditStatsFileName(auditFile) {
|
|
|
8859
9237
|
return auditFile.replace(/\.jsonl$/, ".stats.json");
|
|
8860
9238
|
}
|
|
8861
9239
|
function readPersisted(path2) {
|
|
8862
|
-
if (!(0,
|
|
9240
|
+
if (!(0, import_node_fs18.existsSync)(path2)) return null;
|
|
8863
9241
|
try {
|
|
8864
|
-
const value = JSON.parse((0,
|
|
9242
|
+
const value = JSON.parse((0, import_node_fs18.readFileSync)(path2, "utf8"));
|
|
8865
9243
|
if (value.version !== SIDECAR_VERSION || !Number.isSafeInteger(value.auditBytes) || (value.auditBytes ?? -1) < 0 || !Number.isSafeInteger(value.requestCount) || (value.requestCount ?? -1) < 0 || !Number.isSafeInteger(value.errorCount) || (value.errorCount ?? -1) < 0 || (value.errorCount ?? 0) > (value.requestCount ?? -1) || typeof value.complete !== "boolean" || value.minTs !== null && !Number.isFinite(value.minTs) || value.maxTs !== null && !Number.isFinite(value.maxTs)) {
|
|
8866
9244
|
return null;
|
|
8867
9245
|
}
|
|
@@ -8871,7 +9249,7 @@ function readPersisted(path2) {
|
|
|
8871
9249
|
}
|
|
8872
9250
|
}
|
|
8873
9251
|
function updateAuditStatsAfterAppend(auditPath, auditBytesBefore, auditBytesAfter, record) {
|
|
8874
|
-
const statsPath = (0,
|
|
9252
|
+
const statsPath = (0, import_node_path11.join)((0, import_node_path11.dirname)(auditPath), auditStatsFileName((0, import_node_path11.basename)(auditPath)));
|
|
8875
9253
|
const previous = auditBytesBefore === 0 ? {
|
|
8876
9254
|
version: SIDECAR_VERSION,
|
|
8877
9255
|
auditBytes: 0,
|
|
@@ -8891,13 +9269,13 @@ function updateAuditStatsAfterAppend(auditPath, auditBytesBefore, auditBytesAfte
|
|
|
8891
9269
|
minTs: previous.minTs === null ? record.ts : Math.min(previous.minTs, record.ts),
|
|
8892
9270
|
maxTs: previous.maxTs === null ? record.ts : Math.max(previous.maxTs, record.ts)
|
|
8893
9271
|
};
|
|
8894
|
-
(0,
|
|
9272
|
+
(0, import_node_fs18.writeFileSync)(statsPath, JSON.stringify(next), "utf8");
|
|
8895
9273
|
}
|
|
8896
9274
|
function queryCovers(stats, from, to) {
|
|
8897
9275
|
return stats.requestCount === 0 || stats.minTs !== null && stats.maxTs !== null && from <= stats.minTs && to >= stats.maxTs;
|
|
8898
9276
|
}
|
|
8899
|
-
function fileOverlaps(
|
|
8900
|
-
const start = auditFileDateMs(
|
|
9277
|
+
function fileOverlaps(name, from, to) {
|
|
9278
|
+
const start = auditFileDateMs(name);
|
|
8901
9279
|
if (start === null) return false;
|
|
8902
9280
|
const date = new Date(start);
|
|
8903
9281
|
const end = new Date(date.getFullYear(), date.getMonth(), date.getDate() + 1).getTime();
|
|
@@ -8949,7 +9327,7 @@ async function scanAuditFile(auditPath, startByte, auditBytes, from, to) {
|
|
|
8949
9327
|
prefixTruncated = false;
|
|
8950
9328
|
};
|
|
8951
9329
|
if (auditBytes > startByte) {
|
|
8952
|
-
const stream = (0,
|
|
9330
|
+
const stream = (0, import_node_fs18.createReadStream)(auditPath, {
|
|
8953
9331
|
start: startByte,
|
|
8954
9332
|
end: auditBytes - 1,
|
|
8955
9333
|
highWaterMark: READ_CHUNK_BYTES
|
|
@@ -9002,21 +9380,27 @@ function mergePersistedStats(previous, appended) {
|
|
|
9002
9380
|
};
|
|
9003
9381
|
}
|
|
9004
9382
|
async function readAuditStats(auditDir, query2 = {}) {
|
|
9005
|
-
if (!(0,
|
|
9383
|
+
if (!(0, import_node_fs18.existsSync)(auditDir)) return { requestCount: 0, errorCount: 0, complete: true };
|
|
9006
9384
|
const from = typeof query2.from === "number" ? query2.from : -Infinity;
|
|
9007
9385
|
const to = typeof query2.to === "number" ? query2.to : Infinity;
|
|
9008
|
-
let
|
|
9386
|
+
let sources;
|
|
9009
9387
|
try {
|
|
9010
|
-
|
|
9388
|
+
sources = (0, import_node_fs18.readdirSync)(auditDir).filter((name) => fileOverlaps(name, from, to)).sort().map(
|
|
9389
|
+
(name) => AUDIT_DAY_DIR_RE.test(name) ? {
|
|
9390
|
+
auditPath: (0, import_node_path11.join)(auditDir, name, AUDIT_META_FILE),
|
|
9391
|
+
statsPath: (0, import_node_path11.join)(auditDir, name, auditStatsFileName(AUDIT_META_FILE))
|
|
9392
|
+
} : {
|
|
9393
|
+
auditPath: (0, import_node_path11.join)(auditDir, name),
|
|
9394
|
+
statsPath: (0, import_node_path11.join)(auditDir, auditStatsFileName(name))
|
|
9395
|
+
}
|
|
9396
|
+
).filter((source) => (0, import_node_fs18.existsSync)(source.auditPath));
|
|
9011
9397
|
} catch {
|
|
9012
9398
|
return { requestCount: 0, errorCount: 0, complete: false };
|
|
9013
9399
|
}
|
|
9014
9400
|
const total = { requestCount: 0, errorCount: 0, complete: true };
|
|
9015
|
-
for (const
|
|
9016
|
-
const auditPath = (0, import_node_path10.join)(auditDir, file);
|
|
9401
|
+
for (const { auditPath, statsPath } of sources) {
|
|
9017
9402
|
try {
|
|
9018
|
-
const auditBytes = (0,
|
|
9019
|
-
const statsPath = (0, import_node_path10.join)(auditDir, auditStatsFileName(file));
|
|
9403
|
+
const auditBytes = (0, import_node_fs18.statSync)(auditPath).size;
|
|
9020
9404
|
const persisted = readPersisted(statsPath);
|
|
9021
9405
|
if (persisted && persisted.complete && persisted.auditBytes === auditBytes && queryCovers(persisted, from, to)) {
|
|
9022
9406
|
total.requestCount += persisted.requestCount;
|
|
@@ -9035,7 +9419,7 @@ async function readAuditStats(auditDir, query2 = {}) {
|
|
|
9035
9419
|
total.errorCount += scanned.filtered.errorCount + (resumable?.errorCount ?? 0);
|
|
9036
9420
|
total.complete = total.complete && scanned.filtered.complete;
|
|
9037
9421
|
const current = resumable ? mergePersistedStats(resumable, scanned.all) : scanned.all;
|
|
9038
|
-
if (current.complete) (0,
|
|
9422
|
+
if (current.complete) (0, import_node_fs18.writeFileSync)(statsPath, JSON.stringify(current), "utf8");
|
|
9039
9423
|
} catch {
|
|
9040
9424
|
total.complete = false;
|
|
9041
9425
|
}
|
|
@@ -9046,6 +9430,7 @@ async function readAuditStats(auditDir, query2 = {}) {
|
|
|
9046
9430
|
// src/audit/AuditPruneSweeper.ts
|
|
9047
9431
|
var DAY_MS = 24 * 60 * 6e4;
|
|
9048
9432
|
var SWEEP_INTERVAL_MS2 = 60 * 6e4;
|
|
9433
|
+
var ARCHIVE_BATCH = 64;
|
|
9049
9434
|
var AuditPruneSweeper = class {
|
|
9050
9435
|
constructor(auditDir, logger, config, intervalMs = SWEEP_INTERVAL_MS2, now = Date.now) {
|
|
9051
9436
|
this.auditDir = auditDir;
|
|
@@ -9061,6 +9446,7 @@ var AuditPruneSweeper = class {
|
|
|
9061
9446
|
now;
|
|
9062
9447
|
timer = null;
|
|
9063
9448
|
sweeping = false;
|
|
9449
|
+
archiving = false;
|
|
9064
9450
|
/** Whether pruning is active (audit enabled). */
|
|
9065
9451
|
get enabled() {
|
|
9066
9452
|
return this.config.enabled;
|
|
@@ -9070,13 +9456,13 @@ var AuditPruneSweeper = class {
|
|
|
9070
9456
|
this.config = config;
|
|
9071
9457
|
}
|
|
9072
9458
|
/**
|
|
9073
|
-
* Arm the
|
|
9074
|
-
*
|
|
9459
|
+
* Arm the interval AND run one pass immediately (boot cleanup). No-op when
|
|
9460
|
+
* audit is disabled (zero regression). Idempotent.
|
|
9075
9461
|
*/
|
|
9076
9462
|
start() {
|
|
9077
9463
|
if (this.timer || !this.config.enabled) return;
|
|
9078
|
-
void this.
|
|
9079
|
-
this.timer = setInterval(() => void this.
|
|
9464
|
+
void this.runOnce();
|
|
9465
|
+
this.timer = setInterval(() => void this.runOnce(), this.intervalMs);
|
|
9080
9466
|
this.timer.unref?.();
|
|
9081
9467
|
}
|
|
9082
9468
|
/** Clear the interval (daemon shutdown / test teardown). Idempotent. */
|
|
@@ -9086,31 +9472,43 @@ var AuditPruneSweeper = class {
|
|
|
9086
9472
|
this.timer = null;
|
|
9087
9473
|
}
|
|
9088
9474
|
}
|
|
9475
|
+
/** Prune first, then archive — never spend CPU compressing a day about to go. */
|
|
9476
|
+
async runOnce() {
|
|
9477
|
+
await this.sweep();
|
|
9478
|
+
await this.archive();
|
|
9479
|
+
}
|
|
9480
|
+
/** The LOCAL-midnight epoch ms of the current day. */
|
|
9481
|
+
todayMidnight() {
|
|
9482
|
+
const today = new Date(this.now());
|
|
9483
|
+
return new Date(today.getFullYear(), today.getMonth(), today.getDate()).getTime();
|
|
9484
|
+
}
|
|
9089
9485
|
/**
|
|
9090
|
-
* One prune:
|
|
9091
|
-
*
|
|
9092
|
-
*
|
|
9486
|
+
* One prune: remove every audit day strictly OLDER than the retention cutoff
|
|
9487
|
+
* (`now - retentionDays` days, at local-midnight granularity). Exposed for
|
|
9488
|
+
* tests; never throws. Returns the number of days removed.
|
|
9093
9489
|
*/
|
|
9094
9490
|
async sweep() {
|
|
9095
9491
|
if (!this.config.enabled || this.sweeping) return 0;
|
|
9096
9492
|
this.sweeping = true;
|
|
9097
9493
|
try {
|
|
9098
|
-
if (!(0,
|
|
9099
|
-
const
|
|
9100
|
-
const todayMidnight = new Date(today.getFullYear(), today.getMonth(), today.getDate()).getTime();
|
|
9101
|
-
const cutoff = todayMidnight - (this.config.retentionDays - 1) * DAY_MS;
|
|
9494
|
+
if (!(0, import_node_fs19.existsSync)(this.auditDir)) return 0;
|
|
9495
|
+
const cutoff = this.todayMidnight() - (this.config.retentionDays - 1) * DAY_MS;
|
|
9102
9496
|
let removed = 0;
|
|
9103
|
-
for (const
|
|
9104
|
-
const dateMs = auditFileDateMs(
|
|
9497
|
+
for (const name of (0, import_node_fs19.readdirSync)(this.auditDir)) {
|
|
9498
|
+
const dateMs = auditFileDateMs(name);
|
|
9105
9499
|
if (dateMs === null || dateMs >= cutoff) continue;
|
|
9106
9500
|
try {
|
|
9107
|
-
|
|
9501
|
+
if (isAuditDayDir(name)) {
|
|
9502
|
+
(0, import_node_fs19.rmSync)((0, import_node_path12.join)(this.auditDir, name), { recursive: true, force: true });
|
|
9503
|
+
} else {
|
|
9504
|
+
(0, import_node_fs19.unlinkSync)((0, import_node_path12.join)(this.auditDir, name));
|
|
9505
|
+
const statsPath = (0, import_node_path12.join)(this.auditDir, auditStatsFileName(name));
|
|
9506
|
+
if ((0, import_node_fs19.existsSync)(statsPath)) (0, import_node_fs19.unlinkSync)(statsPath);
|
|
9507
|
+
}
|
|
9108
9508
|
removed += 1;
|
|
9109
|
-
const statsPath = (0, import_node_path11.join)(this.auditDir, auditStatsFileName(file));
|
|
9110
|
-
if ((0, import_node_fs18.existsSync)(statsPath)) (0, import_node_fs18.unlinkSync)(statsPath);
|
|
9111
9509
|
} catch (error) {
|
|
9112
|
-
this.logger.warn("[AuditPruneSweeper] failed to
|
|
9113
|
-
|
|
9510
|
+
this.logger.warn("[AuditPruneSweeper] failed to remove expired audit day", {
|
|
9511
|
+
name,
|
|
9114
9512
|
error: error instanceof Error ? error.message : String(error)
|
|
9115
9513
|
});
|
|
9116
9514
|
}
|
|
@@ -9126,59 +9524,349 @@ var AuditPruneSweeper = class {
|
|
|
9126
9524
|
this.sweeping = false;
|
|
9127
9525
|
}
|
|
9128
9526
|
}
|
|
9527
|
+
/**
|
|
9528
|
+
* Gzip the body shards of every CLOSED day (anything before today). Today is
|
|
9529
|
+
* deliberately left as plain text so it stays greppable while it is the day you
|
|
9530
|
+
* are debugging. Exposed for tests; never throws. Returns shards compressed.
|
|
9531
|
+
*/
|
|
9532
|
+
async archive() {
|
|
9533
|
+
if (!this.config.enabled || this.archiving) return 0;
|
|
9534
|
+
this.archiving = true;
|
|
9535
|
+
try {
|
|
9536
|
+
if (!(0, import_node_fs19.existsSync)(this.auditDir)) return 0;
|
|
9537
|
+
const today = this.todayMidnight();
|
|
9538
|
+
let compressed = 0;
|
|
9539
|
+
for (const name of (0, import_node_fs19.readdirSync)(this.auditDir)) {
|
|
9540
|
+
if (compressed >= ARCHIVE_BATCH) break;
|
|
9541
|
+
const dateMs = auditFileDateMs(name);
|
|
9542
|
+
if (dateMs === null || dateMs >= today || !isAuditDayDir(name)) continue;
|
|
9543
|
+
const dayPath = (0, import_node_path12.join)(this.auditDir, name);
|
|
9544
|
+
try {
|
|
9545
|
+
const compaction = compactAuditDay(dayPath);
|
|
9546
|
+
if (compaction.shards > 0) {
|
|
9547
|
+
this.logger.debug("audit cross-session compaction complete", {
|
|
9548
|
+
day: name,
|
|
9549
|
+
shards: compaction.shards,
|
|
9550
|
+
anchors: compaction.anchors,
|
|
9551
|
+
savedBytes: compaction.savedBytes
|
|
9552
|
+
});
|
|
9553
|
+
}
|
|
9554
|
+
} catch (error) {
|
|
9555
|
+
this.logger.warn("[AuditPruneSweeper] cross-session compaction failed", {
|
|
9556
|
+
day: name,
|
|
9557
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9558
|
+
});
|
|
9559
|
+
}
|
|
9560
|
+
compressed += await this.archiveDay(
|
|
9561
|
+
(0, import_node_path12.join)(dayPath, AUDIT_BODIES_DIR),
|
|
9562
|
+
ARCHIVE_BATCH - compressed
|
|
9563
|
+
);
|
|
9564
|
+
}
|
|
9565
|
+
if (compressed > 0) this.logger.debug("audit archive complete", { compressed });
|
|
9566
|
+
return compressed;
|
|
9567
|
+
} catch (error) {
|
|
9568
|
+
this.logger.warn("audit archive pass failed", {
|
|
9569
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9570
|
+
});
|
|
9571
|
+
return 0;
|
|
9572
|
+
} finally {
|
|
9573
|
+
this.archiving = false;
|
|
9574
|
+
}
|
|
9575
|
+
}
|
|
9576
|
+
/** Gzip up to `budget` plain shards in one day's `bodies/` directory. */
|
|
9577
|
+
async archiveDay(bodiesPath, budget) {
|
|
9578
|
+
let shards;
|
|
9579
|
+
try {
|
|
9580
|
+
shards = (0, import_node_fs19.readdirSync)(bodiesPath).filter((file) => file.endsWith(".jsonl"));
|
|
9581
|
+
} catch {
|
|
9582
|
+
return 0;
|
|
9583
|
+
}
|
|
9584
|
+
let compressed = 0;
|
|
9585
|
+
for (const shard of shards) {
|
|
9586
|
+
if (compressed >= budget) break;
|
|
9587
|
+
const source = (0, import_node_path12.join)(bodiesPath, shard);
|
|
9588
|
+
const target = `${source}.gz`;
|
|
9589
|
+
try {
|
|
9590
|
+
if ((0, import_node_fs19.existsSync)(target)) {
|
|
9591
|
+
(0, import_node_fs19.unlinkSync)(source);
|
|
9592
|
+
continue;
|
|
9593
|
+
}
|
|
9594
|
+
await (0, import_promises2.pipeline)((0, import_node_fs19.createReadStream)(source), (0, import_node_zlib.createGzip)(), (0, import_node_fs19.createWriteStream)(target));
|
|
9595
|
+
(0, import_node_fs19.unlinkSync)(source);
|
|
9596
|
+
compressed += 1;
|
|
9597
|
+
} catch (error) {
|
|
9598
|
+
try {
|
|
9599
|
+
if ((0, import_node_fs19.existsSync)(target)) (0, import_node_fs19.unlinkSync)(target);
|
|
9600
|
+
} catch {
|
|
9601
|
+
}
|
|
9602
|
+
this.logger.warn("[AuditPruneSweeper] failed to archive audit body shard", {
|
|
9603
|
+
shard,
|
|
9604
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9605
|
+
});
|
|
9606
|
+
}
|
|
9607
|
+
}
|
|
9608
|
+
return compressed;
|
|
9609
|
+
}
|
|
9129
9610
|
};
|
|
9130
9611
|
|
|
9131
|
-
// src/audit/
|
|
9132
|
-
var
|
|
9133
|
-
var
|
|
9134
|
-
var
|
|
9135
|
-
|
|
9136
|
-
|
|
9137
|
-
|
|
9138
|
-
|
|
9612
|
+
// src/audit/auditBodyReader.ts
|
|
9613
|
+
var import_node_fs21 = require("fs");
|
|
9614
|
+
var import_node_path13 = require("path");
|
|
9615
|
+
var import_node_zlib2 = require("zlib");
|
|
9616
|
+
|
|
9617
|
+
// src/audit/auditJsonl.ts
|
|
9618
|
+
var import_node_fs20 = require("fs");
|
|
9619
|
+
var WINDOW_BYTES = 1 << 20;
|
|
9620
|
+
var MAX_LINE_BYTES = 32 * 1024 * 1024;
|
|
9621
|
+
var NEWLINE = 10;
|
|
9622
|
+
function forEachLineFromTail(path2, onLine) {
|
|
9623
|
+
let fd;
|
|
9624
|
+
let end;
|
|
9625
|
+
try {
|
|
9626
|
+
end = (0, import_node_fs20.statSync)(path2).size;
|
|
9627
|
+
if (end === 0) return;
|
|
9628
|
+
fd = (0, import_node_fs20.openSync)(path2, "r");
|
|
9629
|
+
} catch {
|
|
9630
|
+
return;
|
|
9631
|
+
}
|
|
9632
|
+
try {
|
|
9633
|
+
let carry = Buffer.alloc(0);
|
|
9634
|
+
while (end > 0) {
|
|
9635
|
+
const start = Math.max(0, end - WINDOW_BYTES);
|
|
9636
|
+
const window = Buffer.allocUnsafe(end - start);
|
|
9637
|
+
let read;
|
|
9638
|
+
try {
|
|
9639
|
+
read = (0, import_node_fs20.readSync)(fd, window, 0, end - start, start);
|
|
9640
|
+
} catch {
|
|
9641
|
+
return;
|
|
9642
|
+
}
|
|
9643
|
+
const chunk = carry.length > 0 ? Buffer.concat([window.subarray(0, read), carry]) : window.subarray(0, read);
|
|
9644
|
+
let lineEnd = chunk.length;
|
|
9645
|
+
let nl = lineEnd > 0 ? chunk.lastIndexOf(NEWLINE, lineEnd - 1) : -1;
|
|
9646
|
+
while (nl >= 0) {
|
|
9647
|
+
if (nl + 1 < lineEnd) {
|
|
9648
|
+
const line = chunk.subarray(nl + 1, lineEnd).toString("utf8").trim();
|
|
9649
|
+
if (line && onLine(line)) return;
|
|
9650
|
+
}
|
|
9651
|
+
lineEnd = nl;
|
|
9652
|
+
nl = lineEnd > 0 ? chunk.lastIndexOf(NEWLINE, lineEnd - 1) : -1;
|
|
9653
|
+
}
|
|
9654
|
+
if (start === 0) {
|
|
9655
|
+
if (lineEnd > 0) {
|
|
9656
|
+
const line = chunk.subarray(0, lineEnd).toString("utf8").trim();
|
|
9657
|
+
if (line) onLine(line);
|
|
9658
|
+
}
|
|
9659
|
+
return;
|
|
9660
|
+
}
|
|
9661
|
+
if (lineEnd > MAX_LINE_BYTES) return;
|
|
9662
|
+
carry = Buffer.from(chunk.subarray(0, lineEnd));
|
|
9663
|
+
end = start;
|
|
9664
|
+
}
|
|
9665
|
+
} finally {
|
|
9666
|
+
try {
|
|
9667
|
+
(0, import_node_fs20.closeSync)(fd);
|
|
9668
|
+
} catch {
|
|
9669
|
+
}
|
|
9670
|
+
}
|
|
9671
|
+
}
|
|
9672
|
+
|
|
9673
|
+
// src/audit/auditBodyReader.ts
|
|
9674
|
+
function candidateDays(auditDir, ts) {
|
|
9675
|
+
if (typeof ts === "number" && Number.isFinite(ts)) {
|
|
9676
|
+
const named = auditDayDirName(ts);
|
|
9677
|
+
if ((0, import_node_fs21.existsSync)((0, import_node_path13.join)(auditDir, named))) return [named];
|
|
9678
|
+
}
|
|
9139
9679
|
try {
|
|
9140
|
-
|
|
9680
|
+
return (0, import_node_fs21.readdirSync)(auditDir).filter(isAuditDayDir).sort().reverse();
|
|
9141
9681
|
} catch {
|
|
9142
9682
|
return [];
|
|
9143
9683
|
}
|
|
9144
|
-
|
|
9145
|
-
|
|
9146
|
-
const
|
|
9147
|
-
|
|
9148
|
-
|
|
9149
|
-
|
|
9684
|
+
}
|
|
9685
|
+
function readShard(auditDir, day, sessionKey) {
|
|
9686
|
+
const base = (0, import_node_path13.join)(auditDir, day, AUDIT_BODIES_DIR, auditBodyFileName(sessionKey));
|
|
9687
|
+
try {
|
|
9688
|
+
if ((0, import_node_fs21.existsSync)(base)) return (0, import_node_fs21.readFileSync)(base, "utf8");
|
|
9689
|
+
const gz = `${base}.gz`;
|
|
9690
|
+
if ((0, import_node_fs21.existsSync)(gz)) return (0, import_node_zlib2.gunzipSync)((0, import_node_fs21.readFileSync)(gz)).toString("utf8");
|
|
9691
|
+
} catch {
|
|
9692
|
+
return null;
|
|
9693
|
+
}
|
|
9694
|
+
return null;
|
|
9695
|
+
}
|
|
9696
|
+
function parseShard(raw) {
|
|
9697
|
+
const entries = /* @__PURE__ */ new Map();
|
|
9698
|
+
for (const line of raw.split("\n")) {
|
|
9699
|
+
const trimmed = line.trim();
|
|
9700
|
+
if (!trimmed) continue;
|
|
9701
|
+
let parsed;
|
|
9150
9702
|
try {
|
|
9151
|
-
|
|
9703
|
+
parsed = JSON.parse(trimmed);
|
|
9152
9704
|
} catch {
|
|
9153
9705
|
continue;
|
|
9154
9706
|
}
|
|
9155
|
-
|
|
9156
|
-
|
|
9157
|
-
|
|
9158
|
-
|
|
9707
|
+
if (isAuditBodyEntry(parsed)) entries.set(parsed.id, parsed);
|
|
9708
|
+
}
|
|
9709
|
+
return entries;
|
|
9710
|
+
}
|
|
9711
|
+
function withDictionary(auditDir, day, entries) {
|
|
9712
|
+
let needed = false;
|
|
9713
|
+
for (const entry of entries.values()) {
|
|
9714
|
+
if (entry.req?.base?.startsWith(DICT_BASE_PREFIX)) {
|
|
9715
|
+
needed = true;
|
|
9716
|
+
break;
|
|
9717
|
+
}
|
|
9718
|
+
}
|
|
9719
|
+
if (!needed) return entries;
|
|
9720
|
+
const base = (0, import_node_path13.join)(auditDir, day, AUDIT_BODIES_DIR, AUDIT_DICT_FILE);
|
|
9721
|
+
let raw = null;
|
|
9722
|
+
try {
|
|
9723
|
+
if ((0, import_node_fs21.existsSync)(base)) raw = (0, import_node_fs21.readFileSync)(base, "utf8");
|
|
9724
|
+
else if ((0, import_node_fs21.existsSync)(`${base}.gz`)) raw = (0, import_node_zlib2.gunzipSync)((0, import_node_fs21.readFileSync)(`${base}.gz`)).toString("utf8");
|
|
9725
|
+
} catch {
|
|
9726
|
+
return entries;
|
|
9727
|
+
}
|
|
9728
|
+
if (raw === null) return entries;
|
|
9729
|
+
for (const [id, entry] of parseShard(raw)) entries.set(id, entry);
|
|
9730
|
+
return entries;
|
|
9731
|
+
}
|
|
9732
|
+
function reconstructRequest(entries, entry) {
|
|
9733
|
+
if (!entry.req) return void 0;
|
|
9734
|
+
const chain = [];
|
|
9735
|
+
const visited = /* @__PURE__ */ new Set();
|
|
9736
|
+
let cursor = entry;
|
|
9737
|
+
while (cursor?.req) {
|
|
9738
|
+
if (visited.has(cursor.id)) return void 0;
|
|
9739
|
+
visited.add(cursor.id);
|
|
9740
|
+
chain.push(cursor);
|
|
9741
|
+
if (cursor.req.base === null) break;
|
|
9742
|
+
cursor = entries.get(cursor.req.base);
|
|
9743
|
+
}
|
|
9744
|
+
const anchor = chain[chain.length - 1];
|
|
9745
|
+
if (!anchor?.req || anchor.req.base !== null) return void 0;
|
|
9746
|
+
let text = anchor.req.ins;
|
|
9747
|
+
for (let i = chain.length - 2; i >= 0; i -= 1) {
|
|
9748
|
+
const delta = chain[i]?.req;
|
|
9749
|
+
if (!delta) return void 0;
|
|
9750
|
+
if (delta.pre > text.length || delta.suf > text.length - delta.pre) return void 0;
|
|
9751
|
+
text = applyBodyDelta(text, delta);
|
|
9752
|
+
}
|
|
9753
|
+
return text;
|
|
9754
|
+
}
|
|
9755
|
+
function readAuditBody(auditDir, query2) {
|
|
9756
|
+
if (!isSafeSessionKey(query2.sessionKey) || !query2.id) return {};
|
|
9757
|
+
if (!(0, import_node_fs21.existsSync)(auditDir)) return {};
|
|
9758
|
+
for (const day of candidateDays(auditDir, query2.ts)) {
|
|
9759
|
+
const raw = readShard(auditDir, day, query2.sessionKey);
|
|
9760
|
+
if (raw === null) continue;
|
|
9761
|
+
const entries = withDictionary(auditDir, day, parseShard(raw));
|
|
9762
|
+
const entry = entries.get(query2.id);
|
|
9763
|
+
if (!entry) continue;
|
|
9764
|
+
const result = {};
|
|
9765
|
+
const requestBody = reconstructRequest(entries, entry);
|
|
9766
|
+
if (requestBody !== void 0) result.requestBody = requestBody;
|
|
9767
|
+
if (entry.res !== void 0) result.responseBody = entry.res;
|
|
9768
|
+
return result;
|
|
9769
|
+
}
|
|
9770
|
+
return readLegacyInlineBody(auditDir, query2.id);
|
|
9771
|
+
}
|
|
9772
|
+
function readLegacyInlineBody(auditDir, id) {
|
|
9773
|
+
let names;
|
|
9774
|
+
try {
|
|
9775
|
+
names = (0, import_node_fs21.readdirSync)(auditDir).filter((name) => AUDIT_FILE_RE.test(name)).sort().reverse();
|
|
9776
|
+
} catch {
|
|
9777
|
+
return {};
|
|
9778
|
+
}
|
|
9779
|
+
const needle = JSON.stringify(id);
|
|
9780
|
+
let found = {};
|
|
9781
|
+
for (const name of names) {
|
|
9782
|
+
forEachLineFromTail((0, import_node_path13.join)(auditDir, name), (line) => {
|
|
9783
|
+
if (!line.includes(needle)) return false;
|
|
9784
|
+
let parsed;
|
|
9159
9785
|
try {
|
|
9160
|
-
|
|
9786
|
+
parsed = JSON.parse(line);
|
|
9161
9787
|
} catch {
|
|
9162
|
-
|
|
9788
|
+
return false;
|
|
9163
9789
|
}
|
|
9164
|
-
|
|
9165
|
-
if (
|
|
9166
|
-
|
|
9167
|
-
|
|
9790
|
+
const record = parsed;
|
|
9791
|
+
if (record.id !== id) return false;
|
|
9792
|
+
const result = {};
|
|
9793
|
+
if (typeof record.requestBody === "string") result.requestBody = record.requestBody;
|
|
9794
|
+
if (typeof record.responseBody === "string") result.responseBody = record.responseBody;
|
|
9795
|
+
found = result;
|
|
9796
|
+
return true;
|
|
9797
|
+
});
|
|
9798
|
+
if (found.requestBody !== void 0 || found.responseBody !== void 0) break;
|
|
9799
|
+
}
|
|
9800
|
+
return found;
|
|
9801
|
+
}
|
|
9802
|
+
|
|
9803
|
+
// src/audit/auditReader.ts
|
|
9804
|
+
var import_node_fs22 = require("fs");
|
|
9805
|
+
var import_node_path14 = require("path");
|
|
9806
|
+
var DEFAULT_LIMIT = 200;
|
|
9807
|
+
var MAX_LIMIT = 2e3;
|
|
9808
|
+
var OVERSCAN = 256;
|
|
9809
|
+
function daySources(auditDir) {
|
|
9810
|
+
let names;
|
|
9811
|
+
try {
|
|
9812
|
+
names = (0, import_node_fs22.readdirSync)(auditDir);
|
|
9813
|
+
} catch {
|
|
9814
|
+
return [];
|
|
9815
|
+
}
|
|
9816
|
+
const sources = [];
|
|
9817
|
+
for (const name of names) {
|
|
9818
|
+
const dateMs = auditFileDateMs(name);
|
|
9819
|
+
if (dateMs === null) continue;
|
|
9820
|
+
if (AUDIT_DAY_DIR_RE.test(name)) {
|
|
9821
|
+
const path2 = (0, import_node_path14.join)(auditDir, name, AUDIT_META_FILE);
|
|
9822
|
+
if ((0, import_node_fs22.existsSync)(path2)) sources.push({ path: path2, dateMs });
|
|
9823
|
+
} else if (AUDIT_FILE_RE.test(name)) {
|
|
9824
|
+
sources.push({ path: (0, import_node_path14.join)(auditDir, name), dateMs });
|
|
9168
9825
|
}
|
|
9169
9826
|
}
|
|
9170
|
-
|
|
9171
|
-
return matched.slice(0, limit);
|
|
9827
|
+
return sources.sort((a, b) => b.dateMs - a.dateMs);
|
|
9172
9828
|
}
|
|
9173
9829
|
function isAuditRecord(value) {
|
|
9174
9830
|
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
9175
9831
|
const r = value;
|
|
9176
9832
|
return typeof r["id"] === "string" && typeof r["ts"] === "number" && typeof r["method"] === "string" && typeof r["path"] === "string" && typeof r["status"] === "number";
|
|
9177
9833
|
}
|
|
9834
|
+
function toMetaRecord(record) {
|
|
9835
|
+
if (record.requestBody === void 0 && record.responseBody === void 0) return record;
|
|
9836
|
+
const { requestBody: _req, responseBody: _res, ...meta } = record;
|
|
9837
|
+
return { ...meta, hasBody: true };
|
|
9838
|
+
}
|
|
9839
|
+
function readAuditRecords(auditDir, query2 = {}) {
|
|
9840
|
+
if (!(0, import_node_fs22.existsSync)(auditDir)) return [];
|
|
9841
|
+
const from = typeof query2.from === "number" ? query2.from : -Infinity;
|
|
9842
|
+
const to = typeof query2.to === "number" ? query2.to : Infinity;
|
|
9843
|
+
const limit = Math.min(MAX_LIMIT, Math.max(1, Math.trunc(query2.limit ?? DEFAULT_LIMIT)));
|
|
9844
|
+
const matched = [];
|
|
9845
|
+
for (const source of daySources(auditDir)) {
|
|
9846
|
+
const before = matched.length;
|
|
9847
|
+
forEachLineFromTail(source.path, (line) => {
|
|
9848
|
+
let parsed;
|
|
9849
|
+
try {
|
|
9850
|
+
parsed = JSON.parse(line);
|
|
9851
|
+
} catch {
|
|
9852
|
+
return false;
|
|
9853
|
+
}
|
|
9854
|
+
if (!isAuditRecord(parsed)) return false;
|
|
9855
|
+
if (query2.keyId !== void 0 && parsed.keyId !== query2.keyId) return false;
|
|
9856
|
+
if (query2.sessionKey !== void 0 && parsed.sessionKey !== query2.sessionKey) return false;
|
|
9857
|
+
if (parsed.ts < from || parsed.ts > to) return false;
|
|
9858
|
+
matched.push(toMetaRecord(parsed));
|
|
9859
|
+
return matched.length - before >= limit + OVERSCAN;
|
|
9860
|
+
});
|
|
9861
|
+
if (matched.length >= limit) break;
|
|
9862
|
+
}
|
|
9863
|
+
matched.sort((a, b) => b.ts - a.ts);
|
|
9864
|
+
return matched.slice(0, limit);
|
|
9865
|
+
}
|
|
9178
9866
|
|
|
9179
9867
|
// src/audit/AuditWriter.ts
|
|
9180
|
-
var
|
|
9181
|
-
var
|
|
9868
|
+
var import_node_fs23 = require("fs");
|
|
9869
|
+
var import_node_path15 = require("path");
|
|
9182
9870
|
var AuditWriter = class {
|
|
9183
9871
|
constructor(auditDir, logger, defer = (fn) => setTimeout(fn, 0)) {
|
|
9184
9872
|
this.auditDir = auditDir;
|
|
@@ -9188,10 +9876,13 @@ var AuditWriter = class {
|
|
|
9188
9876
|
auditDir;
|
|
9189
9877
|
logger;
|
|
9190
9878
|
defer;
|
|
9191
|
-
|
|
9879
|
+
/** Day directories already created this process (avoids an mkdir per record). */
|
|
9880
|
+
ensuredDirs = /* @__PURE__ */ new Set();
|
|
9881
|
+
/** Per-session encoding bases. Memory-only; a miss simply writes a full snapshot. */
|
|
9882
|
+
bases = new SessionBaseCache();
|
|
9192
9883
|
/**
|
|
9193
|
-
* Enqueue one record
|
|
9194
|
-
*
|
|
9884
|
+
* Enqueue one record. Returns IMMEDIATELY (fire-and-forget); every fs write and
|
|
9885
|
+
* the delta encoding happen on the deferred tick. A failure is logged, never thrown.
|
|
9195
9886
|
*/
|
|
9196
9887
|
record(record) {
|
|
9197
9888
|
this.defer(() => {
|
|
@@ -9204,25 +9895,41 @@ var AuditWriter = class {
|
|
|
9204
9895
|
}
|
|
9205
9896
|
});
|
|
9206
9897
|
}
|
|
9898
|
+
/** Drop all retained encoding bases (config reload / shutdown / test teardown). */
|
|
9899
|
+
reset() {
|
|
9900
|
+
this.bases.clear();
|
|
9901
|
+
this.ensuredDirs.clear();
|
|
9902
|
+
}
|
|
9207
9903
|
/**
|
|
9208
|
-
* Append synchronously — the awaitable form tests use to assert
|
|
9209
|
-
*
|
|
9210
|
-
* store's lazy file creation).
|
|
9904
|
+
* Append synchronously — the awaitable form tests use to assert a line landed.
|
|
9905
|
+
* Writes the metadata line first (canonical), then the body shard.
|
|
9211
9906
|
*/
|
|
9212
9907
|
appendNow(record) {
|
|
9213
|
-
|
|
9214
|
-
|
|
9215
|
-
|
|
9216
|
-
|
|
9217
|
-
|
|
9218
|
-
|
|
9219
|
-
|
|
9220
|
-
(
|
|
9908
|
+
const dayDir = auditDayDirName(record.ts);
|
|
9909
|
+
const dayPath = this.ensureDir((0, import_node_path15.join)(this.auditDir, dayDir));
|
|
9910
|
+
this.appendMeta(dayPath, record);
|
|
9911
|
+
this.appendBody(dayPath, dayDir, record);
|
|
9912
|
+
}
|
|
9913
|
+
/** Create a directory once per process and remember it. */
|
|
9914
|
+
ensureDir(path2) {
|
|
9915
|
+
if (!this.ensuredDirs.has(path2)) {
|
|
9916
|
+
(0, import_node_fs23.mkdirSync)(path2, { recursive: true });
|
|
9917
|
+
this.ensuredDirs.add(path2);
|
|
9918
|
+
}
|
|
9919
|
+
return path2;
|
|
9920
|
+
}
|
|
9921
|
+
/** Write the body-free metadata line + refresh the exact-count sidecar. */
|
|
9922
|
+
appendMeta(dayPath, record) {
|
|
9923
|
+
const { requestBody: _req, responseBody: _res, ...meta } = record;
|
|
9924
|
+
const file = (0, import_node_path15.join)(dayPath, AUDIT_META_FILE);
|
|
9925
|
+
const line = JSON.stringify(meta) + "\n";
|
|
9926
|
+
const bytesBefore = (0, import_node_fs23.existsSync)(file) ? (0, import_node_fs23.statSync)(file).size : 0;
|
|
9927
|
+
(0, import_node_fs23.appendFileSync)(file, line, "utf8");
|
|
9221
9928
|
try {
|
|
9222
9929
|
updateAuditStatsAfterAppend(
|
|
9223
9930
|
file,
|
|
9224
|
-
|
|
9225
|
-
|
|
9931
|
+
bytesBefore,
|
|
9932
|
+
bytesBefore + Buffer.byteLength(line, "utf8"),
|
|
9226
9933
|
record
|
|
9227
9934
|
);
|
|
9228
9935
|
} catch (error) {
|
|
@@ -9231,12 +9938,39 @@ var AuditWriter = class {
|
|
|
9231
9938
|
});
|
|
9232
9939
|
}
|
|
9233
9940
|
}
|
|
9941
|
+
/**
|
|
9942
|
+
* Write the delta-encoded body shard for one record. A no-op when nothing was
|
|
9943
|
+
* captured or when the session key is missing/unsafe — in which case the body
|
|
9944
|
+
* is dropped rather than written to an unvalidated path.
|
|
9945
|
+
*/
|
|
9946
|
+
appendBody(dayPath, dayDir, record) {
|
|
9947
|
+
if (record.requestBody === void 0 && record.responseBody === void 0) return;
|
|
9948
|
+
const sessionKey = record.sessionKey;
|
|
9949
|
+
if (!isSafeSessionKey(sessionKey)) {
|
|
9950
|
+
this.logger.warn("[AuditWriter] dropping audit body with no usable session key", {
|
|
9951
|
+
id: record.id
|
|
9952
|
+
});
|
|
9953
|
+
return;
|
|
9954
|
+
}
|
|
9955
|
+
try {
|
|
9956
|
+
const line = encodeBodyEntry(record, sessionKey, dayDir, this.bases);
|
|
9957
|
+
if (line === null) return;
|
|
9958
|
+
const bodiesPath = this.ensureDir((0, import_node_path15.join)(dayPath, AUDIT_BODIES_DIR));
|
|
9959
|
+
(0, import_node_fs23.appendFileSync)((0, import_node_path15.join)(bodiesPath, auditBodyFileName(sessionKey)), line + "\n", "utf8");
|
|
9960
|
+
} catch (error) {
|
|
9961
|
+
this.bases.forget(sessionKey);
|
|
9962
|
+
this.logger.warn("[AuditWriter] failed to append audit body shard", {
|
|
9963
|
+
id: record.id,
|
|
9964
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9965
|
+
});
|
|
9966
|
+
}
|
|
9967
|
+
}
|
|
9234
9968
|
};
|
|
9235
9969
|
|
|
9236
9970
|
// src/billing/BillingPublisher.ts
|
|
9237
|
-
var
|
|
9971
|
+
var import_node_fs24 = require("fs");
|
|
9238
9972
|
var import_node_crypto13 = require("crypto");
|
|
9239
|
-
var
|
|
9973
|
+
var import_node_path16 = require("path");
|
|
9240
9974
|
var import_upstreamFetch6 = require("@omnicross/core/pipeline/upstreamFetch");
|
|
9241
9975
|
|
|
9242
9976
|
// src/billing/billingFiles.ts
|
|
@@ -9307,8 +10041,8 @@ var BillingPublisher = class {
|
|
|
9307
10041
|
*/
|
|
9308
10042
|
appendNow(event) {
|
|
9309
10043
|
this.ensureDir();
|
|
9310
|
-
const file = (0,
|
|
9311
|
-
(0,
|
|
10044
|
+
const file = (0, import_node_path16.join)(this.billingDir, billingFileName(event.ts));
|
|
10045
|
+
(0, import_node_fs24.appendFileSync)(file, JSON.stringify(event) + "\n", "utf8");
|
|
9312
10046
|
}
|
|
9313
10047
|
/**
|
|
9314
10048
|
* One best-effort delivery attempt for an event ALREADY in the ledger. POSTs the
|
|
@@ -9357,8 +10091,8 @@ var BillingPublisher = class {
|
|
|
9357
10091
|
markDelivered(event) {
|
|
9358
10092
|
try {
|
|
9359
10093
|
this.ensureDir();
|
|
9360
|
-
const file = (0,
|
|
9361
|
-
(0,
|
|
10094
|
+
const file = (0, import_node_path16.join)(this.billingDir, deliveredFileName(event.ts));
|
|
10095
|
+
(0, import_node_fs24.appendFileSync)(file, JSON.stringify({ id: event.id, deliveredAt: this.now() }) + "\n", "utf8");
|
|
9362
10096
|
} catch (error) {
|
|
9363
10097
|
this.logger.warn("[BillingPublisher] failed to append delivery marker", {
|
|
9364
10098
|
error: error instanceof Error ? error.message : String(error)
|
|
@@ -9367,20 +10101,20 @@ var BillingPublisher = class {
|
|
|
9367
10101
|
}
|
|
9368
10102
|
ensureDir() {
|
|
9369
10103
|
if (this.dirEnsured) return;
|
|
9370
|
-
(0,
|
|
10104
|
+
(0, import_node_fs24.mkdirSync)(this.billingDir, { recursive: true });
|
|
9371
10105
|
this.dirEnsured = true;
|
|
9372
10106
|
}
|
|
9373
10107
|
};
|
|
9374
10108
|
|
|
9375
10109
|
// src/billing/billingReader.ts
|
|
9376
|
-
var
|
|
9377
|
-
var
|
|
10110
|
+
var import_node_fs25 = require("fs");
|
|
10111
|
+
var import_node_path17 = require("path");
|
|
9378
10112
|
function readBillingLedger(billingDir) {
|
|
9379
10113
|
const view = { events: [], deliveredIds: /* @__PURE__ */ new Set() };
|
|
9380
|
-
if (!(0,
|
|
10114
|
+
if (!(0, import_node_fs25.existsSync)(billingDir)) return view;
|
|
9381
10115
|
let files;
|
|
9382
10116
|
try {
|
|
9383
|
-
files = (0,
|
|
10117
|
+
files = (0, import_node_fs25.readdirSync)(billingDir);
|
|
9384
10118
|
} catch {
|
|
9385
10119
|
return view;
|
|
9386
10120
|
}
|
|
@@ -9411,7 +10145,7 @@ function readBillingStatus(billingDir) {
|
|
|
9411
10145
|
function parseLines(dir, file) {
|
|
9412
10146
|
let raw;
|
|
9413
10147
|
try {
|
|
9414
|
-
raw = (0,
|
|
10148
|
+
raw = (0, import_node_fs25.readFileSync)((0, import_node_path17.join)(dir, file), "utf8");
|
|
9415
10149
|
} catch {
|
|
9416
10150
|
return [];
|
|
9417
10151
|
}
|
|
@@ -9897,7 +10631,7 @@ function buildDaemon(config, paths) {
|
|
|
9897
10631
|
}
|
|
9898
10632
|
);
|
|
9899
10633
|
const pricingStore = new JsonPricingStore(defaultPricingPath(paths.configPath));
|
|
9900
|
-
const pricingEngine = new
|
|
10634
|
+
const pricingEngine = new import_usage2.PricingEngine(pricingStore, logger, {
|
|
9901
10635
|
// Catalog egress follows the same global/env proxy policy as every other
|
|
9902
10636
|
// daemon upstream call; no provider/account override applies here.
|
|
9903
10637
|
fetchImpl: ((input, init) => (0, import_upstreamFetch8.fetchUpstream)(String(input), init ?? {}))
|
|
@@ -9913,8 +10647,10 @@ function buildDaemon(config, paths) {
|
|
|
9913
10647
|
async (providerId, model) => await pricingEngine.getEntry(providerId, model) !== null
|
|
9914
10648
|
);
|
|
9915
10649
|
const keySpendTracker = new import_outbound_api5.KeySpendTracker(usageEventStore);
|
|
9916
|
-
const
|
|
9917
|
-
|
|
10650
|
+
const usageThroughput = (0, import_usage2.getSharedUsageThroughputTracker)();
|
|
10651
|
+
const usageRecorder = new import_usage2.UsageRecorder(usageEventStore, pricingEngine, logger, {
|
|
10652
|
+
onRecord: (apiKeyId, costUsd, at) => keySpendTracker.add(apiKeyId, costUsd, at),
|
|
10653
|
+
onEvent: (row, at) => usageThroughput.record(row, at)
|
|
9918
10654
|
});
|
|
9919
10655
|
const providerProxy = (0, import_provider_proxy4.getProviderProxy)({ llmConfig, apiKeyPool, usageRecorder });
|
|
9920
10656
|
const routeLeaseManager = new import_provider_proxy4.RouteLeaseManager(
|
|
@@ -10063,6 +10799,11 @@ function buildDaemon(config, paths) {
|
|
|
10063
10799
|
// NEVER unauth, NEVER on `/health`. Routed in `AdminServer` (not `adminApi.ts`).
|
|
10064
10800
|
auditReader: (query2) => readAuditRecords(auditDir, query2),
|
|
10065
10801
|
auditStatsReader: (query2) => readAuditStats(auditDir, query2),
|
|
10802
|
+
// audit-store-sharding: bodies live in per-session shards, so opening ONE
|
|
10803
|
+
// record's payload is a separate authed call that replays its delta chain.
|
|
10804
|
+
auditBodyReader: (query2) => readAuditBody(auditDir, query2),
|
|
10805
|
+
// audit-store-sharding D8: the manual counterpart to the daily pass.
|
|
10806
|
+
auditCompactor: () => compactAllClosedAuditDays(auditDir),
|
|
10066
10807
|
// billing-event-stream: the AUTHED `GET /admin/api/billing-status` returns the
|
|
10067
10808
|
// secret-free total/delivered/pending counts of the durable ledger.
|
|
10068
10809
|
billingStatusReader: () => readBillingStatus(billingDir)
|
|
@@ -10136,11 +10877,12 @@ function resetDaemonSingletonsForTests() {
|
|
|
10136
10877
|
(0, import_SubscriptionIdentityStore3.__resetSharedIdentityStoreForTests)();
|
|
10137
10878
|
(0, import_AccountAllowanceStore4.__resetSharedAccountAllowanceStoreForTests)();
|
|
10138
10879
|
(0, import_AccountAllowanceScheduling5.__resetSharedAccountAllowanceSchedulingForTests)();
|
|
10880
|
+
(0, import_usage2.__resetSharedUsageThroughputTrackerForTests)();
|
|
10139
10881
|
}
|
|
10140
10882
|
function isTokensStoreReadable(tokensPath) {
|
|
10141
10883
|
try {
|
|
10142
|
-
if (!(0,
|
|
10143
|
-
(0,
|
|
10884
|
+
if (!(0, import_node_fs26.existsSync)(tokensPath)) return true;
|
|
10885
|
+
(0, import_node_fs26.accessSync)(tokensPath, import_node_fs26.constants.R_OK);
|
|
10144
10886
|
return true;
|
|
10145
10887
|
} catch {
|
|
10146
10888
|
return false;
|