@omnicross/daemon 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +1312 -369
- package/dist/cli.js +1309 -361
- package/dist/index.cjs +849 -107
- package/dist/index.d.cts +147 -38
- package/dist/index.d.ts +147 -38
- package/dist/index.js +847 -100
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/bootstrap.ts
|
|
2
|
-
import { accessSync, constants as fsConstants, existsSync as
|
|
2
|
+
import { accessSync, constants as fsConstants, existsSync as existsSync21 } from "fs";
|
|
3
3
|
import { DEFAULT_AUDIT_CONFIG } from "@omnicross/contracts/audit-types";
|
|
4
4
|
import { DEFAULT_BILLING_CONFIG } from "@omnicross/contracts/billing-types";
|
|
5
5
|
import { getGeminiCodeAssistProjectResolver } from "@omnicross/core/auth/GeminiCodeAssistProjectResolver";
|
|
@@ -33,7 +33,12 @@ import {
|
|
|
33
33
|
} from "@omnicross/core/provider-proxy";
|
|
34
34
|
import { routeLeaseDescriptorPort } from "@omnicross/cli-launcher";
|
|
35
35
|
import { KeySpendTracker } from "@omnicross/core/outbound-api";
|
|
36
|
-
import {
|
|
36
|
+
import {
|
|
37
|
+
__resetSharedUsageThroughputTrackerForTests,
|
|
38
|
+
getSharedUsageThroughputTracker as getSharedUsageThroughputTracker2,
|
|
39
|
+
PricingEngine,
|
|
40
|
+
UsageRecorder
|
|
41
|
+
} from "@omnicross/core/usage";
|
|
37
42
|
import {
|
|
38
43
|
setSubscriptionAccountService,
|
|
39
44
|
setSubscriptionProviderRegistry,
|
|
@@ -637,6 +642,37 @@ function handleAuditQuery(req, res, reader) {
|
|
|
637
642
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
638
643
|
res.end(JSON.stringify({ records }));
|
|
639
644
|
}
|
|
645
|
+
function handleAuditBodyQuery(req, res, reader) {
|
|
646
|
+
const url = new URL(req.url ?? "/", "http://localhost");
|
|
647
|
+
const id = url.searchParams.get("id")?.trim();
|
|
648
|
+
const sessionKey = url.searchParams.get("session")?.trim();
|
|
649
|
+
if (!id || !sessionKey) {
|
|
650
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
651
|
+
res.end(JSON.stringify({ error: "id and session are required" }));
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
const query2 = { id, sessionKey };
|
|
655
|
+
const ts = intParam(url.searchParams.get("ts"));
|
|
656
|
+
if (ts !== void 0) query2.ts = ts;
|
|
657
|
+
const body = reader ? reader(query2) : {};
|
|
658
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
659
|
+
res.end(JSON.stringify(body));
|
|
660
|
+
}
|
|
661
|
+
function handleAuditCompact(res, compact) {
|
|
662
|
+
if (!compact) {
|
|
663
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
664
|
+
res.end(JSON.stringify({ days: 0, shards: 0, savedBytes: 0 }));
|
|
665
|
+
return;
|
|
666
|
+
}
|
|
667
|
+
try {
|
|
668
|
+
const result = compact();
|
|
669
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
670
|
+
res.end(JSON.stringify(result));
|
|
671
|
+
} catch (error) {
|
|
672
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
673
|
+
res.end(JSON.stringify({ error: error instanceof Error ? error.message : "compaction failed" }));
|
|
674
|
+
}
|
|
675
|
+
}
|
|
640
676
|
async function handleAuditStatsQuery(req, res, reader) {
|
|
641
677
|
const url = new URL(req.url ?? "/", "http://localhost");
|
|
642
678
|
const query2 = {};
|
|
@@ -3646,6 +3682,7 @@ function applyAuditConfig(config) {
|
|
|
3646
3682
|
} else {
|
|
3647
3683
|
setAuditCaptureConfig(null);
|
|
3648
3684
|
setAuditSink(null);
|
|
3685
|
+
writer?.reset();
|
|
3649
3686
|
if (sweeper) {
|
|
3650
3687
|
if (config) sweeper.configure(config);
|
|
3651
3688
|
sweeper.dispose();
|
|
@@ -3656,6 +3693,7 @@ function resetAuditRuntimeForTests() {
|
|
|
3656
3693
|
setAuditCaptureConfig(null);
|
|
3657
3694
|
setAuditSink(null);
|
|
3658
3695
|
setUpstreamTracePath(null);
|
|
3696
|
+
writer?.reset();
|
|
3659
3697
|
if (sweeper) sweeper.dispose();
|
|
3660
3698
|
writer = null;
|
|
3661
3699
|
sweeper = null;
|
|
@@ -4247,6 +4285,7 @@ async function handleImport(body, deps) {
|
|
|
4247
4285
|
}
|
|
4248
4286
|
|
|
4249
4287
|
// src/admin/usagePricing.ts
|
|
4288
|
+
import { getSharedUsageThroughputTracker } from "@omnicross/core/usage";
|
|
4250
4289
|
var err4 = (status, message) => ({
|
|
4251
4290
|
status,
|
|
4252
4291
|
body: { error: { type: "admin_api_error", message } }
|
|
@@ -4272,6 +4311,9 @@ var BUCKET_SPAN_MS = {
|
|
|
4272
4311
|
};
|
|
4273
4312
|
var MAX_TIMESERIES_BUCKETS = 2e3;
|
|
4274
4313
|
async function handleUsageGet(view, query2, deps) {
|
|
4314
|
+
if (view === "throughput") {
|
|
4315
|
+
return { status: 200, body: getSharedUsageThroughputTracker().snapshot() };
|
|
4316
|
+
}
|
|
4275
4317
|
const range = parseRange(query2);
|
|
4276
4318
|
if (!isRange(range)) return range;
|
|
4277
4319
|
switch (view) {
|
|
@@ -6003,7 +6045,7 @@ async function handleUiStatic(req, res, urlPath, uiDist) {
|
|
|
6003
6045
|
}
|
|
6004
6046
|
|
|
6005
6047
|
// src/admin/version.ts
|
|
6006
|
-
var DAEMON_VERSION = true ? "0.1.
|
|
6048
|
+
var DAEMON_VERSION = true ? "0.1.10" : "0.0.0-dev";
|
|
6007
6049
|
|
|
6008
6050
|
// src/admin/AdminServer.ts
|
|
6009
6051
|
var LOOPBACK_ADDR = "127.0.0.1";
|
|
@@ -6115,6 +6157,14 @@ var AdminServer = class {
|
|
|
6115
6157
|
await handleAuditStatsQuery(req, res, this.deps.auditStatsReader);
|
|
6116
6158
|
return;
|
|
6117
6159
|
}
|
|
6160
|
+
if (path2 === "/admin/api/audit/body" && (req.method === "GET" || req.method === "HEAD")) {
|
|
6161
|
+
handleAuditBodyQuery(req, res, this.deps.auditBodyReader);
|
|
6162
|
+
return;
|
|
6163
|
+
}
|
|
6164
|
+
if (path2 === "/admin/api/audit/compact" && req.method === "POST") {
|
|
6165
|
+
handleAuditCompact(res, this.deps.auditCompactor);
|
|
6166
|
+
return;
|
|
6167
|
+
}
|
|
6118
6168
|
if (path2 === "/admin/api/billing-status" && (req.method === "GET" || req.method === "HEAD")) {
|
|
6119
6169
|
handleBillingStatus(res, this.deps.billingStatusReader);
|
|
6120
6170
|
return;
|
|
@@ -8866,18 +8916,214 @@ var AccountHealthSweeper = class {
|
|
|
8866
8916
|
};
|
|
8867
8917
|
|
|
8868
8918
|
// src/audit/AuditPruneSweeper.ts
|
|
8869
|
-
import { existsSync as
|
|
8870
|
-
import { join as
|
|
8919
|
+
import { createReadStream as createReadStream2, createWriteStream as createWriteStream2, existsSync as existsSync16, readdirSync as readdirSync3, rmSync as rmSync4, unlinkSync as unlinkSync4 } from "fs";
|
|
8920
|
+
import { join as join8 } from "path";
|
|
8921
|
+
import { pipeline } from "stream/promises";
|
|
8922
|
+
import { createGzip } from "zlib";
|
|
8923
|
+
|
|
8924
|
+
// src/audit/auditDictionary.ts
|
|
8925
|
+
import { existsSync as existsSync14, readdirSync, readFileSync as readFileSync14, renameSync as renameSync5, unlinkSync as unlinkSync3, writeFileSync as writeFileSync12 } from "fs";
|
|
8926
|
+
import { join as join6 } from "path";
|
|
8927
|
+
|
|
8928
|
+
// src/audit/auditBodyStore.ts
|
|
8929
|
+
var ANCHOR_EVERY = 64;
|
|
8930
|
+
var ANCHOR_DELTA_RATIO = 0.75;
|
|
8931
|
+
var DIVERGED_PREFIX_RATIO = 0.25;
|
|
8932
|
+
var MAX_BASE_CHARS = 4e6;
|
|
8933
|
+
var CACHE_BUDGET_CHARS = 16e6;
|
|
8934
|
+
var CACHE_MAX_SESSIONS = 32;
|
|
8935
|
+
var MAX_BASES_PER_SESSION = 4;
|
|
8936
|
+
var isHighSurrogate = (code) => code >= 55296 && code <= 56319;
|
|
8937
|
+
var isLowSurrogate = (code) => code >= 56320 && code <= 57343;
|
|
8938
|
+
function computeBodyDelta(prev, next) {
|
|
8939
|
+
const shortest = Math.min(prev.length, next.length);
|
|
8940
|
+
let pre = 0;
|
|
8941
|
+
while (pre < shortest && prev.charCodeAt(pre) === next.charCodeAt(pre)) pre += 1;
|
|
8942
|
+
if (pre > 0 && isHighSurrogate(prev.charCodeAt(pre - 1))) pre -= 1;
|
|
8943
|
+
const maxSuf = shortest - pre;
|
|
8944
|
+
let suf = 0;
|
|
8945
|
+
while (suf < maxSuf && prev.charCodeAt(prev.length - 1 - suf) === next.charCodeAt(next.length - 1 - suf)) {
|
|
8946
|
+
suf += 1;
|
|
8947
|
+
}
|
|
8948
|
+
if (suf > 0 && isLowSurrogate(prev.charCodeAt(prev.length - suf))) suf -= 1;
|
|
8949
|
+
return { pre, suf, ins: next.slice(pre, next.length - suf) };
|
|
8950
|
+
}
|
|
8951
|
+
function applyBodyDelta(prev, delta) {
|
|
8952
|
+
const head = delta.pre > 0 ? prev.slice(0, delta.pre) : "";
|
|
8953
|
+
const tail = delta.suf > 0 ? prev.slice(prev.length - delta.suf) : "";
|
|
8954
|
+
return head + delta.ins + tail;
|
|
8955
|
+
}
|
|
8956
|
+
var SessionBaseCache = class {
|
|
8957
|
+
constructor(maxSessions = CACHE_MAX_SESSIONS, budgetChars = CACHE_BUDGET_CHARS, maxBaseChars = MAX_BASE_CHARS, maxHeads = MAX_BASES_PER_SESSION) {
|
|
8958
|
+
this.maxSessions = maxSessions;
|
|
8959
|
+
this.budgetChars = budgetChars;
|
|
8960
|
+
this.maxBaseChars = maxBaseChars;
|
|
8961
|
+
this.maxHeads = maxHeads;
|
|
8962
|
+
}
|
|
8963
|
+
maxSessions;
|
|
8964
|
+
budgetChars;
|
|
8965
|
+
maxBaseChars;
|
|
8966
|
+
maxHeads;
|
|
8967
|
+
/** Session key to its retained heads, most-recent first. */
|
|
8968
|
+
entries = /* @__PURE__ */ new Map();
|
|
8969
|
+
chars = 0;
|
|
8970
|
+
/** Retained sessions (tests + diagnostics). */
|
|
8971
|
+
get size() {
|
|
8972
|
+
return this.entries.size;
|
|
8973
|
+
}
|
|
8974
|
+
/** A session's retained heads, most-recent first. Refreshes LRU recency. */
|
|
8975
|
+
get(sessionKey) {
|
|
8976
|
+
const found = this.entries.get(sessionKey);
|
|
8977
|
+
if (!found) return [];
|
|
8978
|
+
this.entries.delete(sessionKey);
|
|
8979
|
+
this.entries.set(sessionKey, found);
|
|
8980
|
+
return found;
|
|
8981
|
+
}
|
|
8982
|
+
/**
|
|
8983
|
+
* Retain `base` as a head of `sessionKey`.
|
|
8984
|
+
*
|
|
8985
|
+
* `replacesId` is the head this turn CONTINUES (its body was preserved whole
|
|
8986
|
+
* inside the new one), which is swapped out so a linear conversation keeps
|
|
8987
|
+
* exactly one head. Omit it when the turn started a distinct stream %s that
|
|
8988
|
+
* head is added alongside, which is what keeps a fork's branches apart.
|
|
8989
|
+
*
|
|
8990
|
+
* A body larger than `maxBaseChars` is not retained: the next turn anchors
|
|
8991
|
+
* rather than letting one oversized session monopolize the budget.
|
|
8992
|
+
*/
|
|
8993
|
+
remember(sessionKey, base, replacesId) {
|
|
8994
|
+
const heads = this.entries.get(sessionKey) ?? [];
|
|
8995
|
+
if (replacesId !== void 0) {
|
|
8996
|
+
const at = heads.findIndex((head) => head.lastId === replacesId);
|
|
8997
|
+
if (at >= 0) {
|
|
8998
|
+
this.chars -= heads[at].text.length;
|
|
8999
|
+
heads.splice(at, 1);
|
|
9000
|
+
}
|
|
9001
|
+
}
|
|
9002
|
+
if (base.text.length <= this.maxBaseChars) {
|
|
9003
|
+
heads.unshift(base);
|
|
9004
|
+
this.chars += base.text.length;
|
|
9005
|
+
}
|
|
9006
|
+
while (heads.length > this.maxHeads) {
|
|
9007
|
+
const dropped = heads.pop();
|
|
9008
|
+
if (dropped) this.chars -= dropped.text.length;
|
|
9009
|
+
}
|
|
9010
|
+
this.entries.delete(sessionKey);
|
|
9011
|
+
if (heads.length > 0) this.entries.set(sessionKey, heads);
|
|
9012
|
+
this.evict();
|
|
9013
|
+
}
|
|
9014
|
+
/** Drop a session's heads (eviction, or a write failure invalidating them). */
|
|
9015
|
+
forget(sessionKey) {
|
|
9016
|
+
const heads = this.entries.get(sessionKey);
|
|
9017
|
+
if (!heads) return;
|
|
9018
|
+
for (const head of heads) this.chars -= head.text.length;
|
|
9019
|
+
this.entries.delete(sessionKey);
|
|
9020
|
+
}
|
|
9021
|
+
/** Drop everything (writer disposal / test teardown). */
|
|
9022
|
+
clear() {
|
|
9023
|
+
this.entries.clear();
|
|
9024
|
+
this.chars = 0;
|
|
9025
|
+
}
|
|
9026
|
+
/** Evict least-recently-used sessions until both bounds hold. */
|
|
9027
|
+
evict() {
|
|
9028
|
+
while (this.entries.size > this.maxSessions || this.chars > this.budgetChars && this.entries.size > 1) {
|
|
9029
|
+
const oldest = this.entries.keys().next();
|
|
9030
|
+
if (oldest.done) break;
|
|
9031
|
+
this.forget(oldest.value);
|
|
9032
|
+
}
|
|
9033
|
+
}
|
|
9034
|
+
};
|
|
9035
|
+
function anchorReason(base, dayDir, delta, nextLength) {
|
|
9036
|
+
if (!base || !delta) return "new";
|
|
9037
|
+
if (base.dayDir !== dayDir) return "day";
|
|
9038
|
+
if (base.chainLen >= ANCHOR_EVERY) return "chain";
|
|
9039
|
+
if (delta.pre < base.text.length * DIVERGED_PREFIX_RATIO) return "diverged";
|
|
9040
|
+
if (delta.ins.length > nextLength * ANCHOR_DELTA_RATIO) return "costly";
|
|
9041
|
+
return null;
|
|
9042
|
+
}
|
|
9043
|
+
function pickBase(heads, next) {
|
|
9044
|
+
let best = null;
|
|
9045
|
+
for (const base of heads) {
|
|
9046
|
+
const delta = computeBodyDelta(base.text, next);
|
|
9047
|
+
if (best !== null && delta.ins.length >= best.delta.ins.length) continue;
|
|
9048
|
+
best = { base, delta, continues: delta.pre + delta.suf >= base.text.length };
|
|
9049
|
+
}
|
|
9050
|
+
return best;
|
|
9051
|
+
}
|
|
9052
|
+
function encodeBodyEntry(record, sessionKey, dayDir, cache) {
|
|
9053
|
+
const requestBody = record.requestBody;
|
|
9054
|
+
const responseBody = record.responseBody;
|
|
9055
|
+
if (requestBody === void 0 && responseBody === void 0) return null;
|
|
9056
|
+
const entry = { id: record.id, ts: record.ts };
|
|
9057
|
+
if (requestBody !== void 0) {
|
|
9058
|
+
const heads = cache.get(sessionKey);
|
|
9059
|
+
const sameDay = heads.filter((head) => head.dayDir === dayDir);
|
|
9060
|
+
const chosen = pickBase(sameDay, requestBody);
|
|
9061
|
+
const reason = heads.length > 0 && sameDay.length === 0 ? "day" : anchorReason(chosen?.base, dayDir, chosen?.delta ?? null, requestBody.length);
|
|
9062
|
+
if (reason !== null) {
|
|
9063
|
+
entry.req = { base: null, anchor: reason, pre: 0, suf: 0, ins: requestBody };
|
|
9064
|
+
cache.remember(
|
|
9065
|
+
sessionKey,
|
|
9066
|
+
{ dayDir, lastId: record.id, text: requestBody, chainLen: 0 },
|
|
9067
|
+
chosen?.continues === true ? chosen.base.lastId : void 0
|
|
9068
|
+
);
|
|
9069
|
+
} else {
|
|
9070
|
+
const picked = chosen;
|
|
9071
|
+
entry.req = {
|
|
9072
|
+
base: picked.base.lastId,
|
|
9073
|
+
...picked.continues ? { cont: true } : {},
|
|
9074
|
+
pre: picked.delta.pre,
|
|
9075
|
+
suf: picked.delta.suf,
|
|
9076
|
+
ins: picked.delta.ins
|
|
9077
|
+
};
|
|
9078
|
+
cache.remember(
|
|
9079
|
+
sessionKey,
|
|
9080
|
+
{ dayDir, lastId: record.id, text: requestBody, chainLen: picked.base.chainLen + 1 },
|
|
9081
|
+
picked.continues ? picked.base.lastId : void 0
|
|
9082
|
+
);
|
|
9083
|
+
}
|
|
9084
|
+
}
|
|
9085
|
+
if (responseBody !== void 0) entry.res = responseBody;
|
|
9086
|
+
return JSON.stringify(entry);
|
|
9087
|
+
}
|
|
9088
|
+
function isAuditBodyEntry(value) {
|
|
9089
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
9090
|
+
const entry = value;
|
|
9091
|
+
if (typeof entry["id"] !== "string" || typeof entry["ts"] !== "number") return false;
|
|
9092
|
+
if (entry["res"] !== void 0 && typeof entry["res"] !== "string") return false;
|
|
9093
|
+
const req = entry["req"];
|
|
9094
|
+
if (req === void 0) return true;
|
|
9095
|
+
if (!req || typeof req !== "object" || Array.isArray(req)) return false;
|
|
9096
|
+
const delta = req;
|
|
9097
|
+
if (delta["anchor"] !== void 0 && typeof delta["anchor"] !== "string") return false;
|
|
9098
|
+
if (delta["cont"] !== void 0 && typeof delta["cont"] !== "boolean") return false;
|
|
9099
|
+
return (delta["base"] === null || typeof delta["base"] === "string") && Number.isSafeInteger(delta["pre"]) && delta["pre"] >= 0 && Number.isSafeInteger(delta["suf"]) && delta["suf"] >= 0 && typeof delta["ins"] === "string";
|
|
9100
|
+
}
|
|
8871
9101
|
|
|
8872
9102
|
// src/audit/auditFiles.ts
|
|
8873
9103
|
var AUDIT_FILE_RE = /^audit-(\d{4})-(\d{2})-(\d{2})\.jsonl$/;
|
|
9104
|
+
var AUDIT_DAY_DIR_RE = /^audit-(\d{4})-(\d{2})-(\d{2})$/;
|
|
9105
|
+
var AUDIT_META_FILE = "meta.jsonl";
|
|
9106
|
+
var AUDIT_BODIES_DIR = "bodies";
|
|
9107
|
+
var AUDIT_SESSION_KEY_RE = /^[0-9a-f]{8,64}$/;
|
|
8874
9108
|
var pad22 = (n) => String(n).padStart(2, "0");
|
|
8875
|
-
|
|
9109
|
+
var localDateStamp = (ts) => {
|
|
8876
9110
|
const d = new Date(ts);
|
|
8877
|
-
return
|
|
9111
|
+
return `${d.getFullYear()}-${pad22(d.getMonth() + 1)}-${pad22(d.getDate())}`;
|
|
9112
|
+
};
|
|
9113
|
+
function auditDayDirName(ts) {
|
|
9114
|
+
return `audit-${localDateStamp(ts)}`;
|
|
9115
|
+
}
|
|
9116
|
+
function isAuditDayDir(name) {
|
|
9117
|
+
return AUDIT_DAY_DIR_RE.test(name);
|
|
9118
|
+
}
|
|
9119
|
+
function isSafeSessionKey(key) {
|
|
9120
|
+
return typeof key === "string" && AUDIT_SESSION_KEY_RE.test(key);
|
|
8878
9121
|
}
|
|
8879
|
-
function
|
|
8880
|
-
|
|
9122
|
+
function auditBodyFileName(sessionKey) {
|
|
9123
|
+
return `${sessionKey}.jsonl`;
|
|
9124
|
+
}
|
|
9125
|
+
function auditFileDateMs(name) {
|
|
9126
|
+
const m = AUDIT_FILE_RE.exec(name) ?? AUDIT_DAY_DIR_RE.exec(name);
|
|
8881
9127
|
if (!m) return null;
|
|
8882
9128
|
const year = Number(m[1]);
|
|
8883
9129
|
const month = Number(m[2]);
|
|
@@ -8889,16 +9135,153 @@ function auditFileDateMs(fileName) {
|
|
|
8889
9135
|
return d.getTime();
|
|
8890
9136
|
}
|
|
8891
9137
|
|
|
9138
|
+
// src/audit/auditDictionary.ts
|
|
9139
|
+
var AUDIT_DICT_FILE = "_dict.jsonl";
|
|
9140
|
+
var DICT_BASE_PREFIX = "dict:";
|
|
9141
|
+
var DICT_CANDIDATES = 3;
|
|
9142
|
+
var MIN_SAVING_RATIO = 0.2;
|
|
9143
|
+
function parseEntries(raw) {
|
|
9144
|
+
const entries = [];
|
|
9145
|
+
for (const line of raw.split("\n")) {
|
|
9146
|
+
const trimmed = line.trim();
|
|
9147
|
+
if (!trimmed) continue;
|
|
9148
|
+
try {
|
|
9149
|
+
const parsed = JSON.parse(trimmed);
|
|
9150
|
+
if (isAuditBodyEntry(parsed)) entries.push(parsed);
|
|
9151
|
+
} catch {
|
|
9152
|
+
}
|
|
9153
|
+
}
|
|
9154
|
+
return entries;
|
|
9155
|
+
}
|
|
9156
|
+
function plainShards(bodiesPath) {
|
|
9157
|
+
try {
|
|
9158
|
+
return readdirSync(bodiesPath).filter(
|
|
9159
|
+
(file) => file.endsWith(".jsonl") && isSafeSessionKey(file.slice(0, -".jsonl".length))
|
|
9160
|
+
);
|
|
9161
|
+
} catch {
|
|
9162
|
+
return [];
|
|
9163
|
+
}
|
|
9164
|
+
}
|
|
9165
|
+
function chooseDictionary(anchors) {
|
|
9166
|
+
if (anchors.length < 2) return null;
|
|
9167
|
+
const total = anchors.reduce((sum, body) => sum + body.length, 0);
|
|
9168
|
+
const candidates = [...anchors].sort((a, b) => b.length - a.length).slice(0, DICT_CANDIDATES);
|
|
9169
|
+
let best = null;
|
|
9170
|
+
for (const candidate of candidates) {
|
|
9171
|
+
let cost = candidate.length;
|
|
9172
|
+
for (const body of anchors) {
|
|
9173
|
+
cost += body === candidate ? 0 : computeBodyDelta(candidate, body).ins.length;
|
|
9174
|
+
}
|
|
9175
|
+
if (best === null || cost < best.cost) best = { body: candidate, cost };
|
|
9176
|
+
}
|
|
9177
|
+
if (best === null) return null;
|
|
9178
|
+
return total - best.cost >= total * MIN_SAVING_RATIO ? best.body : null;
|
|
9179
|
+
}
|
|
9180
|
+
var EMPTY = { shards: 0, anchors: 0, savedBytes: 0 };
|
|
9181
|
+
function compactAuditDay(dayPath) {
|
|
9182
|
+
const bodiesPath = join6(dayPath, AUDIT_BODIES_DIR);
|
|
9183
|
+
if (!existsSync14(bodiesPath)) return EMPTY;
|
|
9184
|
+
const dictPath = join6(bodiesPath, AUDIT_DICT_FILE);
|
|
9185
|
+
if (existsSync14(dictPath) || existsSync14(`${dictPath}.gz`)) return EMPTY;
|
|
9186
|
+
const shardFiles = plainShards(bodiesPath);
|
|
9187
|
+
if (shardFiles.length < 2) return EMPTY;
|
|
9188
|
+
const loaded = /* @__PURE__ */ new Map();
|
|
9189
|
+
const anchors = [];
|
|
9190
|
+
for (const file of shardFiles) {
|
|
9191
|
+
let entries;
|
|
9192
|
+
try {
|
|
9193
|
+
entries = parseEntries(readFileSync14(join6(bodiesPath, file), "utf8"));
|
|
9194
|
+
} catch {
|
|
9195
|
+
continue;
|
|
9196
|
+
}
|
|
9197
|
+
loaded.set(file, entries);
|
|
9198
|
+
for (const entry of entries) {
|
|
9199
|
+
if (entry.req && entry.req.base === null) anchors.push(entry.req.ins);
|
|
9200
|
+
}
|
|
9201
|
+
}
|
|
9202
|
+
if (anchors.length < 2) return EMPTY;
|
|
9203
|
+
const dictionary = chooseDictionary(anchors);
|
|
9204
|
+
if (dictionary === null) return EMPTY;
|
|
9205
|
+
const dictEntry = {
|
|
9206
|
+
id: `${DICT_BASE_PREFIX}0`,
|
|
9207
|
+
ts: 0,
|
|
9208
|
+
req: { base: null, anchor: "dict", pre: 0, suf: 0, ins: dictionary }
|
|
9209
|
+
};
|
|
9210
|
+
writeFileSync12(dictPath, JSON.stringify(dictEntry) + "\n", "utf8");
|
|
9211
|
+
const result = { shards: 0, anchors: 0, savedBytes: 0 };
|
|
9212
|
+
for (const [file, entries] of loaded) {
|
|
9213
|
+
let changed = false;
|
|
9214
|
+
let saved = 0;
|
|
9215
|
+
const rewritten = entries.map((entry) => {
|
|
9216
|
+
if (!entry.req || entry.req.base !== null || entry.req.ins === dictionary) return entry;
|
|
9217
|
+
const delta = computeBodyDelta(dictionary, entry.req.ins);
|
|
9218
|
+
if (delta.ins.length >= entry.req.ins.length) return entry;
|
|
9219
|
+
changed = true;
|
|
9220
|
+
saved += entry.req.ins.length - delta.ins.length;
|
|
9221
|
+
return {
|
|
9222
|
+
...entry,
|
|
9223
|
+
req: { base: dictEntry.id, pre: delta.pre, suf: delta.suf, ins: delta.ins }
|
|
9224
|
+
};
|
|
9225
|
+
});
|
|
9226
|
+
if (!changed) continue;
|
|
9227
|
+
const target = join6(bodiesPath, file);
|
|
9228
|
+
const temp = `${target}.compacting`;
|
|
9229
|
+
try {
|
|
9230
|
+
writeFileSync12(temp, rewritten.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8");
|
|
9231
|
+
renameSync5(temp, target);
|
|
9232
|
+
} catch {
|
|
9233
|
+
try {
|
|
9234
|
+
if (existsSync14(temp)) unlinkSync3(temp);
|
|
9235
|
+
} catch {
|
|
9236
|
+
}
|
|
9237
|
+
continue;
|
|
9238
|
+
}
|
|
9239
|
+
result.shards += 1;
|
|
9240
|
+
result.anchors += rewritten.filter((e) => e.req?.base === dictEntry.id).length;
|
|
9241
|
+
result.savedBytes += saved;
|
|
9242
|
+
}
|
|
9243
|
+
if (result.shards === 0) {
|
|
9244
|
+
try {
|
|
9245
|
+
unlinkSync3(dictPath);
|
|
9246
|
+
} catch {
|
|
9247
|
+
}
|
|
9248
|
+
}
|
|
9249
|
+
return result;
|
|
9250
|
+
}
|
|
9251
|
+
function compactAllClosedAuditDays(auditDir, now = Date.now) {
|
|
9252
|
+
const run = { days: 0, shards: 0, savedBytes: 0 };
|
|
9253
|
+
if (!existsSync14(auditDir)) return run;
|
|
9254
|
+
const today = auditDayDirName(now());
|
|
9255
|
+
let names;
|
|
9256
|
+
try {
|
|
9257
|
+
names = readdirSync(auditDir).filter(isAuditDayDir).sort();
|
|
9258
|
+
} catch {
|
|
9259
|
+
return run;
|
|
9260
|
+
}
|
|
9261
|
+
for (const name of names) {
|
|
9262
|
+
if (name === today) continue;
|
|
9263
|
+
try {
|
|
9264
|
+
const result = compactAuditDay(join6(auditDir, name));
|
|
9265
|
+
if (result.shards === 0) continue;
|
|
9266
|
+
run.days += 1;
|
|
9267
|
+
run.shards += result.shards;
|
|
9268
|
+
run.savedBytes += result.savedBytes;
|
|
9269
|
+
} catch {
|
|
9270
|
+
}
|
|
9271
|
+
}
|
|
9272
|
+
return run;
|
|
9273
|
+
}
|
|
9274
|
+
|
|
8892
9275
|
// src/audit/auditStats.ts
|
|
8893
9276
|
import {
|
|
8894
9277
|
createReadStream,
|
|
8895
|
-
existsSync as
|
|
8896
|
-
readFileSync as
|
|
8897
|
-
readdirSync,
|
|
9278
|
+
existsSync as existsSync15,
|
|
9279
|
+
readFileSync as readFileSync15,
|
|
9280
|
+
readdirSync as readdirSync2,
|
|
8898
9281
|
statSync as statSync3,
|
|
8899
|
-
writeFileSync as
|
|
9282
|
+
writeFileSync as writeFileSync13
|
|
8900
9283
|
} from "fs";
|
|
8901
|
-
import { basename, dirname as dirname7, join as
|
|
9284
|
+
import { basename, dirname as dirname7, join as join7 } from "path";
|
|
8902
9285
|
var SIDECAR_VERSION = 1;
|
|
8903
9286
|
var META_PREFIX_BYTES = 64 * 1024;
|
|
8904
9287
|
var READ_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
@@ -8906,9 +9289,9 @@ function auditStatsFileName(auditFile) {
|
|
|
8906
9289
|
return auditFile.replace(/\.jsonl$/, ".stats.json");
|
|
8907
9290
|
}
|
|
8908
9291
|
function readPersisted(path2) {
|
|
8909
|
-
if (!
|
|
9292
|
+
if (!existsSync15(path2)) return null;
|
|
8910
9293
|
try {
|
|
8911
|
-
const value = JSON.parse(
|
|
9294
|
+
const value = JSON.parse(readFileSync15(path2, "utf8"));
|
|
8912
9295
|
if (value.version !== SIDECAR_VERSION || !Number.isSafeInteger(value.auditBytes) || (value.auditBytes ?? -1) < 0 || !Number.isSafeInteger(value.requestCount) || (value.requestCount ?? -1) < 0 || !Number.isSafeInteger(value.errorCount) || (value.errorCount ?? -1) < 0 || (value.errorCount ?? 0) > (value.requestCount ?? -1) || typeof value.complete !== "boolean" || value.minTs !== null && !Number.isFinite(value.minTs) || value.maxTs !== null && !Number.isFinite(value.maxTs)) {
|
|
8913
9296
|
return null;
|
|
8914
9297
|
}
|
|
@@ -8918,7 +9301,7 @@ function readPersisted(path2) {
|
|
|
8918
9301
|
}
|
|
8919
9302
|
}
|
|
8920
9303
|
function updateAuditStatsAfterAppend(auditPath, auditBytesBefore, auditBytesAfter, record) {
|
|
8921
|
-
const statsPath =
|
|
9304
|
+
const statsPath = join7(dirname7(auditPath), auditStatsFileName(basename(auditPath)));
|
|
8922
9305
|
const previous = auditBytesBefore === 0 ? {
|
|
8923
9306
|
version: SIDECAR_VERSION,
|
|
8924
9307
|
auditBytes: 0,
|
|
@@ -8938,13 +9321,13 @@ function updateAuditStatsAfterAppend(auditPath, auditBytesBefore, auditBytesAfte
|
|
|
8938
9321
|
minTs: previous.minTs === null ? record.ts : Math.min(previous.minTs, record.ts),
|
|
8939
9322
|
maxTs: previous.maxTs === null ? record.ts : Math.max(previous.maxTs, record.ts)
|
|
8940
9323
|
};
|
|
8941
|
-
|
|
9324
|
+
writeFileSync13(statsPath, JSON.stringify(next), "utf8");
|
|
8942
9325
|
}
|
|
8943
9326
|
function queryCovers(stats, from, to) {
|
|
8944
9327
|
return stats.requestCount === 0 || stats.minTs !== null && stats.maxTs !== null && from <= stats.minTs && to >= stats.maxTs;
|
|
8945
9328
|
}
|
|
8946
|
-
function fileOverlaps(
|
|
8947
|
-
const start = auditFileDateMs(
|
|
9329
|
+
function fileOverlaps(name, from, to) {
|
|
9330
|
+
const start = auditFileDateMs(name);
|
|
8948
9331
|
if (start === null) return false;
|
|
8949
9332
|
const date = new Date(start);
|
|
8950
9333
|
const end = new Date(date.getFullYear(), date.getMonth(), date.getDate() + 1).getTime();
|
|
@@ -9049,21 +9432,27 @@ function mergePersistedStats(previous, appended) {
|
|
|
9049
9432
|
};
|
|
9050
9433
|
}
|
|
9051
9434
|
async function readAuditStats(auditDir, query2 = {}) {
|
|
9052
|
-
if (!
|
|
9435
|
+
if (!existsSync15(auditDir)) return { requestCount: 0, errorCount: 0, complete: true };
|
|
9053
9436
|
const from = typeof query2.from === "number" ? query2.from : -Infinity;
|
|
9054
9437
|
const to = typeof query2.to === "number" ? query2.to : Infinity;
|
|
9055
|
-
let
|
|
9438
|
+
let sources;
|
|
9056
9439
|
try {
|
|
9057
|
-
|
|
9440
|
+
sources = readdirSync2(auditDir).filter((name) => fileOverlaps(name, from, to)).sort().map(
|
|
9441
|
+
(name) => AUDIT_DAY_DIR_RE.test(name) ? {
|
|
9442
|
+
auditPath: join7(auditDir, name, AUDIT_META_FILE),
|
|
9443
|
+
statsPath: join7(auditDir, name, auditStatsFileName(AUDIT_META_FILE))
|
|
9444
|
+
} : {
|
|
9445
|
+
auditPath: join7(auditDir, name),
|
|
9446
|
+
statsPath: join7(auditDir, auditStatsFileName(name))
|
|
9447
|
+
}
|
|
9448
|
+
).filter((source) => existsSync15(source.auditPath));
|
|
9058
9449
|
} catch {
|
|
9059
9450
|
return { requestCount: 0, errorCount: 0, complete: false };
|
|
9060
9451
|
}
|
|
9061
9452
|
const total = { requestCount: 0, errorCount: 0, complete: true };
|
|
9062
|
-
for (const
|
|
9063
|
-
const auditPath = join6(auditDir, file);
|
|
9453
|
+
for (const { auditPath, statsPath } of sources) {
|
|
9064
9454
|
try {
|
|
9065
9455
|
const auditBytes = statSync3(auditPath).size;
|
|
9066
|
-
const statsPath = join6(auditDir, auditStatsFileName(file));
|
|
9067
9456
|
const persisted = readPersisted(statsPath);
|
|
9068
9457
|
if (persisted && persisted.complete && persisted.auditBytes === auditBytes && queryCovers(persisted, from, to)) {
|
|
9069
9458
|
total.requestCount += persisted.requestCount;
|
|
@@ -9082,7 +9471,7 @@ async function readAuditStats(auditDir, query2 = {}) {
|
|
|
9082
9471
|
total.errorCount += scanned.filtered.errorCount + (resumable?.errorCount ?? 0);
|
|
9083
9472
|
total.complete = total.complete && scanned.filtered.complete;
|
|
9084
9473
|
const current = resumable ? mergePersistedStats(resumable, scanned.all) : scanned.all;
|
|
9085
|
-
if (current.complete)
|
|
9474
|
+
if (current.complete) writeFileSync13(statsPath, JSON.stringify(current), "utf8");
|
|
9086
9475
|
} catch {
|
|
9087
9476
|
total.complete = false;
|
|
9088
9477
|
}
|
|
@@ -9093,6 +9482,7 @@ async function readAuditStats(auditDir, query2 = {}) {
|
|
|
9093
9482
|
// src/audit/AuditPruneSweeper.ts
|
|
9094
9483
|
var DAY_MS = 24 * 60 * 6e4;
|
|
9095
9484
|
var SWEEP_INTERVAL_MS2 = 60 * 6e4;
|
|
9485
|
+
var ARCHIVE_BATCH = 64;
|
|
9096
9486
|
var AuditPruneSweeper = class {
|
|
9097
9487
|
constructor(auditDir, logger, config, intervalMs = SWEEP_INTERVAL_MS2, now = Date.now) {
|
|
9098
9488
|
this.auditDir = auditDir;
|
|
@@ -9108,6 +9498,7 @@ var AuditPruneSweeper = class {
|
|
|
9108
9498
|
now;
|
|
9109
9499
|
timer = null;
|
|
9110
9500
|
sweeping = false;
|
|
9501
|
+
archiving = false;
|
|
9111
9502
|
/** Whether pruning is active (audit enabled). */
|
|
9112
9503
|
get enabled() {
|
|
9113
9504
|
return this.config.enabled;
|
|
@@ -9117,13 +9508,13 @@ var AuditPruneSweeper = class {
|
|
|
9117
9508
|
this.config = config;
|
|
9118
9509
|
}
|
|
9119
9510
|
/**
|
|
9120
|
-
* Arm the
|
|
9121
|
-
*
|
|
9511
|
+
* Arm the interval AND run one pass immediately (boot cleanup). No-op when
|
|
9512
|
+
* audit is disabled (zero regression). Idempotent.
|
|
9122
9513
|
*/
|
|
9123
9514
|
start() {
|
|
9124
9515
|
if (this.timer || !this.config.enabled) return;
|
|
9125
|
-
void this.
|
|
9126
|
-
this.timer = setInterval(() => void this.
|
|
9516
|
+
void this.runOnce();
|
|
9517
|
+
this.timer = setInterval(() => void this.runOnce(), this.intervalMs);
|
|
9127
9518
|
this.timer.unref?.();
|
|
9128
9519
|
}
|
|
9129
9520
|
/** Clear the interval (daemon shutdown / test teardown). Idempotent. */
|
|
@@ -9133,31 +9524,43 @@ var AuditPruneSweeper = class {
|
|
|
9133
9524
|
this.timer = null;
|
|
9134
9525
|
}
|
|
9135
9526
|
}
|
|
9527
|
+
/** Prune first, then archive — never spend CPU compressing a day about to go. */
|
|
9528
|
+
async runOnce() {
|
|
9529
|
+
await this.sweep();
|
|
9530
|
+
await this.archive();
|
|
9531
|
+
}
|
|
9532
|
+
/** The LOCAL-midnight epoch ms of the current day. */
|
|
9533
|
+
todayMidnight() {
|
|
9534
|
+
const today = new Date(this.now());
|
|
9535
|
+
return new Date(today.getFullYear(), today.getMonth(), today.getDate()).getTime();
|
|
9536
|
+
}
|
|
9136
9537
|
/**
|
|
9137
|
-
* One prune:
|
|
9138
|
-
*
|
|
9139
|
-
*
|
|
9538
|
+
* One prune: remove every audit day strictly OLDER than the retention cutoff
|
|
9539
|
+
* (`now - retentionDays` days, at local-midnight granularity). Exposed for
|
|
9540
|
+
* tests; never throws. Returns the number of days removed.
|
|
9140
9541
|
*/
|
|
9141
9542
|
async sweep() {
|
|
9142
9543
|
if (!this.config.enabled || this.sweeping) return 0;
|
|
9143
9544
|
this.sweeping = true;
|
|
9144
9545
|
try {
|
|
9145
|
-
if (!
|
|
9146
|
-
const
|
|
9147
|
-
const todayMidnight = new Date(today.getFullYear(), today.getMonth(), today.getDate()).getTime();
|
|
9148
|
-
const cutoff = todayMidnight - (this.config.retentionDays - 1) * DAY_MS;
|
|
9546
|
+
if (!existsSync16(this.auditDir)) return 0;
|
|
9547
|
+
const cutoff = this.todayMidnight() - (this.config.retentionDays - 1) * DAY_MS;
|
|
9149
9548
|
let removed = 0;
|
|
9150
|
-
for (const
|
|
9151
|
-
const dateMs = auditFileDateMs(
|
|
9549
|
+
for (const name of readdirSync3(this.auditDir)) {
|
|
9550
|
+
const dateMs = auditFileDateMs(name);
|
|
9152
9551
|
if (dateMs === null || dateMs >= cutoff) continue;
|
|
9153
9552
|
try {
|
|
9154
|
-
|
|
9553
|
+
if (isAuditDayDir(name)) {
|
|
9554
|
+
rmSync4(join8(this.auditDir, name), { recursive: true, force: true });
|
|
9555
|
+
} else {
|
|
9556
|
+
unlinkSync4(join8(this.auditDir, name));
|
|
9557
|
+
const statsPath = join8(this.auditDir, auditStatsFileName(name));
|
|
9558
|
+
if (existsSync16(statsPath)) unlinkSync4(statsPath);
|
|
9559
|
+
}
|
|
9155
9560
|
removed += 1;
|
|
9156
|
-
const statsPath = join7(this.auditDir, auditStatsFileName(file));
|
|
9157
|
-
if (existsSync15(statsPath)) unlinkSync3(statsPath);
|
|
9158
9561
|
} catch (error) {
|
|
9159
|
-
this.logger.warn("[AuditPruneSweeper] failed to
|
|
9160
|
-
|
|
9562
|
+
this.logger.warn("[AuditPruneSweeper] failed to remove expired audit day", {
|
|
9563
|
+
name,
|
|
9161
9564
|
error: error instanceof Error ? error.message : String(error)
|
|
9162
9565
|
});
|
|
9163
9566
|
}
|
|
@@ -9173,59 +9576,349 @@ var AuditPruneSweeper = class {
|
|
|
9173
9576
|
this.sweeping = false;
|
|
9174
9577
|
}
|
|
9175
9578
|
}
|
|
9579
|
+
/**
|
|
9580
|
+
* Gzip the body shards of every CLOSED day (anything before today). Today is
|
|
9581
|
+
* deliberately left as plain text so it stays greppable while it is the day you
|
|
9582
|
+
* are debugging. Exposed for tests; never throws. Returns shards compressed.
|
|
9583
|
+
*/
|
|
9584
|
+
async archive() {
|
|
9585
|
+
if (!this.config.enabled || this.archiving) return 0;
|
|
9586
|
+
this.archiving = true;
|
|
9587
|
+
try {
|
|
9588
|
+
if (!existsSync16(this.auditDir)) return 0;
|
|
9589
|
+
const today = this.todayMidnight();
|
|
9590
|
+
let compressed = 0;
|
|
9591
|
+
for (const name of readdirSync3(this.auditDir)) {
|
|
9592
|
+
if (compressed >= ARCHIVE_BATCH) break;
|
|
9593
|
+
const dateMs = auditFileDateMs(name);
|
|
9594
|
+
if (dateMs === null || dateMs >= today || !isAuditDayDir(name)) continue;
|
|
9595
|
+
const dayPath = join8(this.auditDir, name);
|
|
9596
|
+
try {
|
|
9597
|
+
const compaction = compactAuditDay(dayPath);
|
|
9598
|
+
if (compaction.shards > 0) {
|
|
9599
|
+
this.logger.debug("audit cross-session compaction complete", {
|
|
9600
|
+
day: name,
|
|
9601
|
+
shards: compaction.shards,
|
|
9602
|
+
anchors: compaction.anchors,
|
|
9603
|
+
savedBytes: compaction.savedBytes
|
|
9604
|
+
});
|
|
9605
|
+
}
|
|
9606
|
+
} catch (error) {
|
|
9607
|
+
this.logger.warn("[AuditPruneSweeper] cross-session compaction failed", {
|
|
9608
|
+
day: name,
|
|
9609
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9610
|
+
});
|
|
9611
|
+
}
|
|
9612
|
+
compressed += await this.archiveDay(
|
|
9613
|
+
join8(dayPath, AUDIT_BODIES_DIR),
|
|
9614
|
+
ARCHIVE_BATCH - compressed
|
|
9615
|
+
);
|
|
9616
|
+
}
|
|
9617
|
+
if (compressed > 0) this.logger.debug("audit archive complete", { compressed });
|
|
9618
|
+
return compressed;
|
|
9619
|
+
} catch (error) {
|
|
9620
|
+
this.logger.warn("audit archive pass failed", {
|
|
9621
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9622
|
+
});
|
|
9623
|
+
return 0;
|
|
9624
|
+
} finally {
|
|
9625
|
+
this.archiving = false;
|
|
9626
|
+
}
|
|
9627
|
+
}
|
|
9628
|
+
/** Gzip up to `budget` plain shards in one day's `bodies/` directory. */
|
|
9629
|
+
async archiveDay(bodiesPath, budget) {
|
|
9630
|
+
let shards;
|
|
9631
|
+
try {
|
|
9632
|
+
shards = readdirSync3(bodiesPath).filter((file) => file.endsWith(".jsonl"));
|
|
9633
|
+
} catch {
|
|
9634
|
+
return 0;
|
|
9635
|
+
}
|
|
9636
|
+
let compressed = 0;
|
|
9637
|
+
for (const shard of shards) {
|
|
9638
|
+
if (compressed >= budget) break;
|
|
9639
|
+
const source = join8(bodiesPath, shard);
|
|
9640
|
+
const target = `${source}.gz`;
|
|
9641
|
+
try {
|
|
9642
|
+
if (existsSync16(target)) {
|
|
9643
|
+
unlinkSync4(source);
|
|
9644
|
+
continue;
|
|
9645
|
+
}
|
|
9646
|
+
await pipeline(createReadStream2(source), createGzip(), createWriteStream2(target));
|
|
9647
|
+
unlinkSync4(source);
|
|
9648
|
+
compressed += 1;
|
|
9649
|
+
} catch (error) {
|
|
9650
|
+
try {
|
|
9651
|
+
if (existsSync16(target)) unlinkSync4(target);
|
|
9652
|
+
} catch {
|
|
9653
|
+
}
|
|
9654
|
+
this.logger.warn("[AuditPruneSweeper] failed to archive audit body shard", {
|
|
9655
|
+
shard,
|
|
9656
|
+
error: error instanceof Error ? error.message : String(error)
|
|
9657
|
+
});
|
|
9658
|
+
}
|
|
9659
|
+
}
|
|
9660
|
+
return compressed;
|
|
9661
|
+
}
|
|
9176
9662
|
};
|
|
9177
9663
|
|
|
9178
|
-
// src/audit/
|
|
9179
|
-
import { existsSync as
|
|
9180
|
-
import { join as
|
|
9181
|
-
|
|
9182
|
-
|
|
9183
|
-
|
|
9184
|
-
|
|
9185
|
-
|
|
9664
|
+
// src/audit/auditBodyReader.ts
|
|
9665
|
+
import { existsSync as existsSync17, readdirSync as readdirSync4, readFileSync as readFileSync16, statSync as statSync5 } from "fs";
|
|
9666
|
+
import { join as join9 } from "path";
|
|
9667
|
+
import { gunzipSync } from "zlib";
|
|
9668
|
+
|
|
9669
|
+
// src/audit/auditJsonl.ts
|
|
9670
|
+
import { closeSync, openSync, readSync, statSync as statSync4 } from "fs";
|
|
9671
|
+
var WINDOW_BYTES = 1 << 20;
|
|
9672
|
+
var MAX_LINE_BYTES = 32 * 1024 * 1024;
|
|
9673
|
+
var NEWLINE = 10;
|
|
9674
|
+
function forEachLineFromTail(path2, onLine) {
|
|
9675
|
+
let fd;
|
|
9676
|
+
let end;
|
|
9677
|
+
try {
|
|
9678
|
+
end = statSync4(path2).size;
|
|
9679
|
+
if (end === 0) return;
|
|
9680
|
+
fd = openSync(path2, "r");
|
|
9681
|
+
} catch {
|
|
9682
|
+
return;
|
|
9683
|
+
}
|
|
9684
|
+
try {
|
|
9685
|
+
let carry = Buffer.alloc(0);
|
|
9686
|
+
while (end > 0) {
|
|
9687
|
+
const start = Math.max(0, end - WINDOW_BYTES);
|
|
9688
|
+
const window = Buffer.allocUnsafe(end - start);
|
|
9689
|
+
let read;
|
|
9690
|
+
try {
|
|
9691
|
+
read = readSync(fd, window, 0, end - start, start);
|
|
9692
|
+
} catch {
|
|
9693
|
+
return;
|
|
9694
|
+
}
|
|
9695
|
+
const chunk = carry.length > 0 ? Buffer.concat([window.subarray(0, read), carry]) : window.subarray(0, read);
|
|
9696
|
+
let lineEnd = chunk.length;
|
|
9697
|
+
let nl = lineEnd > 0 ? chunk.lastIndexOf(NEWLINE, lineEnd - 1) : -1;
|
|
9698
|
+
while (nl >= 0) {
|
|
9699
|
+
if (nl + 1 < lineEnd) {
|
|
9700
|
+
const line = chunk.subarray(nl + 1, lineEnd).toString("utf8").trim();
|
|
9701
|
+
if (line && onLine(line)) return;
|
|
9702
|
+
}
|
|
9703
|
+
lineEnd = nl;
|
|
9704
|
+
nl = lineEnd > 0 ? chunk.lastIndexOf(NEWLINE, lineEnd - 1) : -1;
|
|
9705
|
+
}
|
|
9706
|
+
if (start === 0) {
|
|
9707
|
+
if (lineEnd > 0) {
|
|
9708
|
+
const line = chunk.subarray(0, lineEnd).toString("utf8").trim();
|
|
9709
|
+
if (line) onLine(line);
|
|
9710
|
+
}
|
|
9711
|
+
return;
|
|
9712
|
+
}
|
|
9713
|
+
if (lineEnd > MAX_LINE_BYTES) return;
|
|
9714
|
+
carry = Buffer.from(chunk.subarray(0, lineEnd));
|
|
9715
|
+
end = start;
|
|
9716
|
+
}
|
|
9717
|
+
} finally {
|
|
9718
|
+
try {
|
|
9719
|
+
closeSync(fd);
|
|
9720
|
+
} catch {
|
|
9721
|
+
}
|
|
9722
|
+
}
|
|
9723
|
+
}
|
|
9724
|
+
|
|
9725
|
+
// src/audit/auditBodyReader.ts
|
|
9726
|
+
function candidateDays(auditDir, ts) {
|
|
9727
|
+
if (typeof ts === "number" && Number.isFinite(ts)) {
|
|
9728
|
+
const named = auditDayDirName(ts);
|
|
9729
|
+
if (existsSync17(join9(auditDir, named))) return [named];
|
|
9730
|
+
}
|
|
9186
9731
|
try {
|
|
9187
|
-
|
|
9732
|
+
return readdirSync4(auditDir).filter(isAuditDayDir).sort().reverse();
|
|
9188
9733
|
} catch {
|
|
9189
9734
|
return [];
|
|
9190
9735
|
}
|
|
9191
|
-
|
|
9192
|
-
|
|
9193
|
-
const
|
|
9194
|
-
|
|
9195
|
-
|
|
9196
|
-
|
|
9736
|
+
}
|
|
9737
|
+
function readShard(auditDir, day, sessionKey) {
|
|
9738
|
+
const base = join9(auditDir, day, AUDIT_BODIES_DIR, auditBodyFileName(sessionKey));
|
|
9739
|
+
try {
|
|
9740
|
+
if (existsSync17(base)) return readFileSync16(base, "utf8");
|
|
9741
|
+
const gz = `${base}.gz`;
|
|
9742
|
+
if (existsSync17(gz)) return gunzipSync(readFileSync16(gz)).toString("utf8");
|
|
9743
|
+
} catch {
|
|
9744
|
+
return null;
|
|
9745
|
+
}
|
|
9746
|
+
return null;
|
|
9747
|
+
}
|
|
9748
|
+
function parseShard(raw) {
|
|
9749
|
+
const entries = /* @__PURE__ */ new Map();
|
|
9750
|
+
for (const line of raw.split("\n")) {
|
|
9751
|
+
const trimmed = line.trim();
|
|
9752
|
+
if (!trimmed) continue;
|
|
9753
|
+
let parsed;
|
|
9197
9754
|
try {
|
|
9198
|
-
|
|
9755
|
+
parsed = JSON.parse(trimmed);
|
|
9199
9756
|
} catch {
|
|
9200
9757
|
continue;
|
|
9201
9758
|
}
|
|
9202
|
-
|
|
9203
|
-
|
|
9204
|
-
|
|
9205
|
-
|
|
9759
|
+
if (isAuditBodyEntry(parsed)) entries.set(parsed.id, parsed);
|
|
9760
|
+
}
|
|
9761
|
+
return entries;
|
|
9762
|
+
}
|
|
9763
|
+
function withDictionary(auditDir, day, entries) {
|
|
9764
|
+
let needed = false;
|
|
9765
|
+
for (const entry of entries.values()) {
|
|
9766
|
+
if (entry.req?.base?.startsWith(DICT_BASE_PREFIX)) {
|
|
9767
|
+
needed = true;
|
|
9768
|
+
break;
|
|
9769
|
+
}
|
|
9770
|
+
}
|
|
9771
|
+
if (!needed) return entries;
|
|
9772
|
+
const base = join9(auditDir, day, AUDIT_BODIES_DIR, AUDIT_DICT_FILE);
|
|
9773
|
+
let raw = null;
|
|
9774
|
+
try {
|
|
9775
|
+
if (existsSync17(base)) raw = readFileSync16(base, "utf8");
|
|
9776
|
+
else if (existsSync17(`${base}.gz`)) raw = gunzipSync(readFileSync16(`${base}.gz`)).toString("utf8");
|
|
9777
|
+
} catch {
|
|
9778
|
+
return entries;
|
|
9779
|
+
}
|
|
9780
|
+
if (raw === null) return entries;
|
|
9781
|
+
for (const [id, entry] of parseShard(raw)) entries.set(id, entry);
|
|
9782
|
+
return entries;
|
|
9783
|
+
}
|
|
9784
|
+
function reconstructRequest(entries, entry) {
|
|
9785
|
+
if (!entry.req) return void 0;
|
|
9786
|
+
const chain = [];
|
|
9787
|
+
const visited = /* @__PURE__ */ new Set();
|
|
9788
|
+
let cursor = entry;
|
|
9789
|
+
while (cursor?.req) {
|
|
9790
|
+
if (visited.has(cursor.id)) return void 0;
|
|
9791
|
+
visited.add(cursor.id);
|
|
9792
|
+
chain.push(cursor);
|
|
9793
|
+
if (cursor.req.base === null) break;
|
|
9794
|
+
cursor = entries.get(cursor.req.base);
|
|
9795
|
+
}
|
|
9796
|
+
const anchor = chain[chain.length - 1];
|
|
9797
|
+
if (!anchor?.req || anchor.req.base !== null) return void 0;
|
|
9798
|
+
let text = anchor.req.ins;
|
|
9799
|
+
for (let i = chain.length - 2; i >= 0; i -= 1) {
|
|
9800
|
+
const delta = chain[i]?.req;
|
|
9801
|
+
if (!delta) return void 0;
|
|
9802
|
+
if (delta.pre > text.length || delta.suf > text.length - delta.pre) return void 0;
|
|
9803
|
+
text = applyBodyDelta(text, delta);
|
|
9804
|
+
}
|
|
9805
|
+
return text;
|
|
9806
|
+
}
|
|
9807
|
+
function readAuditBody(auditDir, query2) {
|
|
9808
|
+
if (!isSafeSessionKey(query2.sessionKey) || !query2.id) return {};
|
|
9809
|
+
if (!existsSync17(auditDir)) return {};
|
|
9810
|
+
for (const day of candidateDays(auditDir, query2.ts)) {
|
|
9811
|
+
const raw = readShard(auditDir, day, query2.sessionKey);
|
|
9812
|
+
if (raw === null) continue;
|
|
9813
|
+
const entries = withDictionary(auditDir, day, parseShard(raw));
|
|
9814
|
+
const entry = entries.get(query2.id);
|
|
9815
|
+
if (!entry) continue;
|
|
9816
|
+
const result = {};
|
|
9817
|
+
const requestBody = reconstructRequest(entries, entry);
|
|
9818
|
+
if (requestBody !== void 0) result.requestBody = requestBody;
|
|
9819
|
+
if (entry.res !== void 0) result.responseBody = entry.res;
|
|
9820
|
+
return result;
|
|
9821
|
+
}
|
|
9822
|
+
return readLegacyInlineBody(auditDir, query2.id);
|
|
9823
|
+
}
|
|
9824
|
+
function readLegacyInlineBody(auditDir, id) {
|
|
9825
|
+
let names;
|
|
9826
|
+
try {
|
|
9827
|
+
names = readdirSync4(auditDir).filter((name) => AUDIT_FILE_RE.test(name)).sort().reverse();
|
|
9828
|
+
} catch {
|
|
9829
|
+
return {};
|
|
9830
|
+
}
|
|
9831
|
+
const needle = JSON.stringify(id);
|
|
9832
|
+
let found = {};
|
|
9833
|
+
for (const name of names) {
|
|
9834
|
+
forEachLineFromTail(join9(auditDir, name), (line) => {
|
|
9835
|
+
if (!line.includes(needle)) return false;
|
|
9836
|
+
let parsed;
|
|
9206
9837
|
try {
|
|
9207
|
-
|
|
9838
|
+
parsed = JSON.parse(line);
|
|
9208
9839
|
} catch {
|
|
9209
|
-
|
|
9840
|
+
return false;
|
|
9210
9841
|
}
|
|
9211
|
-
|
|
9212
|
-
if (
|
|
9213
|
-
|
|
9214
|
-
|
|
9842
|
+
const record = parsed;
|
|
9843
|
+
if (record.id !== id) return false;
|
|
9844
|
+
const result = {};
|
|
9845
|
+
if (typeof record.requestBody === "string") result.requestBody = record.requestBody;
|
|
9846
|
+
if (typeof record.responseBody === "string") result.responseBody = record.responseBody;
|
|
9847
|
+
found = result;
|
|
9848
|
+
return true;
|
|
9849
|
+
});
|
|
9850
|
+
if (found.requestBody !== void 0 || found.responseBody !== void 0) break;
|
|
9851
|
+
}
|
|
9852
|
+
return found;
|
|
9853
|
+
}
|
|
9854
|
+
|
|
9855
|
+
// src/audit/auditReader.ts
|
|
9856
|
+
import { existsSync as existsSync18, readdirSync as readdirSync5 } from "fs";
|
|
9857
|
+
import { join as join10 } from "path";
|
|
9858
|
+
var DEFAULT_LIMIT = 200;
|
|
9859
|
+
var MAX_LIMIT = 2e3;
|
|
9860
|
+
var OVERSCAN = 256;
|
|
9861
|
+
function daySources(auditDir) {
|
|
9862
|
+
let names;
|
|
9863
|
+
try {
|
|
9864
|
+
names = readdirSync5(auditDir);
|
|
9865
|
+
} catch {
|
|
9866
|
+
return [];
|
|
9867
|
+
}
|
|
9868
|
+
const sources = [];
|
|
9869
|
+
for (const name of names) {
|
|
9870
|
+
const dateMs = auditFileDateMs(name);
|
|
9871
|
+
if (dateMs === null) continue;
|
|
9872
|
+
if (AUDIT_DAY_DIR_RE.test(name)) {
|
|
9873
|
+
const path2 = join10(auditDir, name, AUDIT_META_FILE);
|
|
9874
|
+
if (existsSync18(path2)) sources.push({ path: path2, dateMs });
|
|
9875
|
+
} else if (AUDIT_FILE_RE.test(name)) {
|
|
9876
|
+
sources.push({ path: join10(auditDir, name), dateMs });
|
|
9215
9877
|
}
|
|
9216
9878
|
}
|
|
9217
|
-
|
|
9218
|
-
return matched.slice(0, limit);
|
|
9879
|
+
return sources.sort((a, b) => b.dateMs - a.dateMs);
|
|
9219
9880
|
}
|
|
9220
9881
|
function isAuditRecord(value) {
|
|
9221
9882
|
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
9222
9883
|
const r = value;
|
|
9223
9884
|
return typeof r["id"] === "string" && typeof r["ts"] === "number" && typeof r["method"] === "string" && typeof r["path"] === "string" && typeof r["status"] === "number";
|
|
9224
9885
|
}
|
|
9886
|
+
function toMetaRecord(record) {
|
|
9887
|
+
if (record.requestBody === void 0 && record.responseBody === void 0) return record;
|
|
9888
|
+
const { requestBody: _req, responseBody: _res, ...meta } = record;
|
|
9889
|
+
return { ...meta, hasBody: true };
|
|
9890
|
+
}
|
|
9891
|
+
function readAuditRecords(auditDir, query2 = {}) {
|
|
9892
|
+
if (!existsSync18(auditDir)) return [];
|
|
9893
|
+
const from = typeof query2.from === "number" ? query2.from : -Infinity;
|
|
9894
|
+
const to = typeof query2.to === "number" ? query2.to : Infinity;
|
|
9895
|
+
const limit = Math.min(MAX_LIMIT, Math.max(1, Math.trunc(query2.limit ?? DEFAULT_LIMIT)));
|
|
9896
|
+
const matched = [];
|
|
9897
|
+
for (const source of daySources(auditDir)) {
|
|
9898
|
+
const before = matched.length;
|
|
9899
|
+
forEachLineFromTail(source.path, (line) => {
|
|
9900
|
+
let parsed;
|
|
9901
|
+
try {
|
|
9902
|
+
parsed = JSON.parse(line);
|
|
9903
|
+
} catch {
|
|
9904
|
+
return false;
|
|
9905
|
+
}
|
|
9906
|
+
if (!isAuditRecord(parsed)) return false;
|
|
9907
|
+
if (query2.keyId !== void 0 && parsed.keyId !== query2.keyId) return false;
|
|
9908
|
+
if (query2.sessionKey !== void 0 && parsed.sessionKey !== query2.sessionKey) return false;
|
|
9909
|
+
if (parsed.ts < from || parsed.ts > to) return false;
|
|
9910
|
+
matched.push(toMetaRecord(parsed));
|
|
9911
|
+
return matched.length - before >= limit + OVERSCAN;
|
|
9912
|
+
});
|
|
9913
|
+
if (matched.length >= limit) break;
|
|
9914
|
+
}
|
|
9915
|
+
matched.sort((a, b) => b.ts - a.ts);
|
|
9916
|
+
return matched.slice(0, limit);
|
|
9917
|
+
}
|
|
9225
9918
|
|
|
9226
9919
|
// src/audit/AuditWriter.ts
|
|
9227
|
-
import { appendFileSync as appendFileSync2, existsSync as
|
|
9228
|
-
import { join as
|
|
9920
|
+
import { appendFileSync as appendFileSync2, existsSync as existsSync19, mkdirSync as mkdirSync5, statSync as statSync6 } from "fs";
|
|
9921
|
+
import { join as join11 } from "path";
|
|
9229
9922
|
var AuditWriter = class {
|
|
9230
9923
|
constructor(auditDir, logger, defer = (fn) => setTimeout(fn, 0)) {
|
|
9231
9924
|
this.auditDir = auditDir;
|
|
@@ -9235,10 +9928,13 @@ var AuditWriter = class {
|
|
|
9235
9928
|
auditDir;
|
|
9236
9929
|
logger;
|
|
9237
9930
|
defer;
|
|
9238
|
-
|
|
9931
|
+
/** Day directories already created this process (avoids an mkdir per record). */
|
|
9932
|
+
ensuredDirs = /* @__PURE__ */ new Set();
|
|
9933
|
+
/** Per-session encoding bases. Memory-only; a miss simply writes a full snapshot. */
|
|
9934
|
+
bases = new SessionBaseCache();
|
|
9239
9935
|
/**
|
|
9240
|
-
* Enqueue one record
|
|
9241
|
-
*
|
|
9936
|
+
* Enqueue one record. Returns IMMEDIATELY (fire-and-forget); every fs write and
|
|
9937
|
+
* the delta encoding happen on the deferred tick. A failure is logged, never thrown.
|
|
9242
9938
|
*/
|
|
9243
9939
|
record(record) {
|
|
9244
9940
|
this.defer(() => {
|
|
@@ -9251,25 +9947,41 @@ var AuditWriter = class {
|
|
|
9251
9947
|
}
|
|
9252
9948
|
});
|
|
9253
9949
|
}
|
|
9950
|
+
/** Drop all retained encoding bases (config reload / shutdown / test teardown). */
|
|
9951
|
+
reset() {
|
|
9952
|
+
this.bases.clear();
|
|
9953
|
+
this.ensuredDirs.clear();
|
|
9954
|
+
}
|
|
9254
9955
|
/**
|
|
9255
|
-
* Append synchronously — the awaitable form tests use to assert
|
|
9256
|
-
*
|
|
9257
|
-
* store's lazy file creation).
|
|
9956
|
+
* Append synchronously — the awaitable form tests use to assert a line landed.
|
|
9957
|
+
* Writes the metadata line first (canonical), then the body shard.
|
|
9258
9958
|
*/
|
|
9259
9959
|
appendNow(record) {
|
|
9260
|
-
|
|
9261
|
-
|
|
9262
|
-
|
|
9263
|
-
|
|
9264
|
-
|
|
9265
|
-
|
|
9266
|
-
|
|
9960
|
+
const dayDir = auditDayDirName(record.ts);
|
|
9961
|
+
const dayPath = this.ensureDir(join11(this.auditDir, dayDir));
|
|
9962
|
+
this.appendMeta(dayPath, record);
|
|
9963
|
+
this.appendBody(dayPath, dayDir, record);
|
|
9964
|
+
}
|
|
9965
|
+
/** Create a directory once per process and remember it. */
|
|
9966
|
+
ensureDir(path2) {
|
|
9967
|
+
if (!this.ensuredDirs.has(path2)) {
|
|
9968
|
+
mkdirSync5(path2, { recursive: true });
|
|
9969
|
+
this.ensuredDirs.add(path2);
|
|
9970
|
+
}
|
|
9971
|
+
return path2;
|
|
9972
|
+
}
|
|
9973
|
+
/** Write the body-free metadata line + refresh the exact-count sidecar. */
|
|
9974
|
+
appendMeta(dayPath, record) {
|
|
9975
|
+
const { requestBody: _req, responseBody: _res, ...meta } = record;
|
|
9976
|
+
const file = join11(dayPath, AUDIT_META_FILE);
|
|
9977
|
+
const line = JSON.stringify(meta) + "\n";
|
|
9978
|
+
const bytesBefore = existsSync19(file) ? statSync6(file).size : 0;
|
|
9267
9979
|
appendFileSync2(file, line, "utf8");
|
|
9268
9980
|
try {
|
|
9269
9981
|
updateAuditStatsAfterAppend(
|
|
9270
9982
|
file,
|
|
9271
|
-
|
|
9272
|
-
|
|
9983
|
+
bytesBefore,
|
|
9984
|
+
bytesBefore + Buffer.byteLength(line, "utf8"),
|
|
9273
9985
|
record
|
|
9274
9986
|
);
|
|
9275
9987
|
} catch (error) {
|
|
@@ -9278,12 +9990,39 @@ var AuditWriter = class {
|
|
|
9278
9990
|
});
|
|
9279
9991
|
}
|
|
9280
9992
|
}
|
|
9993
|
+
/**
|
|
9994
|
+
* Write the delta-encoded body shard for one record. A no-op when nothing was
|
|
9995
|
+
* captured or when the session key is missing/unsafe — in which case the body
|
|
9996
|
+
* is dropped rather than written to an unvalidated path.
|
|
9997
|
+
*/
|
|
9998
|
+
appendBody(dayPath, dayDir, record) {
|
|
9999
|
+
if (record.requestBody === void 0 && record.responseBody === void 0) return;
|
|
10000
|
+
const sessionKey = record.sessionKey;
|
|
10001
|
+
if (!isSafeSessionKey(sessionKey)) {
|
|
10002
|
+
this.logger.warn("[AuditWriter] dropping audit body with no usable session key", {
|
|
10003
|
+
id: record.id
|
|
10004
|
+
});
|
|
10005
|
+
return;
|
|
10006
|
+
}
|
|
10007
|
+
try {
|
|
10008
|
+
const line = encodeBodyEntry(record, sessionKey, dayDir, this.bases);
|
|
10009
|
+
if (line === null) return;
|
|
10010
|
+
const bodiesPath = this.ensureDir(join11(dayPath, AUDIT_BODIES_DIR));
|
|
10011
|
+
appendFileSync2(join11(bodiesPath, auditBodyFileName(sessionKey)), line + "\n", "utf8");
|
|
10012
|
+
} catch (error) {
|
|
10013
|
+
this.bases.forget(sessionKey);
|
|
10014
|
+
this.logger.warn("[AuditWriter] failed to append audit body shard", {
|
|
10015
|
+
id: record.id,
|
|
10016
|
+
error: error instanceof Error ? error.message : String(error)
|
|
10017
|
+
});
|
|
10018
|
+
}
|
|
10019
|
+
}
|
|
9281
10020
|
};
|
|
9282
10021
|
|
|
9283
10022
|
// src/billing/BillingPublisher.ts
|
|
9284
10023
|
import { appendFileSync as appendFileSync3, mkdirSync as mkdirSync6 } from "fs";
|
|
9285
10024
|
import { createHmac } from "crypto";
|
|
9286
|
-
import { join as
|
|
10025
|
+
import { join as join12 } from "path";
|
|
9287
10026
|
import { fetchUpstream as fetchUpstream5 } from "@omnicross/core/pipeline/upstreamFetch";
|
|
9288
10027
|
|
|
9289
10028
|
// src/billing/billingFiles.ts
|
|
@@ -9354,7 +10093,7 @@ var BillingPublisher = class {
|
|
|
9354
10093
|
*/
|
|
9355
10094
|
appendNow(event) {
|
|
9356
10095
|
this.ensureDir();
|
|
9357
|
-
const file =
|
|
10096
|
+
const file = join12(this.billingDir, billingFileName(event.ts));
|
|
9358
10097
|
appendFileSync3(file, JSON.stringify(event) + "\n", "utf8");
|
|
9359
10098
|
}
|
|
9360
10099
|
/**
|
|
@@ -9404,7 +10143,7 @@ var BillingPublisher = class {
|
|
|
9404
10143
|
markDelivered(event) {
|
|
9405
10144
|
try {
|
|
9406
10145
|
this.ensureDir();
|
|
9407
|
-
const file =
|
|
10146
|
+
const file = join12(this.billingDir, deliveredFileName(event.ts));
|
|
9408
10147
|
appendFileSync3(file, JSON.stringify({ id: event.id, deliveredAt: this.now() }) + "\n", "utf8");
|
|
9409
10148
|
} catch (error) {
|
|
9410
10149
|
this.logger.warn("[BillingPublisher] failed to append delivery marker", {
|
|
@@ -9420,14 +10159,14 @@ var BillingPublisher = class {
|
|
|
9420
10159
|
};
|
|
9421
10160
|
|
|
9422
10161
|
// src/billing/billingReader.ts
|
|
9423
|
-
import { existsSync as
|
|
9424
|
-
import { join as
|
|
10162
|
+
import { existsSync as existsSync20, readdirSync as readdirSync6, readFileSync as readFileSync17 } from "fs";
|
|
10163
|
+
import { join as join13 } from "path";
|
|
9425
10164
|
function readBillingLedger(billingDir) {
|
|
9426
10165
|
const view = { events: [], deliveredIds: /* @__PURE__ */ new Set() };
|
|
9427
|
-
if (!
|
|
10166
|
+
if (!existsSync20(billingDir)) return view;
|
|
9428
10167
|
let files;
|
|
9429
10168
|
try {
|
|
9430
|
-
files =
|
|
10169
|
+
files = readdirSync6(billingDir);
|
|
9431
10170
|
} catch {
|
|
9432
10171
|
return view;
|
|
9433
10172
|
}
|
|
@@ -9458,7 +10197,7 @@ function readBillingStatus(billingDir) {
|
|
|
9458
10197
|
function parseLines(dir, file) {
|
|
9459
10198
|
let raw;
|
|
9460
10199
|
try {
|
|
9461
|
-
raw =
|
|
10200
|
+
raw = readFileSync17(join13(dir, file), "utf8");
|
|
9462
10201
|
} catch {
|
|
9463
10202
|
return [];
|
|
9464
10203
|
}
|
|
@@ -9962,8 +10701,10 @@ function buildDaemon(config, paths) {
|
|
|
9962
10701
|
async (providerId, model) => await pricingEngine.getEntry(providerId, model) !== null
|
|
9963
10702
|
);
|
|
9964
10703
|
const keySpendTracker = new KeySpendTracker(usageEventStore);
|
|
10704
|
+
const usageThroughput = getSharedUsageThroughputTracker2();
|
|
9965
10705
|
const usageRecorder = new UsageRecorder(usageEventStore, pricingEngine, logger, {
|
|
9966
|
-
onRecord: (apiKeyId, costUsd, at) => keySpendTracker.add(apiKeyId, costUsd, at)
|
|
10706
|
+
onRecord: (apiKeyId, costUsd, at) => keySpendTracker.add(apiKeyId, costUsd, at),
|
|
10707
|
+
onEvent: (row, at) => usageThroughput.record(row, at)
|
|
9967
10708
|
});
|
|
9968
10709
|
const providerProxy = getProviderProxy({ llmConfig, apiKeyPool, usageRecorder });
|
|
9969
10710
|
const routeLeaseManager = new RouteLeaseManager(
|
|
@@ -10112,6 +10853,11 @@ function buildDaemon(config, paths) {
|
|
|
10112
10853
|
// NEVER unauth, NEVER on `/health`. Routed in `AdminServer` (not `adminApi.ts`).
|
|
10113
10854
|
auditReader: (query2) => readAuditRecords(auditDir, query2),
|
|
10114
10855
|
auditStatsReader: (query2) => readAuditStats(auditDir, query2),
|
|
10856
|
+
// audit-store-sharding: bodies live in per-session shards, so opening ONE
|
|
10857
|
+
// record's payload is a separate authed call that replays its delta chain.
|
|
10858
|
+
auditBodyReader: (query2) => readAuditBody(auditDir, query2),
|
|
10859
|
+
// audit-store-sharding D8: the manual counterpart to the daily pass.
|
|
10860
|
+
auditCompactor: () => compactAllClosedAuditDays(auditDir),
|
|
10115
10861
|
// billing-event-stream: the AUTHED `GET /admin/api/billing-status` returns the
|
|
10116
10862
|
// secret-free total/delivered/pending counts of the durable ledger.
|
|
10117
10863
|
billingStatusReader: () => readBillingStatus(billingDir)
|
|
@@ -10185,10 +10931,11 @@ function resetDaemonSingletonsForTests() {
|
|
|
10185
10931
|
__resetSharedIdentityStoreForTests();
|
|
10186
10932
|
__resetSharedAccountAllowanceStoreForTests();
|
|
10187
10933
|
__resetSharedAccountAllowanceSchedulingForTests();
|
|
10934
|
+
__resetSharedUsageThroughputTrackerForTests();
|
|
10188
10935
|
}
|
|
10189
10936
|
function isTokensStoreReadable(tokensPath) {
|
|
10190
10937
|
try {
|
|
10191
|
-
if (!
|
|
10938
|
+
if (!existsSync21(tokensPath)) return true;
|
|
10192
10939
|
accessSync(tokensPath, fsConstants.R_OK);
|
|
10193
10940
|
return true;
|
|
10194
10941
|
} catch {
|