@omnicross/daemon 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  // src/bootstrap.ts
2
- import { accessSync, constants as fsConstants, existsSync as existsSync19 } from "fs";
2
+ import { accessSync, constants as fsConstants, existsSync as existsSync21 } from "fs";
3
3
  import { DEFAULT_AUDIT_CONFIG } from "@omnicross/contracts/audit-types";
4
4
  import { DEFAULT_BILLING_CONFIG } from "@omnicross/contracts/billing-types";
5
5
  import { getGeminiCodeAssistProjectResolver } from "@omnicross/core/auth/GeminiCodeAssistProjectResolver";
@@ -33,7 +33,12 @@ import {
33
33
  } from "@omnicross/core/provider-proxy";
34
34
  import { routeLeaseDescriptorPort } from "@omnicross/cli-launcher";
35
35
  import { KeySpendTracker } from "@omnicross/core/outbound-api";
36
- import { PricingEngine, UsageRecorder } from "@omnicross/core/usage";
36
+ import {
37
+ __resetSharedUsageThroughputTrackerForTests,
38
+ getSharedUsageThroughputTracker as getSharedUsageThroughputTracker2,
39
+ PricingEngine,
40
+ UsageRecorder
41
+ } from "@omnicross/core/usage";
37
42
  import {
38
43
  setSubscriptionAccountService,
39
44
  setSubscriptionProviderRegistry,
@@ -637,6 +642,37 @@ function handleAuditQuery(req, res, reader) {
637
642
  res.writeHead(200, { "Content-Type": "application/json" });
638
643
  res.end(JSON.stringify({ records }));
639
644
  }
645
+ function handleAuditBodyQuery(req, res, reader) {
646
+ const url = new URL(req.url ?? "/", "http://localhost");
647
+ const id = url.searchParams.get("id")?.trim();
648
+ const sessionKey = url.searchParams.get("session")?.trim();
649
+ if (!id || !sessionKey) {
650
+ res.writeHead(400, { "Content-Type": "application/json" });
651
+ res.end(JSON.stringify({ error: "id and session are required" }));
652
+ return;
653
+ }
654
+ const query2 = { id, sessionKey };
655
+ const ts = intParam(url.searchParams.get("ts"));
656
+ if (ts !== void 0) query2.ts = ts;
657
+ const body = reader ? reader(query2) : {};
658
+ res.writeHead(200, { "Content-Type": "application/json" });
659
+ res.end(JSON.stringify(body));
660
+ }
661
+ function handleAuditCompact(res, compact) {
662
+ if (!compact) {
663
+ res.writeHead(200, { "Content-Type": "application/json" });
664
+ res.end(JSON.stringify({ days: 0, shards: 0, savedBytes: 0 }));
665
+ return;
666
+ }
667
+ try {
668
+ const result = compact();
669
+ res.writeHead(200, { "Content-Type": "application/json" });
670
+ res.end(JSON.stringify(result));
671
+ } catch (error) {
672
+ res.writeHead(500, { "Content-Type": "application/json" });
673
+ res.end(JSON.stringify({ error: error instanceof Error ? error.message : "compaction failed" }));
674
+ }
675
+ }
640
676
  async function handleAuditStatsQuery(req, res, reader) {
641
677
  const url = new URL(req.url ?? "/", "http://localhost");
642
678
  const query2 = {};
@@ -3646,6 +3682,7 @@ function applyAuditConfig(config) {
3646
3682
  } else {
3647
3683
  setAuditCaptureConfig(null);
3648
3684
  setAuditSink(null);
3685
+ writer?.reset();
3649
3686
  if (sweeper) {
3650
3687
  if (config) sweeper.configure(config);
3651
3688
  sweeper.dispose();
@@ -3656,6 +3693,7 @@ function resetAuditRuntimeForTests() {
3656
3693
  setAuditCaptureConfig(null);
3657
3694
  setAuditSink(null);
3658
3695
  setUpstreamTracePath(null);
3696
+ writer?.reset();
3659
3697
  if (sweeper) sweeper.dispose();
3660
3698
  writer = null;
3661
3699
  sweeper = null;
@@ -4247,6 +4285,7 @@ async function handleImport(body, deps) {
4247
4285
  }
4248
4286
 
4249
4287
  // src/admin/usagePricing.ts
4288
+ import { getSharedUsageThroughputTracker } from "@omnicross/core/usage";
4250
4289
  var err4 = (status, message) => ({
4251
4290
  status,
4252
4291
  body: { error: { type: "admin_api_error", message } }
@@ -4272,6 +4311,9 @@ var BUCKET_SPAN_MS = {
4272
4311
  };
4273
4312
  var MAX_TIMESERIES_BUCKETS = 2e3;
4274
4313
  async function handleUsageGet(view, query2, deps) {
4314
+ if (view === "throughput") {
4315
+ return { status: 200, body: getSharedUsageThroughputTracker().snapshot() };
4316
+ }
4275
4317
  const range = parseRange(query2);
4276
4318
  if (!isRange(range)) return range;
4277
4319
  switch (view) {
@@ -6003,7 +6045,7 @@ async function handleUiStatic(req, res, urlPath, uiDist) {
6003
6045
  }
6004
6046
 
6005
6047
  // src/admin/version.ts
6006
- var DAEMON_VERSION = true ? "0.1.9" : "0.0.0-dev";
6048
+ var DAEMON_VERSION = true ? "0.1.10" : "0.0.0-dev";
6007
6049
 
6008
6050
  // src/admin/AdminServer.ts
6009
6051
  var LOOPBACK_ADDR = "127.0.0.1";
@@ -6115,6 +6157,14 @@ var AdminServer = class {
6115
6157
  await handleAuditStatsQuery(req, res, this.deps.auditStatsReader);
6116
6158
  return;
6117
6159
  }
6160
+ if (path2 === "/admin/api/audit/body" && (req.method === "GET" || req.method === "HEAD")) {
6161
+ handleAuditBodyQuery(req, res, this.deps.auditBodyReader);
6162
+ return;
6163
+ }
6164
+ if (path2 === "/admin/api/audit/compact" && req.method === "POST") {
6165
+ handleAuditCompact(res, this.deps.auditCompactor);
6166
+ return;
6167
+ }
6118
6168
  if (path2 === "/admin/api/billing-status" && (req.method === "GET" || req.method === "HEAD")) {
6119
6169
  handleBillingStatus(res, this.deps.billingStatusReader);
6120
6170
  return;
@@ -8866,18 +8916,214 @@ var AccountHealthSweeper = class {
8866
8916
  };
8867
8917
 
8868
8918
  // src/audit/AuditPruneSweeper.ts
8869
- import { existsSync as existsSync15, readdirSync as readdirSync2, unlinkSync as unlinkSync3 } from "fs";
8870
- import { join as join7 } from "path";
8919
+ import { createReadStream as createReadStream2, createWriteStream as createWriteStream2, existsSync as existsSync16, readdirSync as readdirSync3, rmSync as rmSync4, unlinkSync as unlinkSync4 } from "fs";
8920
+ import { join as join8 } from "path";
8921
+ import { pipeline } from "stream/promises";
8922
+ import { createGzip } from "zlib";
8923
+
8924
+ // src/audit/auditDictionary.ts
8925
+ import { existsSync as existsSync14, readdirSync, readFileSync as readFileSync14, renameSync as renameSync5, unlinkSync as unlinkSync3, writeFileSync as writeFileSync12 } from "fs";
8926
+ import { join as join6 } from "path";
8927
+
8928
+ // src/audit/auditBodyStore.ts
8929
+ var ANCHOR_EVERY = 64;
8930
+ var ANCHOR_DELTA_RATIO = 0.75;
8931
+ var DIVERGED_PREFIX_RATIO = 0.25;
8932
+ var MAX_BASE_CHARS = 4e6;
8933
+ var CACHE_BUDGET_CHARS = 16e6;
8934
+ var CACHE_MAX_SESSIONS = 32;
8935
+ var MAX_BASES_PER_SESSION = 4;
8936
+ var isHighSurrogate = (code) => code >= 55296 && code <= 56319;
8937
+ var isLowSurrogate = (code) => code >= 56320 && code <= 57343;
8938
+ function computeBodyDelta(prev, next) {
8939
+ const shortest = Math.min(prev.length, next.length);
8940
+ let pre = 0;
8941
+ while (pre < shortest && prev.charCodeAt(pre) === next.charCodeAt(pre)) pre += 1;
8942
+ if (pre > 0 && isHighSurrogate(prev.charCodeAt(pre - 1))) pre -= 1;
8943
+ const maxSuf = shortest - pre;
8944
+ let suf = 0;
8945
+ while (suf < maxSuf && prev.charCodeAt(prev.length - 1 - suf) === next.charCodeAt(next.length - 1 - suf)) {
8946
+ suf += 1;
8947
+ }
8948
+ if (suf > 0 && isLowSurrogate(prev.charCodeAt(prev.length - suf))) suf -= 1;
8949
+ return { pre, suf, ins: next.slice(pre, next.length - suf) };
8950
+ }
8951
+ function applyBodyDelta(prev, delta) {
8952
+ const head = delta.pre > 0 ? prev.slice(0, delta.pre) : "";
8953
+ const tail = delta.suf > 0 ? prev.slice(prev.length - delta.suf) : "";
8954
+ return head + delta.ins + tail;
8955
+ }
8956
+ var SessionBaseCache = class {
8957
+ constructor(maxSessions = CACHE_MAX_SESSIONS, budgetChars = CACHE_BUDGET_CHARS, maxBaseChars = MAX_BASE_CHARS, maxHeads = MAX_BASES_PER_SESSION) {
8958
+ this.maxSessions = maxSessions;
8959
+ this.budgetChars = budgetChars;
8960
+ this.maxBaseChars = maxBaseChars;
8961
+ this.maxHeads = maxHeads;
8962
+ }
8963
+ maxSessions;
8964
+ budgetChars;
8965
+ maxBaseChars;
8966
+ maxHeads;
8967
+ /** Session key to its retained heads, most-recent first. */
8968
+ entries = /* @__PURE__ */ new Map();
8969
+ chars = 0;
8970
+ /** Retained sessions (tests + diagnostics). */
8971
+ get size() {
8972
+ return this.entries.size;
8973
+ }
8974
+ /** A session's retained heads, most-recent first. Refreshes LRU recency. */
8975
+ get(sessionKey) {
8976
+ const found = this.entries.get(sessionKey);
8977
+ if (!found) return [];
8978
+ this.entries.delete(sessionKey);
8979
+ this.entries.set(sessionKey, found);
8980
+ return found;
8981
+ }
8982
+ /**
8983
+ * Retain `base` as a head of `sessionKey`.
8984
+ *
8985
+ * `replacesId` is the head this turn CONTINUES (its body was preserved whole
8986
+ * inside the new one), which is swapped out so a linear conversation keeps
8987
+ * exactly one head. Omit it when the turn started a distinct stream %s that
8988
+ * head is added alongside, which is what keeps a fork's branches apart.
8989
+ *
8990
+ * A body larger than `maxBaseChars` is not retained: the next turn anchors
8991
+ * rather than letting one oversized session monopolize the budget.
8992
+ */
8993
+ remember(sessionKey, base, replacesId) {
8994
+ const heads = this.entries.get(sessionKey) ?? [];
8995
+ if (replacesId !== void 0) {
8996
+ const at = heads.findIndex((head) => head.lastId === replacesId);
8997
+ if (at >= 0) {
8998
+ this.chars -= heads[at].text.length;
8999
+ heads.splice(at, 1);
9000
+ }
9001
+ }
9002
+ if (base.text.length <= this.maxBaseChars) {
9003
+ heads.unshift(base);
9004
+ this.chars += base.text.length;
9005
+ }
9006
+ while (heads.length > this.maxHeads) {
9007
+ const dropped = heads.pop();
9008
+ if (dropped) this.chars -= dropped.text.length;
9009
+ }
9010
+ this.entries.delete(sessionKey);
9011
+ if (heads.length > 0) this.entries.set(sessionKey, heads);
9012
+ this.evict();
9013
+ }
9014
+ /** Drop a session's heads (eviction, or a write failure invalidating them). */
9015
+ forget(sessionKey) {
9016
+ const heads = this.entries.get(sessionKey);
9017
+ if (!heads) return;
9018
+ for (const head of heads) this.chars -= head.text.length;
9019
+ this.entries.delete(sessionKey);
9020
+ }
9021
+ /** Drop everything (writer disposal / test teardown). */
9022
+ clear() {
9023
+ this.entries.clear();
9024
+ this.chars = 0;
9025
+ }
9026
+ /** Evict least-recently-used sessions until both bounds hold. */
9027
+ evict() {
9028
+ while (this.entries.size > this.maxSessions || this.chars > this.budgetChars && this.entries.size > 1) {
9029
+ const oldest = this.entries.keys().next();
9030
+ if (oldest.done) break;
9031
+ this.forget(oldest.value);
9032
+ }
9033
+ }
9034
+ };
9035
+ function anchorReason(base, dayDir, delta, nextLength) {
9036
+ if (!base || !delta) return "new";
9037
+ if (base.dayDir !== dayDir) return "day";
9038
+ if (base.chainLen >= ANCHOR_EVERY) return "chain";
9039
+ if (delta.pre < base.text.length * DIVERGED_PREFIX_RATIO) return "diverged";
9040
+ if (delta.ins.length > nextLength * ANCHOR_DELTA_RATIO) return "costly";
9041
+ return null;
9042
+ }
9043
+ function pickBase(heads, next) {
9044
+ let best = null;
9045
+ for (const base of heads) {
9046
+ const delta = computeBodyDelta(base.text, next);
9047
+ if (best !== null && delta.ins.length >= best.delta.ins.length) continue;
9048
+ best = { base, delta, continues: delta.pre + delta.suf >= base.text.length };
9049
+ }
9050
+ return best;
9051
+ }
9052
+ function encodeBodyEntry(record, sessionKey, dayDir, cache) {
9053
+ const requestBody = record.requestBody;
9054
+ const responseBody = record.responseBody;
9055
+ if (requestBody === void 0 && responseBody === void 0) return null;
9056
+ const entry = { id: record.id, ts: record.ts };
9057
+ if (requestBody !== void 0) {
9058
+ const heads = cache.get(sessionKey);
9059
+ const sameDay = heads.filter((head) => head.dayDir === dayDir);
9060
+ const chosen = pickBase(sameDay, requestBody);
9061
+ const reason = heads.length > 0 && sameDay.length === 0 ? "day" : anchorReason(chosen?.base, dayDir, chosen?.delta ?? null, requestBody.length);
9062
+ if (reason !== null) {
9063
+ entry.req = { base: null, anchor: reason, pre: 0, suf: 0, ins: requestBody };
9064
+ cache.remember(
9065
+ sessionKey,
9066
+ { dayDir, lastId: record.id, text: requestBody, chainLen: 0 },
9067
+ chosen?.continues === true ? chosen.base.lastId : void 0
9068
+ );
9069
+ } else {
9070
+ const picked = chosen;
9071
+ entry.req = {
9072
+ base: picked.base.lastId,
9073
+ ...picked.continues ? { cont: true } : {},
9074
+ pre: picked.delta.pre,
9075
+ suf: picked.delta.suf,
9076
+ ins: picked.delta.ins
9077
+ };
9078
+ cache.remember(
9079
+ sessionKey,
9080
+ { dayDir, lastId: record.id, text: requestBody, chainLen: picked.base.chainLen + 1 },
9081
+ picked.continues ? picked.base.lastId : void 0
9082
+ );
9083
+ }
9084
+ }
9085
+ if (responseBody !== void 0) entry.res = responseBody;
9086
+ return JSON.stringify(entry);
9087
+ }
9088
+ function isAuditBodyEntry(value) {
9089
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
9090
+ const entry = value;
9091
+ if (typeof entry["id"] !== "string" || typeof entry["ts"] !== "number") return false;
9092
+ if (entry["res"] !== void 0 && typeof entry["res"] !== "string") return false;
9093
+ const req = entry["req"];
9094
+ if (req === void 0) return true;
9095
+ if (!req || typeof req !== "object" || Array.isArray(req)) return false;
9096
+ const delta = req;
9097
+ if (delta["anchor"] !== void 0 && typeof delta["anchor"] !== "string") return false;
9098
+ if (delta["cont"] !== void 0 && typeof delta["cont"] !== "boolean") return false;
9099
+ return (delta["base"] === null || typeof delta["base"] === "string") && Number.isSafeInteger(delta["pre"]) && delta["pre"] >= 0 && Number.isSafeInteger(delta["suf"]) && delta["suf"] >= 0 && typeof delta["ins"] === "string";
9100
+ }
8871
9101
 
8872
9102
  // src/audit/auditFiles.ts
8873
9103
  var AUDIT_FILE_RE = /^audit-(\d{4})-(\d{2})-(\d{2})\.jsonl$/;
9104
+ var AUDIT_DAY_DIR_RE = /^audit-(\d{4})-(\d{2})-(\d{2})$/;
9105
+ var AUDIT_META_FILE = "meta.jsonl";
9106
+ var AUDIT_BODIES_DIR = "bodies";
9107
+ var AUDIT_SESSION_KEY_RE = /^[0-9a-f]{8,64}$/;
8874
9108
  var pad22 = (n) => String(n).padStart(2, "0");
8875
- function auditFileName(ts) {
9109
+ var localDateStamp = (ts) => {
8876
9110
  const d = new Date(ts);
8877
- return `audit-${d.getFullYear()}-${pad22(d.getMonth() + 1)}-${pad22(d.getDate())}.jsonl`;
9111
+ return `${d.getFullYear()}-${pad22(d.getMonth() + 1)}-${pad22(d.getDate())}`;
9112
+ };
9113
+ function auditDayDirName(ts) {
9114
+ return `audit-${localDateStamp(ts)}`;
9115
+ }
9116
+ function isAuditDayDir(name) {
9117
+ return AUDIT_DAY_DIR_RE.test(name);
9118
+ }
9119
+ function isSafeSessionKey(key) {
9120
+ return typeof key === "string" && AUDIT_SESSION_KEY_RE.test(key);
8878
9121
  }
8879
- function auditFileDateMs(fileName) {
8880
- const m = AUDIT_FILE_RE.exec(fileName);
9122
+ function auditBodyFileName(sessionKey) {
9123
+ return `${sessionKey}.jsonl`;
9124
+ }
9125
+ function auditFileDateMs(name) {
9126
+ const m = AUDIT_FILE_RE.exec(name) ?? AUDIT_DAY_DIR_RE.exec(name);
8881
9127
  if (!m) return null;
8882
9128
  const year = Number(m[1]);
8883
9129
  const month = Number(m[2]);
@@ -8889,16 +9135,153 @@ function auditFileDateMs(fileName) {
8889
9135
  return d.getTime();
8890
9136
  }
8891
9137
 
9138
+ // src/audit/auditDictionary.ts
9139
+ var AUDIT_DICT_FILE = "_dict.jsonl";
9140
+ var DICT_BASE_PREFIX = "dict:";
9141
+ var DICT_CANDIDATES = 3;
9142
+ var MIN_SAVING_RATIO = 0.2;
9143
+ function parseEntries(raw) {
9144
+ const entries = [];
9145
+ for (const line of raw.split("\n")) {
9146
+ const trimmed = line.trim();
9147
+ if (!trimmed) continue;
9148
+ try {
9149
+ const parsed = JSON.parse(trimmed);
9150
+ if (isAuditBodyEntry(parsed)) entries.push(parsed);
9151
+ } catch {
9152
+ }
9153
+ }
9154
+ return entries;
9155
+ }
9156
+ function plainShards(bodiesPath) {
9157
+ try {
9158
+ return readdirSync(bodiesPath).filter(
9159
+ (file) => file.endsWith(".jsonl") && isSafeSessionKey(file.slice(0, -".jsonl".length))
9160
+ );
9161
+ } catch {
9162
+ return [];
9163
+ }
9164
+ }
9165
+ function chooseDictionary(anchors) {
9166
+ if (anchors.length < 2) return null;
9167
+ const total = anchors.reduce((sum, body) => sum + body.length, 0);
9168
+ const candidates = [...anchors].sort((a, b) => b.length - a.length).slice(0, DICT_CANDIDATES);
9169
+ let best = null;
9170
+ for (const candidate of candidates) {
9171
+ let cost = candidate.length;
9172
+ for (const body of anchors) {
9173
+ cost += body === candidate ? 0 : computeBodyDelta(candidate, body).ins.length;
9174
+ }
9175
+ if (best === null || cost < best.cost) best = { body: candidate, cost };
9176
+ }
9177
+ if (best === null) return null;
9178
+ return total - best.cost >= total * MIN_SAVING_RATIO ? best.body : null;
9179
+ }
9180
+ var EMPTY = { shards: 0, anchors: 0, savedBytes: 0 };
9181
+ function compactAuditDay(dayPath) {
9182
+ const bodiesPath = join6(dayPath, AUDIT_BODIES_DIR);
9183
+ if (!existsSync14(bodiesPath)) return EMPTY;
9184
+ const dictPath = join6(bodiesPath, AUDIT_DICT_FILE);
9185
+ if (existsSync14(dictPath) || existsSync14(`${dictPath}.gz`)) return EMPTY;
9186
+ const shardFiles = plainShards(bodiesPath);
9187
+ if (shardFiles.length < 2) return EMPTY;
9188
+ const loaded = /* @__PURE__ */ new Map();
9189
+ const anchors = [];
9190
+ for (const file of shardFiles) {
9191
+ let entries;
9192
+ try {
9193
+ entries = parseEntries(readFileSync14(join6(bodiesPath, file), "utf8"));
9194
+ } catch {
9195
+ continue;
9196
+ }
9197
+ loaded.set(file, entries);
9198
+ for (const entry of entries) {
9199
+ if (entry.req && entry.req.base === null) anchors.push(entry.req.ins);
9200
+ }
9201
+ }
9202
+ if (anchors.length < 2) return EMPTY;
9203
+ const dictionary = chooseDictionary(anchors);
9204
+ if (dictionary === null) return EMPTY;
9205
+ const dictEntry = {
9206
+ id: `${DICT_BASE_PREFIX}0`,
9207
+ ts: 0,
9208
+ req: { base: null, anchor: "dict", pre: 0, suf: 0, ins: dictionary }
9209
+ };
9210
+ writeFileSync12(dictPath, JSON.stringify(dictEntry) + "\n", "utf8");
9211
+ const result = { shards: 0, anchors: 0, savedBytes: 0 };
9212
+ for (const [file, entries] of loaded) {
9213
+ let changed = false;
9214
+ let saved = 0;
9215
+ const rewritten = entries.map((entry) => {
9216
+ if (!entry.req || entry.req.base !== null || entry.req.ins === dictionary) return entry;
9217
+ const delta = computeBodyDelta(dictionary, entry.req.ins);
9218
+ if (delta.ins.length >= entry.req.ins.length) return entry;
9219
+ changed = true;
9220
+ saved += entry.req.ins.length - delta.ins.length;
9221
+ return {
9222
+ ...entry,
9223
+ req: { base: dictEntry.id, pre: delta.pre, suf: delta.suf, ins: delta.ins }
9224
+ };
9225
+ });
9226
+ if (!changed) continue;
9227
+ const target = join6(bodiesPath, file);
9228
+ const temp = `${target}.compacting`;
9229
+ try {
9230
+ writeFileSync12(temp, rewritten.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8");
9231
+ renameSync5(temp, target);
9232
+ } catch {
9233
+ try {
9234
+ if (existsSync14(temp)) unlinkSync3(temp);
9235
+ } catch {
9236
+ }
9237
+ continue;
9238
+ }
9239
+ result.shards += 1;
9240
+ result.anchors += rewritten.filter((e) => e.req?.base === dictEntry.id).length;
9241
+ result.savedBytes += saved;
9242
+ }
9243
+ if (result.shards === 0) {
9244
+ try {
9245
+ unlinkSync3(dictPath);
9246
+ } catch {
9247
+ }
9248
+ }
9249
+ return result;
9250
+ }
9251
+ function compactAllClosedAuditDays(auditDir, now = Date.now) {
9252
+ const run = { days: 0, shards: 0, savedBytes: 0 };
9253
+ if (!existsSync14(auditDir)) return run;
9254
+ const today = auditDayDirName(now());
9255
+ let names;
9256
+ try {
9257
+ names = readdirSync(auditDir).filter(isAuditDayDir).sort();
9258
+ } catch {
9259
+ return run;
9260
+ }
9261
+ for (const name of names) {
9262
+ if (name === today) continue;
9263
+ try {
9264
+ const result = compactAuditDay(join6(auditDir, name));
9265
+ if (result.shards === 0) continue;
9266
+ run.days += 1;
9267
+ run.shards += result.shards;
9268
+ run.savedBytes += result.savedBytes;
9269
+ } catch {
9270
+ }
9271
+ }
9272
+ return run;
9273
+ }
9274
+
8892
9275
  // src/audit/auditStats.ts
8893
9276
  import {
8894
9277
  createReadStream,
8895
- existsSync as existsSync14,
8896
- readFileSync as readFileSync14,
8897
- readdirSync,
9278
+ existsSync as existsSync15,
9279
+ readFileSync as readFileSync15,
9280
+ readdirSync as readdirSync2,
8898
9281
  statSync as statSync3,
8899
- writeFileSync as writeFileSync12
9282
+ writeFileSync as writeFileSync13
8900
9283
  } from "fs";
8901
- import { basename, dirname as dirname7, join as join6 } from "path";
9284
+ import { basename, dirname as dirname7, join as join7 } from "path";
8902
9285
  var SIDECAR_VERSION = 1;
8903
9286
  var META_PREFIX_BYTES = 64 * 1024;
8904
9287
  var READ_CHUNK_BYTES = 4 * 1024 * 1024;
@@ -8906,9 +9289,9 @@ function auditStatsFileName(auditFile) {
8906
9289
  return auditFile.replace(/\.jsonl$/, ".stats.json");
8907
9290
  }
8908
9291
  function readPersisted(path2) {
8909
- if (!existsSync14(path2)) return null;
9292
+ if (!existsSync15(path2)) return null;
8910
9293
  try {
8911
- const value = JSON.parse(readFileSync14(path2, "utf8"));
9294
+ const value = JSON.parse(readFileSync15(path2, "utf8"));
8912
9295
  if (value.version !== SIDECAR_VERSION || !Number.isSafeInteger(value.auditBytes) || (value.auditBytes ?? -1) < 0 || !Number.isSafeInteger(value.requestCount) || (value.requestCount ?? -1) < 0 || !Number.isSafeInteger(value.errorCount) || (value.errorCount ?? -1) < 0 || (value.errorCount ?? 0) > (value.requestCount ?? -1) || typeof value.complete !== "boolean" || value.minTs !== null && !Number.isFinite(value.minTs) || value.maxTs !== null && !Number.isFinite(value.maxTs)) {
8913
9296
  return null;
8914
9297
  }
@@ -8918,7 +9301,7 @@ function readPersisted(path2) {
8918
9301
  }
8919
9302
  }
8920
9303
  function updateAuditStatsAfterAppend(auditPath, auditBytesBefore, auditBytesAfter, record) {
8921
- const statsPath = join6(dirname7(auditPath), auditStatsFileName(basename(auditPath)));
9304
+ const statsPath = join7(dirname7(auditPath), auditStatsFileName(basename(auditPath)));
8922
9305
  const previous = auditBytesBefore === 0 ? {
8923
9306
  version: SIDECAR_VERSION,
8924
9307
  auditBytes: 0,
@@ -8938,13 +9321,13 @@ function updateAuditStatsAfterAppend(auditPath, auditBytesBefore, auditBytesAfte
8938
9321
  minTs: previous.minTs === null ? record.ts : Math.min(previous.minTs, record.ts),
8939
9322
  maxTs: previous.maxTs === null ? record.ts : Math.max(previous.maxTs, record.ts)
8940
9323
  };
8941
- writeFileSync12(statsPath, JSON.stringify(next), "utf8");
9324
+ writeFileSync13(statsPath, JSON.stringify(next), "utf8");
8942
9325
  }
8943
9326
  function queryCovers(stats, from, to) {
8944
9327
  return stats.requestCount === 0 || stats.minTs !== null && stats.maxTs !== null && from <= stats.minTs && to >= stats.maxTs;
8945
9328
  }
8946
- function fileOverlaps(file, from, to) {
8947
- const start = auditFileDateMs(file);
9329
+ function fileOverlaps(name, from, to) {
9330
+ const start = auditFileDateMs(name);
8948
9331
  if (start === null) return false;
8949
9332
  const date = new Date(start);
8950
9333
  const end = new Date(date.getFullYear(), date.getMonth(), date.getDate() + 1).getTime();
@@ -9049,21 +9432,27 @@ function mergePersistedStats(previous, appended) {
9049
9432
  };
9050
9433
  }
9051
9434
  async function readAuditStats(auditDir, query2 = {}) {
9052
- if (!existsSync14(auditDir)) return { requestCount: 0, errorCount: 0, complete: true };
9435
+ if (!existsSync15(auditDir)) return { requestCount: 0, errorCount: 0, complete: true };
9053
9436
  const from = typeof query2.from === "number" ? query2.from : -Infinity;
9054
9437
  const to = typeof query2.to === "number" ? query2.to : Infinity;
9055
- let files;
9438
+ let sources;
9056
9439
  try {
9057
- files = readdirSync(auditDir).filter((file) => AUDIT_FILE_RE.test(file) && fileOverlaps(file, from, to)).sort();
9440
+ sources = readdirSync2(auditDir).filter((name) => fileOverlaps(name, from, to)).sort().map(
9441
+ (name) => AUDIT_DAY_DIR_RE.test(name) ? {
9442
+ auditPath: join7(auditDir, name, AUDIT_META_FILE),
9443
+ statsPath: join7(auditDir, name, auditStatsFileName(AUDIT_META_FILE))
9444
+ } : {
9445
+ auditPath: join7(auditDir, name),
9446
+ statsPath: join7(auditDir, auditStatsFileName(name))
9447
+ }
9448
+ ).filter((source) => existsSync15(source.auditPath));
9058
9449
  } catch {
9059
9450
  return { requestCount: 0, errorCount: 0, complete: false };
9060
9451
  }
9061
9452
  const total = { requestCount: 0, errorCount: 0, complete: true };
9062
- for (const file of files) {
9063
- const auditPath = join6(auditDir, file);
9453
+ for (const { auditPath, statsPath } of sources) {
9064
9454
  try {
9065
9455
  const auditBytes = statSync3(auditPath).size;
9066
- const statsPath = join6(auditDir, auditStatsFileName(file));
9067
9456
  const persisted = readPersisted(statsPath);
9068
9457
  if (persisted && persisted.complete && persisted.auditBytes === auditBytes && queryCovers(persisted, from, to)) {
9069
9458
  total.requestCount += persisted.requestCount;
@@ -9082,7 +9471,7 @@ async function readAuditStats(auditDir, query2 = {}) {
9082
9471
  total.errorCount += scanned.filtered.errorCount + (resumable?.errorCount ?? 0);
9083
9472
  total.complete = total.complete && scanned.filtered.complete;
9084
9473
  const current = resumable ? mergePersistedStats(resumable, scanned.all) : scanned.all;
9085
- if (current.complete) writeFileSync12(statsPath, JSON.stringify(current), "utf8");
9474
+ if (current.complete) writeFileSync13(statsPath, JSON.stringify(current), "utf8");
9086
9475
  } catch {
9087
9476
  total.complete = false;
9088
9477
  }
@@ -9093,6 +9482,7 @@ async function readAuditStats(auditDir, query2 = {}) {
9093
9482
  // src/audit/AuditPruneSweeper.ts
9094
9483
  var DAY_MS = 24 * 60 * 6e4;
9095
9484
  var SWEEP_INTERVAL_MS2 = 60 * 6e4;
9485
+ var ARCHIVE_BATCH = 64;
9096
9486
  var AuditPruneSweeper = class {
9097
9487
  constructor(auditDir, logger, config, intervalMs = SWEEP_INTERVAL_MS2, now = Date.now) {
9098
9488
  this.auditDir = auditDir;
@@ -9108,6 +9498,7 @@ var AuditPruneSweeper = class {
9108
9498
  now;
9109
9499
  timer = null;
9110
9500
  sweeping = false;
9501
+ archiving = false;
9111
9502
  /** Whether pruning is active (audit enabled). */
9112
9503
  get enabled() {
9113
9504
  return this.config.enabled;
@@ -9117,13 +9508,13 @@ var AuditPruneSweeper = class {
9117
9508
  this.config = config;
9118
9509
  }
9119
9510
  /**
9120
- * Arm the prune interval AND run one prune immediately (boot cleanup). No-op
9121
- * when audit is disabled (zero regression). Idempotent.
9511
+ * Arm the interval AND run one pass immediately (boot cleanup). No-op when
9512
+ * audit is disabled (zero regression). Idempotent.
9122
9513
  */
9123
9514
  start() {
9124
9515
  if (this.timer || !this.config.enabled) return;
9125
- void this.sweep();
9126
- this.timer = setInterval(() => void this.sweep(), this.intervalMs);
9516
+ void this.runOnce();
9517
+ this.timer = setInterval(() => void this.runOnce(), this.intervalMs);
9127
9518
  this.timer.unref?.();
9128
9519
  }
9129
9520
  /** Clear the interval (daemon shutdown / test teardown). Idempotent. */
@@ -9133,31 +9524,43 @@ var AuditPruneSweeper = class {
9133
9524
  this.timer = null;
9134
9525
  }
9135
9526
  }
9527
+ /** Prune first, then archive — never spend CPU compressing a day about to go. */
9528
+ async runOnce() {
9529
+ await this.sweep();
9530
+ await this.archive();
9531
+ }
9532
+ /** The LOCAL-midnight epoch ms of the current day. */
9533
+ todayMidnight() {
9534
+ const today = new Date(this.now());
9535
+ return new Date(today.getFullYear(), today.getMonth(), today.getDate()).getTime();
9536
+ }
9136
9537
  /**
9137
- * One prune: unlink every audit date file strictly OLDER than the retention
9138
- * cutoff (`now - retentionDays` days, at local-midnight granularity). Exposed
9139
- * for tests; never throws. Returns the number of files removed.
9538
+ * One prune: remove every audit day strictly OLDER than the retention cutoff
9539
+ * (`now - retentionDays` days, at local-midnight granularity). Exposed for
9540
+ * tests; never throws. Returns the number of days removed.
9140
9541
  */
9141
9542
  async sweep() {
9142
9543
  if (!this.config.enabled || this.sweeping) return 0;
9143
9544
  this.sweeping = true;
9144
9545
  try {
9145
- if (!existsSync15(this.auditDir)) return 0;
9146
- const today = new Date(this.now());
9147
- const todayMidnight = new Date(today.getFullYear(), today.getMonth(), today.getDate()).getTime();
9148
- const cutoff = todayMidnight - (this.config.retentionDays - 1) * DAY_MS;
9546
+ if (!existsSync16(this.auditDir)) return 0;
9547
+ const cutoff = this.todayMidnight() - (this.config.retentionDays - 1) * DAY_MS;
9149
9548
  let removed = 0;
9150
- for (const file of readdirSync2(this.auditDir)) {
9151
- const dateMs = auditFileDateMs(file);
9549
+ for (const name of readdirSync3(this.auditDir)) {
9550
+ const dateMs = auditFileDateMs(name);
9152
9551
  if (dateMs === null || dateMs >= cutoff) continue;
9153
9552
  try {
9154
- unlinkSync3(join7(this.auditDir, file));
9553
+ if (isAuditDayDir(name)) {
9554
+ rmSync4(join8(this.auditDir, name), { recursive: true, force: true });
9555
+ } else {
9556
+ unlinkSync4(join8(this.auditDir, name));
9557
+ const statsPath = join8(this.auditDir, auditStatsFileName(name));
9558
+ if (existsSync16(statsPath)) unlinkSync4(statsPath);
9559
+ }
9155
9560
  removed += 1;
9156
- const statsPath = join7(this.auditDir, auditStatsFileName(file));
9157
- if (existsSync15(statsPath)) unlinkSync3(statsPath);
9158
9561
  } catch (error) {
9159
- this.logger.warn("[AuditPruneSweeper] failed to unlink expired audit file", {
9160
- file,
9562
+ this.logger.warn("[AuditPruneSweeper] failed to remove expired audit day", {
9563
+ name,
9161
9564
  error: error instanceof Error ? error.message : String(error)
9162
9565
  });
9163
9566
  }
@@ -9173,59 +9576,349 @@ var AuditPruneSweeper = class {
9173
9576
  this.sweeping = false;
9174
9577
  }
9175
9578
  }
9579
+ /**
9580
+ * Gzip the body shards of every CLOSED day (anything before today). Today is
9581
+ * deliberately left as plain text so it stays greppable while it is the day you
9582
+ * are debugging. Exposed for tests; never throws. Returns shards compressed.
9583
+ */
9584
+ async archive() {
9585
+ if (!this.config.enabled || this.archiving) return 0;
9586
+ this.archiving = true;
9587
+ try {
9588
+ if (!existsSync16(this.auditDir)) return 0;
9589
+ const today = this.todayMidnight();
9590
+ let compressed = 0;
9591
+ for (const name of readdirSync3(this.auditDir)) {
9592
+ if (compressed >= ARCHIVE_BATCH) break;
9593
+ const dateMs = auditFileDateMs(name);
9594
+ if (dateMs === null || dateMs >= today || !isAuditDayDir(name)) continue;
9595
+ const dayPath = join8(this.auditDir, name);
9596
+ try {
9597
+ const compaction = compactAuditDay(dayPath);
9598
+ if (compaction.shards > 0) {
9599
+ this.logger.debug("audit cross-session compaction complete", {
9600
+ day: name,
9601
+ shards: compaction.shards,
9602
+ anchors: compaction.anchors,
9603
+ savedBytes: compaction.savedBytes
9604
+ });
9605
+ }
9606
+ } catch (error) {
9607
+ this.logger.warn("[AuditPruneSweeper] cross-session compaction failed", {
9608
+ day: name,
9609
+ error: error instanceof Error ? error.message : String(error)
9610
+ });
9611
+ }
9612
+ compressed += await this.archiveDay(
9613
+ join8(dayPath, AUDIT_BODIES_DIR),
9614
+ ARCHIVE_BATCH - compressed
9615
+ );
9616
+ }
9617
+ if (compressed > 0) this.logger.debug("audit archive complete", { compressed });
9618
+ return compressed;
9619
+ } catch (error) {
9620
+ this.logger.warn("audit archive pass failed", {
9621
+ error: error instanceof Error ? error.message : String(error)
9622
+ });
9623
+ return 0;
9624
+ } finally {
9625
+ this.archiving = false;
9626
+ }
9627
+ }
9628
+ /** Gzip up to `budget` plain shards in one day's `bodies/` directory. */
9629
+ async archiveDay(bodiesPath, budget) {
9630
+ let shards;
9631
+ try {
9632
+ shards = readdirSync3(bodiesPath).filter((file) => file.endsWith(".jsonl"));
9633
+ } catch {
9634
+ return 0;
9635
+ }
9636
+ let compressed = 0;
9637
+ for (const shard of shards) {
9638
+ if (compressed >= budget) break;
9639
+ const source = join8(bodiesPath, shard);
9640
+ const target = `${source}.gz`;
9641
+ try {
9642
+ if (existsSync16(target)) {
9643
+ unlinkSync4(source);
9644
+ continue;
9645
+ }
9646
+ await pipeline(createReadStream2(source), createGzip(), createWriteStream2(target));
9647
+ unlinkSync4(source);
9648
+ compressed += 1;
9649
+ } catch (error) {
9650
+ try {
9651
+ if (existsSync16(target)) unlinkSync4(target);
9652
+ } catch {
9653
+ }
9654
+ this.logger.warn("[AuditPruneSweeper] failed to archive audit body shard", {
9655
+ shard,
9656
+ error: error instanceof Error ? error.message : String(error)
9657
+ });
9658
+ }
9659
+ }
9660
+ return compressed;
9661
+ }
9176
9662
  };
9177
9663
 
9178
- // src/audit/auditReader.ts
9179
- import { existsSync as existsSync16, readdirSync as readdirSync3, readFileSync as readFileSync15 } from "fs";
9180
- import { join as join8 } from "path";
9181
- var DEFAULT_LIMIT = 200;
9182
- var MAX_LIMIT = 2e3;
9183
- function readAuditRecords(auditDir, query2 = {}) {
9184
- if (!existsSync16(auditDir)) return [];
9185
- let files;
9664
+ // src/audit/auditBodyReader.ts
9665
+ import { existsSync as existsSync17, readdirSync as readdirSync4, readFileSync as readFileSync16, statSync as statSync5 } from "fs";
9666
+ import { join as join9 } from "path";
9667
+ import { gunzipSync } from "zlib";
9668
+
9669
+ // src/audit/auditJsonl.ts
9670
+ import { closeSync, openSync, readSync, statSync as statSync4 } from "fs";
9671
+ var WINDOW_BYTES = 1 << 20;
9672
+ var MAX_LINE_BYTES = 32 * 1024 * 1024;
9673
+ var NEWLINE = 10;
9674
+ function forEachLineFromTail(path2, onLine) {
9675
+ let fd;
9676
+ let end;
9677
+ try {
9678
+ end = statSync4(path2).size;
9679
+ if (end === 0) return;
9680
+ fd = openSync(path2, "r");
9681
+ } catch {
9682
+ return;
9683
+ }
9684
+ try {
9685
+ let carry = Buffer.alloc(0);
9686
+ while (end > 0) {
9687
+ const start = Math.max(0, end - WINDOW_BYTES);
9688
+ const window = Buffer.allocUnsafe(end - start);
9689
+ let read;
9690
+ try {
9691
+ read = readSync(fd, window, 0, end - start, start);
9692
+ } catch {
9693
+ return;
9694
+ }
9695
+ const chunk = carry.length > 0 ? Buffer.concat([window.subarray(0, read), carry]) : window.subarray(0, read);
9696
+ let lineEnd = chunk.length;
9697
+ let nl = lineEnd > 0 ? chunk.lastIndexOf(NEWLINE, lineEnd - 1) : -1;
9698
+ while (nl >= 0) {
9699
+ if (nl + 1 < lineEnd) {
9700
+ const line = chunk.subarray(nl + 1, lineEnd).toString("utf8").trim();
9701
+ if (line && onLine(line)) return;
9702
+ }
9703
+ lineEnd = nl;
9704
+ nl = lineEnd > 0 ? chunk.lastIndexOf(NEWLINE, lineEnd - 1) : -1;
9705
+ }
9706
+ if (start === 0) {
9707
+ if (lineEnd > 0) {
9708
+ const line = chunk.subarray(0, lineEnd).toString("utf8").trim();
9709
+ if (line) onLine(line);
9710
+ }
9711
+ return;
9712
+ }
9713
+ if (lineEnd > MAX_LINE_BYTES) return;
9714
+ carry = Buffer.from(chunk.subarray(0, lineEnd));
9715
+ end = start;
9716
+ }
9717
+ } finally {
9718
+ try {
9719
+ closeSync(fd);
9720
+ } catch {
9721
+ }
9722
+ }
9723
+ }
9724
+
9725
+ // src/audit/auditBodyReader.ts
9726
+ function candidateDays(auditDir, ts) {
9727
+ if (typeof ts === "number" && Number.isFinite(ts)) {
9728
+ const named = auditDayDirName(ts);
9729
+ if (existsSync17(join9(auditDir, named))) return [named];
9730
+ }
9186
9731
  try {
9187
- files = readdirSync3(auditDir).filter((f) => AUDIT_FILE_RE.test(f));
9732
+ return readdirSync4(auditDir).filter(isAuditDayDir).sort().reverse();
9188
9733
  } catch {
9189
9734
  return [];
9190
9735
  }
9191
- const from = typeof query2.from === "number" ? query2.from : -Infinity;
9192
- const to = typeof query2.to === "number" ? query2.to : Infinity;
9193
- const limit = Math.min(MAX_LIMIT, Math.max(1, Math.trunc(query2.limit ?? DEFAULT_LIMIT)));
9194
- const matched = [];
9195
- for (const file of files.sort().reverse()) {
9196
- let raw;
9736
+ }
9737
+ function readShard(auditDir, day, sessionKey) {
9738
+ const base = join9(auditDir, day, AUDIT_BODIES_DIR, auditBodyFileName(sessionKey));
9739
+ try {
9740
+ if (existsSync17(base)) return readFileSync16(base, "utf8");
9741
+ const gz = `${base}.gz`;
9742
+ if (existsSync17(gz)) return gunzipSync(readFileSync16(gz)).toString("utf8");
9743
+ } catch {
9744
+ return null;
9745
+ }
9746
+ return null;
9747
+ }
9748
+ function parseShard(raw) {
9749
+ const entries = /* @__PURE__ */ new Map();
9750
+ for (const line of raw.split("\n")) {
9751
+ const trimmed = line.trim();
9752
+ if (!trimmed) continue;
9753
+ let parsed;
9197
9754
  try {
9198
- raw = readFileSync15(join8(auditDir, file), "utf8");
9755
+ parsed = JSON.parse(trimmed);
9199
9756
  } catch {
9200
9757
  continue;
9201
9758
  }
9202
- for (const line of raw.split("\n")) {
9203
- const trimmed = line.trim();
9204
- if (!trimmed) continue;
9205
- let rec;
9759
+ if (isAuditBodyEntry(parsed)) entries.set(parsed.id, parsed);
9760
+ }
9761
+ return entries;
9762
+ }
9763
+ function withDictionary(auditDir, day, entries) {
9764
+ let needed = false;
9765
+ for (const entry of entries.values()) {
9766
+ if (entry.req?.base?.startsWith(DICT_BASE_PREFIX)) {
9767
+ needed = true;
9768
+ break;
9769
+ }
9770
+ }
9771
+ if (!needed) return entries;
9772
+ const base = join9(auditDir, day, AUDIT_BODIES_DIR, AUDIT_DICT_FILE);
9773
+ let raw = null;
9774
+ try {
9775
+ if (existsSync17(base)) raw = readFileSync16(base, "utf8");
9776
+ else if (existsSync17(`${base}.gz`)) raw = gunzipSync(readFileSync16(`${base}.gz`)).toString("utf8");
9777
+ } catch {
9778
+ return entries;
9779
+ }
9780
+ if (raw === null) return entries;
9781
+ for (const [id, entry] of parseShard(raw)) entries.set(id, entry);
9782
+ return entries;
9783
+ }
9784
+ function reconstructRequest(entries, entry) {
9785
+ if (!entry.req) return void 0;
9786
+ const chain = [];
9787
+ const visited = /* @__PURE__ */ new Set();
9788
+ let cursor = entry;
9789
+ while (cursor?.req) {
9790
+ if (visited.has(cursor.id)) return void 0;
9791
+ visited.add(cursor.id);
9792
+ chain.push(cursor);
9793
+ if (cursor.req.base === null) break;
9794
+ cursor = entries.get(cursor.req.base);
9795
+ }
9796
+ const anchor = chain[chain.length - 1];
9797
+ if (!anchor?.req || anchor.req.base !== null) return void 0;
9798
+ let text = anchor.req.ins;
9799
+ for (let i = chain.length - 2; i >= 0; i -= 1) {
9800
+ const delta = chain[i]?.req;
9801
+ if (!delta) return void 0;
9802
+ if (delta.pre > text.length || delta.suf > text.length - delta.pre) return void 0;
9803
+ text = applyBodyDelta(text, delta);
9804
+ }
9805
+ return text;
9806
+ }
9807
+ function readAuditBody(auditDir, query2) {
9808
+ if (!isSafeSessionKey(query2.sessionKey) || !query2.id) return {};
9809
+ if (!existsSync17(auditDir)) return {};
9810
+ for (const day of candidateDays(auditDir, query2.ts)) {
9811
+ const raw = readShard(auditDir, day, query2.sessionKey);
9812
+ if (raw === null) continue;
9813
+ const entries = withDictionary(auditDir, day, parseShard(raw));
9814
+ const entry = entries.get(query2.id);
9815
+ if (!entry) continue;
9816
+ const result = {};
9817
+ const requestBody = reconstructRequest(entries, entry);
9818
+ if (requestBody !== void 0) result.requestBody = requestBody;
9819
+ if (entry.res !== void 0) result.responseBody = entry.res;
9820
+ return result;
9821
+ }
9822
+ return readLegacyInlineBody(auditDir, query2.id);
9823
+ }
9824
+ function readLegacyInlineBody(auditDir, id) {
9825
+ let names;
9826
+ try {
9827
+ names = readdirSync4(auditDir).filter((name) => AUDIT_FILE_RE.test(name)).sort().reverse();
9828
+ } catch {
9829
+ return {};
9830
+ }
9831
+ const needle = JSON.stringify(id);
9832
+ let found = {};
9833
+ for (const name of names) {
9834
+ forEachLineFromTail(join9(auditDir, name), (line) => {
9835
+ if (!line.includes(needle)) return false;
9836
+ let parsed;
9206
9837
  try {
9207
- rec = JSON.parse(trimmed);
9838
+ parsed = JSON.parse(line);
9208
9839
  } catch {
9209
- continue;
9840
+ return false;
9210
9841
  }
9211
- if (!isAuditRecord(rec)) continue;
9212
- if (query2.keyId !== void 0 && rec.keyId !== query2.keyId) continue;
9213
- if (rec.ts < from || rec.ts > to) continue;
9214
- matched.push(rec);
9842
+ const record = parsed;
9843
+ if (record.id !== id) return false;
9844
+ const result = {};
9845
+ if (typeof record.requestBody === "string") result.requestBody = record.requestBody;
9846
+ if (typeof record.responseBody === "string") result.responseBody = record.responseBody;
9847
+ found = result;
9848
+ return true;
9849
+ });
9850
+ if (found.requestBody !== void 0 || found.responseBody !== void 0) break;
9851
+ }
9852
+ return found;
9853
+ }
9854
+
9855
+ // src/audit/auditReader.ts
9856
+ import { existsSync as existsSync18, readdirSync as readdirSync5 } from "fs";
9857
+ import { join as join10 } from "path";
9858
+ var DEFAULT_LIMIT = 200;
9859
+ var MAX_LIMIT = 2e3;
9860
+ var OVERSCAN = 256;
9861
+ function daySources(auditDir) {
9862
+ let names;
9863
+ try {
9864
+ names = readdirSync5(auditDir);
9865
+ } catch {
9866
+ return [];
9867
+ }
9868
+ const sources = [];
9869
+ for (const name of names) {
9870
+ const dateMs = auditFileDateMs(name);
9871
+ if (dateMs === null) continue;
9872
+ if (AUDIT_DAY_DIR_RE.test(name)) {
9873
+ const path2 = join10(auditDir, name, AUDIT_META_FILE);
9874
+ if (existsSync18(path2)) sources.push({ path: path2, dateMs });
9875
+ } else if (AUDIT_FILE_RE.test(name)) {
9876
+ sources.push({ path: join10(auditDir, name), dateMs });
9215
9877
  }
9216
9878
  }
9217
- matched.sort((a, b) => b.ts - a.ts);
9218
- return matched.slice(0, limit);
9879
+ return sources.sort((a, b) => b.dateMs - a.dateMs);
9219
9880
  }
9220
9881
  function isAuditRecord(value) {
9221
9882
  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
9222
9883
  const r = value;
9223
9884
  return typeof r["id"] === "string" && typeof r["ts"] === "number" && typeof r["method"] === "string" && typeof r["path"] === "string" && typeof r["status"] === "number";
9224
9885
  }
9886
+ function toMetaRecord(record) {
9887
+ if (record.requestBody === void 0 && record.responseBody === void 0) return record;
9888
+ const { requestBody: _req, responseBody: _res, ...meta } = record;
9889
+ return { ...meta, hasBody: true };
9890
+ }
9891
+ function readAuditRecords(auditDir, query2 = {}) {
9892
+ if (!existsSync18(auditDir)) return [];
9893
+ const from = typeof query2.from === "number" ? query2.from : -Infinity;
9894
+ const to = typeof query2.to === "number" ? query2.to : Infinity;
9895
+ const limit = Math.min(MAX_LIMIT, Math.max(1, Math.trunc(query2.limit ?? DEFAULT_LIMIT)));
9896
+ const matched = [];
9897
+ for (const source of daySources(auditDir)) {
9898
+ const before = matched.length;
9899
+ forEachLineFromTail(source.path, (line) => {
9900
+ let parsed;
9901
+ try {
9902
+ parsed = JSON.parse(line);
9903
+ } catch {
9904
+ return false;
9905
+ }
9906
+ if (!isAuditRecord(parsed)) return false;
9907
+ if (query2.keyId !== void 0 && parsed.keyId !== query2.keyId) return false;
9908
+ if (query2.sessionKey !== void 0 && parsed.sessionKey !== query2.sessionKey) return false;
9909
+ if (parsed.ts < from || parsed.ts > to) return false;
9910
+ matched.push(toMetaRecord(parsed));
9911
+ return matched.length - before >= limit + OVERSCAN;
9912
+ });
9913
+ if (matched.length >= limit) break;
9914
+ }
9915
+ matched.sort((a, b) => b.ts - a.ts);
9916
+ return matched.slice(0, limit);
9917
+ }
9225
9918
 
9226
9919
  // src/audit/AuditWriter.ts
9227
- import { appendFileSync as appendFileSync2, existsSync as existsSync17, mkdirSync as mkdirSync5, statSync as statSync4 } from "fs";
9228
- import { join as join9 } from "path";
9920
+ import { appendFileSync as appendFileSync2, existsSync as existsSync19, mkdirSync as mkdirSync5, statSync as statSync6 } from "fs";
9921
+ import { join as join11 } from "path";
9229
9922
  var AuditWriter = class {
9230
9923
  constructor(auditDir, logger, defer = (fn) => setTimeout(fn, 0)) {
9231
9924
  this.auditDir = auditDir;
@@ -9235,10 +9928,13 @@ var AuditWriter = class {
9235
9928
  auditDir;
9236
9929
  logger;
9237
9930
  defer;
9238
- dirEnsured = false;
9931
+ /** Day directories already created this process (avoids an mkdir per record). */
9932
+ ensuredDirs = /* @__PURE__ */ new Set();
9933
+ /** Per-session encoding bases. Memory-only; a miss simply writes a full snapshot. */
9934
+ bases = new SessionBaseCache();
9239
9935
  /**
9240
- * Enqueue one record for append. Returns IMMEDIATELY (fire-and-forget); the fs
9241
- * write happens on the deferred tick. A failure is logged, never thrown.
9936
+ * Enqueue one record. Returns IMMEDIATELY (fire-and-forget); every fs write and
9937
+ * the delta encoding happen on the deferred tick. A failure is logged, never thrown.
9242
9938
  */
9243
9939
  record(record) {
9244
9940
  this.defer(() => {
@@ -9251,25 +9947,41 @@ var AuditWriter = class {
9251
9947
  }
9252
9948
  });
9253
9949
  }
9950
+ /** Drop all retained encoding bases (config reload / shutdown / test teardown). */
9951
+ reset() {
9952
+ this.bases.clear();
9953
+ this.ensuredDirs.clear();
9954
+ }
9254
9955
  /**
9255
- * Append synchronously — the awaitable form tests use to assert the line landed.
9256
- * Ensures the `audit/` directory exists on first write (lazy, like the usage
9257
- * store's lazy file creation).
9956
+ * Append synchronously — the awaitable form tests use to assert a line landed.
9957
+ * Writes the metadata line first (canonical), then the body shard.
9258
9958
  */
9259
9959
  appendNow(record) {
9260
- if (!this.dirEnsured) {
9261
- mkdirSync5(this.auditDir, { recursive: true });
9262
- this.dirEnsured = true;
9263
- }
9264
- const file = join9(this.auditDir, auditFileName(record.ts));
9265
- const line = JSON.stringify(record) + "\n";
9266
- const auditBytesBefore = existsSync17(file) ? statSync4(file).size : 0;
9960
+ const dayDir = auditDayDirName(record.ts);
9961
+ const dayPath = this.ensureDir(join11(this.auditDir, dayDir));
9962
+ this.appendMeta(dayPath, record);
9963
+ this.appendBody(dayPath, dayDir, record);
9964
+ }
9965
+ /** Create a directory once per process and remember it. */
9966
+ ensureDir(path2) {
9967
+ if (!this.ensuredDirs.has(path2)) {
9968
+ mkdirSync5(path2, { recursive: true });
9969
+ this.ensuredDirs.add(path2);
9970
+ }
9971
+ return path2;
9972
+ }
9973
+ /** Write the body-free metadata line + refresh the exact-count sidecar. */
9974
+ appendMeta(dayPath, record) {
9975
+ const { requestBody: _req, responseBody: _res, ...meta } = record;
9976
+ const file = join11(dayPath, AUDIT_META_FILE);
9977
+ const line = JSON.stringify(meta) + "\n";
9978
+ const bytesBefore = existsSync19(file) ? statSync6(file).size : 0;
9267
9979
  appendFileSync2(file, line, "utf8");
9268
9980
  try {
9269
9981
  updateAuditStatsAfterAppend(
9270
9982
  file,
9271
- auditBytesBefore,
9272
- auditBytesBefore + Buffer.byteLength(line, "utf8"),
9983
+ bytesBefore,
9984
+ bytesBefore + Buffer.byteLength(line, "utf8"),
9273
9985
  record
9274
9986
  );
9275
9987
  } catch (error) {
@@ -9278,12 +9990,39 @@ var AuditWriter = class {
9278
9990
  });
9279
9991
  }
9280
9992
  }
9993
+ /**
9994
+ * Write the delta-encoded body shard for one record. A no-op when nothing was
9995
+ * captured or when the session key is missing/unsafe — in which case the body
9996
+ * is dropped rather than written to an unvalidated path.
9997
+ */
9998
+ appendBody(dayPath, dayDir, record) {
9999
+ if (record.requestBody === void 0 && record.responseBody === void 0) return;
10000
+ const sessionKey = record.sessionKey;
10001
+ if (!isSafeSessionKey(sessionKey)) {
10002
+ this.logger.warn("[AuditWriter] dropping audit body with no usable session key", {
10003
+ id: record.id
10004
+ });
10005
+ return;
10006
+ }
10007
+ try {
10008
+ const line = encodeBodyEntry(record, sessionKey, dayDir, this.bases);
10009
+ if (line === null) return;
10010
+ const bodiesPath = this.ensureDir(join11(dayPath, AUDIT_BODIES_DIR));
10011
+ appendFileSync2(join11(bodiesPath, auditBodyFileName(sessionKey)), line + "\n", "utf8");
10012
+ } catch (error) {
10013
+ this.bases.forget(sessionKey);
10014
+ this.logger.warn("[AuditWriter] failed to append audit body shard", {
10015
+ id: record.id,
10016
+ error: error instanceof Error ? error.message : String(error)
10017
+ });
10018
+ }
10019
+ }
9281
10020
  };
9282
10021
 
9283
10022
  // src/billing/BillingPublisher.ts
9284
10023
  import { appendFileSync as appendFileSync3, mkdirSync as mkdirSync6 } from "fs";
9285
10024
  import { createHmac } from "crypto";
9286
- import { join as join10 } from "path";
10025
+ import { join as join12 } from "path";
9287
10026
  import { fetchUpstream as fetchUpstream5 } from "@omnicross/core/pipeline/upstreamFetch";
9288
10027
 
9289
10028
  // src/billing/billingFiles.ts
@@ -9354,7 +10093,7 @@ var BillingPublisher = class {
9354
10093
  */
9355
10094
  appendNow(event) {
9356
10095
  this.ensureDir();
9357
- const file = join10(this.billingDir, billingFileName(event.ts));
10096
+ const file = join12(this.billingDir, billingFileName(event.ts));
9358
10097
  appendFileSync3(file, JSON.stringify(event) + "\n", "utf8");
9359
10098
  }
9360
10099
  /**
@@ -9404,7 +10143,7 @@ var BillingPublisher = class {
9404
10143
  markDelivered(event) {
9405
10144
  try {
9406
10145
  this.ensureDir();
9407
- const file = join10(this.billingDir, deliveredFileName(event.ts));
10146
+ const file = join12(this.billingDir, deliveredFileName(event.ts));
9408
10147
  appendFileSync3(file, JSON.stringify({ id: event.id, deliveredAt: this.now() }) + "\n", "utf8");
9409
10148
  } catch (error) {
9410
10149
  this.logger.warn("[BillingPublisher] failed to append delivery marker", {
@@ -9420,14 +10159,14 @@ var BillingPublisher = class {
9420
10159
  };
9421
10160
 
9422
10161
  // src/billing/billingReader.ts
9423
- import { existsSync as existsSync18, readdirSync as readdirSync4, readFileSync as readFileSync16 } from "fs";
9424
- import { join as join11 } from "path";
10162
+ import { existsSync as existsSync20, readdirSync as readdirSync6, readFileSync as readFileSync17 } from "fs";
10163
+ import { join as join13 } from "path";
9425
10164
  function readBillingLedger(billingDir) {
9426
10165
  const view = { events: [], deliveredIds: /* @__PURE__ */ new Set() };
9427
- if (!existsSync18(billingDir)) return view;
10166
+ if (!existsSync20(billingDir)) return view;
9428
10167
  let files;
9429
10168
  try {
9430
- files = readdirSync4(billingDir);
10169
+ files = readdirSync6(billingDir);
9431
10170
  } catch {
9432
10171
  return view;
9433
10172
  }
@@ -9458,7 +10197,7 @@ function readBillingStatus(billingDir) {
9458
10197
  function parseLines(dir, file) {
9459
10198
  let raw;
9460
10199
  try {
9461
- raw = readFileSync16(join11(dir, file), "utf8");
10200
+ raw = readFileSync17(join13(dir, file), "utf8");
9462
10201
  } catch {
9463
10202
  return [];
9464
10203
  }
@@ -9962,8 +10701,10 @@ function buildDaemon(config, paths) {
9962
10701
  async (providerId, model) => await pricingEngine.getEntry(providerId, model) !== null
9963
10702
  );
9964
10703
  const keySpendTracker = new KeySpendTracker(usageEventStore);
10704
+ const usageThroughput = getSharedUsageThroughputTracker2();
9965
10705
  const usageRecorder = new UsageRecorder(usageEventStore, pricingEngine, logger, {
9966
- onRecord: (apiKeyId, costUsd, at) => keySpendTracker.add(apiKeyId, costUsd, at)
10706
+ onRecord: (apiKeyId, costUsd, at) => keySpendTracker.add(apiKeyId, costUsd, at),
10707
+ onEvent: (row, at) => usageThroughput.record(row, at)
9967
10708
  });
9968
10709
  const providerProxy = getProviderProxy({ llmConfig, apiKeyPool, usageRecorder });
9969
10710
  const routeLeaseManager = new RouteLeaseManager(
@@ -10112,6 +10853,11 @@ function buildDaemon(config, paths) {
10112
10853
  // NEVER unauth, NEVER on `/health`. Routed in `AdminServer` (not `adminApi.ts`).
10113
10854
  auditReader: (query2) => readAuditRecords(auditDir, query2),
10114
10855
  auditStatsReader: (query2) => readAuditStats(auditDir, query2),
10856
+ // audit-store-sharding: bodies live in per-session shards, so opening ONE
10857
+ // record's payload is a separate authed call that replays its delta chain.
10858
+ auditBodyReader: (query2) => readAuditBody(auditDir, query2),
10859
+ // audit-store-sharding D8: the manual counterpart to the daily pass.
10860
+ auditCompactor: () => compactAllClosedAuditDays(auditDir),
10115
10861
  // billing-event-stream: the AUTHED `GET /admin/api/billing-status` returns the
10116
10862
  // secret-free total/delivered/pending counts of the durable ledger.
10117
10863
  billingStatusReader: () => readBillingStatus(billingDir)
@@ -10185,10 +10931,11 @@ function resetDaemonSingletonsForTests() {
10185
10931
  __resetSharedIdentityStoreForTests();
10186
10932
  __resetSharedAccountAllowanceStoreForTests();
10187
10933
  __resetSharedAccountAllowanceSchedulingForTests();
10934
+ __resetSharedUsageThroughputTrackerForTests();
10188
10935
  }
10189
10936
  function isTokensStoreReadable(tokensPath) {
10190
10937
  try {
10191
- if (!existsSync19(tokensPath)) return true;
10938
+ if (!existsSync21(tokensPath)) return true;
10192
10939
  accessSync(tokensPath, fsConstants.R_OK);
10193
10940
  return true;
10194
10941
  } catch {