@omnicross/daemon 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  // src/bootstrap.ts
2
- import { accessSync, constants as fsConstants, existsSync as existsSync19 } from "fs";
2
+ import { accessSync, constants as fsConstants, existsSync as existsSync21 } from "fs";
3
3
  import { DEFAULT_AUDIT_CONFIG } from "@omnicross/contracts/audit-types";
4
4
  import { DEFAULT_BILLING_CONFIG } from "@omnicross/contracts/billing-types";
5
5
  import { getGeminiCodeAssistProjectResolver } from "@omnicross/core/auth/GeminiCodeAssistProjectResolver";
@@ -33,7 +33,12 @@ import {
33
33
  } from "@omnicross/core/provider-proxy";
34
34
  import { routeLeaseDescriptorPort } from "@omnicross/cli-launcher";
35
35
  import { KeySpendTracker } from "@omnicross/core/outbound-api";
36
- import { PricingEngine, UsageRecorder } from "@omnicross/core/usage";
36
+ import {
37
+ __resetSharedUsageThroughputTrackerForTests,
38
+ getSharedUsageThroughputTracker as getSharedUsageThroughputTracker2,
39
+ PricingEngine,
40
+ UsageRecorder
41
+ } from "@omnicross/core/usage";
37
42
  import {
38
43
  setSubscriptionAccountService,
39
44
  setSubscriptionProviderRegistry,
@@ -637,6 +642,37 @@ function handleAuditQuery(req, res, reader) {
637
642
  res.writeHead(200, { "Content-Type": "application/json" });
638
643
  res.end(JSON.stringify({ records }));
639
644
  }
645
+ function handleAuditBodyQuery(req, res, reader) {
646
+ const url = new URL(req.url ?? "/", "http://localhost");
647
+ const id = url.searchParams.get("id")?.trim();
648
+ const sessionKey = url.searchParams.get("session")?.trim();
649
+ if (!id || !sessionKey) {
650
+ res.writeHead(400, { "Content-Type": "application/json" });
651
+ res.end(JSON.stringify({ error: "id and session are required" }));
652
+ return;
653
+ }
654
+ const query2 = { id, sessionKey };
655
+ const ts = intParam(url.searchParams.get("ts"));
656
+ if (ts !== void 0) query2.ts = ts;
657
+ const body = reader ? reader(query2) : {};
658
+ res.writeHead(200, { "Content-Type": "application/json" });
659
+ res.end(JSON.stringify(body));
660
+ }
661
+ function handleAuditCompact(res, compact) {
662
+ if (!compact) {
663
+ res.writeHead(200, { "Content-Type": "application/json" });
664
+ res.end(JSON.stringify({ days: 0, shards: 0, savedBytes: 0 }));
665
+ return;
666
+ }
667
+ try {
668
+ const result = compact();
669
+ res.writeHead(200, { "Content-Type": "application/json" });
670
+ res.end(JSON.stringify(result));
671
+ } catch (error) {
672
+ res.writeHead(500, { "Content-Type": "application/json" });
673
+ res.end(JSON.stringify({ error: error instanceof Error ? error.message : "compaction failed" }));
674
+ }
675
+ }
640
676
  async function handleAuditStatsQuery(req, res, reader) {
641
677
  const url = new URL(req.url ?? "/", "http://localhost");
642
678
  const query2 = {};
@@ -1264,6 +1300,35 @@ function validateApiKeys(raw) {
1264
1300
  }
1265
1301
  return out.length > 0 ? out : void 0;
1266
1302
  }
1303
+ var THINK_LEVELS = /* @__PURE__ */ new Set([
1304
+ "none",
1305
+ "minimal",
1306
+ "low",
1307
+ "medium",
1308
+ "high",
1309
+ "xhigh",
1310
+ "max"
1311
+ ]);
1312
+ function validateThinkingLevels(raw) {
1313
+ if (!Array.isArray(raw)) return void 0;
1314
+ if (!raw.every((level) => typeof level === "string" && THINK_LEVELS.has(level))) {
1315
+ return void 0;
1316
+ }
1317
+ return [...raw];
1318
+ }
1319
+ function validateThinkingTokenLimit(raw) {
1320
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) return void 0;
1321
+ const bounds = raw;
1322
+ const min = bounds["min"];
1323
+ const max = bounds["max"];
1324
+ if (typeof min !== "number" || !Number.isFinite(min) || !Number.isInteger(min) || min < 0) {
1325
+ return void 0;
1326
+ }
1327
+ if (typeof max !== "number" || !Number.isFinite(max) || !Number.isInteger(max) || max < min) {
1328
+ return void 0;
1329
+ }
1330
+ return { min, max };
1331
+ }
1267
1332
  function validateModelConfigs(raw) {
1268
1333
  if (!Array.isArray(raw)) return void 0;
1269
1334
  const out = [];
@@ -1278,6 +1343,10 @@ function validateModelConfigs(raw) {
1278
1343
  if (typeof m["enabled"] === "boolean") entry.enabled = m["enabled"];
1279
1344
  if (typeof m["vision"] === "boolean") entry.vision = m["vision"];
1280
1345
  if (typeof m["reasoning"] === "boolean") entry.reasoning = m["reasoning"];
1346
+ const thinkingLevels = validateThinkingLevels(m["thinkingLevels"]);
1347
+ if (thinkingLevels) entry.thinkingLevels = thinkingLevels;
1348
+ const thinkingTokenLimit = validateThinkingTokenLimit(m["thinkingTokenLimit"]);
1349
+ if (thinkingTokenLimit) entry.thinkingTokenLimit = thinkingTokenLimit;
1281
1350
  out.push(entry);
1282
1351
  }
1283
1352
  return out.length > 0 ? out : void 0;
@@ -3613,6 +3682,7 @@ function applyAuditConfig(config) {
3613
3682
  } else {
3614
3683
  setAuditCaptureConfig(null);
3615
3684
  setAuditSink(null);
3685
+ writer?.reset();
3616
3686
  if (sweeper) {
3617
3687
  if (config) sweeper.configure(config);
3618
3688
  sweeper.dispose();
@@ -3623,6 +3693,7 @@ function resetAuditRuntimeForTests() {
3623
3693
  setAuditCaptureConfig(null);
3624
3694
  setAuditSink(null);
3625
3695
  setUpstreamTracePath(null);
3696
+ writer?.reset();
3626
3697
  if (sweeper) sweeper.dispose();
3627
3698
  writer = null;
3628
3699
  sweeper = null;
@@ -4214,6 +4285,7 @@ async function handleImport(body, deps) {
4214
4285
  }
4215
4286
 
4216
4287
  // src/admin/usagePricing.ts
4288
+ import { getSharedUsageThroughputTracker } from "@omnicross/core/usage";
4217
4289
  var err4 = (status, message) => ({
4218
4290
  status,
4219
4291
  body: { error: { type: "admin_api_error", message } }
@@ -4239,6 +4311,9 @@ var BUCKET_SPAN_MS = {
4239
4311
  };
4240
4312
  var MAX_TIMESERIES_BUCKETS = 2e3;
4241
4313
  async function handleUsageGet(view, query2, deps) {
4314
+ if (view === "throughput") {
4315
+ return { status: 200, body: getSharedUsageThroughputTracker().snapshot() };
4316
+ }
4242
4317
  const range = parseRange(query2);
4243
4318
  if (!isRange(range)) return range;
4244
4319
  switch (view) {
@@ -5044,6 +5119,12 @@ function parseModelConfigsInput(raw, existing) {
5044
5119
  else if (typeof prior?.vision === "boolean") entry.vision = prior.vision;
5045
5120
  if (typeof m["reasoning"] === "boolean") entry.reasoning = m["reasoning"];
5046
5121
  else if (typeof prior?.reasoning === "boolean") entry.reasoning = prior.reasoning;
5122
+ const thinkingLevels = validateThinkingLevels(m["thinkingLevels"]);
5123
+ if (thinkingLevels) entry.thinkingLevels = thinkingLevels;
5124
+ else if (prior?.thinkingLevels) entry.thinkingLevels = prior.thinkingLevels;
5125
+ const thinkingTokenLimit = validateThinkingTokenLimit(m["thinkingTokenLimit"]);
5126
+ if (thinkingTokenLimit) entry.thinkingTokenLimit = thinkingTokenLimit;
5127
+ else if (prior?.thinkingTokenLimit) entry.thinkingTokenLimit = prior.thinkingTokenLimit;
5047
5128
  out.push(entry);
5048
5129
  }
5049
5130
  return out.length > 0 ? out : void 0;
@@ -5964,7 +6045,7 @@ async function handleUiStatic(req, res, urlPath, uiDist) {
5964
6045
  }
5965
6046
 
5966
6047
  // src/admin/version.ts
5967
- var DAEMON_VERSION = true ? "0.1.8" : "0.0.0-dev";
6048
+ var DAEMON_VERSION = true ? "0.1.10" : "0.0.0-dev";
5968
6049
 
5969
6050
  // src/admin/AdminServer.ts
5970
6051
  var LOOPBACK_ADDR = "127.0.0.1";
@@ -6076,6 +6157,14 @@ var AdminServer = class {
6076
6157
  await handleAuditStatsQuery(req, res, this.deps.auditStatsReader);
6077
6158
  return;
6078
6159
  }
6160
+ if (path2 === "/admin/api/audit/body" && (req.method === "GET" || req.method === "HEAD")) {
6161
+ handleAuditBodyQuery(req, res, this.deps.auditBodyReader);
6162
+ return;
6163
+ }
6164
+ if (path2 === "/admin/api/audit/compact" && req.method === "POST") {
6165
+ handleAuditCompact(res, this.deps.auditCompactor);
6166
+ return;
6167
+ }
6079
6168
  if (path2 === "/admin/api/billing-status" && (req.method === "GET" || req.method === "HEAD")) {
6080
6169
  handleBillingStatus(res, this.deps.billingStatusReader);
6081
6170
  return;
@@ -6535,6 +6624,15 @@ function toLLMProvider(row) {
6535
6624
  api_base_url: row.baseUrl,
6536
6625
  api_key: resolvePreferredApiKey(row),
6537
6626
  models,
6627
+ modelConfigs: row.modelConfigs?.map((config) => ({
6628
+ id: config.id,
6629
+ name: config.name ?? config.id,
6630
+ enabled: config.enabled ?? true,
6631
+ vision: config.vision,
6632
+ reasoning: config.reasoning,
6633
+ thinkingLevels: config.thinkingLevels,
6634
+ thinkingTokenLimit: config.thinkingTokenLimit
6635
+ })),
6538
6636
  enabled: true,
6539
6637
  transformer,
6540
6638
  // app-parity-2 child 3: POPULATE the coding-plan endpoint onto the core
@@ -8818,18 +8916,214 @@ var AccountHealthSweeper = class {
8818
8916
  };
8819
8917
 
8820
8918
  // src/audit/AuditPruneSweeper.ts
8821
- import { existsSync as existsSync15, readdirSync as readdirSync2, unlinkSync as unlinkSync3 } from "fs";
8822
- import { join as join7 } from "path";
8919
+ import { createReadStream as createReadStream2, createWriteStream as createWriteStream2, existsSync as existsSync16, readdirSync as readdirSync3, rmSync as rmSync4, unlinkSync as unlinkSync4 } from "fs";
8920
+ import { join as join8 } from "path";
8921
+ import { pipeline } from "stream/promises";
8922
+ import { createGzip } from "zlib";
8923
+
8924
+ // src/audit/auditDictionary.ts
8925
+ import { existsSync as existsSync14, readdirSync, readFileSync as readFileSync14, renameSync as renameSync5, unlinkSync as unlinkSync3, writeFileSync as writeFileSync12 } from "fs";
8926
+ import { join as join6 } from "path";
8927
+
8928
+ // src/audit/auditBodyStore.ts
8929
+ var ANCHOR_EVERY = 64;
8930
+ var ANCHOR_DELTA_RATIO = 0.75;
8931
+ var DIVERGED_PREFIX_RATIO = 0.25;
8932
+ var MAX_BASE_CHARS = 4e6;
8933
+ var CACHE_BUDGET_CHARS = 16e6;
8934
+ var CACHE_MAX_SESSIONS = 32;
8935
+ var MAX_BASES_PER_SESSION = 4;
8936
+ var isHighSurrogate = (code) => code >= 55296 && code <= 56319;
8937
+ var isLowSurrogate = (code) => code >= 56320 && code <= 57343;
8938
+ function computeBodyDelta(prev, next) {
8939
+ const shortest = Math.min(prev.length, next.length);
8940
+ let pre = 0;
8941
+ while (pre < shortest && prev.charCodeAt(pre) === next.charCodeAt(pre)) pre += 1;
8942
+ if (pre > 0 && isHighSurrogate(prev.charCodeAt(pre - 1))) pre -= 1;
8943
+ const maxSuf = shortest - pre;
8944
+ let suf = 0;
8945
+ while (suf < maxSuf && prev.charCodeAt(prev.length - 1 - suf) === next.charCodeAt(next.length - 1 - suf)) {
8946
+ suf += 1;
8947
+ }
8948
+ if (suf > 0 && isLowSurrogate(prev.charCodeAt(prev.length - suf))) suf -= 1;
8949
+ return { pre, suf, ins: next.slice(pre, next.length - suf) };
8950
+ }
8951
+ function applyBodyDelta(prev, delta) {
8952
+ const head = delta.pre > 0 ? prev.slice(0, delta.pre) : "";
8953
+ const tail = delta.suf > 0 ? prev.slice(prev.length - delta.suf) : "";
8954
+ return head + delta.ins + tail;
8955
+ }
8956
+ var SessionBaseCache = class {
8957
+ constructor(maxSessions = CACHE_MAX_SESSIONS, budgetChars = CACHE_BUDGET_CHARS, maxBaseChars = MAX_BASE_CHARS, maxHeads = MAX_BASES_PER_SESSION) {
8958
+ this.maxSessions = maxSessions;
8959
+ this.budgetChars = budgetChars;
8960
+ this.maxBaseChars = maxBaseChars;
8961
+ this.maxHeads = maxHeads;
8962
+ }
8963
+ maxSessions;
8964
+ budgetChars;
8965
+ maxBaseChars;
8966
+ maxHeads;
8967
+ /** Session key to its retained heads, most-recent first. */
8968
+ entries = /* @__PURE__ */ new Map();
8969
+ chars = 0;
8970
+ /** Retained sessions (tests + diagnostics). */
8971
+ get size() {
8972
+ return this.entries.size;
8973
+ }
8974
+ /** A session's retained heads, most-recent first. Refreshes LRU recency. */
8975
+ get(sessionKey) {
8976
+ const found = this.entries.get(sessionKey);
8977
+ if (!found) return [];
8978
+ this.entries.delete(sessionKey);
8979
+ this.entries.set(sessionKey, found);
8980
+ return found;
8981
+ }
8982
+ /**
8983
+ * Retain `base` as a head of `sessionKey`.
8984
+ *
8985
+ * `replacesId` is the head this turn CONTINUES (its body was preserved whole
8986
+ * inside the new one), which is swapped out so a linear conversation keeps
8987
+ * exactly one head. Omit it when the turn started a distinct stream %s that
8988
+ * head is added alongside, which is what keeps a fork's branches apart.
8989
+ *
8990
+ * A body larger than `maxBaseChars` is not retained: the next turn anchors
8991
+ * rather than letting one oversized session monopolize the budget.
8992
+ */
8993
+ remember(sessionKey, base, replacesId) {
8994
+ const heads = this.entries.get(sessionKey) ?? [];
8995
+ if (replacesId !== void 0) {
8996
+ const at = heads.findIndex((head) => head.lastId === replacesId);
8997
+ if (at >= 0) {
8998
+ this.chars -= heads[at].text.length;
8999
+ heads.splice(at, 1);
9000
+ }
9001
+ }
9002
+ if (base.text.length <= this.maxBaseChars) {
9003
+ heads.unshift(base);
9004
+ this.chars += base.text.length;
9005
+ }
9006
+ while (heads.length > this.maxHeads) {
9007
+ const dropped = heads.pop();
9008
+ if (dropped) this.chars -= dropped.text.length;
9009
+ }
9010
+ this.entries.delete(sessionKey);
9011
+ if (heads.length > 0) this.entries.set(sessionKey, heads);
9012
+ this.evict();
9013
+ }
9014
+ /** Drop a session's heads (eviction, or a write failure invalidating them). */
9015
+ forget(sessionKey) {
9016
+ const heads = this.entries.get(sessionKey);
9017
+ if (!heads) return;
9018
+ for (const head of heads) this.chars -= head.text.length;
9019
+ this.entries.delete(sessionKey);
9020
+ }
9021
+ /** Drop everything (writer disposal / test teardown). */
9022
+ clear() {
9023
+ this.entries.clear();
9024
+ this.chars = 0;
9025
+ }
9026
+ /** Evict least-recently-used sessions until both bounds hold. */
9027
+ evict() {
9028
+ while (this.entries.size > this.maxSessions || this.chars > this.budgetChars && this.entries.size > 1) {
9029
+ const oldest = this.entries.keys().next();
9030
+ if (oldest.done) break;
9031
+ this.forget(oldest.value);
9032
+ }
9033
+ }
9034
+ };
9035
+ function anchorReason(base, dayDir, delta, nextLength) {
9036
+ if (!base || !delta) return "new";
9037
+ if (base.dayDir !== dayDir) return "day";
9038
+ if (base.chainLen >= ANCHOR_EVERY) return "chain";
9039
+ if (delta.pre < base.text.length * DIVERGED_PREFIX_RATIO) return "diverged";
9040
+ if (delta.ins.length > nextLength * ANCHOR_DELTA_RATIO) return "costly";
9041
+ return null;
9042
+ }
9043
+ function pickBase(heads, next) {
9044
+ let best = null;
9045
+ for (const base of heads) {
9046
+ const delta = computeBodyDelta(base.text, next);
9047
+ if (best !== null && delta.ins.length >= best.delta.ins.length) continue;
9048
+ best = { base, delta, continues: delta.pre + delta.suf >= base.text.length };
9049
+ }
9050
+ return best;
9051
+ }
9052
+ function encodeBodyEntry(record, sessionKey, dayDir, cache) {
9053
+ const requestBody = record.requestBody;
9054
+ const responseBody = record.responseBody;
9055
+ if (requestBody === void 0 && responseBody === void 0) return null;
9056
+ const entry = { id: record.id, ts: record.ts };
9057
+ if (requestBody !== void 0) {
9058
+ const heads = cache.get(sessionKey);
9059
+ const sameDay = heads.filter((head) => head.dayDir === dayDir);
9060
+ const chosen = pickBase(sameDay, requestBody);
9061
+ const reason = heads.length > 0 && sameDay.length === 0 ? "day" : anchorReason(chosen?.base, dayDir, chosen?.delta ?? null, requestBody.length);
9062
+ if (reason !== null) {
9063
+ entry.req = { base: null, anchor: reason, pre: 0, suf: 0, ins: requestBody };
9064
+ cache.remember(
9065
+ sessionKey,
9066
+ { dayDir, lastId: record.id, text: requestBody, chainLen: 0 },
9067
+ chosen?.continues === true ? chosen.base.lastId : void 0
9068
+ );
9069
+ } else {
9070
+ const picked = chosen;
9071
+ entry.req = {
9072
+ base: picked.base.lastId,
9073
+ ...picked.continues ? { cont: true } : {},
9074
+ pre: picked.delta.pre,
9075
+ suf: picked.delta.suf,
9076
+ ins: picked.delta.ins
9077
+ };
9078
+ cache.remember(
9079
+ sessionKey,
9080
+ { dayDir, lastId: record.id, text: requestBody, chainLen: picked.base.chainLen + 1 },
9081
+ picked.continues ? picked.base.lastId : void 0
9082
+ );
9083
+ }
9084
+ }
9085
+ if (responseBody !== void 0) entry.res = responseBody;
9086
+ return JSON.stringify(entry);
9087
+ }
9088
+ function isAuditBodyEntry(value) {
9089
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
9090
+ const entry = value;
9091
+ if (typeof entry["id"] !== "string" || typeof entry["ts"] !== "number") return false;
9092
+ if (entry["res"] !== void 0 && typeof entry["res"] !== "string") return false;
9093
+ const req = entry["req"];
9094
+ if (req === void 0) return true;
9095
+ if (!req || typeof req !== "object" || Array.isArray(req)) return false;
9096
+ const delta = req;
9097
+ if (delta["anchor"] !== void 0 && typeof delta["anchor"] !== "string") return false;
9098
+ if (delta["cont"] !== void 0 && typeof delta["cont"] !== "boolean") return false;
9099
+ return (delta["base"] === null || typeof delta["base"] === "string") && Number.isSafeInteger(delta["pre"]) && delta["pre"] >= 0 && Number.isSafeInteger(delta["suf"]) && delta["suf"] >= 0 && typeof delta["ins"] === "string";
9100
+ }
8823
9101
 
8824
9102
  // src/audit/auditFiles.ts
8825
9103
  var AUDIT_FILE_RE = /^audit-(\d{4})-(\d{2})-(\d{2})\.jsonl$/;
9104
+ var AUDIT_DAY_DIR_RE = /^audit-(\d{4})-(\d{2})-(\d{2})$/;
9105
+ var AUDIT_META_FILE = "meta.jsonl";
9106
+ var AUDIT_BODIES_DIR = "bodies";
9107
+ var AUDIT_SESSION_KEY_RE = /^[0-9a-f]{8,64}$/;
8826
9108
  var pad22 = (n) => String(n).padStart(2, "0");
8827
- function auditFileName(ts) {
9109
+ var localDateStamp = (ts) => {
8828
9110
  const d = new Date(ts);
8829
- return `audit-${d.getFullYear()}-${pad22(d.getMonth() + 1)}-${pad22(d.getDate())}.jsonl`;
9111
+ return `${d.getFullYear()}-${pad22(d.getMonth() + 1)}-${pad22(d.getDate())}`;
9112
+ };
9113
+ function auditDayDirName(ts) {
9114
+ return `audit-${localDateStamp(ts)}`;
9115
+ }
9116
+ function isAuditDayDir(name) {
9117
+ return AUDIT_DAY_DIR_RE.test(name);
9118
+ }
9119
+ function isSafeSessionKey(key) {
9120
+ return typeof key === "string" && AUDIT_SESSION_KEY_RE.test(key);
8830
9121
  }
8831
- function auditFileDateMs(fileName) {
8832
- const m = AUDIT_FILE_RE.exec(fileName);
9122
+ function auditBodyFileName(sessionKey) {
9123
+ return `${sessionKey}.jsonl`;
9124
+ }
9125
+ function auditFileDateMs(name) {
9126
+ const m = AUDIT_FILE_RE.exec(name) ?? AUDIT_DAY_DIR_RE.exec(name);
8833
9127
  if (!m) return null;
8834
9128
  const year = Number(m[1]);
8835
9129
  const month = Number(m[2]);
@@ -8841,16 +9135,153 @@ function auditFileDateMs(fileName) {
8841
9135
  return d.getTime();
8842
9136
  }
8843
9137
 
9138
+ // src/audit/auditDictionary.ts
9139
+ var AUDIT_DICT_FILE = "_dict.jsonl";
9140
+ var DICT_BASE_PREFIX = "dict:";
9141
+ var DICT_CANDIDATES = 3;
9142
+ var MIN_SAVING_RATIO = 0.2;
9143
+ function parseEntries(raw) {
9144
+ const entries = [];
9145
+ for (const line of raw.split("\n")) {
9146
+ const trimmed = line.trim();
9147
+ if (!trimmed) continue;
9148
+ try {
9149
+ const parsed = JSON.parse(trimmed);
9150
+ if (isAuditBodyEntry(parsed)) entries.push(parsed);
9151
+ } catch {
9152
+ }
9153
+ }
9154
+ return entries;
9155
+ }
9156
+ function plainShards(bodiesPath) {
9157
+ try {
9158
+ return readdirSync(bodiesPath).filter(
9159
+ (file) => file.endsWith(".jsonl") && isSafeSessionKey(file.slice(0, -".jsonl".length))
9160
+ );
9161
+ } catch {
9162
+ return [];
9163
+ }
9164
+ }
9165
+ function chooseDictionary(anchors) {
9166
+ if (anchors.length < 2) return null;
9167
+ const total = anchors.reduce((sum, body) => sum + body.length, 0);
9168
+ const candidates = [...anchors].sort((a, b) => b.length - a.length).slice(0, DICT_CANDIDATES);
9169
+ let best = null;
9170
+ for (const candidate of candidates) {
9171
+ let cost = candidate.length;
9172
+ for (const body of anchors) {
9173
+ cost += body === candidate ? 0 : computeBodyDelta(candidate, body).ins.length;
9174
+ }
9175
+ if (best === null || cost < best.cost) best = { body: candidate, cost };
9176
+ }
9177
+ if (best === null) return null;
9178
+ return total - best.cost >= total * MIN_SAVING_RATIO ? best.body : null;
9179
+ }
9180
+ var EMPTY = { shards: 0, anchors: 0, savedBytes: 0 };
9181
+ function compactAuditDay(dayPath) {
9182
+ const bodiesPath = join6(dayPath, AUDIT_BODIES_DIR);
9183
+ if (!existsSync14(bodiesPath)) return EMPTY;
9184
+ const dictPath = join6(bodiesPath, AUDIT_DICT_FILE);
9185
+ if (existsSync14(dictPath) || existsSync14(`${dictPath}.gz`)) return EMPTY;
9186
+ const shardFiles = plainShards(bodiesPath);
9187
+ if (shardFiles.length < 2) return EMPTY;
9188
+ const loaded = /* @__PURE__ */ new Map();
9189
+ const anchors = [];
9190
+ for (const file of shardFiles) {
9191
+ let entries;
9192
+ try {
9193
+ entries = parseEntries(readFileSync14(join6(bodiesPath, file), "utf8"));
9194
+ } catch {
9195
+ continue;
9196
+ }
9197
+ loaded.set(file, entries);
9198
+ for (const entry of entries) {
9199
+ if (entry.req && entry.req.base === null) anchors.push(entry.req.ins);
9200
+ }
9201
+ }
9202
+ if (anchors.length < 2) return EMPTY;
9203
+ const dictionary = chooseDictionary(anchors);
9204
+ if (dictionary === null) return EMPTY;
9205
+ const dictEntry = {
9206
+ id: `${DICT_BASE_PREFIX}0`,
9207
+ ts: 0,
9208
+ req: { base: null, anchor: "dict", pre: 0, suf: 0, ins: dictionary }
9209
+ };
9210
+ writeFileSync12(dictPath, JSON.stringify(dictEntry) + "\n", "utf8");
9211
+ const result = { shards: 0, anchors: 0, savedBytes: 0 };
9212
+ for (const [file, entries] of loaded) {
9213
+ let changed = false;
9214
+ let saved = 0;
9215
+ const rewritten = entries.map((entry) => {
9216
+ if (!entry.req || entry.req.base !== null || entry.req.ins === dictionary) return entry;
9217
+ const delta = computeBodyDelta(dictionary, entry.req.ins);
9218
+ if (delta.ins.length >= entry.req.ins.length) return entry;
9219
+ changed = true;
9220
+ saved += entry.req.ins.length - delta.ins.length;
9221
+ return {
9222
+ ...entry,
9223
+ req: { base: dictEntry.id, pre: delta.pre, suf: delta.suf, ins: delta.ins }
9224
+ };
9225
+ });
9226
+ if (!changed) continue;
9227
+ const target = join6(bodiesPath, file);
9228
+ const temp = `${target}.compacting`;
9229
+ try {
9230
+ writeFileSync12(temp, rewritten.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8");
9231
+ renameSync5(temp, target);
9232
+ } catch {
9233
+ try {
9234
+ if (existsSync14(temp)) unlinkSync3(temp);
9235
+ } catch {
9236
+ }
9237
+ continue;
9238
+ }
9239
+ result.shards += 1;
9240
+ result.anchors += rewritten.filter((e) => e.req?.base === dictEntry.id).length;
9241
+ result.savedBytes += saved;
9242
+ }
9243
+ if (result.shards === 0) {
9244
+ try {
9245
+ unlinkSync3(dictPath);
9246
+ } catch {
9247
+ }
9248
+ }
9249
+ return result;
9250
+ }
9251
+ function compactAllClosedAuditDays(auditDir, now = Date.now) {
9252
+ const run = { days: 0, shards: 0, savedBytes: 0 };
9253
+ if (!existsSync14(auditDir)) return run;
9254
+ const today = auditDayDirName(now());
9255
+ let names;
9256
+ try {
9257
+ names = readdirSync(auditDir).filter(isAuditDayDir).sort();
9258
+ } catch {
9259
+ return run;
9260
+ }
9261
+ for (const name of names) {
9262
+ if (name === today) continue;
9263
+ try {
9264
+ const result = compactAuditDay(join6(auditDir, name));
9265
+ if (result.shards === 0) continue;
9266
+ run.days += 1;
9267
+ run.shards += result.shards;
9268
+ run.savedBytes += result.savedBytes;
9269
+ } catch {
9270
+ }
9271
+ }
9272
+ return run;
9273
+ }
9274
+
8844
9275
  // src/audit/auditStats.ts
8845
9276
  import {
8846
9277
  createReadStream,
8847
- existsSync as existsSync14,
8848
- readFileSync as readFileSync14,
8849
- readdirSync,
9278
+ existsSync as existsSync15,
9279
+ readFileSync as readFileSync15,
9280
+ readdirSync as readdirSync2,
8850
9281
  statSync as statSync3,
8851
- writeFileSync as writeFileSync12
9282
+ writeFileSync as writeFileSync13
8852
9283
  } from "fs";
8853
- import { basename, dirname as dirname7, join as join6 } from "path";
9284
+ import { basename, dirname as dirname7, join as join7 } from "path";
8854
9285
  var SIDECAR_VERSION = 1;
8855
9286
  var META_PREFIX_BYTES = 64 * 1024;
8856
9287
  var READ_CHUNK_BYTES = 4 * 1024 * 1024;
@@ -8858,9 +9289,9 @@ function auditStatsFileName(auditFile) {
8858
9289
  return auditFile.replace(/\.jsonl$/, ".stats.json");
8859
9290
  }
8860
9291
  function readPersisted(path2) {
8861
- if (!existsSync14(path2)) return null;
9292
+ if (!existsSync15(path2)) return null;
8862
9293
  try {
8863
- const value = JSON.parse(readFileSync14(path2, "utf8"));
9294
+ const value = JSON.parse(readFileSync15(path2, "utf8"));
8864
9295
  if (value.version !== SIDECAR_VERSION || !Number.isSafeInteger(value.auditBytes) || (value.auditBytes ?? -1) < 0 || !Number.isSafeInteger(value.requestCount) || (value.requestCount ?? -1) < 0 || !Number.isSafeInteger(value.errorCount) || (value.errorCount ?? -1) < 0 || (value.errorCount ?? 0) > (value.requestCount ?? -1) || typeof value.complete !== "boolean" || value.minTs !== null && !Number.isFinite(value.minTs) || value.maxTs !== null && !Number.isFinite(value.maxTs)) {
8865
9296
  return null;
8866
9297
  }
@@ -8870,7 +9301,7 @@ function readPersisted(path2) {
8870
9301
  }
8871
9302
  }
8872
9303
  function updateAuditStatsAfterAppend(auditPath, auditBytesBefore, auditBytesAfter, record) {
8873
- const statsPath = join6(dirname7(auditPath), auditStatsFileName(basename(auditPath)));
9304
+ const statsPath = join7(dirname7(auditPath), auditStatsFileName(basename(auditPath)));
8874
9305
  const previous = auditBytesBefore === 0 ? {
8875
9306
  version: SIDECAR_VERSION,
8876
9307
  auditBytes: 0,
@@ -8890,13 +9321,13 @@ function updateAuditStatsAfterAppend(auditPath, auditBytesBefore, auditBytesAfte
8890
9321
  minTs: previous.minTs === null ? record.ts : Math.min(previous.minTs, record.ts),
8891
9322
  maxTs: previous.maxTs === null ? record.ts : Math.max(previous.maxTs, record.ts)
8892
9323
  };
8893
- writeFileSync12(statsPath, JSON.stringify(next), "utf8");
9324
+ writeFileSync13(statsPath, JSON.stringify(next), "utf8");
8894
9325
  }
8895
9326
  function queryCovers(stats, from, to) {
8896
9327
  return stats.requestCount === 0 || stats.minTs !== null && stats.maxTs !== null && from <= stats.minTs && to >= stats.maxTs;
8897
9328
  }
8898
- function fileOverlaps(file, from, to) {
8899
- const start = auditFileDateMs(file);
9329
+ function fileOverlaps(name, from, to) {
9330
+ const start = auditFileDateMs(name);
8900
9331
  if (start === null) return false;
8901
9332
  const date = new Date(start);
8902
9333
  const end = new Date(date.getFullYear(), date.getMonth(), date.getDate() + 1).getTime();
@@ -9001,21 +9432,27 @@ function mergePersistedStats(previous, appended) {
9001
9432
  };
9002
9433
  }
9003
9434
  async function readAuditStats(auditDir, query2 = {}) {
9004
- if (!existsSync14(auditDir)) return { requestCount: 0, errorCount: 0, complete: true };
9435
+ if (!existsSync15(auditDir)) return { requestCount: 0, errorCount: 0, complete: true };
9005
9436
  const from = typeof query2.from === "number" ? query2.from : -Infinity;
9006
9437
  const to = typeof query2.to === "number" ? query2.to : Infinity;
9007
- let files;
9438
+ let sources;
9008
9439
  try {
9009
- files = readdirSync(auditDir).filter((file) => AUDIT_FILE_RE.test(file) && fileOverlaps(file, from, to)).sort();
9440
+ sources = readdirSync2(auditDir).filter((name) => fileOverlaps(name, from, to)).sort().map(
9441
+ (name) => AUDIT_DAY_DIR_RE.test(name) ? {
9442
+ auditPath: join7(auditDir, name, AUDIT_META_FILE),
9443
+ statsPath: join7(auditDir, name, auditStatsFileName(AUDIT_META_FILE))
9444
+ } : {
9445
+ auditPath: join7(auditDir, name),
9446
+ statsPath: join7(auditDir, auditStatsFileName(name))
9447
+ }
9448
+ ).filter((source) => existsSync15(source.auditPath));
9010
9449
  } catch {
9011
9450
  return { requestCount: 0, errorCount: 0, complete: false };
9012
9451
  }
9013
9452
  const total = { requestCount: 0, errorCount: 0, complete: true };
9014
- for (const file of files) {
9015
- const auditPath = join6(auditDir, file);
9453
+ for (const { auditPath, statsPath } of sources) {
9016
9454
  try {
9017
9455
  const auditBytes = statSync3(auditPath).size;
9018
- const statsPath = join6(auditDir, auditStatsFileName(file));
9019
9456
  const persisted = readPersisted(statsPath);
9020
9457
  if (persisted && persisted.complete && persisted.auditBytes === auditBytes && queryCovers(persisted, from, to)) {
9021
9458
  total.requestCount += persisted.requestCount;
@@ -9034,7 +9471,7 @@ async function readAuditStats(auditDir, query2 = {}) {
9034
9471
  total.errorCount += scanned.filtered.errorCount + (resumable?.errorCount ?? 0);
9035
9472
  total.complete = total.complete && scanned.filtered.complete;
9036
9473
  const current = resumable ? mergePersistedStats(resumable, scanned.all) : scanned.all;
9037
- if (current.complete) writeFileSync12(statsPath, JSON.stringify(current), "utf8");
9474
+ if (current.complete) writeFileSync13(statsPath, JSON.stringify(current), "utf8");
9038
9475
  } catch {
9039
9476
  total.complete = false;
9040
9477
  }
@@ -9045,6 +9482,7 @@ async function readAuditStats(auditDir, query2 = {}) {
9045
9482
  // src/audit/AuditPruneSweeper.ts
9046
9483
  var DAY_MS = 24 * 60 * 6e4;
9047
9484
  var SWEEP_INTERVAL_MS2 = 60 * 6e4;
9485
+ var ARCHIVE_BATCH = 64;
9048
9486
  var AuditPruneSweeper = class {
9049
9487
  constructor(auditDir, logger, config, intervalMs = SWEEP_INTERVAL_MS2, now = Date.now) {
9050
9488
  this.auditDir = auditDir;
@@ -9060,6 +9498,7 @@ var AuditPruneSweeper = class {
9060
9498
  now;
9061
9499
  timer = null;
9062
9500
  sweeping = false;
9501
+ archiving = false;
9063
9502
  /** Whether pruning is active (audit enabled). */
9064
9503
  get enabled() {
9065
9504
  return this.config.enabled;
@@ -9069,13 +9508,13 @@ var AuditPruneSweeper = class {
9069
9508
  this.config = config;
9070
9509
  }
9071
9510
  /**
9072
- * Arm the prune interval AND run one prune immediately (boot cleanup). No-op
9073
- * when audit is disabled (zero regression). Idempotent.
9511
+ * Arm the interval AND run one pass immediately (boot cleanup). No-op when
9512
+ * audit is disabled (zero regression). Idempotent.
9074
9513
  */
9075
9514
  start() {
9076
9515
  if (this.timer || !this.config.enabled) return;
9077
- void this.sweep();
9078
- this.timer = setInterval(() => void this.sweep(), this.intervalMs);
9516
+ void this.runOnce();
9517
+ this.timer = setInterval(() => void this.runOnce(), this.intervalMs);
9079
9518
  this.timer.unref?.();
9080
9519
  }
9081
9520
  /** Clear the interval (daemon shutdown / test teardown). Idempotent. */
@@ -9085,31 +9524,43 @@ var AuditPruneSweeper = class {
9085
9524
  this.timer = null;
9086
9525
  }
9087
9526
  }
9527
+ /** Prune first, then archive — never spend CPU compressing a day about to go. */
9528
+ async runOnce() {
9529
+ await this.sweep();
9530
+ await this.archive();
9531
+ }
9532
+ /** The LOCAL-midnight epoch ms of the current day. */
9533
+ todayMidnight() {
9534
+ const today = new Date(this.now());
9535
+ return new Date(today.getFullYear(), today.getMonth(), today.getDate()).getTime();
9536
+ }
9088
9537
  /**
9089
- * One prune: unlink every audit date file strictly OLDER than the retention
9090
- * cutoff (`now - retentionDays` days, at local-midnight granularity). Exposed
9091
- * for tests; never throws. Returns the number of files removed.
9538
+ * One prune: remove every audit day strictly OLDER than the retention cutoff
9539
+ * (`now - retentionDays` days, at local-midnight granularity). Exposed for
9540
+ * tests; never throws. Returns the number of days removed.
9092
9541
  */
9093
9542
  async sweep() {
9094
9543
  if (!this.config.enabled || this.sweeping) return 0;
9095
9544
  this.sweeping = true;
9096
9545
  try {
9097
- if (!existsSync15(this.auditDir)) return 0;
9098
- const today = new Date(this.now());
9099
- const todayMidnight = new Date(today.getFullYear(), today.getMonth(), today.getDate()).getTime();
9100
- const cutoff = todayMidnight - (this.config.retentionDays - 1) * DAY_MS;
9546
+ if (!existsSync16(this.auditDir)) return 0;
9547
+ const cutoff = this.todayMidnight() - (this.config.retentionDays - 1) * DAY_MS;
9101
9548
  let removed = 0;
9102
- for (const file of readdirSync2(this.auditDir)) {
9103
- const dateMs = auditFileDateMs(file);
9549
+ for (const name of readdirSync3(this.auditDir)) {
9550
+ const dateMs = auditFileDateMs(name);
9104
9551
  if (dateMs === null || dateMs >= cutoff) continue;
9105
9552
  try {
9106
- unlinkSync3(join7(this.auditDir, file));
9553
+ if (isAuditDayDir(name)) {
9554
+ rmSync4(join8(this.auditDir, name), { recursive: true, force: true });
9555
+ } else {
9556
+ unlinkSync4(join8(this.auditDir, name));
9557
+ const statsPath = join8(this.auditDir, auditStatsFileName(name));
9558
+ if (existsSync16(statsPath)) unlinkSync4(statsPath);
9559
+ }
9107
9560
  removed += 1;
9108
- const statsPath = join7(this.auditDir, auditStatsFileName(file));
9109
- if (existsSync15(statsPath)) unlinkSync3(statsPath);
9110
9561
  } catch (error) {
9111
- this.logger.warn("[AuditPruneSweeper] failed to unlink expired audit file", {
9112
- file,
9562
+ this.logger.warn("[AuditPruneSweeper] failed to remove expired audit day", {
9563
+ name,
9113
9564
  error: error instanceof Error ? error.message : String(error)
9114
9565
  });
9115
9566
  }
@@ -9125,59 +9576,349 @@ var AuditPruneSweeper = class {
9125
9576
  this.sweeping = false;
9126
9577
  }
9127
9578
  }
9579
+ /**
9580
+ * Gzip the body shards of every CLOSED day (anything before today). Today is
9581
+ * deliberately left as plain text so it stays greppable while it is the day you
9582
+ * are debugging. Exposed for tests; never throws. Returns shards compressed.
9583
+ */
9584
+ async archive() {
9585
+ if (!this.config.enabled || this.archiving) return 0;
9586
+ this.archiving = true;
9587
+ try {
9588
+ if (!existsSync16(this.auditDir)) return 0;
9589
+ const today = this.todayMidnight();
9590
+ let compressed = 0;
9591
+ for (const name of readdirSync3(this.auditDir)) {
9592
+ if (compressed >= ARCHIVE_BATCH) break;
9593
+ const dateMs = auditFileDateMs(name);
9594
+ if (dateMs === null || dateMs >= today || !isAuditDayDir(name)) continue;
9595
+ const dayPath = join8(this.auditDir, name);
9596
+ try {
9597
+ const compaction = compactAuditDay(dayPath);
9598
+ if (compaction.shards > 0) {
9599
+ this.logger.debug("audit cross-session compaction complete", {
9600
+ day: name,
9601
+ shards: compaction.shards,
9602
+ anchors: compaction.anchors,
9603
+ savedBytes: compaction.savedBytes
9604
+ });
9605
+ }
9606
+ } catch (error) {
9607
+ this.logger.warn("[AuditPruneSweeper] cross-session compaction failed", {
9608
+ day: name,
9609
+ error: error instanceof Error ? error.message : String(error)
9610
+ });
9611
+ }
9612
+ compressed += await this.archiveDay(
9613
+ join8(dayPath, AUDIT_BODIES_DIR),
9614
+ ARCHIVE_BATCH - compressed
9615
+ );
9616
+ }
9617
+ if (compressed > 0) this.logger.debug("audit archive complete", { compressed });
9618
+ return compressed;
9619
+ } catch (error) {
9620
+ this.logger.warn("audit archive pass failed", {
9621
+ error: error instanceof Error ? error.message : String(error)
9622
+ });
9623
+ return 0;
9624
+ } finally {
9625
+ this.archiving = false;
9626
+ }
9627
+ }
9628
+ /** Gzip up to `budget` plain shards in one day's `bodies/` directory. */
9629
+ async archiveDay(bodiesPath, budget) {
9630
+ let shards;
9631
+ try {
9632
+ shards = readdirSync3(bodiesPath).filter((file) => file.endsWith(".jsonl"));
9633
+ } catch {
9634
+ return 0;
9635
+ }
9636
+ let compressed = 0;
9637
+ for (const shard of shards) {
9638
+ if (compressed >= budget) break;
9639
+ const source = join8(bodiesPath, shard);
9640
+ const target = `${source}.gz`;
9641
+ try {
9642
+ if (existsSync16(target)) {
9643
+ unlinkSync4(source);
9644
+ continue;
9645
+ }
9646
+ await pipeline(createReadStream2(source), createGzip(), createWriteStream2(target));
9647
+ unlinkSync4(source);
9648
+ compressed += 1;
9649
+ } catch (error) {
9650
+ try {
9651
+ if (existsSync16(target)) unlinkSync4(target);
9652
+ } catch {
9653
+ }
9654
+ this.logger.warn("[AuditPruneSweeper] failed to archive audit body shard", {
9655
+ shard,
9656
+ error: error instanceof Error ? error.message : String(error)
9657
+ });
9658
+ }
9659
+ }
9660
+ return compressed;
9661
+ }
9128
9662
  };
9129
9663
 
9130
- // src/audit/auditReader.ts
9131
- import { existsSync as existsSync16, readdirSync as readdirSync3, readFileSync as readFileSync15 } from "fs";
9132
- import { join as join8 } from "path";
9133
- var DEFAULT_LIMIT = 200;
9134
- var MAX_LIMIT = 2e3;
9135
- function readAuditRecords(auditDir, query2 = {}) {
9136
- if (!existsSync16(auditDir)) return [];
9137
- let files;
9664
+ // src/audit/auditBodyReader.ts
9665
+ import { existsSync as existsSync17, readdirSync as readdirSync4, readFileSync as readFileSync16, statSync as statSync5 } from "fs";
9666
+ import { join as join9 } from "path";
9667
+ import { gunzipSync } from "zlib";
9668
+
9669
+ // src/audit/auditJsonl.ts
9670
+ import { closeSync, openSync, readSync, statSync as statSync4 } from "fs";
9671
+ var WINDOW_BYTES = 1 << 20;
9672
+ var MAX_LINE_BYTES = 32 * 1024 * 1024;
9673
+ var NEWLINE = 10;
9674
+ function forEachLineFromTail(path2, onLine) {
9675
+ let fd;
9676
+ let end;
9677
+ try {
9678
+ end = statSync4(path2).size;
9679
+ if (end === 0) return;
9680
+ fd = openSync(path2, "r");
9681
+ } catch {
9682
+ return;
9683
+ }
9684
+ try {
9685
+ let carry = Buffer.alloc(0);
9686
+ while (end > 0) {
9687
+ const start = Math.max(0, end - WINDOW_BYTES);
9688
+ const window = Buffer.allocUnsafe(end - start);
9689
+ let read;
9690
+ try {
9691
+ read = readSync(fd, window, 0, end - start, start);
9692
+ } catch {
9693
+ return;
9694
+ }
9695
+ const chunk = carry.length > 0 ? Buffer.concat([window.subarray(0, read), carry]) : window.subarray(0, read);
9696
+ let lineEnd = chunk.length;
9697
+ let nl = lineEnd > 0 ? chunk.lastIndexOf(NEWLINE, lineEnd - 1) : -1;
9698
+ while (nl >= 0) {
9699
+ if (nl + 1 < lineEnd) {
9700
+ const line = chunk.subarray(nl + 1, lineEnd).toString("utf8").trim();
9701
+ if (line && onLine(line)) return;
9702
+ }
9703
+ lineEnd = nl;
9704
+ nl = lineEnd > 0 ? chunk.lastIndexOf(NEWLINE, lineEnd - 1) : -1;
9705
+ }
9706
+ if (start === 0) {
9707
+ if (lineEnd > 0) {
9708
+ const line = chunk.subarray(0, lineEnd).toString("utf8").trim();
9709
+ if (line) onLine(line);
9710
+ }
9711
+ return;
9712
+ }
9713
+ if (lineEnd > MAX_LINE_BYTES) return;
9714
+ carry = Buffer.from(chunk.subarray(0, lineEnd));
9715
+ end = start;
9716
+ }
9717
+ } finally {
9718
+ try {
9719
+ closeSync(fd);
9720
+ } catch {
9721
+ }
9722
+ }
9723
+ }
9724
+
9725
+ // src/audit/auditBodyReader.ts
9726
+ function candidateDays(auditDir, ts) {
9727
+ if (typeof ts === "number" && Number.isFinite(ts)) {
9728
+ const named = auditDayDirName(ts);
9729
+ if (existsSync17(join9(auditDir, named))) return [named];
9730
+ }
9138
9731
  try {
9139
- files = readdirSync3(auditDir).filter((f) => AUDIT_FILE_RE.test(f));
9732
+ return readdirSync4(auditDir).filter(isAuditDayDir).sort().reverse();
9140
9733
  } catch {
9141
9734
  return [];
9142
9735
  }
9143
- const from = typeof query2.from === "number" ? query2.from : -Infinity;
9144
- const to = typeof query2.to === "number" ? query2.to : Infinity;
9145
- const limit = Math.min(MAX_LIMIT, Math.max(1, Math.trunc(query2.limit ?? DEFAULT_LIMIT)));
9146
- const matched = [];
9147
- for (const file of files.sort().reverse()) {
9148
- let raw;
9736
+ }
9737
+ function readShard(auditDir, day, sessionKey) {
9738
+ const base = join9(auditDir, day, AUDIT_BODIES_DIR, auditBodyFileName(sessionKey));
9739
+ try {
9740
+ if (existsSync17(base)) return readFileSync16(base, "utf8");
9741
+ const gz = `${base}.gz`;
9742
+ if (existsSync17(gz)) return gunzipSync(readFileSync16(gz)).toString("utf8");
9743
+ } catch {
9744
+ return null;
9745
+ }
9746
+ return null;
9747
+ }
9748
+ function parseShard(raw) {
9749
+ const entries = /* @__PURE__ */ new Map();
9750
+ for (const line of raw.split("\n")) {
9751
+ const trimmed = line.trim();
9752
+ if (!trimmed) continue;
9753
+ let parsed;
9149
9754
  try {
9150
- raw = readFileSync15(join8(auditDir, file), "utf8");
9755
+ parsed = JSON.parse(trimmed);
9151
9756
  } catch {
9152
9757
  continue;
9153
9758
  }
9154
- for (const line of raw.split("\n")) {
9155
- const trimmed = line.trim();
9156
- if (!trimmed) continue;
9157
- let rec;
9759
+ if (isAuditBodyEntry(parsed)) entries.set(parsed.id, parsed);
9760
+ }
9761
+ return entries;
9762
+ }
9763
+ function withDictionary(auditDir, day, entries) {
9764
+ let needed = false;
9765
+ for (const entry of entries.values()) {
9766
+ if (entry.req?.base?.startsWith(DICT_BASE_PREFIX)) {
9767
+ needed = true;
9768
+ break;
9769
+ }
9770
+ }
9771
+ if (!needed) return entries;
9772
+ const base = join9(auditDir, day, AUDIT_BODIES_DIR, AUDIT_DICT_FILE);
9773
+ let raw = null;
9774
+ try {
9775
+ if (existsSync17(base)) raw = readFileSync16(base, "utf8");
9776
+ else if (existsSync17(`${base}.gz`)) raw = gunzipSync(readFileSync16(`${base}.gz`)).toString("utf8");
9777
+ } catch {
9778
+ return entries;
9779
+ }
9780
+ if (raw === null) return entries;
9781
+ for (const [id, entry] of parseShard(raw)) entries.set(id, entry);
9782
+ return entries;
9783
+ }
9784
+ function reconstructRequest(entries, entry) {
9785
+ if (!entry.req) return void 0;
9786
+ const chain = [];
9787
+ const visited = /* @__PURE__ */ new Set();
9788
+ let cursor = entry;
9789
+ while (cursor?.req) {
9790
+ if (visited.has(cursor.id)) return void 0;
9791
+ visited.add(cursor.id);
9792
+ chain.push(cursor);
9793
+ if (cursor.req.base === null) break;
9794
+ cursor = entries.get(cursor.req.base);
9795
+ }
9796
+ const anchor = chain[chain.length - 1];
9797
+ if (!anchor?.req || anchor.req.base !== null) return void 0;
9798
+ let text = anchor.req.ins;
9799
+ for (let i = chain.length - 2; i >= 0; i -= 1) {
9800
+ const delta = chain[i]?.req;
9801
+ if (!delta) return void 0;
9802
+ if (delta.pre > text.length || delta.suf > text.length - delta.pre) return void 0;
9803
+ text = applyBodyDelta(text, delta);
9804
+ }
9805
+ return text;
9806
+ }
9807
+ function readAuditBody(auditDir, query2) {
9808
+ if (!isSafeSessionKey(query2.sessionKey) || !query2.id) return {};
9809
+ if (!existsSync17(auditDir)) return {};
9810
+ for (const day of candidateDays(auditDir, query2.ts)) {
9811
+ const raw = readShard(auditDir, day, query2.sessionKey);
9812
+ if (raw === null) continue;
9813
+ const entries = withDictionary(auditDir, day, parseShard(raw));
9814
+ const entry = entries.get(query2.id);
9815
+ if (!entry) continue;
9816
+ const result = {};
9817
+ const requestBody = reconstructRequest(entries, entry);
9818
+ if (requestBody !== void 0) result.requestBody = requestBody;
9819
+ if (entry.res !== void 0) result.responseBody = entry.res;
9820
+ return result;
9821
+ }
9822
+ return readLegacyInlineBody(auditDir, query2.id);
9823
+ }
9824
+ function readLegacyInlineBody(auditDir, id) {
9825
+ let names;
9826
+ try {
9827
+ names = readdirSync4(auditDir).filter((name) => AUDIT_FILE_RE.test(name)).sort().reverse();
9828
+ } catch {
9829
+ return {};
9830
+ }
9831
+ const needle = JSON.stringify(id);
9832
+ let found = {};
9833
+ for (const name of names) {
9834
+ forEachLineFromTail(join9(auditDir, name), (line) => {
9835
+ if (!line.includes(needle)) return false;
9836
+ let parsed;
9158
9837
  try {
9159
- rec = JSON.parse(trimmed);
9838
+ parsed = JSON.parse(line);
9160
9839
  } catch {
9161
- continue;
9840
+ return false;
9162
9841
  }
9163
- if (!isAuditRecord(rec)) continue;
9164
- if (query2.keyId !== void 0 && rec.keyId !== query2.keyId) continue;
9165
- if (rec.ts < from || rec.ts > to) continue;
9166
- matched.push(rec);
9842
+ const record = parsed;
9843
+ if (record.id !== id) return false;
9844
+ const result = {};
9845
+ if (typeof record.requestBody === "string") result.requestBody = record.requestBody;
9846
+ if (typeof record.responseBody === "string") result.responseBody = record.responseBody;
9847
+ found = result;
9848
+ return true;
9849
+ });
9850
+ if (found.requestBody !== void 0 || found.responseBody !== void 0) break;
9851
+ }
9852
+ return found;
9853
+ }
9854
+
9855
+ // src/audit/auditReader.ts
9856
+ import { existsSync as existsSync18, readdirSync as readdirSync5 } from "fs";
9857
+ import { join as join10 } from "path";
9858
+ var DEFAULT_LIMIT = 200;
9859
+ var MAX_LIMIT = 2e3;
9860
+ var OVERSCAN = 256;
9861
+ function daySources(auditDir) {
9862
+ let names;
9863
+ try {
9864
+ names = readdirSync5(auditDir);
9865
+ } catch {
9866
+ return [];
9867
+ }
9868
+ const sources = [];
9869
+ for (const name of names) {
9870
+ const dateMs = auditFileDateMs(name);
9871
+ if (dateMs === null) continue;
9872
+ if (AUDIT_DAY_DIR_RE.test(name)) {
9873
+ const path2 = join10(auditDir, name, AUDIT_META_FILE);
9874
+ if (existsSync18(path2)) sources.push({ path: path2, dateMs });
9875
+ } else if (AUDIT_FILE_RE.test(name)) {
9876
+ sources.push({ path: join10(auditDir, name), dateMs });
9167
9877
  }
9168
9878
  }
9169
- matched.sort((a, b) => b.ts - a.ts);
9170
- return matched.slice(0, limit);
9879
+ return sources.sort((a, b) => b.dateMs - a.dateMs);
9171
9880
  }
9172
9881
  function isAuditRecord(value) {
9173
9882
  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
9174
9883
  const r = value;
9175
9884
  return typeof r["id"] === "string" && typeof r["ts"] === "number" && typeof r["method"] === "string" && typeof r["path"] === "string" && typeof r["status"] === "number";
9176
9885
  }
9886
+ function toMetaRecord(record) {
9887
+ if (record.requestBody === void 0 && record.responseBody === void 0) return record;
9888
+ const { requestBody: _req, responseBody: _res, ...meta } = record;
9889
+ return { ...meta, hasBody: true };
9890
+ }
9891
+ function readAuditRecords(auditDir, query2 = {}) {
9892
+ if (!existsSync18(auditDir)) return [];
9893
+ const from = typeof query2.from === "number" ? query2.from : -Infinity;
9894
+ const to = typeof query2.to === "number" ? query2.to : Infinity;
9895
+ const limit = Math.min(MAX_LIMIT, Math.max(1, Math.trunc(query2.limit ?? DEFAULT_LIMIT)));
9896
+ const matched = [];
9897
+ for (const source of daySources(auditDir)) {
9898
+ const before = matched.length;
9899
+ forEachLineFromTail(source.path, (line) => {
9900
+ let parsed;
9901
+ try {
9902
+ parsed = JSON.parse(line);
9903
+ } catch {
9904
+ return false;
9905
+ }
9906
+ if (!isAuditRecord(parsed)) return false;
9907
+ if (query2.keyId !== void 0 && parsed.keyId !== query2.keyId) return false;
9908
+ if (query2.sessionKey !== void 0 && parsed.sessionKey !== query2.sessionKey) return false;
9909
+ if (parsed.ts < from || parsed.ts > to) return false;
9910
+ matched.push(toMetaRecord(parsed));
9911
+ return matched.length - before >= limit + OVERSCAN;
9912
+ });
9913
+ if (matched.length >= limit) break;
9914
+ }
9915
+ matched.sort((a, b) => b.ts - a.ts);
9916
+ return matched.slice(0, limit);
9917
+ }
9177
9918
 
9178
9919
  // src/audit/AuditWriter.ts
9179
- import { appendFileSync as appendFileSync2, existsSync as existsSync17, mkdirSync as mkdirSync5, statSync as statSync4 } from "fs";
9180
- import { join as join9 } from "path";
9920
+ import { appendFileSync as appendFileSync2, existsSync as existsSync19, mkdirSync as mkdirSync5, statSync as statSync6 } from "fs";
9921
+ import { join as join11 } from "path";
9181
9922
  var AuditWriter = class {
9182
9923
  constructor(auditDir, logger, defer = (fn) => setTimeout(fn, 0)) {
9183
9924
  this.auditDir = auditDir;
@@ -9187,10 +9928,13 @@ var AuditWriter = class {
9187
9928
  auditDir;
9188
9929
  logger;
9189
9930
  defer;
9190
- dirEnsured = false;
9931
+ /** Day directories already created this process (avoids an mkdir per record). */
9932
+ ensuredDirs = /* @__PURE__ */ new Set();
9933
+ /** Per-session encoding bases. Memory-only; a miss simply writes a full snapshot. */
9934
+ bases = new SessionBaseCache();
9191
9935
  /**
9192
- * Enqueue one record for append. Returns IMMEDIATELY (fire-and-forget); the fs
9193
- * write happens on the deferred tick. A failure is logged, never thrown.
9936
+ * Enqueue one record. Returns IMMEDIATELY (fire-and-forget); every fs write and
9937
+ * the delta encoding happen on the deferred tick. A failure is logged, never thrown.
9194
9938
  */
9195
9939
  record(record) {
9196
9940
  this.defer(() => {
@@ -9203,25 +9947,41 @@ var AuditWriter = class {
9203
9947
  }
9204
9948
  });
9205
9949
  }
9950
+ /** Drop all retained encoding bases (config reload / shutdown / test teardown). */
9951
+ reset() {
9952
+ this.bases.clear();
9953
+ this.ensuredDirs.clear();
9954
+ }
9206
9955
  /**
9207
- * Append synchronously — the awaitable form tests use to assert the line landed.
9208
- * Ensures the `audit/` directory exists on first write (lazy, like the usage
9209
- * store's lazy file creation).
9956
+ * Append synchronously — the awaitable form tests use to assert a line landed.
9957
+ * Writes the metadata line first (canonical), then the body shard.
9210
9958
  */
9211
9959
  appendNow(record) {
9212
- if (!this.dirEnsured) {
9213
- mkdirSync5(this.auditDir, { recursive: true });
9214
- this.dirEnsured = true;
9215
- }
9216
- const file = join9(this.auditDir, auditFileName(record.ts));
9217
- const line = JSON.stringify(record) + "\n";
9218
- const auditBytesBefore = existsSync17(file) ? statSync4(file).size : 0;
9960
+ const dayDir = auditDayDirName(record.ts);
9961
+ const dayPath = this.ensureDir(join11(this.auditDir, dayDir));
9962
+ this.appendMeta(dayPath, record);
9963
+ this.appendBody(dayPath, dayDir, record);
9964
+ }
9965
+ /** Create a directory once per process and remember it. */
9966
+ ensureDir(path2) {
9967
+ if (!this.ensuredDirs.has(path2)) {
9968
+ mkdirSync5(path2, { recursive: true });
9969
+ this.ensuredDirs.add(path2);
9970
+ }
9971
+ return path2;
9972
+ }
9973
+ /** Write the body-free metadata line + refresh the exact-count sidecar. */
9974
+ appendMeta(dayPath, record) {
9975
+ const { requestBody: _req, responseBody: _res, ...meta } = record;
9976
+ const file = join11(dayPath, AUDIT_META_FILE);
9977
+ const line = JSON.stringify(meta) + "\n";
9978
+ const bytesBefore = existsSync19(file) ? statSync6(file).size : 0;
9219
9979
  appendFileSync2(file, line, "utf8");
9220
9980
  try {
9221
9981
  updateAuditStatsAfterAppend(
9222
9982
  file,
9223
- auditBytesBefore,
9224
- auditBytesBefore + Buffer.byteLength(line, "utf8"),
9983
+ bytesBefore,
9984
+ bytesBefore + Buffer.byteLength(line, "utf8"),
9225
9985
  record
9226
9986
  );
9227
9987
  } catch (error) {
@@ -9230,12 +9990,39 @@ var AuditWriter = class {
9230
9990
  });
9231
9991
  }
9232
9992
  }
9993
+ /**
9994
+ * Write the delta-encoded body shard for one record. A no-op when nothing was
9995
+ * captured or when the session key is missing/unsafe — in which case the body
9996
+ * is dropped rather than written to an unvalidated path.
9997
+ */
9998
+ appendBody(dayPath, dayDir, record) {
9999
+ if (record.requestBody === void 0 && record.responseBody === void 0) return;
10000
+ const sessionKey = record.sessionKey;
10001
+ if (!isSafeSessionKey(sessionKey)) {
10002
+ this.logger.warn("[AuditWriter] dropping audit body with no usable session key", {
10003
+ id: record.id
10004
+ });
10005
+ return;
10006
+ }
10007
+ try {
10008
+ const line = encodeBodyEntry(record, sessionKey, dayDir, this.bases);
10009
+ if (line === null) return;
10010
+ const bodiesPath = this.ensureDir(join11(dayPath, AUDIT_BODIES_DIR));
10011
+ appendFileSync2(join11(bodiesPath, auditBodyFileName(sessionKey)), line + "\n", "utf8");
10012
+ } catch (error) {
10013
+ this.bases.forget(sessionKey);
10014
+ this.logger.warn("[AuditWriter] failed to append audit body shard", {
10015
+ id: record.id,
10016
+ error: error instanceof Error ? error.message : String(error)
10017
+ });
10018
+ }
10019
+ }
9233
10020
  };
9234
10021
 
9235
10022
  // src/billing/BillingPublisher.ts
9236
10023
  import { appendFileSync as appendFileSync3, mkdirSync as mkdirSync6 } from "fs";
9237
10024
  import { createHmac } from "crypto";
9238
- import { join as join10 } from "path";
10025
+ import { join as join12 } from "path";
9239
10026
  import { fetchUpstream as fetchUpstream5 } from "@omnicross/core/pipeline/upstreamFetch";
9240
10027
 
9241
10028
  // src/billing/billingFiles.ts
@@ -9306,7 +10093,7 @@ var BillingPublisher = class {
9306
10093
  */
9307
10094
  appendNow(event) {
9308
10095
  this.ensureDir();
9309
- const file = join10(this.billingDir, billingFileName(event.ts));
10096
+ const file = join12(this.billingDir, billingFileName(event.ts));
9310
10097
  appendFileSync3(file, JSON.stringify(event) + "\n", "utf8");
9311
10098
  }
9312
10099
  /**
@@ -9356,7 +10143,7 @@ var BillingPublisher = class {
9356
10143
  markDelivered(event) {
9357
10144
  try {
9358
10145
  this.ensureDir();
9359
- const file = join10(this.billingDir, deliveredFileName(event.ts));
10146
+ const file = join12(this.billingDir, deliveredFileName(event.ts));
9360
10147
  appendFileSync3(file, JSON.stringify({ id: event.id, deliveredAt: this.now() }) + "\n", "utf8");
9361
10148
  } catch (error) {
9362
10149
  this.logger.warn("[BillingPublisher] failed to append delivery marker", {
@@ -9372,14 +10159,14 @@ var BillingPublisher = class {
9372
10159
  };
9373
10160
 
9374
10161
  // src/billing/billingReader.ts
9375
- import { existsSync as existsSync18, readdirSync as readdirSync4, readFileSync as readFileSync16 } from "fs";
9376
- import { join as join11 } from "path";
10162
+ import { existsSync as existsSync20, readdirSync as readdirSync6, readFileSync as readFileSync17 } from "fs";
10163
+ import { join as join13 } from "path";
9377
10164
  function readBillingLedger(billingDir) {
9378
10165
  const view = { events: [], deliveredIds: /* @__PURE__ */ new Set() };
9379
- if (!existsSync18(billingDir)) return view;
10166
+ if (!existsSync20(billingDir)) return view;
9380
10167
  let files;
9381
10168
  try {
9382
- files = readdirSync4(billingDir);
10169
+ files = readdirSync6(billingDir);
9383
10170
  } catch {
9384
10171
  return view;
9385
10172
  }
@@ -9410,7 +10197,7 @@ function readBillingStatus(billingDir) {
9410
10197
  function parseLines(dir, file) {
9411
10198
  let raw;
9412
10199
  try {
9413
- raw = readFileSync16(join11(dir, file), "utf8");
10200
+ raw = readFileSync17(join13(dir, file), "utf8");
9414
10201
  } catch {
9415
10202
  return [];
9416
10203
  }
@@ -9914,8 +10701,10 @@ function buildDaemon(config, paths) {
9914
10701
  async (providerId, model) => await pricingEngine.getEntry(providerId, model) !== null
9915
10702
  );
9916
10703
  const keySpendTracker = new KeySpendTracker(usageEventStore);
10704
+ const usageThroughput = getSharedUsageThroughputTracker2();
9917
10705
  const usageRecorder = new UsageRecorder(usageEventStore, pricingEngine, logger, {
9918
- onRecord: (apiKeyId, costUsd, at) => keySpendTracker.add(apiKeyId, costUsd, at)
10706
+ onRecord: (apiKeyId, costUsd, at) => keySpendTracker.add(apiKeyId, costUsd, at),
10707
+ onEvent: (row, at) => usageThroughput.record(row, at)
9919
10708
  });
9920
10709
  const providerProxy = getProviderProxy({ llmConfig, apiKeyPool, usageRecorder });
9921
10710
  const routeLeaseManager = new RouteLeaseManager(
@@ -10064,6 +10853,11 @@ function buildDaemon(config, paths) {
10064
10853
  // NEVER unauth, NEVER on `/health`. Routed in `AdminServer` (not `adminApi.ts`).
10065
10854
  auditReader: (query2) => readAuditRecords(auditDir, query2),
10066
10855
  auditStatsReader: (query2) => readAuditStats(auditDir, query2),
10856
+ // audit-store-sharding: bodies live in per-session shards, so opening ONE
10857
+ // record's payload is a separate authed call that replays its delta chain.
10858
+ auditBodyReader: (query2) => readAuditBody(auditDir, query2),
10859
+ // audit-store-sharding D8: the manual counterpart to the daily pass.
10860
+ auditCompactor: () => compactAllClosedAuditDays(auditDir),
10067
10861
  // billing-event-stream: the AUTHED `GET /admin/api/billing-status` returns the
10068
10862
  // secret-free total/delivered/pending counts of the durable ledger.
10069
10863
  billingStatusReader: () => readBillingStatus(billingDir)
@@ -10137,10 +10931,11 @@ function resetDaemonSingletonsForTests() {
10137
10931
  __resetSharedIdentityStoreForTests();
10138
10932
  __resetSharedAccountAllowanceStoreForTests();
10139
10933
  __resetSharedAccountAllowanceSchedulingForTests();
10934
+ __resetSharedUsageThroughputTrackerForTests();
10140
10935
  }
10141
10936
  function isTokensStoreReadable(tokensPath) {
10142
10937
  try {
10143
- if (!existsSync19(tokensPath)) return true;
10938
+ if (!existsSync21(tokensPath)) return true;
10144
10939
  accessSync(tokensPath, fsConstants.R_OK);
10145
10940
  return true;
10146
10941
  } catch {