@omnicross/daemon 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -15,7 +15,8 @@ import { SubscriptionIdentityStore } from '@omnicross/core/provider-proxy/identi
15
15
  import { LoggingConfig, HealthReport } from '@omnicross/contracts/health-logging-types';
16
16
  import { SubscriptionAccountHealth } from '@omnicross/core/pipeline/SubscriptionAccountHealth';
17
17
  import { fetchUpstream } from '@omnicross/core/pipeline/upstreamFetch';
18
- import { AuditRecord, AuditStats, AuditConfig } from '@omnicross/contracts/audit-types';
18
+ import { ThinkLevel } from '@omnicross/contracts/completion-types';
19
+ import { AuditRecord, AuditStats, AuditBodyResult, AuditConfig } from '@omnicross/contracts/audit-types';
19
20
  import { BillingDeliveryStatus, BillingConfig, BillingEvent } from '@omnicross/contracts/billing-types';
20
21
  import http from 'node:http';
21
22
  import { LLMProvider, AgentDefaultModels, GlobalModelParameters } from '@omnicross/contracts/llm-config';
@@ -145,11 +146,11 @@ interface DaemonApiKeyEntry {
145
146
  /**
146
147
  * Per-model metadata subset (app-parity child 2). A hand-authored SUBSET of the
147
148
  * app's `ModelConfig` (`app/src/shared-types/llm-config.ts`), carrying ONLY the
148
- * named-five fields the daemon stores + round-trips, keyed by the model `id`:
149
- * `name` (display name), `enabled`, `group`, `vision`, `reasoning`. The wider
149
+ * allowlisted fields the daemon stores + round-trips, keyed by the model `id`:
150
+ * display metadata plus target thinking capabilities. The wider
150
151
  * `ModelConfig` fields the discovery flow may send (`category`/`contextLength`/
151
152
  * `maxTokens`/`functionCall`/`webSearch`/`completionSettings`/`openRouterProvider`/
152
- * `thinkingLevels`/…) are NOT in this allowlist — they are DROPPED by
153
+ * fields are NOT in this allowlist — they are DROPPED by
153
154
  * deny-by-default (`validateModelConfigs`/`parseModelConfigsInput`).
154
155
  *
155
156
  * ENFORCEMENT (app-parity-2 child 2): `enabled` is now a DISCOVERY/advertisement
@@ -158,8 +159,8 @@ interface DaemonApiKeyEntry {
158
159
  * advertisement, NOT a hard per-request block (core does not validate a requested
159
160
  * model against `models[]`, so a hardcoded disabled model id still reaches the
160
161
  * upstream, which rejects it). The admin management view (`toProviderView`) still
161
- * lists ALL models. The other fields (`name`/`group`/`vision`/`reasoning`) remain
162
- * display-only metadata (no core per-model capability binding on the BYO path).
162
+ * lists ALL models. `name`/`group`/`vision`/`reasoning` remain display metadata;
163
+ * thinking capabilities are projected into core for request negotiation.
163
164
  */
164
165
  interface DaemonModelConfig {
165
166
  /** Model id — the metadata key (parallels an entry in the flat `models[]`). */
@@ -175,6 +176,13 @@ interface DaemonModelConfig {
175
176
  vision?: boolean;
176
177
  /** Reasoning-capable hint (display only; not consumed by routing). */
177
178
  reasoning?: boolean;
179
+ /** Discrete reasoning efforts accepted by this target model. */
180
+ thinkingLevels?: ThinkLevel[];
181
+ /** Valid legacy thinking-budget bounds for this target model. */
182
+ thinkingTokenLimit?: {
183
+ min: number;
184
+ max: number;
185
+ };
178
186
  }
179
187
  /**
180
188
  * One transformer chain entry (app-parity child 5). Mirrors the app's
@@ -1413,11 +1421,53 @@ declare class AccountHealthProbeScheduler implements AccountProbeHistoryReader {
1413
1421
  }
1414
1422
 
1415
1423
  /**
1416
- * auditReaderread + filter the date-rotated audit store (request-audit-log,
1417
- * design D4/D6). Backs the AUTHED admin query only (the records carry IP/UA +
1418
- * possibly bodies). Reads the relevant `audit-*.jsonl` files, parses defensively
1419
- * (a torn final line never poisons a query), filters by key id + time window, and
1420
- * returns NEWEST-FIRST up to a bounded limit.
1424
+ * auditBodyReaderreconstruct captured bodies out of the per-session shards
1425
+ * (audit-store-sharding, design D4).
1426
+ *
1427
+ * Bodies no longer sit inline on the metadata line, so reading one is an explicit
1428
+ * second step: locate the session shard, replay its delta chain, hand back the
1429
+ * original text. Backs the AUTHED admin body query and the `omnicross audit` CLI
1430
+ * — never an unauthenticated surface (a body can hold prompts and PII).
1431
+ *
1432
+ * Shards are read whole. That is bounded on purpose: a shard holds ONE session's
1433
+ * deltas, which is exactly the thing the delta encoding keeps small.
1434
+ *
1435
+ * Both storage forms are handled transparently: today's plain `.jsonl` (kept
1436
+ * greppable on disk) and an archived `.jsonl.gz` from a rolled-over day.
1437
+ *
1438
+ * @module @omnicross/daemon/audit/auditBodyReader
1439
+ */
1440
+
1441
+ /** Locate one record's bodies. `ts` narrows the search to a single day directory. */
1442
+ interface AuditBodyQuery {
1443
+ /** The audit record id whose bodies to reconstruct. */
1444
+ id: string;
1445
+ /** The record's session key (its shard). */
1446
+ sessionKey: string;
1447
+ /** The record's timestamp, if known — skips scanning every retained day. */
1448
+ ts?: number;
1449
+ }
1450
+
1451
+ /**
1452
+ * auditReader — read + filter the audit store (request-audit-log design D4/D6,
1453
+ * re-laid-out by audit-store-sharding design D5). Backs the AUTHED admin query
1454
+ * only (records carry IP/UA). Returns NEWEST-FIRST up to a bounded limit.
1455
+ *
1456
+ * Reads BOTH on-disk layouts: the current `audit-YYYY-MM-DD/meta.jsonl` and the
1457
+ * legacy flat `audit-YYYY-MM-DD.jsonl` left behind by an older daemon (still
1458
+ * queryable until TTL prunes it — there is no migration step).
1459
+ *
1460
+ * BOUNDED BY CONSTRUCTION. Days are visited newest-first and each file is walked
1461
+ * BACKWARDS from its tail, so a query touches only as much of the store as it
1462
+ * needs. Two stops apply:
1463
+ * - cross-day: once `limit` rows are in hand, no older day is opened at all.
1464
+ * Exact, because day files never overlap in date.
1465
+ * - within-day: at most `limit + OVERSCAN` rows are taken from one file. The
1466
+ * overscan absorbs the slight append-order skew from a slow request landing
1467
+ * after a fast one that started later.
1468
+ *
1469
+ * A returned record NEVER carries a body — those live in the per-session shards
1470
+ * and are fetched one at a time. `hasBody` says whether one exists.
1421
1471
  *
1422
1472
  * @module @omnicross/daemon/audit/auditReader
1423
1473
  */
@@ -1426,6 +1476,8 @@ declare class AccountHealthProbeScheduler implements AccountProbeHistoryReader {
1426
1476
  interface AuditQuery {
1427
1477
  /** Restrict to one outbound key id. */
1428
1478
  keyId?: string;
1479
+ /** Restrict to one conversation-session key. */
1480
+ sessionKey?: string;
1429
1481
  /** Inclusive lower bound (epoch ms). */
1430
1482
  from?: number;
1431
1483
  /** Inclusive upper bound (epoch ms). */
@@ -1450,9 +1502,9 @@ interface AuditStatsQuery {
1450
1502
  * auditQueryApi — the AUTHED `GET /admin/api/audit?keyId=&from=&to=&limit=`
1451
1503
  * handler (request-audit-log, design D6).
1452
1504
  *
1453
- * Audit records carry client IP / user-agent (PII) and, when body capture is on,
1454
- * redacted bodies — so unlike the coarse `/health` boolean they are served ONLY
1455
- * behind the admin auth gate. This lives in its OWN helper module (the
1505
+ * Audit records carry client IP / user-agent (PII), and the sibling body query
1506
+ * returns redacted request/response snapshots — so unlike the coarse `/health`
1507
+ * boolean they are served ONLY behind the admin auth gate. This lives in its OWN helper module (the
1456
1508
  * #4/#8/#10 helper-module convention) so `adminApi.ts` — at its line cap — is not
1457
1509
  * touched: `AdminServer.dispatch` routes the path here directly, AFTER its auth
1458
1510
  * gate. NEVER unauthenticated, NEVER surfaced on `/health`.
@@ -1467,6 +1519,14 @@ interface AuditStatsQuery {
1467
1519
  type AuditQueryReader = (query: AuditQuery) => AuditRecord[];
1468
1520
  /** Metadata-only aggregate reader used by the overview. */
1469
1521
  type AuditStatsReader = (query: AuditStatsQuery) => Promise<AuditStats> | AuditStats;
1522
+ /** Reconstructs ONE record's bodies from the per-session shard store. */
1523
+ type AuditBodyReader = (query: AuditBodyQuery) => AuditBodyResult;
1524
+ /** Runs the cross-session body compaction over every closed day. */
1525
+ type AuditCompactor = () => {
1526
+ days: number;
1527
+ shards: number;
1528
+ savedBytes: number;
1529
+ };
1470
1530
 
1471
1531
  /**
1472
1532
  * billingStatusApi — the AUTHED `GET /admin/api/billing-status` handler
@@ -2124,6 +2184,13 @@ interface AdminServerDeps extends AdminApiDeps {
2124
2184
  auditReader?: AuditQueryReader;
2125
2185
  /** Metadata-only audit aggregate used by the overview error-rate metric. */
2126
2186
  auditStatsReader?: AuditStatsReader;
2187
+ /**
2188
+ * Reconstructs one record's bodies from the per-session shard store
2189
+ * (audit-store-sharding). Absent when audit was never enabled.
2190
+ */
2191
+ auditBodyReader?: AuditBodyReader;
2192
+ /** Runs cross-session body compaction on demand (audit-store-sharding D8). */
2193
+ auditCompactor?: AuditCompactor;
2127
2194
  /**
2128
2195
  * OPTIONAL billing delivery-status reader (billing-event-stream, design D5).
2129
2196
  * When wired (bootstrap → the ledger dir), the AUTHED `GET /admin/api/billing-status`
@@ -2287,17 +2354,28 @@ declare class AccountHealthSweeper {
2287
2354
  }
2288
2355
 
2289
2356
  /**
2290
- * AuditPruneSweeper — the TTL prune for the audit store (request-audit-log,
2291
- * design D4). Deletes whole `audit-YYYY-MM-DD.jsonl` files whose date is older
2292
- * than `retentionDays` — a cheap file UNLINK, never a line-level rewrite of a
2293
- * live file (which jsonl makes awkward). So the store never grows unbounded and
2294
- * TTL is O(files).
2357
+ * AuditPruneSweeper — retention + archiving for the audit store
2358
+ * (request-audit-log design D4, extended by audit-store-sharding design D6).
2359
+ *
2360
+ * Two jobs run on the same hourly tick:
2361
+ *
2362
+ * - PRUNE. Remove every day older than `retentionDays`. A current day is a
2363
+ * DIRECTORY (removed recursively), a legacy day is a flat file (unlinked with
2364
+ * its stats sidecar). Either way it stays a whole-day removal — never a
2365
+ * line-level rewrite of a live file, which jsonl makes awkward.
2366
+ * - ARCHIVE. First run the cross-session compaction pass (see
2367
+ * {@link compactAuditDay}), then gzip the body shards of any day that is no
2368
+ * longer today. The
2369
+ * current day stays PLAIN TEXT so it can still be tailed and grepped while
2370
+ * debugging; once a day is closed nothing appends to it again, so rewriting
2371
+ * and compressing it is safe and buys a large multiple on top of the delta.
2372
+ * `meta.jsonl` is NEVER compressed — it is the query hot path.
2295
2373
  *
2296
2374
  * Modeled on the #8 `AccountHealthProbeScheduler` / `AccountHealthSweeper`:
2297
2375
  * `start()` arms an `unref()`ed interval, `dispose()` clears it, a single-sweep
2298
- * re-entrancy guard prevents overlap. A prune ALSO runs once at boot (`start`
2299
- * fires an immediate sweep). Disabled/zero-retention config ⇒ armed-off no-op
2300
- * (byte-identical zero regression). Never throws.
2376
+ * re-entrancy guard prevents overlap. Both jobs also run once at boot. Disabled
2377
+ * config means armed-off, so audit-off is byte-identical zero regression. Never
2378
+ * throws.
2301
2379
  *
2302
2380
  * @module @omnicross/daemon/audit/AuditPruneSweeper
2303
2381
  */
@@ -2311,6 +2389,7 @@ declare class AuditPruneSweeper {
2311
2389
  private readonly now;
2312
2390
  private timer;
2313
2391
  private sweeping;
2392
+ private archiving;
2314
2393
  constructor(auditDir: string, logger: Logger, config: AuditConfig, intervalMs?: number,
2315
2394
  /** Injectable clock (ms) for deterministic tests. */
2316
2395
  now?: () => number);
@@ -2319,32 +2398,56 @@ declare class AuditPruneSweeper {
2319
2398
  /** Re-apply config to the live instance (boot + admin PUT hot-reload). */
2320
2399
  configure(config: AuditConfig): void;
2321
2400
  /**
2322
- * Arm the prune interval AND run one prune immediately (boot cleanup). No-op
2323
- * when audit is disabled (zero regression). Idempotent.
2401
+ * Arm the interval AND run one pass immediately (boot cleanup). No-op when
2402
+ * audit is disabled (zero regression). Idempotent.
2324
2403
  */
2325
2404
  start(): void;
2326
2405
  /** Clear the interval (daemon shutdown / test teardown). Idempotent. */
2327
2406
  dispose(): void;
2407
+ /** Prune first, then archive — never spend CPU compressing a day about to go. */
2408
+ private runOnce;
2409
+ /** The LOCAL-midnight epoch ms of the current day. */
2410
+ private todayMidnight;
2328
2411
  /**
2329
- * One prune: unlink every audit date file strictly OLDER than the retention
2330
- * cutoff (`now - retentionDays` days, at local-midnight granularity). Exposed
2331
- * for tests; never throws. Returns the number of files removed.
2412
+ * One prune: remove every audit day strictly OLDER than the retention cutoff
2413
+ * (`now - retentionDays` days, at local-midnight granularity). Exposed for
2414
+ * tests; never throws. Returns the number of days removed.
2332
2415
  */
2333
2416
  sweep(): Promise<number>;
2417
+ /**
2418
+ * Gzip the body shards of every CLOSED day (anything before today). Today is
2419
+ * deliberately left as plain text so it stays greppable while it is the day you
2420
+ * are debugging. Exposed for tests; never throws. Returns shards compressed.
2421
+ */
2422
+ archive(): Promise<number>;
2423
+ /** Gzip up to `budget` plain shards in one day's `bodies/` directory. */
2424
+ private archiveDay;
2334
2425
  }
2335
2426
 
2336
2427
  /**
2337
- * AuditWriter — the daemon's file-backed audit sink (request-audit-log, design
2338
- * D4/D5). Registered as `@omnicross/core`'s audit sink when audit is enabled; its
2339
- * {@link record} is what `recordAudit` hands each assembled record to.
2428
+ * AuditWriter — the daemon's file-backed audit sink (request-audit-log design
2429
+ * D4/D5, re-laid-out by audit-store-sharding design D2). Registered as
2430
+ * `@omnicross/core`'s audit sink when audit is enabled; its {@link record} is
2431
+ * what `recordAudit` hands each assembled record to.
2340
2432
  *
2341
- * FIRE-AND-FORGET (hard constraint): {@link record} DEFERS the fs append off the
2433
+ * FIRE-AND-FORGET (hard constraint): {@link record} DEFERS all fs work off the
2342
2434
  * caller's stack (an injectable `defer`, default a zero-delay timer — the
2343
2435
  * `UsageRecorder` precedent) and returns immediately, so the relay response path
2344
- * never waits on disk I/O. A write error is swallowed + logged (a failing audit
2345
- * store must never affect a relay). Each record is appended as ONE JSON line to
2346
- * `audit/audit-YYYY-MM-DD.jsonl` (the record's LOCAL date), matching the
2347
- * `usage-events.jsonl` pattern — no new dependency, TTL is a whole-file unlink.
2436
+ * never waits on disk I/O. Delta encoding rides that same deferred tick and is
2437
+ * pure memory, so it can never stall a response either. A write error is
2438
+ * swallowed + logged (a failing audit store must never affect a relay).
2439
+ *
2440
+ * Each record is split across TWO destinations under `audit/audit-YYYY-MM-DD/`:
2441
+ *
2442
+ * - `meta.jsonl` — one small JSON line WITHOUT bodies. This is what queries and
2443
+ * the stats sidecar read, so a listing no longer pays for megabytes of prompt.
2444
+ * - `bodies/<sessionKey>.jsonl` — the captured bodies, sharded per conversation
2445
+ * and delta-encoded against the session's previous turn.
2446
+ *
2447
+ * The two writes are INDEPENDENT: the metadata line is the canonical record, so a
2448
+ * body-shard failure is logged and the metadata still lands. When a shard write
2449
+ * fails the session's encoding base is dropped, otherwise the next turn would
2450
+ * chain a delta onto a base no reader can find.
2348
2451
  *
2349
2452
  * @module @omnicross/daemon/audit/AuditWriter
2350
2453
  */
@@ -2352,23 +2455,37 @@ declare class AuditPruneSweeper {
2352
2455
  declare class AuditWriter {
2353
2456
  private readonly auditDir;
2354
2457
  private readonly logger;
2355
- /** Deferral used by `record()` to schedule the append off the caller's path. */
2458
+ /** Deferral used by `record()` to schedule the writes off the caller's path. */
2356
2459
  private readonly defer;
2357
- private dirEnsured;
2460
+ /** Day directories already created this process (avoids an mkdir per record). */
2461
+ private readonly ensuredDirs;
2462
+ /** Per-session encoding bases. Memory-only; a miss simply writes a full snapshot. */
2463
+ private readonly bases;
2358
2464
  constructor(auditDir: string, logger: Logger,
2359
- /** Deferral used by `record()` to schedule the append off the caller's path. */
2465
+ /** Deferral used by `record()` to schedule the writes off the caller's path. */
2360
2466
  defer?: (fn: () => void) => void);
2361
2467
  /**
2362
- * Enqueue one record for append. Returns IMMEDIATELY (fire-and-forget); the fs
2363
- * write happens on the deferred tick. A failure is logged, never thrown.
2468
+ * Enqueue one record. Returns IMMEDIATELY (fire-and-forget); every fs write and
2469
+ * the delta encoding happen on the deferred tick. A failure is logged, never thrown.
2364
2470
  */
2365
2471
  record(record: AuditRecord): void;
2472
+ /** Drop all retained encoding bases (config reload / shutdown / test teardown). */
2473
+ reset(): void;
2366
2474
  /**
2367
- * Append synchronously — the awaitable form tests use to assert the line landed.
2368
- * Ensures the `audit/` directory exists on first write (lazy, like the usage
2369
- * store's lazy file creation).
2475
+ * Append synchronously — the awaitable form tests use to assert a line landed.
2476
+ * Writes the metadata line first (canonical), then the body shard.
2370
2477
  */
2371
2478
  appendNow(record: AuditRecord): void;
2479
+ /** Create a directory once per process and remember it. */
2480
+ private ensureDir;
2481
+ /** Write the body-free metadata line + refresh the exact-count sidecar. */
2482
+ private appendMeta;
2483
+ /**
2484
+ * Write the delta-encoded body shard for one record. A no-op when nothing was
2485
+ * captured or when the session key is missing/unsafe — in which case the body
2486
+ * is dropped rather than written to an unvalidated path.
2487
+ */
2488
+ private appendBody;
2372
2489
  }
2373
2490
 
2374
2491
  /**
package/dist/index.d.ts CHANGED
@@ -15,7 +15,8 @@ import { SubscriptionIdentityStore } from '@omnicross/core/provider-proxy/identi
15
15
  import { LoggingConfig, HealthReport } from '@omnicross/contracts/health-logging-types';
16
16
  import { SubscriptionAccountHealth } from '@omnicross/core/pipeline/SubscriptionAccountHealth';
17
17
  import { fetchUpstream } from '@omnicross/core/pipeline/upstreamFetch';
18
- import { AuditRecord, AuditStats, AuditConfig } from '@omnicross/contracts/audit-types';
18
+ import { ThinkLevel } from '@omnicross/contracts/completion-types';
19
+ import { AuditRecord, AuditStats, AuditBodyResult, AuditConfig } from '@omnicross/contracts/audit-types';
19
20
  import { BillingDeliveryStatus, BillingConfig, BillingEvent } from '@omnicross/contracts/billing-types';
20
21
  import http from 'node:http';
21
22
  import { LLMProvider, AgentDefaultModels, GlobalModelParameters } from '@omnicross/contracts/llm-config';
@@ -145,11 +146,11 @@ interface DaemonApiKeyEntry {
145
146
  /**
146
147
  * Per-model metadata subset (app-parity child 2). A hand-authored SUBSET of the
147
148
  * app's `ModelConfig` (`app/src/shared-types/llm-config.ts`), carrying ONLY the
148
- * named-five fields the daemon stores + round-trips, keyed by the model `id`:
149
- * `name` (display name), `enabled`, `group`, `vision`, `reasoning`. The wider
149
+ * allowlisted fields the daemon stores + round-trips, keyed by the model `id`:
150
+ * display metadata plus target thinking capabilities. The wider
150
151
  * `ModelConfig` fields the discovery flow may send (`category`/`contextLength`/
151
152
  * `maxTokens`/`functionCall`/`webSearch`/`completionSettings`/`openRouterProvider`/
152
- * `thinkingLevels`/…) are NOT in this allowlist — they are DROPPED by
153
+ * fields are NOT in this allowlist — they are DROPPED by
153
154
  * deny-by-default (`validateModelConfigs`/`parseModelConfigsInput`).
154
155
  *
155
156
  * ENFORCEMENT (app-parity-2 child 2): `enabled` is now a DISCOVERY/advertisement
@@ -158,8 +159,8 @@ interface DaemonApiKeyEntry {
158
159
  * advertisement, NOT a hard per-request block (core does not validate a requested
159
160
  * model against `models[]`, so a hardcoded disabled model id still reaches the
160
161
  * upstream, which rejects it). The admin management view (`toProviderView`) still
161
- * lists ALL models. The other fields (`name`/`group`/`vision`/`reasoning`) remain
162
- * display-only metadata (no core per-model capability binding on the BYO path).
162
+ * lists ALL models. `name`/`group`/`vision`/`reasoning` remain display metadata;
163
+ * thinking capabilities are projected into core for request negotiation.
163
164
  */
164
165
  interface DaemonModelConfig {
165
166
  /** Model id — the metadata key (parallels an entry in the flat `models[]`). */
@@ -175,6 +176,13 @@ interface DaemonModelConfig {
175
176
  vision?: boolean;
176
177
  /** Reasoning-capable hint (display only; not consumed by routing). */
177
178
  reasoning?: boolean;
179
+ /** Discrete reasoning efforts accepted by this target model. */
180
+ thinkingLevels?: ThinkLevel[];
181
+ /** Valid legacy thinking-budget bounds for this target model. */
182
+ thinkingTokenLimit?: {
183
+ min: number;
184
+ max: number;
185
+ };
178
186
  }
179
187
  /**
180
188
  * One transformer chain entry (app-parity child 5). Mirrors the app's
@@ -1413,11 +1421,53 @@ declare class AccountHealthProbeScheduler implements AccountProbeHistoryReader {
1413
1421
  }
1414
1422
 
1415
1423
  /**
1416
- * auditReaderread + filter the date-rotated audit store (request-audit-log,
1417
- * design D4/D6). Backs the AUTHED admin query only (the records carry IP/UA +
1418
- * possibly bodies). Reads the relevant `audit-*.jsonl` files, parses defensively
1419
- * (a torn final line never poisons a query), filters by key id + time window, and
1420
- * returns NEWEST-FIRST up to a bounded limit.
1424
+ * auditBodyReaderreconstruct captured bodies out of the per-session shards
1425
+ * (audit-store-sharding, design D4).
1426
+ *
1427
+ * Bodies no longer sit inline on the metadata line, so reading one is an explicit
1428
+ * second step: locate the session shard, replay its delta chain, hand back the
1429
+ * original text. Backs the AUTHED admin body query and the `omnicross audit` CLI
1430
+ * — never an unauthenticated surface (a body can hold prompts and PII).
1431
+ *
1432
+ * Shards are read whole. That is bounded on purpose: a shard holds ONE session's
1433
+ * deltas, which is exactly the thing the delta encoding keeps small.
1434
+ *
1435
+ * Both storage forms are handled transparently: today's plain `.jsonl` (kept
1436
+ * greppable on disk) and an archived `.jsonl.gz` from a rolled-over day.
1437
+ *
1438
+ * @module @omnicross/daemon/audit/auditBodyReader
1439
+ */
1440
+
1441
+ /** Locate one record's bodies. `ts` narrows the search to a single day directory. */
1442
+ interface AuditBodyQuery {
1443
+ /** The audit record id whose bodies to reconstruct. */
1444
+ id: string;
1445
+ /** The record's session key (its shard). */
1446
+ sessionKey: string;
1447
+ /** The record's timestamp, if known — skips scanning every retained day. */
1448
+ ts?: number;
1449
+ }
1450
+
1451
+ /**
1452
+ * auditReader — read + filter the audit store (request-audit-log design D4/D6,
1453
+ * re-laid-out by audit-store-sharding design D5). Backs the AUTHED admin query
1454
+ * only (records carry IP/UA). Returns NEWEST-FIRST up to a bounded limit.
1455
+ *
1456
+ * Reads BOTH on-disk layouts: the current `audit-YYYY-MM-DD/meta.jsonl` and the
1457
+ * legacy flat `audit-YYYY-MM-DD.jsonl` left behind by an older daemon (still
1458
+ * queryable until TTL prunes it — there is no migration step).
1459
+ *
1460
+ * BOUNDED BY CONSTRUCTION. Days are visited newest-first and each file is walked
1461
+ * BACKWARDS from its tail, so a query touches only as much of the store as it
1462
+ * needs. Two stops apply:
1463
+ * - cross-day: once `limit` rows are in hand, no older day is opened at all.
1464
+ * Exact, because day files never overlap in date.
1465
+ * - within-day: at most `limit + OVERSCAN` rows are taken from one file. The
1466
+ * overscan absorbs the slight append-order skew from a slow request landing
1467
+ * after a fast one that started later.
1468
+ *
1469
+ * A returned record NEVER carries a body — those live in the per-session shards
1470
+ * and are fetched one at a time. `hasBody` says whether one exists.
1421
1471
  *
1422
1472
  * @module @omnicross/daemon/audit/auditReader
1423
1473
  */
@@ -1426,6 +1476,8 @@ declare class AccountHealthProbeScheduler implements AccountProbeHistoryReader {
1426
1476
  interface AuditQuery {
1427
1477
  /** Restrict to one outbound key id. */
1428
1478
  keyId?: string;
1479
+ /** Restrict to one conversation-session key. */
1480
+ sessionKey?: string;
1429
1481
  /** Inclusive lower bound (epoch ms). */
1430
1482
  from?: number;
1431
1483
  /** Inclusive upper bound (epoch ms). */
@@ -1450,9 +1502,9 @@ interface AuditStatsQuery {
1450
1502
  * auditQueryApi — the AUTHED `GET /admin/api/audit?keyId=&from=&to=&limit=`
1451
1503
  * handler (request-audit-log, design D6).
1452
1504
  *
1453
- * Audit records carry client IP / user-agent (PII) and, when body capture is on,
1454
- * redacted bodies — so unlike the coarse `/health` boolean they are served ONLY
1455
- * behind the admin auth gate. This lives in its OWN helper module (the
1505
+ * Audit records carry client IP / user-agent (PII), and the sibling body query
1506
+ * returns redacted request/response snapshots — so unlike the coarse `/health`
1507
+ * boolean they are served ONLY behind the admin auth gate. This lives in its OWN helper module (the
1456
1508
  * #4/#8/#10 helper-module convention) so `adminApi.ts` — at its line cap — is not
1457
1509
  * touched: `AdminServer.dispatch` routes the path here directly, AFTER its auth
1458
1510
  * gate. NEVER unauthenticated, NEVER surfaced on `/health`.
@@ -1467,6 +1519,14 @@ interface AuditStatsQuery {
1467
1519
  type AuditQueryReader = (query: AuditQuery) => AuditRecord[];
1468
1520
  /** Metadata-only aggregate reader used by the overview. */
1469
1521
  type AuditStatsReader = (query: AuditStatsQuery) => Promise<AuditStats> | AuditStats;
1522
+ /** Reconstructs ONE record's bodies from the per-session shard store. */
1523
+ type AuditBodyReader = (query: AuditBodyQuery) => AuditBodyResult;
1524
+ /** Runs the cross-session body compaction over every closed day. */
1525
+ type AuditCompactor = () => {
1526
+ days: number;
1527
+ shards: number;
1528
+ savedBytes: number;
1529
+ };
1470
1530
 
1471
1531
  /**
1472
1532
  * billingStatusApi — the AUTHED `GET /admin/api/billing-status` handler
@@ -2124,6 +2184,13 @@ interface AdminServerDeps extends AdminApiDeps {
2124
2184
  auditReader?: AuditQueryReader;
2125
2185
  /** Metadata-only audit aggregate used by the overview error-rate metric. */
2126
2186
  auditStatsReader?: AuditStatsReader;
2187
+ /**
2188
+ * Reconstructs one record's bodies from the per-session shard store
2189
+ * (audit-store-sharding). Absent when audit was never enabled.
2190
+ */
2191
+ auditBodyReader?: AuditBodyReader;
2192
+ /** Runs cross-session body compaction on demand (audit-store-sharding D8). */
2193
+ auditCompactor?: AuditCompactor;
2127
2194
  /**
2128
2195
  * OPTIONAL billing delivery-status reader (billing-event-stream, design D5).
2129
2196
  * When wired (bootstrap → the ledger dir), the AUTHED `GET /admin/api/billing-status`
@@ -2287,17 +2354,28 @@ declare class AccountHealthSweeper {
2287
2354
  }
2288
2355
 
2289
2356
  /**
2290
- * AuditPruneSweeper — the TTL prune for the audit store (request-audit-log,
2291
- * design D4). Deletes whole `audit-YYYY-MM-DD.jsonl` files whose date is older
2292
- * than `retentionDays` — a cheap file UNLINK, never a line-level rewrite of a
2293
- * live file (which jsonl makes awkward). So the store never grows unbounded and
2294
- * TTL is O(files).
2357
+ * AuditPruneSweeper — retention + archiving for the audit store
2358
+ * (request-audit-log design D4, extended by audit-store-sharding design D6).
2359
+ *
2360
+ * Two jobs run on the same hourly tick:
2361
+ *
2362
+ * - PRUNE. Remove every day older than `retentionDays`. A current day is a
2363
+ * DIRECTORY (removed recursively), a legacy day is a flat file (unlinked with
2364
+ * its stats sidecar). Either way it stays a whole-day removal — never a
2365
+ * line-level rewrite of a live file, which jsonl makes awkward.
2366
+ * - ARCHIVE. First run the cross-session compaction pass (see
2367
+ * {@link compactAuditDay}), then gzip the body shards of any day that is no
2368
+ * longer today. The
2369
+ * current day stays PLAIN TEXT so it can still be tailed and grepped while
2370
+ * debugging; once a day is closed nothing appends to it again, so rewriting
2371
+ * and compressing it is safe and buys a large multiple on top of the delta.
2372
+ * `meta.jsonl` is NEVER compressed — it is the query hot path.
2295
2373
  *
2296
2374
  * Modeled on the #8 `AccountHealthProbeScheduler` / `AccountHealthSweeper`:
2297
2375
  * `start()` arms an `unref()`ed interval, `dispose()` clears it, a single-sweep
2298
- * re-entrancy guard prevents overlap. A prune ALSO runs once at boot (`start`
2299
- * fires an immediate sweep). Disabled/zero-retention config ⇒ armed-off no-op
2300
- * (byte-identical zero regression). Never throws.
2376
+ * re-entrancy guard prevents overlap. Both jobs also run once at boot. Disabled
2377
+ * config means armed-off, so audit-off is byte-identical zero regression. Never
2378
+ * throws.
2301
2379
  *
2302
2380
  * @module @omnicross/daemon/audit/AuditPruneSweeper
2303
2381
  */
@@ -2311,6 +2389,7 @@ declare class AuditPruneSweeper {
2311
2389
  private readonly now;
2312
2390
  private timer;
2313
2391
  private sweeping;
2392
+ private archiving;
2314
2393
  constructor(auditDir: string, logger: Logger, config: AuditConfig, intervalMs?: number,
2315
2394
  /** Injectable clock (ms) for deterministic tests. */
2316
2395
  now?: () => number);
@@ -2319,32 +2398,56 @@ declare class AuditPruneSweeper {
2319
2398
  /** Re-apply config to the live instance (boot + admin PUT hot-reload). */
2320
2399
  configure(config: AuditConfig): void;
2321
2400
  /**
2322
- * Arm the prune interval AND run one prune immediately (boot cleanup). No-op
2323
- * when audit is disabled (zero regression). Idempotent.
2401
+ * Arm the interval AND run one pass immediately (boot cleanup). No-op when
2402
+ * audit is disabled (zero regression). Idempotent.
2324
2403
  */
2325
2404
  start(): void;
2326
2405
  /** Clear the interval (daemon shutdown / test teardown). Idempotent. */
2327
2406
  dispose(): void;
2407
+ /** Prune first, then archive — never spend CPU compressing a day about to go. */
2408
+ private runOnce;
2409
+ /** The LOCAL-midnight epoch ms of the current day. */
2410
+ private todayMidnight;
2328
2411
  /**
2329
- * One prune: unlink every audit date file strictly OLDER than the retention
2330
- * cutoff (`now - retentionDays` days, at local-midnight granularity). Exposed
2331
- * for tests; never throws. Returns the number of files removed.
2412
+ * One prune: remove every audit day strictly OLDER than the retention cutoff
2413
+ * (`now - retentionDays` days, at local-midnight granularity). Exposed for
2414
+ * tests; never throws. Returns the number of days removed.
2332
2415
  */
2333
2416
  sweep(): Promise<number>;
2417
+ /**
2418
+ * Gzip the body shards of every CLOSED day (anything before today). Today is
2419
+ * deliberately left as plain text so it stays greppable while it is the day you
2420
+ * are debugging. Exposed for tests; never throws. Returns shards compressed.
2421
+ */
2422
+ archive(): Promise<number>;
2423
+ /** Gzip up to `budget` plain shards in one day's `bodies/` directory. */
2424
+ private archiveDay;
2334
2425
  }
2335
2426
 
2336
2427
  /**
2337
- * AuditWriter — the daemon's file-backed audit sink (request-audit-log, design
2338
- * D4/D5). Registered as `@omnicross/core`'s audit sink when audit is enabled; its
2339
- * {@link record} is what `recordAudit` hands each assembled record to.
2428
+ * AuditWriter — the daemon's file-backed audit sink (request-audit-log design
2429
+ * D4/D5, re-laid-out by audit-store-sharding design D2). Registered as
2430
+ * `@omnicross/core`'s audit sink when audit is enabled; its {@link record} is
2431
+ * what `recordAudit` hands each assembled record to.
2340
2432
  *
2341
- * FIRE-AND-FORGET (hard constraint): {@link record} DEFERS the fs append off the
2433
+ * FIRE-AND-FORGET (hard constraint): {@link record} DEFERS all fs work off the
2342
2434
  * caller's stack (an injectable `defer`, default a zero-delay timer — the
2343
2435
  * `UsageRecorder` precedent) and returns immediately, so the relay response path
2344
- * never waits on disk I/O. A write error is swallowed + logged (a failing audit
2345
- * store must never affect a relay). Each record is appended as ONE JSON line to
2346
- * `audit/audit-YYYY-MM-DD.jsonl` (the record's LOCAL date), matching the
2347
- * `usage-events.jsonl` pattern — no new dependency, TTL is a whole-file unlink.
2436
+ * never waits on disk I/O. Delta encoding rides that same deferred tick and is
2437
+ * pure memory, so it can never stall a response either. A write error is
2438
+ * swallowed + logged (a failing audit store must never affect a relay).
2439
+ *
2440
+ * Each record is split across TWO destinations under `audit/audit-YYYY-MM-DD/`:
2441
+ *
2442
+ * - `meta.jsonl` — one small JSON line WITHOUT bodies. This is what queries and
2443
+ * the stats sidecar read, so a listing no longer pays for megabytes of prompt.
2444
+ * - `bodies/<sessionKey>.jsonl` — the captured bodies, sharded per conversation
2445
+ * and delta-encoded against the session's previous turn.
2446
+ *
2447
+ * The two writes are INDEPENDENT: the metadata line is the canonical record, so a
2448
+ * body-shard failure is logged and the metadata still lands. When a shard write
2449
+ * fails the session's encoding base is dropped, otherwise the next turn would
2450
+ * chain a delta onto a base no reader can find.
2348
2451
  *
2349
2452
  * @module @omnicross/daemon/audit/AuditWriter
2350
2453
  */
@@ -2352,23 +2455,37 @@ declare class AuditPruneSweeper {
2352
2455
  declare class AuditWriter {
2353
2456
  private readonly auditDir;
2354
2457
  private readonly logger;
2355
- /** Deferral used by `record()` to schedule the append off the caller's path. */
2458
+ /** Deferral used by `record()` to schedule the writes off the caller's path. */
2356
2459
  private readonly defer;
2357
- private dirEnsured;
2460
+ /** Day directories already created this process (avoids an mkdir per record). */
2461
+ private readonly ensuredDirs;
2462
+ /** Per-session encoding bases. Memory-only; a miss simply writes a full snapshot. */
2463
+ private readonly bases;
2358
2464
  constructor(auditDir: string, logger: Logger,
2359
- /** Deferral used by `record()` to schedule the append off the caller's path. */
2465
+ /** Deferral used by `record()` to schedule the writes off the caller's path. */
2360
2466
  defer?: (fn: () => void) => void);
2361
2467
  /**
2362
- * Enqueue one record for append. Returns IMMEDIATELY (fire-and-forget); the fs
2363
- * write happens on the deferred tick. A failure is logged, never thrown.
2468
+ * Enqueue one record. Returns IMMEDIATELY (fire-and-forget); every fs write and
2469
+ * the delta encoding happen on the deferred tick. A failure is logged, never thrown.
2364
2470
  */
2365
2471
  record(record: AuditRecord): void;
2472
+ /** Drop all retained encoding bases (config reload / shutdown / test teardown). */
2473
+ reset(): void;
2366
2474
  /**
2367
- * Append synchronously — the awaitable form tests use to assert the line landed.
2368
- * Ensures the `audit/` directory exists on first write (lazy, like the usage
2369
- * store's lazy file creation).
2475
+ * Append synchronously — the awaitable form tests use to assert a line landed.
2476
+ * Writes the metadata line first (canonical), then the body shard.
2370
2477
  */
2371
2478
  appendNow(record: AuditRecord): void;
2479
+ /** Create a directory once per process and remember it. */
2480
+ private ensureDir;
2481
+ /** Write the body-free metadata line + refresh the exact-count sidecar. */
2482
+ private appendMeta;
2483
+ /**
2484
+ * Write the delta-encoded body shard for one record. A no-op when nothing was
2485
+ * captured or when the session key is missing/unsafe — in which case the body
2486
+ * is dropped rather than written to an unvalidated path.
2487
+ */
2488
+ private appendBody;
2372
2489
  }
2373
2490
 
2374
2491
  /**