@omnicross/daemon 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +1312 -369
- package/dist/cli.js +1309 -361
- package/dist/index.cjs +849 -107
- package/dist/index.d.cts +147 -38
- package/dist/index.d.ts +147 -38
- package/dist/index.js +847 -100
- package/package.json +2 -2
package/dist/index.d.cts
CHANGED
|
@@ -16,7 +16,7 @@ import { LoggingConfig, HealthReport } from '@omnicross/contracts/health-logging
|
|
|
16
16
|
import { SubscriptionAccountHealth } from '@omnicross/core/pipeline/SubscriptionAccountHealth';
|
|
17
17
|
import { fetchUpstream } from '@omnicross/core/pipeline/upstreamFetch';
|
|
18
18
|
import { ThinkLevel } from '@omnicross/contracts/completion-types';
|
|
19
|
-
import { AuditRecord, AuditStats, AuditConfig } from '@omnicross/contracts/audit-types';
|
|
19
|
+
import { AuditRecord, AuditStats, AuditBodyResult, AuditConfig } from '@omnicross/contracts/audit-types';
|
|
20
20
|
import { BillingDeliveryStatus, BillingConfig, BillingEvent } from '@omnicross/contracts/billing-types';
|
|
21
21
|
import http from 'node:http';
|
|
22
22
|
import { LLMProvider, AgentDefaultModels, GlobalModelParameters } from '@omnicross/contracts/llm-config';
|
|
@@ -1421,11 +1421,53 @@ declare class AccountHealthProbeScheduler implements AccountProbeHistoryReader {
|
|
|
1421
1421
|
}
|
|
1422
1422
|
|
|
1423
1423
|
/**
|
|
1424
|
-
*
|
|
1425
|
-
* design D4
|
|
1426
|
-
*
|
|
1427
|
-
*
|
|
1428
|
-
*
|
|
1424
|
+
* auditBodyReader — reconstruct captured bodies out of the per-session shards
|
|
1425
|
+
* (audit-store-sharding, design D4).
|
|
1426
|
+
*
|
|
1427
|
+
* Bodies no longer sit inline on the metadata line, so reading one is an explicit
|
|
1428
|
+
* second step: locate the session shard, replay its delta chain, hand back the
|
|
1429
|
+
* original text. Backs the AUTHED admin body query and the `omnicross audit` CLI
|
|
1430
|
+
* — never an unauthenticated surface (a body can hold prompts and PII).
|
|
1431
|
+
*
|
|
1432
|
+
* Shards are read whole. That is bounded on purpose: a shard holds ONE session's
|
|
1433
|
+
* deltas, which is exactly the thing the delta encoding keeps small.
|
|
1434
|
+
*
|
|
1435
|
+
* Both storage forms are handled transparently: today's plain `.jsonl` (kept
|
|
1436
|
+
* greppable on disk) and an archived `.jsonl.gz` from a rolled-over day.
|
|
1437
|
+
*
|
|
1438
|
+
* @module @omnicross/daemon/audit/auditBodyReader
|
|
1439
|
+
*/
|
|
1440
|
+
|
|
1441
|
+
/** Locate one record's bodies. `ts` narrows the search to a single day directory. */
|
|
1442
|
+
interface AuditBodyQuery {
|
|
1443
|
+
/** The audit record id whose bodies to reconstruct. */
|
|
1444
|
+
id: string;
|
|
1445
|
+
/** The record's session key (its shard). */
|
|
1446
|
+
sessionKey: string;
|
|
1447
|
+
/** The record's timestamp, if known — skips scanning every retained day. */
|
|
1448
|
+
ts?: number;
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
/**
|
|
1452
|
+
* auditReader — read + filter the audit store (request-audit-log design D4/D6,
|
|
1453
|
+
* re-laid-out by audit-store-sharding design D5). Backs the AUTHED admin query
|
|
1454
|
+
* only (records carry IP/UA). Returns NEWEST-FIRST up to a bounded limit.
|
|
1455
|
+
*
|
|
1456
|
+
* Reads BOTH on-disk layouts: the current `audit-YYYY-MM-DD/meta.jsonl` and the
|
|
1457
|
+
* legacy flat `audit-YYYY-MM-DD.jsonl` left behind by an older daemon (still
|
|
1458
|
+
* queryable until TTL prunes it — there is no migration step).
|
|
1459
|
+
*
|
|
1460
|
+
* BOUNDED BY CONSTRUCTION. Days are visited newest-first and each file is walked
|
|
1461
|
+
* BACKWARDS from its tail, so a query touches only as much of the store as it
|
|
1462
|
+
* needs. Two stops apply:
|
|
1463
|
+
* - cross-day: once `limit` rows are in hand, no older day is opened at all.
|
|
1464
|
+
* Exact, because day files never overlap in date.
|
|
1465
|
+
* - within-day: at most `limit + OVERSCAN` rows are taken from one file. The
|
|
1466
|
+
* overscan absorbs the slight append-order skew from a slow request landing
|
|
1467
|
+
* after a fast one that started later.
|
|
1468
|
+
*
|
|
1469
|
+
* A returned record NEVER carries a body — those live in the per-session shards
|
|
1470
|
+
* and are fetched one at a time. `hasBody` says whether one exists.
|
|
1429
1471
|
*
|
|
1430
1472
|
* @module @omnicross/daemon/audit/auditReader
|
|
1431
1473
|
*/
|
|
@@ -1434,6 +1476,8 @@ declare class AccountHealthProbeScheduler implements AccountProbeHistoryReader {
|
|
|
1434
1476
|
interface AuditQuery {
|
|
1435
1477
|
/** Restrict to one outbound key id. */
|
|
1436
1478
|
keyId?: string;
|
|
1479
|
+
/** Restrict to one conversation-session key. */
|
|
1480
|
+
sessionKey?: string;
|
|
1437
1481
|
/** Inclusive lower bound (epoch ms). */
|
|
1438
1482
|
from?: number;
|
|
1439
1483
|
/** Inclusive upper bound (epoch ms). */
|
|
@@ -1458,9 +1502,9 @@ interface AuditStatsQuery {
|
|
|
1458
1502
|
* auditQueryApi — the AUTHED `GET /admin/api/audit?keyId=&from=&to=&limit=`
|
|
1459
1503
|
* handler (request-audit-log, design D6).
|
|
1460
1504
|
*
|
|
1461
|
-
* Audit records carry client IP / user-agent (PII) and
|
|
1462
|
-
* redacted
|
|
1463
|
-
* behind the admin auth gate. This lives in its OWN helper module (the
|
|
1505
|
+
* Audit records carry client IP / user-agent (PII), and the sibling body query
|
|
1506
|
+
* returns redacted request/response snapshots — so unlike the coarse `/health`
|
|
1507
|
+
* boolean they are served ONLY behind the admin auth gate. This lives in its OWN helper module (the
|
|
1464
1508
|
* #4/#8/#10 helper-module convention) so `adminApi.ts` — at its line cap — is not
|
|
1465
1509
|
* touched: `AdminServer.dispatch` routes the path here directly, AFTER its auth
|
|
1466
1510
|
* gate. NEVER unauthenticated, NEVER surfaced on `/health`.
|
|
@@ -1475,6 +1519,14 @@ interface AuditStatsQuery {
|
|
|
1475
1519
|
type AuditQueryReader = (query: AuditQuery) => AuditRecord[];
|
|
1476
1520
|
/** Metadata-only aggregate reader used by the overview. */
|
|
1477
1521
|
type AuditStatsReader = (query: AuditStatsQuery) => Promise<AuditStats> | AuditStats;
|
|
1522
|
+
/** Reconstructs ONE record's bodies from the per-session shard store. */
|
|
1523
|
+
type AuditBodyReader = (query: AuditBodyQuery) => AuditBodyResult;
|
|
1524
|
+
/** Runs the cross-session body compaction over every closed day. */
|
|
1525
|
+
type AuditCompactor = () => {
|
|
1526
|
+
days: number;
|
|
1527
|
+
shards: number;
|
|
1528
|
+
savedBytes: number;
|
|
1529
|
+
};
|
|
1478
1530
|
|
|
1479
1531
|
/**
|
|
1480
1532
|
* billingStatusApi — the AUTHED `GET /admin/api/billing-status` handler
|
|
@@ -2132,6 +2184,13 @@ interface AdminServerDeps extends AdminApiDeps {
|
|
|
2132
2184
|
auditReader?: AuditQueryReader;
|
|
2133
2185
|
/** Metadata-only audit aggregate used by the overview error-rate metric. */
|
|
2134
2186
|
auditStatsReader?: AuditStatsReader;
|
|
2187
|
+
/**
|
|
2188
|
+
* Reconstructs one record's bodies from the per-session shard store
|
|
2189
|
+
* (audit-store-sharding). Absent when audit was never enabled.
|
|
2190
|
+
*/
|
|
2191
|
+
auditBodyReader?: AuditBodyReader;
|
|
2192
|
+
/** Runs cross-session body compaction on demand (audit-store-sharding D8). */
|
|
2193
|
+
auditCompactor?: AuditCompactor;
|
|
2135
2194
|
/**
|
|
2136
2195
|
* OPTIONAL billing delivery-status reader (billing-event-stream, design D5).
|
|
2137
2196
|
* When wired (bootstrap → the ledger dir), the AUTHED `GET /admin/api/billing-status`
|
|
@@ -2295,17 +2354,28 @@ declare class AccountHealthSweeper {
|
|
|
2295
2354
|
}
|
|
2296
2355
|
|
|
2297
2356
|
/**
|
|
2298
|
-
* AuditPruneSweeper —
|
|
2299
|
-
* design D4
|
|
2300
|
-
*
|
|
2301
|
-
*
|
|
2302
|
-
*
|
|
2357
|
+
* AuditPruneSweeper — retention + archiving for the audit store
|
|
2358
|
+
* (request-audit-log design D4, extended by audit-store-sharding design D6).
|
|
2359
|
+
*
|
|
2360
|
+
* Two jobs run on the same hourly tick:
|
|
2361
|
+
*
|
|
2362
|
+
* - PRUNE. Remove every day older than `retentionDays`. A current day is a
|
|
2363
|
+
* DIRECTORY (removed recursively), a legacy day is a flat file (unlinked with
|
|
2364
|
+
* its stats sidecar). Either way it stays a whole-day removal — never a
|
|
2365
|
+
* line-level rewrite of a live file, which jsonl makes awkward.
|
|
2366
|
+
* - ARCHIVE. First run the cross-session compaction pass (see
|
|
2367
|
+
* {@link compactAuditDay}), then gzip the body shards of any day that is no
|
|
2368
|
+
* longer today. The
|
|
2369
|
+
* current day stays PLAIN TEXT so it can still be tailed and grepped while
|
|
2370
|
+
* debugging; once a day is closed nothing appends to it again, so rewriting
|
|
2371
|
+
* and compressing it is safe and buys a large multiple on top of the delta.
|
|
2372
|
+
* `meta.jsonl` is NEVER compressed — it is the query hot path.
|
|
2303
2373
|
*
|
|
2304
2374
|
* Modeled on the #8 `AccountHealthProbeScheduler` / `AccountHealthSweeper`:
|
|
2305
2375
|
* `start()` arms an `unref()`ed interval, `dispose()` clears it, a single-sweep
|
|
2306
|
-
* re-entrancy guard prevents overlap.
|
|
2307
|
-
*
|
|
2308
|
-
*
|
|
2376
|
+
* re-entrancy guard prevents overlap. Both jobs also run once at boot. Disabled
|
|
2377
|
+
* config means armed-off, so audit-off is byte-identical zero regression. Never
|
|
2378
|
+
* throws.
|
|
2309
2379
|
*
|
|
2310
2380
|
* @module @omnicross/daemon/audit/AuditPruneSweeper
|
|
2311
2381
|
*/
|
|
@@ -2319,6 +2389,7 @@ declare class AuditPruneSweeper {
|
|
|
2319
2389
|
private readonly now;
|
|
2320
2390
|
private timer;
|
|
2321
2391
|
private sweeping;
|
|
2392
|
+
private archiving;
|
|
2322
2393
|
constructor(auditDir: string, logger: Logger, config: AuditConfig, intervalMs?: number,
|
|
2323
2394
|
/** Injectable clock (ms) for deterministic tests. */
|
|
2324
2395
|
now?: () => number);
|
|
@@ -2327,32 +2398,56 @@ declare class AuditPruneSweeper {
|
|
|
2327
2398
|
/** Re-apply config to the live instance (boot + admin PUT hot-reload). */
|
|
2328
2399
|
configure(config: AuditConfig): void;
|
|
2329
2400
|
/**
|
|
2330
|
-
* Arm the
|
|
2331
|
-
*
|
|
2401
|
+
* Arm the interval AND run one pass immediately (boot cleanup). No-op when
|
|
2402
|
+
* audit is disabled (zero regression). Idempotent.
|
|
2332
2403
|
*/
|
|
2333
2404
|
start(): void;
|
|
2334
2405
|
/** Clear the interval (daemon shutdown / test teardown). Idempotent. */
|
|
2335
2406
|
dispose(): void;
|
|
2407
|
+
/** Prune first, then archive — never spend CPU compressing a day about to go. */
|
|
2408
|
+
private runOnce;
|
|
2409
|
+
/** The LOCAL-midnight epoch ms of the current day. */
|
|
2410
|
+
private todayMidnight;
|
|
2336
2411
|
/**
|
|
2337
|
-
* One prune:
|
|
2338
|
-
*
|
|
2339
|
-
*
|
|
2412
|
+
* One prune: remove every audit day strictly OLDER than the retention cutoff
|
|
2413
|
+
* (`now - retentionDays` days, at local-midnight granularity). Exposed for
|
|
2414
|
+
* tests; never throws. Returns the number of days removed.
|
|
2340
2415
|
*/
|
|
2341
2416
|
sweep(): Promise<number>;
|
|
2417
|
+
/**
|
|
2418
|
+
* Gzip the body shards of every CLOSED day (anything before today). Today is
|
|
2419
|
+
* deliberately left as plain text so it stays greppable while it is the day you
|
|
2420
|
+
* are debugging. Exposed for tests; never throws. Returns shards compressed.
|
|
2421
|
+
*/
|
|
2422
|
+
archive(): Promise<number>;
|
|
2423
|
+
/** Gzip up to `budget` plain shards in one day's `bodies/` directory. */
|
|
2424
|
+
private archiveDay;
|
|
2342
2425
|
}
|
|
2343
2426
|
|
|
2344
2427
|
/**
|
|
2345
|
-
* AuditWriter — the daemon's file-backed audit sink (request-audit-log
|
|
2346
|
-
* D4/D5
|
|
2347
|
-
*
|
|
2428
|
+
* AuditWriter — the daemon's file-backed audit sink (request-audit-log design
|
|
2429
|
+
* D4/D5, re-laid-out by audit-store-sharding design D2). Registered as
|
|
2430
|
+
* `@omnicross/core`'s audit sink when audit is enabled; its {@link record} is
|
|
2431
|
+
* what `recordAudit` hands each assembled record to.
|
|
2348
2432
|
*
|
|
2349
|
-
* FIRE-AND-FORGET (hard constraint): {@link record} DEFERS
|
|
2433
|
+
* FIRE-AND-FORGET (hard constraint): {@link record} DEFERS all fs work off the
|
|
2350
2434
|
* caller's stack (an injectable `defer`, default a zero-delay timer — the
|
|
2351
2435
|
* `UsageRecorder` precedent) and returns immediately, so the relay response path
|
|
2352
|
-
* never waits on disk I/O.
|
|
2353
|
-
*
|
|
2354
|
-
*
|
|
2355
|
-
*
|
|
2436
|
+
* never waits on disk I/O. Delta encoding rides that same deferred tick and is
|
|
2437
|
+
* pure memory, so it can never stall a response either. A write error is
|
|
2438
|
+
* swallowed + logged (a failing audit store must never affect a relay).
|
|
2439
|
+
*
|
|
2440
|
+
* Each record is split across TWO destinations under `audit/audit-YYYY-MM-DD/`:
|
|
2441
|
+
*
|
|
2442
|
+
* - `meta.jsonl` — one small JSON line WITHOUT bodies. This is what queries and
|
|
2443
|
+
* the stats sidecar read, so a listing no longer pays for megabytes of prompt.
|
|
2444
|
+
* - `bodies/<sessionKey>.jsonl` — the captured bodies, sharded per conversation
|
|
2445
|
+
* and delta-encoded against the session's previous turn.
|
|
2446
|
+
*
|
|
2447
|
+
* The two writes are INDEPENDENT: the metadata line is the canonical record, so a
|
|
2448
|
+
* body-shard failure is logged and the metadata still lands. When a shard write
|
|
2449
|
+
* fails the session's encoding base is dropped, otherwise the next turn would
|
|
2450
|
+
* chain a delta onto a base no reader can find.
|
|
2356
2451
|
*
|
|
2357
2452
|
* @module @omnicross/daemon/audit/AuditWriter
|
|
2358
2453
|
*/
|
|
@@ -2360,23 +2455,37 @@ declare class AuditPruneSweeper {
|
|
|
2360
2455
|
declare class AuditWriter {
|
|
2361
2456
|
private readonly auditDir;
|
|
2362
2457
|
private readonly logger;
|
|
2363
|
-
/** Deferral used by `record()` to schedule the
|
|
2458
|
+
/** Deferral used by `record()` to schedule the writes off the caller's path. */
|
|
2364
2459
|
private readonly defer;
|
|
2365
|
-
|
|
2460
|
+
/** Day directories already created this process (avoids an mkdir per record). */
|
|
2461
|
+
private readonly ensuredDirs;
|
|
2462
|
+
/** Per-session encoding bases. Memory-only; a miss simply writes a full snapshot. */
|
|
2463
|
+
private readonly bases;
|
|
2366
2464
|
constructor(auditDir: string, logger: Logger,
|
|
2367
|
-
/** Deferral used by `record()` to schedule the
|
|
2465
|
+
/** Deferral used by `record()` to schedule the writes off the caller's path. */
|
|
2368
2466
|
defer?: (fn: () => void) => void);
|
|
2369
2467
|
/**
|
|
2370
|
-
* Enqueue one record
|
|
2371
|
-
*
|
|
2468
|
+
* Enqueue one record. Returns IMMEDIATELY (fire-and-forget); every fs write and
|
|
2469
|
+
* the delta encoding happen on the deferred tick. A failure is logged, never thrown.
|
|
2372
2470
|
*/
|
|
2373
2471
|
record(record: AuditRecord): void;
|
|
2472
|
+
/** Drop all retained encoding bases (config reload / shutdown / test teardown). */
|
|
2473
|
+
reset(): void;
|
|
2374
2474
|
/**
|
|
2375
|
-
* Append synchronously — the awaitable form tests use to assert
|
|
2376
|
-
*
|
|
2377
|
-
* store's lazy file creation).
|
|
2475
|
+
* Append synchronously — the awaitable form tests use to assert a line landed.
|
|
2476
|
+
* Writes the metadata line first (canonical), then the body shard.
|
|
2378
2477
|
*/
|
|
2379
2478
|
appendNow(record: AuditRecord): void;
|
|
2479
|
+
/** Create a directory once per process and remember it. */
|
|
2480
|
+
private ensureDir;
|
|
2481
|
+
/** Write the body-free metadata line + refresh the exact-count sidecar. */
|
|
2482
|
+
private appendMeta;
|
|
2483
|
+
/**
|
|
2484
|
+
* Write the delta-encoded body shard for one record. A no-op when nothing was
|
|
2485
|
+
* captured or when the session key is missing/unsafe — in which case the body
|
|
2486
|
+
* is dropped rather than written to an unvalidated path.
|
|
2487
|
+
*/
|
|
2488
|
+
private appendBody;
|
|
2380
2489
|
}
|
|
2381
2490
|
|
|
2382
2491
|
/**
|
package/dist/index.d.ts
CHANGED
|
@@ -16,7 +16,7 @@ import { LoggingConfig, HealthReport } from '@omnicross/contracts/health-logging
|
|
|
16
16
|
import { SubscriptionAccountHealth } from '@omnicross/core/pipeline/SubscriptionAccountHealth';
|
|
17
17
|
import { fetchUpstream } from '@omnicross/core/pipeline/upstreamFetch';
|
|
18
18
|
import { ThinkLevel } from '@omnicross/contracts/completion-types';
|
|
19
|
-
import { AuditRecord, AuditStats, AuditConfig } from '@omnicross/contracts/audit-types';
|
|
19
|
+
import { AuditRecord, AuditStats, AuditBodyResult, AuditConfig } from '@omnicross/contracts/audit-types';
|
|
20
20
|
import { BillingDeliveryStatus, BillingConfig, BillingEvent } from '@omnicross/contracts/billing-types';
|
|
21
21
|
import http from 'node:http';
|
|
22
22
|
import { LLMProvider, AgentDefaultModels, GlobalModelParameters } from '@omnicross/contracts/llm-config';
|
|
@@ -1421,11 +1421,53 @@ declare class AccountHealthProbeScheduler implements AccountProbeHistoryReader {
|
|
|
1421
1421
|
}
|
|
1422
1422
|
|
|
1423
1423
|
/**
|
|
1424
|
-
*
|
|
1425
|
-
* design D4
|
|
1426
|
-
*
|
|
1427
|
-
*
|
|
1428
|
-
*
|
|
1424
|
+
* auditBodyReader — reconstruct captured bodies out of the per-session shards
|
|
1425
|
+
* (audit-store-sharding, design D4).
|
|
1426
|
+
*
|
|
1427
|
+
* Bodies no longer sit inline on the metadata line, so reading one is an explicit
|
|
1428
|
+
* second step: locate the session shard, replay its delta chain, hand back the
|
|
1429
|
+
* original text. Backs the AUTHED admin body query and the `omnicross audit` CLI
|
|
1430
|
+
* — never an unauthenticated surface (a body can hold prompts and PII).
|
|
1431
|
+
*
|
|
1432
|
+
* Shards are read whole. That is bounded on purpose: a shard holds ONE session's
|
|
1433
|
+
* deltas, which is exactly the thing the delta encoding keeps small.
|
|
1434
|
+
*
|
|
1435
|
+
* Both storage forms are handled transparently: today's plain `.jsonl` (kept
|
|
1436
|
+
* greppable on disk) and an archived `.jsonl.gz` from a rolled-over day.
|
|
1437
|
+
*
|
|
1438
|
+
* @module @omnicross/daemon/audit/auditBodyReader
|
|
1439
|
+
*/
|
|
1440
|
+
|
|
1441
|
+
/** Locate one record's bodies. `ts` narrows the search to a single day directory. */
|
|
1442
|
+
interface AuditBodyQuery {
|
|
1443
|
+
/** The audit record id whose bodies to reconstruct. */
|
|
1444
|
+
id: string;
|
|
1445
|
+
/** The record's session key (its shard). */
|
|
1446
|
+
sessionKey: string;
|
|
1447
|
+
/** The record's timestamp, if known — skips scanning every retained day. */
|
|
1448
|
+
ts?: number;
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
/**
|
|
1452
|
+
* auditReader — read + filter the audit store (request-audit-log design D4/D6,
|
|
1453
|
+
* re-laid-out by audit-store-sharding design D5). Backs the AUTHED admin query
|
|
1454
|
+
* only (records carry IP/UA). Returns NEWEST-FIRST up to a bounded limit.
|
|
1455
|
+
*
|
|
1456
|
+
* Reads BOTH on-disk layouts: the current `audit-YYYY-MM-DD/meta.jsonl` and the
|
|
1457
|
+
* legacy flat `audit-YYYY-MM-DD.jsonl` left behind by an older daemon (still
|
|
1458
|
+
* queryable until TTL prunes it — there is no migration step).
|
|
1459
|
+
*
|
|
1460
|
+
* BOUNDED BY CONSTRUCTION. Days are visited newest-first and each file is walked
|
|
1461
|
+
* BACKWARDS from its tail, so a query touches only as much of the store as it
|
|
1462
|
+
* needs. Two stops apply:
|
|
1463
|
+
* - cross-day: once `limit` rows are in hand, no older day is opened at all.
|
|
1464
|
+
* Exact, because day files never overlap in date.
|
|
1465
|
+
* - within-day: at most `limit + OVERSCAN` rows are taken from one file. The
|
|
1466
|
+
* overscan absorbs the slight append-order skew from a slow request landing
|
|
1467
|
+
* after a fast one that started later.
|
|
1468
|
+
*
|
|
1469
|
+
* A returned record NEVER carries a body — those live in the per-session shards
|
|
1470
|
+
* and are fetched one at a time. `hasBody` says whether one exists.
|
|
1429
1471
|
*
|
|
1430
1472
|
* @module @omnicross/daemon/audit/auditReader
|
|
1431
1473
|
*/
|
|
@@ -1434,6 +1476,8 @@ declare class AccountHealthProbeScheduler implements AccountProbeHistoryReader {
|
|
|
1434
1476
|
interface AuditQuery {
|
|
1435
1477
|
/** Restrict to one outbound key id. */
|
|
1436
1478
|
keyId?: string;
|
|
1479
|
+
/** Restrict to one conversation-session key. */
|
|
1480
|
+
sessionKey?: string;
|
|
1437
1481
|
/** Inclusive lower bound (epoch ms). */
|
|
1438
1482
|
from?: number;
|
|
1439
1483
|
/** Inclusive upper bound (epoch ms). */
|
|
@@ -1458,9 +1502,9 @@ interface AuditStatsQuery {
|
|
|
1458
1502
|
* auditQueryApi — the AUTHED `GET /admin/api/audit?keyId=&from=&to=&limit=`
|
|
1459
1503
|
* handler (request-audit-log, design D6).
|
|
1460
1504
|
*
|
|
1461
|
-
* Audit records carry client IP / user-agent (PII) and
|
|
1462
|
-
* redacted
|
|
1463
|
-
* behind the admin auth gate. This lives in its OWN helper module (the
|
|
1505
|
+
* Audit records carry client IP / user-agent (PII), and the sibling body query
|
|
1506
|
+
* returns redacted request/response snapshots — so unlike the coarse `/health`
|
|
1507
|
+
* boolean they are served ONLY behind the admin auth gate. This lives in its OWN helper module (the
|
|
1464
1508
|
* #4/#8/#10 helper-module convention) so `adminApi.ts` — at its line cap — is not
|
|
1465
1509
|
* touched: `AdminServer.dispatch` routes the path here directly, AFTER its auth
|
|
1466
1510
|
* gate. NEVER unauthenticated, NEVER surfaced on `/health`.
|
|
@@ -1475,6 +1519,14 @@ interface AuditStatsQuery {
|
|
|
1475
1519
|
type AuditQueryReader = (query: AuditQuery) => AuditRecord[];
|
|
1476
1520
|
/** Metadata-only aggregate reader used by the overview. */
|
|
1477
1521
|
type AuditStatsReader = (query: AuditStatsQuery) => Promise<AuditStats> | AuditStats;
|
|
1522
|
+
/** Reconstructs ONE record's bodies from the per-session shard store. */
|
|
1523
|
+
type AuditBodyReader = (query: AuditBodyQuery) => AuditBodyResult;
|
|
1524
|
+
/** Runs the cross-session body compaction over every closed day. */
|
|
1525
|
+
type AuditCompactor = () => {
|
|
1526
|
+
days: number;
|
|
1527
|
+
shards: number;
|
|
1528
|
+
savedBytes: number;
|
|
1529
|
+
};
|
|
1478
1530
|
|
|
1479
1531
|
/**
|
|
1480
1532
|
* billingStatusApi — the AUTHED `GET /admin/api/billing-status` handler
|
|
@@ -2132,6 +2184,13 @@ interface AdminServerDeps extends AdminApiDeps {
|
|
|
2132
2184
|
auditReader?: AuditQueryReader;
|
|
2133
2185
|
/** Metadata-only audit aggregate used by the overview error-rate metric. */
|
|
2134
2186
|
auditStatsReader?: AuditStatsReader;
|
|
2187
|
+
/**
|
|
2188
|
+
* Reconstructs one record's bodies from the per-session shard store
|
|
2189
|
+
* (audit-store-sharding). Absent when audit was never enabled.
|
|
2190
|
+
*/
|
|
2191
|
+
auditBodyReader?: AuditBodyReader;
|
|
2192
|
+
/** Runs cross-session body compaction on demand (audit-store-sharding D8). */
|
|
2193
|
+
auditCompactor?: AuditCompactor;
|
|
2135
2194
|
/**
|
|
2136
2195
|
* OPTIONAL billing delivery-status reader (billing-event-stream, design D5).
|
|
2137
2196
|
* When wired (bootstrap → the ledger dir), the AUTHED `GET /admin/api/billing-status`
|
|
@@ -2295,17 +2354,28 @@ declare class AccountHealthSweeper {
|
|
|
2295
2354
|
}
|
|
2296
2355
|
|
|
2297
2356
|
/**
|
|
2298
|
-
* AuditPruneSweeper —
|
|
2299
|
-
* design D4
|
|
2300
|
-
*
|
|
2301
|
-
*
|
|
2302
|
-
*
|
|
2357
|
+
* AuditPruneSweeper — retention + archiving for the audit store
|
|
2358
|
+
* (request-audit-log design D4, extended by audit-store-sharding design D6).
|
|
2359
|
+
*
|
|
2360
|
+
* Two jobs run on the same hourly tick:
|
|
2361
|
+
*
|
|
2362
|
+
* - PRUNE. Remove every day older than `retentionDays`. A current day is a
|
|
2363
|
+
* DIRECTORY (removed recursively), a legacy day is a flat file (unlinked with
|
|
2364
|
+
* its stats sidecar). Either way it stays a whole-day removal — never a
|
|
2365
|
+
* line-level rewrite of a live file, which jsonl makes awkward.
|
|
2366
|
+
* - ARCHIVE. First run the cross-session compaction pass (see
|
|
2367
|
+
* {@link compactAuditDay}), then gzip the body shards of any day that is no
|
|
2368
|
+
* longer today. The
|
|
2369
|
+
* current day stays PLAIN TEXT so it can still be tailed and grepped while
|
|
2370
|
+
* debugging; once a day is closed nothing appends to it again, so rewriting
|
|
2371
|
+
* and compressing it is safe and buys a large multiple on top of the delta.
|
|
2372
|
+
* `meta.jsonl` is NEVER compressed — it is the query hot path.
|
|
2303
2373
|
*
|
|
2304
2374
|
* Modeled on the #8 `AccountHealthProbeScheduler` / `AccountHealthSweeper`:
|
|
2305
2375
|
* `start()` arms an `unref()`ed interval, `dispose()` clears it, a single-sweep
|
|
2306
|
-
* re-entrancy guard prevents overlap.
|
|
2307
|
-
*
|
|
2308
|
-
*
|
|
2376
|
+
* re-entrancy guard prevents overlap. Both jobs also run once at boot. Disabled
|
|
2377
|
+
* config means armed-off, so audit-off is byte-identical zero regression. Never
|
|
2378
|
+
* throws.
|
|
2309
2379
|
*
|
|
2310
2380
|
* @module @omnicross/daemon/audit/AuditPruneSweeper
|
|
2311
2381
|
*/
|
|
@@ -2319,6 +2389,7 @@ declare class AuditPruneSweeper {
|
|
|
2319
2389
|
private readonly now;
|
|
2320
2390
|
private timer;
|
|
2321
2391
|
private sweeping;
|
|
2392
|
+
private archiving;
|
|
2322
2393
|
constructor(auditDir: string, logger: Logger, config: AuditConfig, intervalMs?: number,
|
|
2323
2394
|
/** Injectable clock (ms) for deterministic tests. */
|
|
2324
2395
|
now?: () => number);
|
|
@@ -2327,32 +2398,56 @@ declare class AuditPruneSweeper {
|
|
|
2327
2398
|
/** Re-apply config to the live instance (boot + admin PUT hot-reload). */
|
|
2328
2399
|
configure(config: AuditConfig): void;
|
|
2329
2400
|
/**
|
|
2330
|
-
* Arm the
|
|
2331
|
-
*
|
|
2401
|
+
* Arm the interval AND run one pass immediately (boot cleanup). No-op when
|
|
2402
|
+
* audit is disabled (zero regression). Idempotent.
|
|
2332
2403
|
*/
|
|
2333
2404
|
start(): void;
|
|
2334
2405
|
/** Clear the interval (daemon shutdown / test teardown). Idempotent. */
|
|
2335
2406
|
dispose(): void;
|
|
2407
|
+
/** Prune first, then archive — never spend CPU compressing a day about to go. */
|
|
2408
|
+
private runOnce;
|
|
2409
|
+
/** The LOCAL-midnight epoch ms of the current day. */
|
|
2410
|
+
private todayMidnight;
|
|
2336
2411
|
/**
|
|
2337
|
-
* One prune:
|
|
2338
|
-
*
|
|
2339
|
-
*
|
|
2412
|
+
* One prune: remove every audit day strictly OLDER than the retention cutoff
|
|
2413
|
+
* (`now - retentionDays` days, at local-midnight granularity). Exposed for
|
|
2414
|
+
* tests; never throws. Returns the number of days removed.
|
|
2340
2415
|
*/
|
|
2341
2416
|
sweep(): Promise<number>;
|
|
2417
|
+
/**
|
|
2418
|
+
* Gzip the body shards of every CLOSED day (anything before today). Today is
|
|
2419
|
+
* deliberately left as plain text so it stays greppable while it is the day you
|
|
2420
|
+
* are debugging. Exposed for tests; never throws. Returns shards compressed.
|
|
2421
|
+
*/
|
|
2422
|
+
archive(): Promise<number>;
|
|
2423
|
+
/** Gzip up to `budget` plain shards in one day's `bodies/` directory. */
|
|
2424
|
+
private archiveDay;
|
|
2342
2425
|
}
|
|
2343
2426
|
|
|
2344
2427
|
/**
|
|
2345
|
-
* AuditWriter — the daemon's file-backed audit sink (request-audit-log
|
|
2346
|
-
* D4/D5
|
|
2347
|
-
*
|
|
2428
|
+
* AuditWriter — the daemon's file-backed audit sink (request-audit-log design
|
|
2429
|
+
* D4/D5, re-laid-out by audit-store-sharding design D2). Registered as
|
|
2430
|
+
* `@omnicross/core`'s audit sink when audit is enabled; its {@link record} is
|
|
2431
|
+
* what `recordAudit` hands each assembled record to.
|
|
2348
2432
|
*
|
|
2349
|
-
* FIRE-AND-FORGET (hard constraint): {@link record} DEFERS
|
|
2433
|
+
* FIRE-AND-FORGET (hard constraint): {@link record} DEFERS all fs work off the
|
|
2350
2434
|
* caller's stack (an injectable `defer`, default a zero-delay timer — the
|
|
2351
2435
|
* `UsageRecorder` precedent) and returns immediately, so the relay response path
|
|
2352
|
-
* never waits on disk I/O.
|
|
2353
|
-
*
|
|
2354
|
-
*
|
|
2355
|
-
*
|
|
2436
|
+
* never waits on disk I/O. Delta encoding rides that same deferred tick and is
|
|
2437
|
+
* pure memory, so it can never stall a response either. A write error is
|
|
2438
|
+
* swallowed + logged (a failing audit store must never affect a relay).
|
|
2439
|
+
*
|
|
2440
|
+
* Each record is split across TWO destinations under `audit/audit-YYYY-MM-DD/`:
|
|
2441
|
+
*
|
|
2442
|
+
* - `meta.jsonl` — one small JSON line WITHOUT bodies. This is what queries and
|
|
2443
|
+
* the stats sidecar read, so a listing no longer pays for megabytes of prompt.
|
|
2444
|
+
* - `bodies/<sessionKey>.jsonl` — the captured bodies, sharded per conversation
|
|
2445
|
+
* and delta-encoded against the session's previous turn.
|
|
2446
|
+
*
|
|
2447
|
+
* The two writes are INDEPENDENT: the metadata line is the canonical record, so a
|
|
2448
|
+
* body-shard failure is logged and the metadata still lands. When a shard write
|
|
2449
|
+
* fails the session's encoding base is dropped, otherwise the next turn would
|
|
2450
|
+
* chain a delta onto a base no reader can find.
|
|
2356
2451
|
*
|
|
2357
2452
|
* @module @omnicross/daemon/audit/AuditWriter
|
|
2358
2453
|
*/
|
|
@@ -2360,23 +2455,37 @@ declare class AuditPruneSweeper {
|
|
|
2360
2455
|
declare class AuditWriter {
|
|
2361
2456
|
private readonly auditDir;
|
|
2362
2457
|
private readonly logger;
|
|
2363
|
-
/** Deferral used by `record()` to schedule the
|
|
2458
|
+
/** Deferral used by `record()` to schedule the writes off the caller's path. */
|
|
2364
2459
|
private readonly defer;
|
|
2365
|
-
|
|
2460
|
+
/** Day directories already created this process (avoids an mkdir per record). */
|
|
2461
|
+
private readonly ensuredDirs;
|
|
2462
|
+
/** Per-session encoding bases. Memory-only; a miss simply writes a full snapshot. */
|
|
2463
|
+
private readonly bases;
|
|
2366
2464
|
constructor(auditDir: string, logger: Logger,
|
|
2367
|
-
/** Deferral used by `record()` to schedule the
|
|
2465
|
+
/** Deferral used by `record()` to schedule the writes off the caller's path. */
|
|
2368
2466
|
defer?: (fn: () => void) => void);
|
|
2369
2467
|
/**
|
|
2370
|
-
* Enqueue one record
|
|
2371
|
-
*
|
|
2468
|
+
* Enqueue one record. Returns IMMEDIATELY (fire-and-forget); every fs write and
|
|
2469
|
+
* the delta encoding happen on the deferred tick. A failure is logged, never thrown.
|
|
2372
2470
|
*/
|
|
2373
2471
|
record(record: AuditRecord): void;
|
|
2472
|
+
/** Drop all retained encoding bases (config reload / shutdown / test teardown). */
|
|
2473
|
+
reset(): void;
|
|
2374
2474
|
/**
|
|
2375
|
-
* Append synchronously — the awaitable form tests use to assert
|
|
2376
|
-
*
|
|
2377
|
-
* store's lazy file creation).
|
|
2475
|
+
* Append synchronously — the awaitable form tests use to assert a line landed.
|
|
2476
|
+
* Writes the metadata line first (canonical), then the body shard.
|
|
2378
2477
|
*/
|
|
2379
2478
|
appendNow(record: AuditRecord): void;
|
|
2479
|
+
/** Create a directory once per process and remember it. */
|
|
2480
|
+
private ensureDir;
|
|
2481
|
+
/** Write the body-free metadata line + refresh the exact-count sidecar. */
|
|
2482
|
+
private appendMeta;
|
|
2483
|
+
/**
|
|
2484
|
+
* Write the delta-encoded body shard for one record. A no-op when nothing was
|
|
2485
|
+
* captured or when the session key is missing/unsafe — in which case the body
|
|
2486
|
+
* is dropped rather than written to an unvalidated path.
|
|
2487
|
+
*/
|
|
2488
|
+
private appendBody;
|
|
2380
2489
|
}
|
|
2381
2490
|
|
|
2382
2491
|
/**
|