@dzhechkov/harness-core 0.3.145 → 0.3.147

package/src/index.ts

@@ -156,11 +156,13 @@ export {
   RECALL_USAGE_LOG_MAX_BYTES,
   RECALL_USAGE_COMPACT_TARGET_BYTES,
   formatRecallUsageRecord,
+  buildRecallUsageRecord,
   parseRecallUsageLog,
   aggregateRecallUsage,
   buildRecallUsageReport,
   shouldCompactRecallUsageLogSize,
   compactRecallUsageLog,
+  compactRecallUsageLogChecked,
 } from './recall-usage.js';
 export type {
   RecallUsageReadRecord,
@@ -171,7 +173,53 @@ export type {
   RecallPatternUsageRef,
   RecallUsagePatternRow,
   RecallUsageReport,
+  RecallUsageRecordInput,
+  CompactRecallUsageOptions,
+  CompactRecallUsageResult,
+  CompactRecallUsageStatus,
 } from './recall-usage.js';
+export {
+  EVENT_CHAIN_SCOPE,
+  EVENT_CHAIN_GENESIS_HASH,
+  EVENT_CHAIN_TAIL_BYTES,
+  EVENT_CHAIN_FIELD_OVERHEAD_BYTES,
+  EVENT_CHAIN_LEDGER_KIND,
+  EVENT_CHAIN_DEFECT_KINDS,
+  fnv1a32,
+  chainHashOf,
+  chainLinesOf,
+  lastChainLine,
+  readTailInfo,
+  appendChainedLines,
+  EMPTY_LOG_TAIL,
+  nextChainFields,
+  withChainFields,
+  chainRecordLines,
+  chainRewrite,
+  defaultEventWeight,
+  eventWeightOfText,
+  verifyEventChain,
+  verifyEventChainText,
+  renderEventChainVerification,
+  rewriteSnapshot,
+  rewriteSnapshotUnchanged,
+  guardedRewrite,
+  DEFAULT_REWRITE_ATTEMPTS,
+} from './event-chain.js';
+export type {
+  ChainFields,
+  LogTail,
+  EventChainLedger,
+  RewriteSnapshot,
+  GuardedRewriteIo,
+  GuardedRewriteResult,
+  GuardedRewriteStatus,
+  RewriteProposal,
+  EventChainDefect,
+  EventChainDefectKind,
+  EventChainVerification,
+  VerifyEventChainOptions,
+} from './event-chain.js';
 export { decideProvenance, environmentCanMintProvenance, publishArgv, discoverPackages, publishPackages, bumpPatch, compareVersions, findUnpackagedSkills, orderByDependencies, syncReadmeVersion } from './publish.js';
 export { fetchAllDownloads } from './downloads.js';
 export type { PackageDownloads, DownloadsReport } from './downloads.js';
@@ -215,6 +263,7 @@ export type { RiskScore, RiskThresholds } from './risk-scoring.js';
 export {
   MODEL_PRICES,
   pricingFor,
+  hasKnownPricing,
   normalizeUsage,
   usageCost,
   invocationCost,
@@ -294,6 +343,43 @@ export {
   readUsageLimits,
   weeklyWindowFor,
 } from './usage.js';
+export { claudeProjectsRoot, rawTokenMixOf, weightedTokensOf } from './usage.js';
+export type { RawTokenMix } from './usage.js';
+// Per-stage cost ledger + reconciliation invariant (feature cost-ledger, ADR-001/002/003).
+export {
+  COST_LEDGER_SCOPE,
+  COST_LEDGER_DEFECT_KINDS,
+  COST_LEDGER_VERDICTS,
+  DEFAULT_COST_LEDGER_EPSILON,
+  extractCostSamples,
+  parseWorkflowRunRecord,
+  buildCostLedger,
+  verifyCostLedgerReport,
+  stageCostAggregates,
+  renderCostLedger,
+  costLedgerJsonl,
+  listCostLedgerRuns,
+  deriveCostLedger,
+  deriveStageCostAggregates,
+  writeCostLedgerJsonl,
+} from './cost-ledger.js';
+export type {
+  CostLedgerDefect,
+  CostLedgerDefectKind,
+  CostLedgerIoOptions,
+  CostLedgerReconciliation,
+  CostLedgerReport,
+  CostLedgerRow,
+  CostLedgerRunRef,
+  CostLedgerSample,
+  CostLedgerVerdict,
+  BuildCostLedgerInput,
+  DeriveCostLedgerOptions,
+  StageCostAggregate,
+  StageSampleSet,
+  WorkflowRunRecord,
+  WorkflowStageEntry,
+} from './cost-ledger.js';
 export type {
   ClaudeUsageModel,
   UsageCalibrationChange,

package/src/operations.ts

@@ -788,6 +788,31 @@ export async function runDoctor(options: { projectRoot: string }): Promise<Docto
     } catch { /* either side absent — covered by other checks */ }
   }
 
+  // 8b. EVIDENCE-CHAIN INTEGRITY (feature event-chain, ADR-001). `.dz/recall-usage.jsonl` and
+  // `.dz/guard-audit.jsonl` are what `dz compounding` and `dz guard promote` decide on; a rewrite
+  // that loses or duplicates a record there is a wrong verdict with no symptom. Deliberately OUTSIDE
+  // the agentdb-writer branch — the evidence base exists whether or not that writer is deployed.
+  // Silent when a log is absent or has never been chained: an unchained file is legal (FR-5), not a
+  // fault, and reporting it would train the reader to ignore this line.
+  try {
+    const { verifyEventChainText, EVENT_CHAIN_SCOPE } = await import('./event-chain.js');
+    for (const rel of ['recall-usage.jsonl', 'guard-audit.jsonl']) {
+      const p = join(root, '.dz', rel);
+      if (!existsSync(p)) continue;
+      const v = verifyEventChainText(readFileSync(p, 'utf-8'));
+      if (v.chained === 0 || v.ok) continue;
+      checks.push({
+        name: `evidence chain (.dz/${rel})`,
+        ok: false,
+        detail:
+          `${v.defects.length} defect(s): ${v.defects.slice(0, 3).map((d) => `${d.kind}@L${d.line}`).join(', ')}` +
+          ` — learning verdicts computed from this log are unsafe. Scope: ${EVENT_CHAIN_SCOPE}`,
+      });
+    }
+  } catch {
+    /* doctor never throws on a diagnostic */
+  }
+
   // 9. Vector-tier mirror divergence (dz-rvf-vector-bridge FR-2/ADR R4). INFORMATIONAL, never an
   // error exit: lexical is the source of truth and `dz consolidate` backfills the mirror. Per
   // Constraint 7 (QR-10) the line reports BOTH counts so a diverged mirror is not misdiagnosed

package/src/recall-usage.ts

@@ -6,9 +6,22 @@
  * into aggregate JSONL rows when it crosses a bounded size. It deliberately knows nothing about the
  * filesystem; callers own reads/writes so the hook and statusline can keep their never-block rules.
  *
+ * Compaction RE-CHAINS everything it writes and records what it measured in its input, so that a
+ * rewrite which counts an event twice fails `verifyEventChain` instead of producing a well-formed
+ * lie — the 2 → 4 → 6 defect below is the reason (see `event-chain.ts`, ADR-002).
+ *
  * @packageDocumentation
  */
 
+import {
+  EVENT_CHAIN_FIELD_OVERHEAD_BYTES,
+  EVENT_CHAIN_LEDGER_KIND,
+  chainRewrite,
+  defaultEventWeight,
+  verifyEventChainText,
+  type EventChainDefect,
+} from './event-chain.js';
+
 export const RECALL_USAGE_LOG_RELATIVE = '.dz/recall-usage.jsonl';
 export const RECALL_USAGE_LOG_MAX_BYTES = 1_048_576;
 export const RECALL_USAGE_COMPACT_TARGET_BYTES = Math.floor(RECALL_USAGE_LOG_MAX_BYTES * 0.75);
@@ -104,7 +117,7 @@ interface Acc {
   totalScore: number;
 }
 
-export function formatRecallUsageRecord(input: {
+export interface RecallUsageRecordInput {
   readonly dzId?: unknown;
   readonly score?: unknown;
   readonly ts?: unknown;
@@ -112,8 +125,18 @@ export function formatRecallUsageRecord(input: {
   readonly runId?: unknown;
   readonly eventId?: unknown;
   readonly queryTruncated?: unknown;
-}): string | undefined {
-  const rec = normalizeReadRecord(input);
+}
+
+/**
+ * The normalized RECORD, before serialization — the writer needs the object so it can hang the
+ * event-chain fields off it (`seq`/`prevHash`, ADR-001) instead of string-splicing a finished line.
+ */
+export function buildRecallUsageRecord(input: RecallUsageRecordInput): RecallUsageReadRecord | undefined {
+  return normalizeReadRecord(input);
+}
+
+export function formatRecallUsageRecord(input: RecallUsageRecordInput): string | undefined {
+  const rec = buildRecallUsageRecord(input);
   return rec === undefined ? undefined : `${JSON.stringify(rec)}\n`;
 }
 
@@ -125,6 +148,10 @@ export function parseRecallUsageLog(text: string): ParsedRecallUsageLog {
     if (trimmed === '') continue;
     try {
       const parsed = JSON.parse(trimmed) as unknown;
+      // The compaction ledger (ADR-002) is chain bookkeeping, not a usage record. Counting it as an
+      // invalid line would make `invalidLines` — a health number the report prints — lie by one per
+      // compaction generation.
+      if (isRecord(parsed) && parsed['kind'] === EVENT_CHAIN_LEDGER_KIND) continue;
       const record = normalizeRecord(parsed);
       if (record === undefined) {
         invalidLines += 1;
@@ -213,10 +240,68 @@ export function shouldCompactRecallUsageLogSize(
   return Number.isFinite(sizeBytes) && sizeBytes > validMax(maxBytes);
 }
 
-export function compactRecallUsageLog(
+export interface CompactRecallUsageOptions {
+  readonly maxBytes?: number;
+  readonly targetBytes?: number;
+  readonly compactedAt?: string;
+  /**
+   * Compact even when the input's chain is already defective. OFF by default and never set by any
+   * automatic caller — see {@link compactRecallUsageLogChecked} for why.
+   */
+  readonly force?: boolean;
+}
+
+export type CompactRecallUsageStatus = 'compacted' | 'refused-dirty' | 'too-large';
+
+export interface CompactRecallUsageResult {
+  readonly status: CompactRecallUsageStatus;
+  /** Empty unless `status === 'compacted'`. */
+  readonly text: string;
+  /** The input defects that caused a refusal. */
+  readonly defects: readonly EventChainDefect[];
+}
+
+/**
+ * Compaction with its verdict attached.
+ *
+ * AM-2 (Codex QE HIGH-2) — A REWRITER MUST NOT LAUNDER. Compaction parses the input, drops what it
+ * cannot read and re-chains from genesis, so a file carrying a `BrokenLink` or a `DoubleCounted`
+ * came out the other side verifying `ok: true`. The strongest evidence check in the system was
+ * being erased by the routine that runs automatically at a size threshold — corruption converted
+ * into a clean chain, with no record that it ever existed.
+ *
+ * So: the input is VERIFIED FIRST, and a defective chained region REFUSES. The pre-chain prefix is
+ * legal and never blocks anything (FR-5); only real defects do.
+ *
+ * ACCEPTED CONSEQUENCE, stated because it is the cost: a log that stays defective stops being
+ * compacted and grows past its cap. That is the right way round — the size cap is a convenience,
+ * the evidence is the product — and it is not silent: `dz doctor` and `dz compounding` both report
+ * the chain defect, and the caller logs the refusal.
+ */
+export function compactRecallUsageLogChecked(
   text: string,
-  opts: { readonly maxBytes?: number; readonly targetBytes?: number; readonly compactedAt?: string } = {},
-): string {
+  opts: CompactRecallUsageOptions = {},
+): CompactRecallUsageResult {
+  if (opts.force !== true) {
+    const v = verifyEventChainText(typeof text === 'string' ? text : '');
+    if (!v.ok) return { status: 'refused-dirty', text: '', defects: v.defects };
+  }
+  const out = compactVerifiedRecallUsageLog(text, opts);
+  return out === ''
+    ? { status: 'too-large', text: '', defects: [] }
+    : { status: 'compacted', text: out, defects: [] };
+}
+
+/**
+ * Back-compatible wrapper: the compacted text, or `''` when the rewrite is REFUSED (a defective
+ * input) or cannot fit. Callers that need to tell those apart use
+ * {@link compactRecallUsageLogChecked}.
+ */
+export function compactRecallUsageLog(text: string, opts: CompactRecallUsageOptions = {}): string {
+  return compactRecallUsageLogChecked(text, opts).text;
+}
+
+function compactVerifiedRecallUsageLog(text: string, opts: CompactRecallUsageOptions): string {
   const maxBytes = validMax(opts.maxBytes ?? RECALL_USAGE_LOG_MAX_BYTES);
   const targetBytes = validTarget(opts.targetBytes ?? Math.floor(maxBytes * 0.75), maxBytes);
   const compactedAt = validTs(opts.compactedAt) ? opts.compactedAt : new Date(0).toISOString();
@@ -233,23 +318,57 @@ export function compactRecallUsageLog(
   const toAggregate = parsed.filter((r) => !retained.has(r as RecallUsageReadRecord));
   const stats = aggregateRecallUsage(toAggregate);
   // Replay rows are budgeted FIRST: they are irreplaceable (the corpus), aggregates are re-derivable.
-  const lines = [...replayRows.map((r) => JSON.stringify(r)), ...stats.map((s) => aggregateLine(s, compactedAt))];
-  let out = joinLines(lines);
-  if (byteLength(out) <= maxBytes) return out;
-
-  const kept: string[] = [];
-  let used = 0;
-  for (const line of lines) {
-    const cost = byteLength(`${line}\n`);
-    if (kept.length > 0 && used + cost > targetBytes) continue;
-    if (cost > maxBytes) continue;
-    if (used + cost <= maxBytes) {
-      kept.push(line);
-      used += cost;
+  const candidates: RecallUsageRecord[] = [...replayRows, ...stats.map((s) => aggregateRecord(s, compactedAt))];
+
+  // The event weight of the INPUT, measured before aggregation. It must come from the input — a
+  // total derived from the output could never disagree with it (ADR-002), which is exactly how a
+  // double-counting rewrite went unseen until a human re-read the code.
+  let sourceEvents = 0;
+  for (const rec of parsed) sourceEvents += defaultEventWeight(rec as unknown as Record<string, unknown>);
+
+  // SELECT, then chain (ADR-002 decision 5): trimming a chain after building it punches holes in it.
+  // Each candidate is charged its own bytes plus a fixed allowance for the chain fields it will get.
+  let selected = candidates;
+  if (byteLength(joinLines(candidates.map(serializeWithChainAllowance))) > maxBytes) {
+    const kept: RecallUsageRecord[] = [];
+    let used = 0;
+    for (const rec of candidates) {
+      const cost = byteLength(`${serializeWithChainAllowance(rec)}\n`);
+      if (kept.length > 0 && used + cost > targetBytes) continue;
+      if (cost > maxBytes) continue;
+      if (used + cost <= maxBytes) {
+        kept.push(rec);
+        used += cost;
+      }
     }
+    selected = kept;
+  }
+
+  // Final shrink: drop from the TAIL and re-chain, so the survivors are always a valid chain.
+  for (;;) {
+    const out = joinLines(chainRewrite(selected, { sourceEvents, droppedEvents: droppedEvents(sourceEvents, selected), compactedAt }));
+    if (byteLength(out) <= maxBytes) return out;
+    if (selected.length === 0) return '';
+    selected = selected.slice(0, -1);
   }
-  out = joinLines(kept);
-  return byteLength(out) <= maxBytes ? out : '';
+}
+
+/**
+ * What the byte budget discarded — so a deliberate trim is never mistaken for lost records.
+ *
+ * The `Math.max(0, …)` is LOAD-BEARING, not defensive tidiness: it is what stops a rewrite that
+ * emits MORE events than it read from explaining its own inflation away with a negative "dropped"
+ * figure. Over-accounting therefore always reaches {@link verifyEventChain} as `DoubleCounted`.
+ */
+function droppedEvents(sourceEvents: number, selected: readonly RecallUsageRecord[]): number {
+  let accounted = 0;
+  for (const rec of selected) accounted += defaultEventWeight(rec as unknown as Record<string, unknown>);
+  return Math.max(0, sourceEvents - accounted);
+}
+
+/** A record's serialized size plus the allowance for the chain fields it will carry. */
+function serializeWithChainAllowance(rec: RecallUsageRecord): string {
+  return JSON.stringify(rec) + ' '.repeat(EVENT_CHAIN_FIELD_OVERHEAD_BYTES);
 }
 
 function normalizeRecord(value: unknown): RecallUsageRecord | undefined {
@@ -356,8 +475,8 @@ function mergeAggregate(byId: Map<string, Acc>, rec: RecallUsageAggregateRecord)
   }
 }
 
-function aggregateLine(stat: RecallUsageStat, compactedAt: string): string {
-  const rec: RecallUsageAggregateRecord = {
+function aggregateRecord(stat: RecallUsageStat, compactedAt: string): RecallUsageAggregateRecord {
+  return {
     kind: 'aggregate',
     dzId: stat.dzId,
     reads: stat.reads,
@@ -367,7 +486,6 @@ function aggregateLine(stat: RecallUsageStat, compactedAt: string): string {
     totalScore: stat.avgScore * stat.reads,
     compactedAt,
   };
-  return JSON.stringify(rec);
 }
 
 function patternRef(p: RecallPatternUsageRef): RecallPatternUsageRef {

package/src/usage.ts

@@ -45,6 +45,77 @@ const MTIME_SLACK_MS = HOUR_MS;
  */
 export const TOKEN_WEIGHTS = { input: 1, cacheWrite: 1.25, cacheWrite1h: 2, cacheRead: 0.1, output: 5 } as const;
 
+/** The four raw token buckets of one `message.usage` record, after clamping. */
+export interface RawTokenMix {
+  readonly input: number;
+  /** Cache-CREATION tokens, already priced at their TTL rate inside {@link weightedTokensOf}. */
+  readonly cacheWrite: number;
+  readonly cacheRead: number;
+  readonly output: number;
+}
+
+function positiveFinite(v: unknown): number {
+  return typeof v === 'number' && isFinite(v) && v > 0 ? v : 0;
+}
+
+/**
+ * The raw token buckets of an Anthropic `message.usage` object, clamped to finite non-negatives.
+ * `cacheWrite` prefers the TTL breakdown (`cache_creation.ephemeral_*`) and falls back to the flat
+ * `cache_creation_input_tokens` — reading only the flat field scored a nested-only record as ZERO.
+ */
+export function rawTokenMixOf(usage: unknown): RawTokenMix {
+  if (typeof usage !== 'object' || usage === null) {
+    return { input: 0, cacheWrite: 0, cacheRead: 0, output: 0 };
+  }
+  const u = usage as Record<string, unknown>;
+  const cc = (typeof u['cache_creation'] === 'object' && u['cache_creation'] !== null
+    ? (u['cache_creation'] as Record<string, unknown>)
+    : {}) as Record<string, unknown>;
+  const c5 = positiveFinite(cc['ephemeral_5m_input_tokens']);
+  const c1h = positiveFinite(cc['ephemeral_1h_input_tokens']);
+  const cacheWrite = c5 + c1h > 0 ? c5 + c1h : positiveFinite(u['cache_creation_input_tokens']);
+  return {
+    input: positiveFinite(u['input_tokens']),
+    cacheWrite,
+    cacheRead: positiveFinite(u['cache_read_input_tokens']),
+    output: positiveFinite(u['output_tokens']),
+  };
+}
+
+/**
+ * THE estimator — cost-weighted "input-equivalent" tokens for one `message.usage` object.
+ *
+ * A flat token sum is 89-99.7% `cache_read` on this machine, which tracks CONVERSATION LENGTH
+ * rather than work done. {@link TOKEN_WEIGHTS} are the published per-token price ratios relative to
+ * base input, so the result tracks consumption instead of context size.
+ *
+ * Extracted verbatim from `computeUsage`'s per-sample arithmetic so that `dz usage` and the
+ * per-stage cost ledger measure the same quantity (feature `cost-ledger`, ADR-002 — the invariant
+ * only means something if both sides use ONE estimator). The return value is UNROUNDED; callers
+ * that need exact integer identities round once at their own extraction point.
+ */
+export function weightedTokensOf(usage: unknown): number {
+  if (typeof usage !== 'object' || usage === null) return 0;
+  const mix = rawTokenMixOf(usage);
+  const u = usage as Record<string, unknown>;
+  const cc =
+    typeof u['cache_creation'] === 'object' && u['cache_creation'] !== null
+      ? (u['cache_creation'] as Record<string, unknown>)
+      : {};
+  const c5 = positiveFinite(cc['ephemeral_5m_input_tokens']);
+  const c1h = positiveFinite(cc['ephemeral_1h_input_tokens']);
+  const cacheWriteCost =
+    c5 + c1h > 0
+      ? c5 * TOKEN_WEIGHTS.cacheWrite + c1h * TOKEN_WEIGHTS.cacheWrite1h
+      : mix.cacheWrite * TOKEN_WEIGHTS.cacheWrite;
+  return (
+    mix.input * TOKEN_WEIGHTS.input +
+    cacheWriteCost +
+    mix.cacheRead * TOKEN_WEIGHTS.cacheRead +
+    mix.output * TOKEN_WEIGHTS.output
+  );
+}
+
 export const CLAUDE_USAGE_MODELS = ['fable', 'opus', 'sonnet', 'haiku'] as const;
 export type ClaudeUsageModel = (typeof CLAUDE_USAGE_MODELS)[number];
 
@@ -230,7 +301,7 @@ export function normalizeClaudeUsageModelKey(raw: unknown): ClaudeUsageModel | n
  * The `~/.claude/projects` root (the account-wide transcript store). Overridable via
  * `DZ_CLAUDE_PROJECTS_ROOT` — used by tests to point at a temp tree. Never throws.
  */
-function claudeProjectsRoot(): string {
+export function claudeProjectsRoot(): string {
   const override = process.env['DZ_CLAUDE_PROJECTS_ROOT'];
   if (typeof override === 'string' && override.length > 0) return override;
   return join(homedir(), '.claude', 'projects');
@@ -414,17 +485,8 @@ function extractSamples(path: string, scanCutoff: number, into: Sample[], seen:
     // tokens": a quantity that tracks consumption instead of context size.
     // Prefer the TTL breakdown when present (5m 1.25x / 1h 2x); fall back to the flat field at the
     // 5m rate. Reading only the flat field scored a nested-only record as ZERO.
-    const c5 = n(usage.cache_creation?.ephemeral_5m_input_tokens);
-    const c1h = n(usage.cache_creation?.ephemeral_1h_input_tokens);
-    const cacheWriteCost =
-      c5 + c1h > 0
-        ? c5 * TOKEN_WEIGHTS.cacheWrite + c1h * TOKEN_WEIGHTS.cacheWrite1h
-        : n(usage.cache_creation_input_tokens) * TOKEN_WEIGHTS.cacheWrite;
-    const tokens =
-      n(usage.input_tokens) * TOKEN_WEIGHTS.input +
-      cacheWriteCost +
-      n(usage.cache_read_input_tokens) * TOKEN_WEIGHTS.cacheRead +
-      n(usage.output_tokens) * TOKEN_WEIGHTS.output;
+    // ONE estimator, shared with the per-stage cost ledger (feature `cost-ledger`, ADR-002).
+    const tokens = weightedTokensOf(usage);
     if (tokens <= 0) continue;
     // Dedup: streamed assistant messages repeat their usage object across chunks.
     const id = typeof rec.message?.id === 'string' ? rec.message.id : '';