harnery 0.35.0 → 0.36.0

package/src/core/semantic/usage.ts

@@ -0,0 +1,272 @@
+import { countTokens } from "gpt-tokenizer/model/gpt-4o";
+import {
+  SEMANTIC_USAGE_SCHEMA_VERSION,
+  SEMANTIC_VISIBLE_USAGE_ESTIMATOR_ID,
+  SEMANTIC_VISIBLE_USAGE_ESTIMATOR_VERSION,
+  type SemanticConfiguredModel,
+  type SemanticHarness,
+  type SemanticInvalidReasonCode,
+  type SemanticUsageReceiptV1,
+  type SemanticUsageToken,
+} from "./contract.ts";
+
+export const SEMANTIC_USAGE_TOKEN_FIELDS = [
+  "input_tokens",
+  "cached_input_tokens",
+  "cache_creation_input_tokens",
+  "output_tokens",
+  "reasoning_tokens",
+  "total_tokens",
+] as const;
+
+export type SemanticUsageTokenField = (typeof SEMANTIC_USAGE_TOKEN_FIELDS)[number];
+export type SemanticUsageTokenTotals = Partial<Record<SemanticUsageTokenField, number>>;
+export type SemanticUsageOutcome = "accepted" | "invalid" | "unavailable" | "deferred";
+
+export interface SemanticUsageEvent {
+  source_harness?: SemanticHarness;
+  configured_model?: SemanticConfiguredModel;
+  resolved_model_id?: string;
+  model_attestation?: "verified" | "requested-only";
+  action?: string;
+  model_call: boolean;
+  usage?: SemanticUsageReceiptV1;
+  invalid_reason_codes?: SemanticInvalidReasonCode[];
+}
+
+export interface SemanticUsageBreakdown {
+  harness?: SemanticHarness;
+  configured_model?: SemanticConfiguredModel;
+  resolved_model_id?: string;
+  model_attestation?: "verified" | "requested-only";
+  call_count: number;
+  outcomes: Record<SemanticUsageOutcome, number>;
+  invalid_reasons: Partial<Record<SemanticInvalidReasonCode, number>>;
+  native_tokens: SemanticUsageTokenTotals;
+  estimated_tokens: SemanticUsageTokenTotals;
+  unreported_calls: number;
+}
+
+export interface SemanticUsageAggregate {
+  call_count: number;
+  outcomes: Record<SemanticUsageOutcome, number>;
+  invalid_reasons: Partial<Record<SemanticInvalidReasonCode, number>>;
+  native_tokens: SemanticUsageTokenTotals;
+  estimated_tokens: SemanticUsageTokenTotals;
+  unreported_calls: number;
+  breakdowns: SemanticUsageBreakdown[];
+}
+
+export function nativeSemanticUsage(
+  values: Partial<Record<SemanticUsageTokenField, unknown>>,
+): SemanticUsageReceiptV1 | undefined {
+  const tokens: Partial<Record<SemanticUsageTokenField, SemanticUsageToken>> = {};
+  for (const field of SEMANTIC_USAGE_TOKEN_FIELDS) {
+    const value = nonNegativeInteger(values[field]);
+    if (value !== undefined) tokens[field] = { value, provenance: "native" };
+  }
+  if (Object.keys(tokens).length === 0) return undefined;
+  return {
+    schema_version: SEMANTIC_USAGE_SCHEMA_VERSION,
+    source: "native",
+    scope: "harness-call",
+    tokens,
+  } as SemanticUsageReceiptV1;
+}
+
+export function estimateVisibleSemanticUsage(
+  prompt: string,
+  response: string,
+): SemanticUsageReceiptV1 {
+  const input = countTokens(prompt);
+  const output = countTokens(response);
+  return {
+    schema_version: SEMANTIC_USAGE_SCHEMA_VERSION,
+    source: "estimated",
+    scope: "visible-payload",
+    estimator: {
+      id: SEMANTIC_VISIBLE_USAGE_ESTIMATOR_ID,
+      version: SEMANTIC_VISIBLE_USAGE_ESTIMATOR_VERSION,
+    },
+    tokens: {
+      input_tokens: { value: input, provenance: "estimated" },
+      output_tokens: { value: output, provenance: "estimated" },
+      total_tokens: { value: input + output, provenance: "estimated" },
+    },
+  };
+}
+
+export function unreportedSemanticUsage(): SemanticUsageReceiptV1 {
+  return {
+    schema_version: SEMANTIC_USAGE_SCHEMA_VERSION,
+    source: "unreported",
+    scope: "unreported",
+  };
+}
+
+export function aggregateSemanticUsage(
+  events: readonly SemanticUsageEvent[],
+): SemanticUsageAggregate {
+  const aggregate = emptySemanticUsageAggregate();
+  const rows = new Map<string, SemanticUsageBreakdown>();
+  for (const event of events) {
+    const outcome = semanticUsageOutcome(event.action);
+    if (outcome) aggregate.outcomes[outcome] += 1;
+    addInvalidReasons(aggregate.invalid_reasons, event.invalid_reason_codes);
+    if (event.model_call) {
+      aggregate.call_count += 1;
+      addReceipt(aggregate, event.usage);
+    }
+    if (!outcome && !event.model_call) continue;
+    const key = [
+      event.source_harness ?? "unattributed",
+      event.configured_model ?? "unattributed",
+      event.resolved_model_id ?? "unresolved",
+      event.model_attestation ?? "unattested",
+    ].join("\u0000");
+    const row = rows.get(key) ?? {
+      ...(event.source_harness ? { harness: event.source_harness } : {}),
+      ...(event.configured_model ? { configured_model: event.configured_model } : {}),
+      ...(event.resolved_model_id ? { resolved_model_id: event.resolved_model_id } : {}),
+      ...(event.model_attestation ? { model_attestation: event.model_attestation } : {}),
+      call_count: 0,
+      outcomes: emptyOutcomes(),
+      invalid_reasons: {},
+      native_tokens: {},
+      estimated_tokens: {},
+      unreported_calls: 0,
+    };
+    if (outcome) row.outcomes[outcome] += 1;
+    addInvalidReasons(row.invalid_reasons, event.invalid_reason_codes);
+    if (event.model_call) {
+      row.call_count += 1;
+      addReceipt(row, event.usage);
+    }
+    rows.set(key, row);
+  }
+  aggregate.breakdowns = [...rows.values()].sort((left, right) =>
+    `${left.harness ?? ""}/${left.configured_model ?? ""}/${left.resolved_model_id ?? ""}`.localeCompare(
+      `${right.harness ?? ""}/${right.configured_model ?? ""}/${right.resolved_model_id ?? ""}`,
+    ),
+  );
+  return aggregate;
+}
+
+export function mergeSemanticUsageAggregates(
+  left: SemanticUsageAggregate,
+  right: SemanticUsageAggregate,
+): SemanticUsageAggregate {
+  const merged = emptySemanticUsageAggregate();
+  merged.call_count = left.call_count + right.call_count;
+  merged.unreported_calls = left.unreported_calls + right.unreported_calls;
+  for (const outcome of Object.keys(merged.outcomes) as SemanticUsageOutcome[]) {
+    merged.outcomes[outcome] = left.outcomes[outcome] + right.outcomes[outcome];
+  }
+  addInvalidReasons(merged.invalid_reasons, left.invalid_reasons);
+  addInvalidReasons(merged.invalid_reasons, right.invalid_reasons);
+  addTotals(merged.native_tokens, left.native_tokens);
+  addTotals(merged.native_tokens, right.native_tokens);
+  addTotals(merged.estimated_tokens, left.estimated_tokens);
+  addTotals(merged.estimated_tokens, right.estimated_tokens);
+  const rows = new Map<string, SemanticUsageBreakdown>();
+  for (const row of [...left.breakdowns, ...right.breakdowns]) {
+    const key = [
+      row.harness ?? "unattributed",
+      row.configured_model ?? "unattributed",
+      row.resolved_model_id ?? "unresolved",
+      row.model_attestation ?? "unattested",
+    ].join("\u0000");
+    const prior = rows.get(key);
+    if (!prior) {
+      rows.set(key, structuredClone(row));
+      continue;
+    }
+    prior.call_count += row.call_count;
+    prior.unreported_calls += row.unreported_calls;
+    addInvalidReasons(prior.invalid_reasons, row.invalid_reasons);
+    for (const outcome of Object.keys(prior.outcomes) as SemanticUsageOutcome[]) {
+      prior.outcomes[outcome] += row.outcomes[outcome];
+    }
+    addTotals(prior.native_tokens, row.native_tokens);
+    addTotals(prior.estimated_tokens, row.estimated_tokens);
+  }
+  merged.breakdowns = [...rows.values()];
+  return merged;
+}
+
+export function emptySemanticUsageAggregate(): SemanticUsageAggregate {
+  return {
+    call_count: 0,
+    outcomes: emptyOutcomes(),
+    invalid_reasons: {},
+    native_tokens: {},
+    estimated_tokens: {},
+    unreported_calls: 0,
+    breakdowns: [],
+  };
+}
+
+function addInvalidReasons(
+  target: Partial<Record<SemanticInvalidReasonCode, number>>,
+  source:
+    | readonly SemanticInvalidReasonCode[]
+    | Partial<Record<SemanticInvalidReasonCode, number>>
+    | undefined,
+): void {
+  if (!source) return;
+  if (Array.isArray(source)) {
+    for (const code of source as readonly SemanticInvalidReasonCode[]) {
+      target[code] = (target[code] ?? 0) + 1;
+    }
+    return;
+  }
+  for (const [code, count] of Object.entries(
+    source as Partial<Record<SemanticInvalidReasonCode, number>>,
+  )) {
+    if (typeof count === "number" && count > 0) {
+      const key = code as SemanticInvalidReasonCode;
+      target[key] = (target[key] ?? 0) + count;
+    }
+  }
+}
+
+function addReceipt(
+  target: Pick<SemanticUsageAggregate, "native_tokens" | "estimated_tokens" | "unreported_calls">,
+  usage: SemanticUsageReceiptV1 | undefined,
+): void {
+  if (!usage || usage.source === "unreported") {
+    target.unreported_calls += 1;
+    return;
+  }
+  for (const field of SEMANTIC_USAGE_TOKEN_FIELDS) {
+    const token = usage.tokens[field as keyof typeof usage.tokens] as
+      | SemanticUsageToken
+      | undefined;
+    if (!token) continue;
+    const totals = token.provenance === "native" ? target.native_tokens : target.estimated_tokens;
+    totals[field] = (totals[field] ?? 0) + token.value;
+  }
+}
+
+function addTotals(target: SemanticUsageTokenTotals, source: SemanticUsageTokenTotals): void {
+  for (const field of SEMANTIC_USAGE_TOKEN_FIELDS) {
+    if (source[field] !== undefined) target[field] = (target[field] ?? 0) + source[field]!;
+  }
+}
+
+function emptyOutcomes(): Record<SemanticUsageOutcome, number> {
+  return { accepted: 0, invalid: 0, unavailable: 0, deferred: 0 };
+}
+
+function semanticUsageOutcome(value: string | undefined): SemanticUsageOutcome | undefined {
+  return value === "accepted" ||
+    value === "invalid" ||
+    value === "unavailable" ||
+    value === "deferred"
+    ? value
+    : undefined;
+}
+
+function nonNegativeInteger(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined;
+}

package/src/core/semantic/validate.ts

@@ -1,11 +1,13 @@
 import { Value } from "@sinclair/typebox/value";
 import {
+  SEMANTIC_INVALID_REASON_CODES,
   SEMANTIC_TAGS,
   type SemanticAgentReadModelV2,
   SemanticAgentReadModelV2Schema,
   type SemanticEvidenceKind,
   type SemanticEvidenceV1,
   type SemanticField,
+  type SemanticInvalidReasonCode,
   type SemanticMeaningV2,
   type SemanticModelReplyV2,
   SemanticModelReplyV2Schema,
@@ -158,6 +160,35 @@ export function validateSemanticModelReply(
   return issues.length ? { ok: false, issues: [...new Set(issues)] } : { ok: true, value: reply };
 }
 
+/** Reduce detailed validator messages to bounded, privacy-safe status categories. */
+export function classifySemanticValidationIssues(
+  issues: readonly string[],
+): SemanticInvalidReasonCode[] {
+  const found = new Set<SemanticInvalidReasonCode>();
+  for (const issue of issues) {
+    if (issue.startsWith("schema:")) found.add("schema");
+    else if (issue === "generation_mismatch" || issue === "evidence_digest_mismatch") {
+      found.add("identity");
+    } else if (issue.includes("citation") || issue === "empty_tags_must_not_cite") {
+      found.add("citation");
+    } else if (issue.includes("must_be_model_synthesis") || issue.includes("prediction")) {
+      found.add("basis");
+    } else if (issue.includes("confidence")) found.add("confidence");
+    else if (issue.includes(":privacy_")) found.add("privacy");
+    else if (
+      issue.includes("unsupported_") ||
+      issue.endsWith(":percent_complete") ||
+      issue.endsWith(":time_remaining") ||
+      issue.endsWith(":emotion") ||
+      issue.endsWith(":quality")
+    ) {
+      found.add("unsupported_claim");
+    } else found.add("unknown");
+  }
+  if (found.size === 0) found.add("unknown");
+  return SEMANTIC_INVALID_REASON_CODES.filter((code) => found.has(code));
+}
+
 export function validateSemanticReadModel(
   input: unknown,
   expected?: Pick<SemanticEvidenceV1, "instance_id" | "generation_id" | "evidence_digest">,

package/src/core/storage/buffer.ts

@@ -0,0 +1,124 @@
+export interface BufferedLogItem<T> {
+  value: T;
+  bytes: number;
+  high_priority: boolean;
+}
+
+export interface BoundedLogBufferOptions<T> {
+  max_bytes: number;
+  max_records: number;
+  high_severity_reserve_bytes: number;
+  flush_interval_ms: number;
+  minimum_high_priority_drain_interval_ms?: number;
+  drain(items: readonly BufferedLogItem<T>[]): Promise<void>;
+  on_drop?(reason: "normal_capacity" | "total_capacity" | "closed", item: T): void;
+  on_drain_error?(error: unknown): void;
+}
+
+export class BoundedLogBuffer<T> {
+  readonly #options: BoundedLogBufferOptions<T>;
+  #items: BufferedLogItem<T>[] = [];
+  #bytes = 0;
+  #closed = false;
+  #timer: ReturnType<typeof setTimeout> | undefined;
+  #draining: Promise<void> = Promise.resolve();
+  #lastHighPriorityDrain = 0;
+
+  constructor(options: BoundedLogBufferOptions<T>) {
+    if (
+      !Number.isSafeInteger(options.max_bytes) ||
+      options.max_bytes <= 0 ||
+      !Number.isSafeInteger(options.max_records) ||
+      options.max_records <= 0 ||
+      options.high_severity_reserve_bytes < 0 ||
+      options.high_severity_reserve_bytes >= options.max_bytes ||
+      options.flush_interval_ms <= 0
+    ) {
+      throw new Error("invalid bounded log buffer policy");
+    }
+    this.#options = options;
+  }
+
+  get size(): { records: number; bytes: number; capacity_bytes: number } {
+    return {
+      records: this.#items.length,
+      bytes: this.#bytes,
+      capacity_bytes: this.#options.max_bytes,
+    };
+  }
+
+  enqueue(item: BufferedLogItem<T>): boolean {
+    if (this.#closed) return this.#drop("closed", item.value);
+    if (!Number.isSafeInteger(item.bytes) || item.bytes <= 0) {
+      throw new Error("buffer item bytes must be a positive safe integer");
+    }
+    const byteLimit = item.high_priority
+      ? this.#options.max_bytes
+      : this.#options.max_bytes - this.#options.high_severity_reserve_bytes;
+    if (this.#items.length >= this.#options.max_records) {
+      return this.#drop(item.high_priority ? "total_capacity" : "normal_capacity", item.value);
+    }
+    if (this.#bytes + item.bytes > byteLimit) {
+      return this.#drop(item.high_priority ? "total_capacity" : "normal_capacity", item.value);
+    }
+    this.#items.push(item);
+    this.#bytes += item.bytes;
+    this.#scheduleTimer();
+    if (item.high_priority) this.#scheduleHighPriorityDrain();
+    return true;
+  }
+
+  async flush(): Promise<void> {
+    this.#cancelTimer();
+    const batch = this.#take();
+    if (batch.length > 0) {
+      this.#draining = this.#draining.then(() => this.#options.drain(batch));
+    }
+    await this.#draining;
+  }
+
+  async close(): Promise<void> {
+    if (this.#closed) return this.#draining;
+    this.#closed = true;
+    await this.flush();
+  }
+
+  #take(): BufferedLogItem<T>[] {
+    const batch = this.#items;
+    this.#items = [];
+    this.#bytes = 0;
+    return batch;
+  }
+
+  #scheduleTimer(): void {
+    if (this.#timer) return;
+    this.#timer = setTimeout(() => {
+      this.#timer = undefined;
+      void this.flush().catch((error) => this.#options.on_drain_error?.(error));
+    }, this.#options.flush_interval_ms);
+    this.#timer.unref?.();
+  }
+
+  #scheduleHighPriorityDrain(): void {
+    const minimum = this.#options.minimum_high_priority_drain_interval_ms ?? 25;
+    const now = Date.now();
+    const delay = Math.max(0, minimum - (now - this.#lastHighPriorityDrain));
+    if (this.#timer) clearTimeout(this.#timer);
+    this.#timer = setTimeout(() => {
+      this.#timer = undefined;
+      this.#lastHighPriorityDrain = Date.now();
+      void this.flush().catch((error) => this.#options.on_drain_error?.(error));
+    }, delay);
+    this.#timer.unref?.();
+  }
+
+  #cancelTimer(): void {
+    if (this.#timer) clearTimeout(this.#timer);
+    this.#timer = undefined;
+  }
+
+  #drop(reason: "normal_capacity" | "total_capacity" | "closed", item: T): false {
+    this.#options.on_drop?.(reason, item);
+    return false;
+  }
+}