@ngockhoale/ukit 3.4.22 → 3.4.24

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 All notable changes to UKit are documented here.
 
+## 3.4.24 - 2026-10-08
+
+**Honest per-category token usage record + report** (C98; capability only — no producer exists yet, so real stores report `no_data`).
+- Schema v1.3 (additive): `model.completed`/`model.failed` payloads may now carry `usage_categories` — the five typed `{value, source}` resources `fresh_input_tokens`, `cache_read_tokens`, `cache_write_tokens`, `output_tokens`, `reasoning_tokens`. Absent categories stay `UNKNOWN`/null — never 0, never estimated. The registry stays append-only; old records remain valid.
+- `routeAdapter` maps provider-reported categories when a producer supplies `usage.categories`; numeric values are accepted only from `PROVIDER` sources, and the legacy aggregate-token mapping is byte-unchanged.
+- New read-only `ukit telemetry usage [--json]`: scans the segment store non-destructively (corrupt segments are skipped and reported, never quarantined by a read), prints a readiness verdict `no_data | aggregate_only | categorized`, per-source totals only (never combined), record counts (not unique runs), and labels `cache.hit/miss` as UKit-internal lookups, not a provider prompt cache. Exit codes: 0 normal, 1 usage error, 2 typed store failure. Creates no files.
+- `readSegments(..., {readOnly: true})` new option: default quarantine behavior is byte-for-byte unchanged.
+- Privacy: the five category keys + `usage_categories` joined the support-projection allowlist and `REDACTION_VERSION` bumped to `df-redact-7` (existing support bundles invalidate once). Counts and source tags only — never content.
+- Not included: a producer that actually records provider token categories (queued Q-C98-1/Q-C98-2 — needs evidence the host exposes usage), any cost/price model or provider identity fields (Q-C98-3). No token-savings or cost claims are made or implied.
+
+## 3.4.23 - 2026-10-08
+
+**Core hook/gate reliability repairs** (C97; each backed by baseline RED, GREEN and independent different-model review).
+- `stale-spec-check`: containment errors for external/traversal targets are caught at the checker boundary instead of crashing the hook chain.
+- `hook-chain-runner`: bounded failure provenance and diagnostics while staying fail-closed.
+- `docContracts`: `checkRenderVars` and `collectScannableDocs` prune isolated worktree checkouts; the walk skips `node_modules` and breaks symlink cycles by realpath (live-repo contract check 12.4s -> 0.3s).
+- Handoff intent: exact command-suffix boundary, `handoff-implement` recognized, torn newest JSONL entry yields unknown instead of an older opposite intent; `handoff-create`-owned runs stay planning-only through Stop/resume.
+- `protect-files`: backslash path spellings get the same protection as slash paths.
+- `land-worktree`: complete base-relative copy-back (committed, staged, deleted, renamed, newline paths).
+- `cleanup-task-worktree`: receipt-gated, recoverable cleanup with source/destination authority containment; no forced removal.
+- Vision: already-pending unread image is idempotent; a corrupt pending marker is retained with a diagnostic.
+- `output-compression`: a subagent's raw tool output no longer charges the parent's compact-pressure session (hard-cap gates unchanged).
+- Not included: shell create-scope detection (TASK-C97-007) and decision-log lock reclaim (TASK-C97-013) failed final review and were reverted to the 3.4.22 behavior.
+- Packed-artifact size cap raised 3.16 MB -> 3.17 MB.
+
 ## 3.4.22 - 2026-10-07
 
 **Windows: `.sh` hooks no longer deny every Edit/Write/Bash.**

package/manifests/platform.full.yaml

@@ -1653,6 +1653,35 @@ items:
     packs:
       - core
 
+  # C97 TASK-C97-009 (MODULE04 FR-M4-004): the task-scoped landing helper that
+  # replaces the unstaged-only cp loop in the fullstack/implement 3c samples.
+  - id: ukit-index-land-worktree-script
+    type: config
+    sourceTemplate: .claude/ukit/index/land-worktree.mjs
+    targetPath: .claude/ukit/index/land-worktree.mjs
+    requires:
+      - ukit-runtime-scripts
+    mergeStrategy: overwrite_with_backup
+    variables: []
+    enabledByDefault: true
+    packs:
+      - core
+
+  # C97 TASK-C97-010 (MODULE04 FR-M4-003/003a): the receipt-gated recoverable
+  # cleanup helper that replaces the force-remove/-D samples in the fullstack,
+  # implement and clear command docs.
+  - id: ukit-index-cleanup-task-worktree-script
+    type: config
+    sourceTemplate: .claude/ukit/index/cleanup-task-worktree.mjs
+    targetPath: .claude/ukit/index/cleanup-task-worktree.mjs
+    requires:
+      - ukit-runtime-scripts
+    mergeStrategy: overwrite_with_backup
+    variables: []
+    enabledByDefault: true
+    packs:
+      - core
+
   - id: ukit-index-stale-spec-check-script
     type: config
     sourceTemplate: .claude/ukit/index/stale-spec-check.mjs

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ngockhoale/ukit",
-  "version": "3.4.22",
+  "version": "3.4.24",
   "description": "Install/update an index-first AI workspace for Claude Code, OpenAI Codex and omp (Oh My Pi).",
   "license": "MIT",
   "type": "module",

package/src/cli/commands/telemetry.js

@@ -3,6 +3,7 @@
 //
 //   collect          ingestStoredTelemetry → flush → maybeRefreshSupport (canary+)
 //   status [--json]  stage, segments, counters, support lag, crashes
+//   usage [--json]   read-only usage-readiness report over the segment store
 //   digest [--json]  rebuildIndex → anomalies + digest markdown
 //   export-support   projectSupport now (explicit user act — works below canary)
 //   import <path>    validateSupportBundle on a received bundle
@@ -29,6 +30,10 @@ import { ingestStoredTelemetry } from '../../core/observability/adapters/ingest.
 import { listSegments, totalSegmentBytes } from '../../core/observability/segments/internal.js';
 import { readSegments } from '../../core/observability/segments/readSegments.js';
 import { rebuildIndex } from '../../core/observability/analytics/rebuild.js';
+import {
+  buildUsageReadiness,
+  renderUsageReadiness,
+} from '../../core/observability/analytics/usageReadiness.js';
 import { renderStoreDigest } from '../../core/observability/analytics/storeDigest.js';
 import { renderTraceDigest } from '../../core/observability/analytics/digest.js';
 import {
@@ -43,8 +48,8 @@ import { isValidOperationId } from '../../core/agentRuntime/eventStore.js';
 import { runEvaluation } from '../../core/observability/evaluation/runner.js';
 
 const HELP_FLAGS = new Set(['--help', '-h', 'help']);
-const SUBCOMMANDS = new Set(['collect', 'status', 'digest', 'export-support', 'export-run', 'import', 'evaluate']);
-const JSON_SUBCOMMANDS = new Set(['status', 'digest', 'evaluate']);
+const SUBCOMMANDS = new Set(['collect', 'status', 'usage', 'digest', 'export-support', 'export-run', 'import', 'evaluate']);
+const JSON_SUBCOMMANDS = new Set(['status', 'usage', 'digest', 'evaluate']);
 
 // The automatic support refresh's canary gate lives in schedule.js via
 // resolveSeamStages (projector seam minimum) — the CLI does not pre-gate.
@@ -61,6 +66,7 @@ function printUsage() {
   console.log('  collect          Ingest stored telemetry into the flight recorder,');
   console.log('                   flush it, and refresh the support view (stage canary+)');
   console.log('  status           Show stage, segments, counters, support lag, crashes');
+  console.log('  usage            Read-only usage readiness report over the segment store');
   console.log('  digest           Rebuild the trace index; print anomalies + digest markdown');
   console.log('  export-support   Write the sanitized UKit Support bundle now');
   console.log('  export-run <op>  Write a sanitized per-run support bundle (V-06)');
@@ -68,7 +74,7 @@ function printUsage() {
   console.log('  evaluate         Run the AI evaluator lane (needs observability.evaluator config)');
   console.log('');
   console.log('Options:');
-  console.log('  --json           JSON output (status, digest, evaluate)');
+  console.log('  --json           JSON output (status, usage, digest, evaluate)');
   console.log('  -h, --help       Show this help');
 }
 
@@ -337,6 +343,59 @@ async function digestCommand({ projectRoot, json }) {
   console.log(out.join('\n'));
 }
 
+// --- usage --------------------------------------------------------------------
+
+// Scan cap (SPEC FR-07): at most 200,000 records feed the report; the reader
+// limit is one higher so a 200001st record can mark the scan as truncated.
+const USAGE_SCAN_LIMIT = 200_000;
+
+// Read-only readiness scan (TASK-C98-006, SPEC §5 FR-07): never getRecorder,
+// never a flush, never mkdir, never a quarantine — the store looks after
+// `usage` exactly as it looked before. Dispatched before loadConfig so the
+// report cannot create a runtime config either.
+async function usageCommand({ projectRoot, json }) {
+  const root = segmentsRoot(projectRoot);
+  const stream = readSegments(root, { readOnly: true, limit: USAGE_SCAN_LIMIT + 1 });
+  const records = [];
+  let truncated = false;
+  for await (const record of stream) {
+    if (records.length === USAGE_SCAN_LIMIT) {
+      truncated = true;
+      break;
+    }
+    records.push(record);
+  }
+  const report = buildUsageReadiness(records, {
+    coverage: stream.coverage,
+    truncated,
+    scanLimit: USAGE_SCAN_LIMIT,
+  });
+  if (json) console.log(JSON.stringify(report));
+  else console.log(renderUsageReadiness(report));
+
+  // Segment-level degradation stays exit 0 — the report itself declares the
+  // gap (unquarantined segments, degraded entries). Only a root-level degrade
+  // (the segments root itself unreadable, e.g. io_enotdir on a file at that
+  // path) is this command's typed failure; the resolveRoot io_enoent on a
+  // not-yet-existing tree is the missing-store success, not a failure
+  // (SPEC FR-07). resolveRoot reports the realpath'd root (macOS /var →
+  // /private/var), so compare against both spellings.
+  const degraded = Array.isArray(stream.coverage.degraded) ? stream.coverage.degraded : [];
+  let realRoot = root;
+  try {
+    realRoot = await fs.realpath(root);
+  } catch {
+    // Root absent (empty store) — the literal spelling is the only one.
+  }
+  const rootDegrade = degraded.find(
+    (e) => e && (e.file === root || e.file === realRoot) && e.reason !== 'io_enoent',
+  );
+  if (rootDegrade) {
+    console.error(`[UKit] telemetry usage failed: ${rootDegrade.reason}`);
+    process.exitCode = 2;
+  }
+}
+
 // --- export-support / import --------------------------------------------------
 
 async function exportSupportCommand({ projectRoot, config }) {
@@ -506,9 +565,13 @@ export async function runTelemetry({ projectRoot, packageRoot, argv = [] }) {
     return;
   }
 
-  const config = await loadConfig(projectRoot);
   const json = flags.includes('--json');
 
+  // Read-only scan — config is not needed and must not be created (FR-07).
+  if (sub === 'usage') return usageCommand({ projectRoot, json });
+
+  const config = await loadConfig(projectRoot);
+
   if (sub === 'collect') return collectCommand({ projectRoot, config });
   if (sub === 'status') return statusCommand({ projectRoot, config, json });
   if (sub === 'digest') return digestCommand({ projectRoot, json });

package/src/cli/index.js

@@ -141,7 +141,7 @@ export async function runCli({ argv, packageRoot, projectRoot, packageVersion })
     console.log('  memory             Inspect shared UKit memory');
     console.log('  playbook           List/show playbooks (project > user > builtin)');
     console.log('  metrics            Telemetry roll-up (route outcomes, failure patterns, memory)');
-    console.log('  telemetry          Flight recorder (collect/status/digest/export-support/import/evaluate)');
+    console.log('  telemetry          Flight recorder (collect/status/usage/digest/export-support/import/evaluate)');
     console.log('  feedback           Record or list wrong-route feedback labels');
     console.log('  self-improve       Run the automatic learn/tune/collect pass now (hooks run it)');
     console.log('  vm                 Agent VM driver (list/show/status/shadow-run/evidence)');

package/src/core/docContracts.js

@@ -92,10 +92,27 @@ async function isDir(absPath) {
 }
 
 // Recursive repo-relative file walk under rootDir/dirRel. `filter` selects
-// files by extension etc. Never throws (unreadable dirs are skipped).
-async function walkFiles(rootDir, dirRel, filter = () => true) {
+// files by extension etc. `pruneDir(rel)` optionally excludes a directory
+// subtree BEFORE descent (default: never prune — unrelated consumers keep the
+// full walk). `dirIs` carries a per-top-level-walk set of descended directory
+// `fs.realpath`s; because `fs.stat` follows symlinks, a symlink cycle (e.g.
+// the live repo's `node_modules/node_modules -> node_modules`, or an
+// exploration worktree symlinked at the repo root) would otherwise grow
+// unbounded — the realpath guard makes every real directory traverse at most
+// once. A directory named `node_modules` is never doc scope, so it is skipped
+// by basename before its installed-tree contents are walked. Never throws
+// (unreadable dirs are skipped).
+async function walkFiles(rootDir, dirRel, filter = () => true, pruneDir = null, dirIs = null) {
+  const dirs = dirIs || new Set();
   const out = [];
   const dirAbs = dirRel ? path.join(rootDir, dirRel) : rootDir;
+  // Seed the starting directory so a child symlink pointing back at it (or at
+  // any ancestor of this walk) is recognized as a cycle immediately.
+  try {
+    dirs.add(await fs.realpath(dirAbs));
+  } catch {
+    /* realpath unavailable — descend without the cycle seed */
+  }
   let names;
   try {
     names = await fs.readdir(dirAbs);
@@ -110,12 +127,36 @@ async function walkFiles(rootDir, dirRel, filter = () => true) {
     } catch {
       continue; // dangling symlink / unreadable
     }
-    if (st.isDirectory()) out.push(...(await walkFiles(rootDir, rel, filter)));
-    else if (filter(name, rel)) out.push(rel);
+    if (st.isDirectory()) {
+      if (pruneDir && pruneDir(rel)) continue;
+      if (name === 'node_modules') continue; // never doc scope — skip the installed tree
+      let real;
+      try {
+        real = await fs.realpath(path.join(rootDir, rel));
+      } catch {
+        continue; // unreadable — nothing to descend
+      }
+      if (dirs.has(real)) continue; // symlink cycle / real directory already walked
+      dirs.add(real);
+      out.push(...(await walkFiles(rootDir, rel, filter, pruneDir, dirs)));
+    } else if (filter(name, rel)) out.push(rel);
   }
   return out;
 }
 
+// SPEC FR-004 (TASK-C97-003): true when `rel` is inside an isolated checkout
+// worktree subtree — `.claude/worktrees/<agent>/` or `.worktrees/<task>/`.
+// Exact path-COMPONENT comparison (not a string prefix), so ordinary lookalike
+// directories such as `.claude/worktrees-not-isolated/` stay in scope.
+function isIsolatedCheckoutSubtree(rel) {
+  const parts = rel.split('/');
+  for (let i = 0; i < parts.length; i++) {
+    if (parts[i] === '.worktrees') return true;
+    if (parts[i] === '.claude' && parts[i + 1] === 'worktrees') return true;
+  }
+  return false;
+}
+
 function makeCheck(id, label, severity, extra) {
   return { id, label, severity, passed: false, ...extra };
 }
@@ -314,8 +355,17 @@ async function checkRenderVars(rootDir, registry) {
     if (typeof e.path !== 'string' || e.class !== 'derived' || e.path.startsWith('template_project/')) continue;
     // Only .md doc surfaces are render targets — scripts/components/templates
     // inside derived dirs legitimately use {{ }} as their own placeholders.
+    // FR-004: descent prunes exact isolated-checkout subtrees BEFORE walking
+    // (render-only prune; raw nested CHANGELOG/docs placeholders belong to
+    // exploration worktrees, not the current checkout). All other .claude
+    // content, lookalike dirs and root instructions stay enforced.
     if (e.path.endsWith('/')) {
-      for (const f of await walkFiles(rootDir, e.path.slice(0, -1), (n) => n.endsWith('.md'))) {
+      for (const f of await walkFiles(
+        rootDir,
+        e.path.slice(0, -1),
+        (n) => n.endsWith('.md'),
+        isIsolatedCheckoutSubtree,
+      )) {
         targets.add(f);
       }
     } else if (e.path.endsWith('.md')) {
@@ -444,7 +494,11 @@ export async function collectScannableDocs(rootDir, registry) {
     if (typeof e.path === 'string' && e.path.endsWith('/')) dirs.add(e.path.slice(0, -1));
   }
   for (const dirRel of dirs) {
-    for (const f of await walkFiles(rootDir, dirRel, (n) => n.endsWith('.md'))) {
+    // FR-004 (TASK-C97-018): prune isolated-checkout subtrees before descent —
+    // exploration worktrees are not part of the current checkout, and their
+    // own node_modules symlink would otherwise drag the whole installed tree
+    // into the walk. `walkFiles` additionally breaks symlink cycles by realpath.
+    for (const f of await walkFiles(rootDir, dirRel, (n) => n.endsWith('.md'), isIsolatedCheckoutSubtree)) {
       files.add(f);
     }
   }

package/src/core/observability/adapters/routeAdapter.js

@@ -29,9 +29,22 @@ import {
   createAdapterContext,
   emitRecord,
 } from './common.js';
+import { USAGE_CATEGORY_KEYS, USAGE_CATEGORIES_FIELD } from '../schema/constants.js';
 
 const RESOURCE_SOURCES = new Set(['PROVIDER', 'LOCAL_TOKENIZER', 'ESTIMATED', 'UNKNOWN']);
 
+// C98 FR-05 producer contract (SPEC §5): `ledger.model.usage.categories`
+// carries camelCase counts normalized by the producer (freshInput/cacheRead/
+// cacheWrite/output/reasoning); the adapter maps them to the schema category
+// keys without interpreting provider-native names.
+const USAGE_CATEGORY_INPUT_KEYS = Object.freeze({
+  freshInput: 'fresh_input_tokens',
+  cacheRead: 'cache_read_tokens',
+  cacheWrite: 'cache_write_tokens',
+  output: 'output_tokens',
+  reasoning: 'reasoning_tokens',
+});
+
 function normalizeRow(row) {
   if (!isPlainObject(row)) return null;
   const audit = isPlainObject(row.audit) ? row.audit : row;
@@ -79,6 +92,29 @@ function modelResource(model) {
   return { value, source };
 }
 
+// C98 FR-05: categories are provider-reported or unknown. A numeric value is
+// accepted only when usage.source === 'PROVIDER'; any other source (incl.
+// missing/UNKNOWN/ESTIMATED/LOCAL_TOKENIZER), absent key, or non-integer
+// value stays `{ value: null, source: 'UNKNOWN' }` — never 0, never derived
+// from the aggregate, never estimated. Returns null when no categories
+// object is supplied, so the payload field is omitted entirely.
+function usageCategories(model) {
+  const usage = isPlainObject(model.usage) ? model.usage : {};
+  if (!isPlainObject(usage.categories)) return null;
+  const categories = {};
+  for (const key of USAGE_CATEGORY_KEYS) {
+    categories[key] = { value: null, source: 'UNKNOWN' };
+  }
+  if (usage.source !== 'PROVIDER') return categories;
+  for (const [inputKey, categoryKey] of Object.entries(USAGE_CATEGORY_INPUT_KEYS)) {
+    const value = usage.categories[inputKey];
+    if (typeof value === 'number' && Number.isInteger(value) && value >= 0) {
+      categories[categoryKey] = { value, source: 'PROVIDER' };
+    }
+  }
+  return categories;
+}
+
 export function adaptRouteOutcomes(rows, options = {}) {
   if (!Array.isArray(rows)) return [];
   const ctx = createAdapterContext({ writerId: 'adapter-route', ...options });
@@ -113,19 +149,22 @@ export function adaptRouteOutcomes(rows, options = {}) {
     const model = isPlainObject(ledger.model) ? ledger.model : null;
     if (model) {
       const resource = modelResource(model);
+      const categories = usageCategories(model);
       const failed = model.failed === true || asString(model.status) === 'failed';
+      const modelPayload = {
+        request_key: requestKey,
+        duration_ms:
+          typeof model.durationMs === 'number' && Number.isFinite(model.durationMs)
+            ? model.durationMs
+            : null,
+        resource,
+        telemetry_complete: resource.source !== 'UNKNOWN',
+      };
+      if (categories) modelPayload[USAGE_CATEGORIES_FIELD] = categories;
       const modelRecord = emitRecord(ctx, {
         semantic_name: failed ? 'model.failed' : 'model.completed',
         wall_time_utc: asIsoTime(audit.ts) ?? asIsoTime(ledger.updatedAt),
-        payload: {
-          request_key: requestKey,
-          duration_ms:
-            typeof model.durationMs === 'number' && Number.isFinite(model.durationMs)
-              ? model.durationMs
-              : null,
-          resource,
-          telemetry_complete: resource.source !== 'UNKNOWN',
-        },
+        payload: modelPayload,
       });
       if (modelRecord) records.push(modelRecord);
     }