@ngockhoale/ukit 1.1.6

package/templates/.claude/ukit/runtime/token-utils.mjs

@@ -0,0 +1,500 @@
+#!/usr/bin/env node
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+export const DEFAULT_PROMPT_CACHE_MAX_ENTRIES = 20;
+export const DEFAULT_COMPACT_HISTORY_MAX_ENTRIES = 50;
+const DEFAULT_MAX_COMPACT_ANCHORS = 3;
+
+const HEADER_RE = /^\s*(?:#{1,6}\s|={3,}|-{3,})/;
+const CODE_FENCE_RE = /^\s*```/;
+const BULLET_RE = /^(\s*[-*]\s+)(.*)$/;
+const COMMAND_RE = /^\s*(?:[$>#]\s*)?(?:node|npm|pnpm|yarn|bun|python(?:3)?|pip|git|docker|kubectl|make|cargo|go|java|pytest|vitest|npx)\b/i;
+const STRUCTURED_LINE_RE = /https?:\/\/|(?:^|[\s(])(?:src|tests?|docs|ukit|\.claude|\.codex|app|lib|packages|scripts)\//i;
+const VERBOSE_REPLACEMENTS = [
+  [/\bplease\b/gi, ''],
+  [/\bremember to\b/gi, ''],
+  [/\bmake sure to\b/gi, 'ensure '],
+  [/\bin order to\b/gi, 'to'],
+  [/\bfor the current task\b/gi, 'for this task'],
+  [/\bcurrent task\b/gi, 'task'],
+  [/\bprevious context block\b/gi, 'context block'],
+  [/\bshould still\b/gi, 'still'],
+];
+
+export function buildRuntimePaths(projectRoot) {
+  const runtimeRoot = path.join(projectRoot, '.ukit');
+  const storageRoot = path.join(runtimeRoot, 'storage');
+  const cacheRoot = path.join(storageRoot, 'cache');
+  const memoryRoot = path.join(storageRoot, 'memory');
+
+  return {
+    runtimeRoot,
+    storageRoot,
+    cacheRoot,
+    memoryRoot,
+    teeCacheDir: path.join(cacheRoot, 'tee'),
+    configPath: path.join(storageRoot, 'config.json'),
+    promptCachePath: path.join(cacheRoot, 'prompt-cache.json'),
+    compactHistoryPath: path.join(cacheRoot, 'compact-history.json'),
+    compactPressurePath: path.join(cacheRoot, 'compact-pressure.json'),
+    outputHistoryPath: path.join(cacheRoot, 'output-history.json'),
+    userMemoryPath: path.join(memoryRoot, 'user.json'),
+    projectsDir: path.join(memoryRoot, 'projects'),
+    sessionsDir: path.join(memoryRoot, 'sessions'),
+  };
+}
+
+export async function readJson(filePath, fallback = null) {
+  try {
+    return JSON.parse(await fs.readFile(filePath, 'utf8'));
+  } catch {
+    return fallback;
+  }
+}
+
+async function writeJson(filePath, value) {
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  await fs.writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
+}
+
+export function buildCompactMachineKey(prefix, payload = {}) {
+  return `${prefix}:${crypto.createHash('sha256').update(JSON.stringify(payload)).digest('base64url')}`;
+}
+
+export function estimateTokenCount(value) {
+  const text = String(value ?? '').trim();
+  if (!text) {
+    return 0;
+  }
+
+  return Math.max(1, Math.ceil(text.length / 4));
+}
+
+function isStructuredLine(line) {
+  return (
+    CODE_FENCE_RE.test(line)
+    || HEADER_RE.test(line)
+    || COMMAND_RE.test(line)
+    || STRUCTURED_LINE_RE.test(line)
+    || line.includes('`')
+  );
+}
+
+function normalizeWhitespace(value) {
+  return value
+    .replace(/\s+\|\s+/g, ' | ')
+    .replace(/\s*[:;]\s*/g, ': ')
+    .replace(/\s*,\s*/g, ', ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+export function compressLine(line) {
+  const raw = String(line ?? '');
+  if (!raw.trim()) {
+    return '';
+  }
+
+  const trimmed = raw.trim();
+  if (isStructuredLine(trimmed)) {
+    return trimmed;
+  }
+
+  const bulletMatch = raw.match(BULLET_RE);
+  const prefix = bulletMatch ? '- ' : '';
+  let next = bulletMatch ? bulletMatch[2].trim() : trimmed;
+
+  for (const [pattern, replacement] of VERBOSE_REPLACEMENTS) {
+    next = next.replace(pattern, replacement);
+  }
+
+  next = normalizeWhitespace(next);
+  return prefix ? `${prefix}${next}` : next;
+}
+
+export function compressMarkdownLines(lines, { maxTokens = 320, maxLines = 12, preserveFirstLine = true } = {}) {
+  const sourceLines = Array.isArray(lines) ? lines : [];
+  const selected = [];
+  const seen = new Set();
+  let usedTokens = 0;
+  let nonEmptyCount = 0;
+  let pendingBlankLine = false;
+
+  for (const [index, line] of sourceLines.entries()) {
+    const compressed = compressLine(line);
+    if (!compressed) {
+      pendingBlankLine = selected.length > 0 && selected[selected.length - 1] !== '';
+      continue;
+    }
+
+    const dedupeKey = compressed.toLowerCase();
+    if (seen.has(dedupeKey)) {
+      continue;
+    }
+
+    const tokens = estimateTokenCount(compressed);
+    const wouldOverflow = selected.length > 0 && (usedTokens + tokens) > maxTokens;
+    const wouldExceedLines = index > 0 && nonEmptyCount >= maxLines;
+    if (wouldOverflow || wouldExceedLines) {
+      break;
+    }
+
+    if (pendingBlankLine && selected.length > 0) {
+      selected.push('');
+      pendingBlankLine = false;
+    }
+
+    selected.push(compressed);
+    seen.add(dedupeKey);
+    usedTokens += tokens;
+    nonEmptyCount += 1;
+  }
+
+  if (preserveFirstLine && selected.length === 0 && sourceLines.length > 0) {
+    const fallback = compressLine(sourceLines[0]);
+    if (fallback) {
+      return [fallback];
+    }
+  }
+
+  return selected;
+}
+
+export function compactContextBlock(
+  lines,
+  {
+    header = null,
+    maxTokens = 320,
+    maxLines = 12,
+    anchorLines = [],
+    anchorPatterns = [],
+    maxAnchors = DEFAULT_MAX_COMPACT_ANCHORS,
+  } = {},
+) {
+  const contentLines = Array.isArray(lines) ? lines.filter((line) => typeof line === 'string') : [];
+  const rawLines = header ? [header, ...contentLines] : [...contentLines];
+  const beforeText = rawLines.join('\n');
+  const normalizedMaxLines = header ? maxLines + 1 : maxLines;
+  const normalizedAnchorLines = resolveAnchorLines(rawLines, {
+    header,
+    anchorLines,
+    anchorPatterns,
+    maxAnchors,
+  });
+  const compactedLines = buildCompactedContextLines(rawLines, {
+    maxTokens,
+    maxLines: normalizedMaxLines,
+    preserveFirstLine: Boolean(header),
+    forceFirstCount: header ? 1 : 0,
+  });
+  let text = compactedLines.join('\n').trim();
+  let validationMode = 'default';
+
+  if (normalizedAnchorLines.length > 0 && findMissingAnchors(text, normalizedAnchorLines).length > 0) {
+    const anchorFirstLines = buildCompactedContextLines(
+      header
+        ? [header, ...normalizedAnchorLines, ...contentLines]
+        : [...normalizedAnchorLines, ...contentLines],
+      {
+        maxTokens,
+        maxLines: normalizedMaxLines,
+        preserveFirstLine: Boolean(header),
+        forceFirstCount: (header ? 1 : 0) + normalizedAnchorLines.length,
+      },
+    );
+    const anchorFirstText = anchorFirstLines.join('\n').trim();
+
+    if (findMissingAnchors(anchorFirstText, normalizedAnchorLines).length === 0) {
+      text = anchorFirstText;
+      validationMode = 'anchor-first';
+    } else {
+      text = buildCompactedContextLines(
+        header
+          ? [header, ...normalizedAnchorLines]
+          : [...normalizedAnchorLines],
+        {
+          maxTokens,
+          maxLines: normalizedMaxLines,
+          preserveFirstLine: Boolean(header),
+          forceFirstCount: (header ? 1 : 0) + normalizedAnchorLines.length,
+        },
+      ).join('\n').trim();
+      validationMode = 'forced-anchors';
+    }
+  }
+
+  return {
+    text,
+    tokensBefore: estimateTokenCount(beforeText),
+    tokensAfter: estimateTokenCount(text),
+    savedTokens: Math.max(0, estimateTokenCount(beforeText) - estimateTokenCount(text)),
+    validationMode,
+    anchorCount: normalizedAnchorLines.length,
+  };
+}
+
+function normalizeAnchorPattern(pattern) {
+  if (pattern instanceof RegExp) {
+    return (line) => pattern.test(line);
+  }
+  if (typeof pattern === 'string' && pattern.trim()) {
+    const prefix = pattern.trim();
+    return (line) => line.startsWith(prefix);
+  }
+  if (typeof pattern === 'function') {
+    return pattern;
+  }
+  return null;
+}
+
+function normalizeLineForDedupe(line) {
+  return String(line ?? '').trim().replace(/\s+/g, ' ').toLowerCase();
+}
+
+function uniqueLines(lines) {
+  const seen = new Set();
+  const unique = [];
+
+  for (const line of lines) {
+    const trimmed = String(line ?? '').trim();
+    const normalized = normalizeLineForDedupe(trimmed);
+    if (!normalized || seen.has(normalized)) {
+      continue;
+    }
+    seen.add(normalized);
+    unique.push(trimmed);
+  }
+
+  return unique;
+}
+
+function resolveAnchorLines(rawLines, { header = null, anchorLines = [], anchorPatterns = [], maxAnchors = DEFAULT_MAX_COMPACT_ANCHORS } = {}) {
+  const predicates = anchorPatterns
+    .map((pattern) => normalizeAnchorPattern(pattern))
+    .filter(Boolean);
+  const explicitAnchorLines = Array.isArray(anchorLines) ? anchorLines : [];
+  const matchedPatternLines = predicates.length > 0
+    ? rawLines.filter((line) => {
+        const trimmed = String(line ?? '').trim();
+        return trimmed && predicates.some((predicate) => predicate(trimmed));
+      })
+    : [];
+
+  return uniqueLines([
+    ...explicitAnchorLines,
+    ...matchedPatternLines,
+  ])
+    .filter((line) => !header || normalizeLineForDedupe(line) !== normalizeLineForDedupe(header))
+    .slice(0, Math.max(0, maxAnchors));
+}
+
+function buildCompactedContextLines(
+  lines,
+  {
+    maxTokens = 320,
+    maxLines = 12,
+    preserveFirstLine = true,
+    forceFirstCount = 0,
+  } = {},
+) {
+  const sourceLines = Array.isArray(lines) ? lines : [];
+  const selected = [];
+  const seen = new Set();
+  let usedTokens = 0;
+  let nonEmptyCount = 0;
+
+  for (const line of sourceLines) {
+    const compressed = compressLine(line);
+    if (!compressed) {
+      continue;
+    }
+
+    const dedupeKey = compressed.toLowerCase();
+    if (seen.has(dedupeKey)) {
+      continue;
+    }
+
+    const tokens = estimateTokenCount(compressed);
+    const forceLine = nonEmptyCount < forceFirstCount;
+    const wouldOverflow = selected.length > 0 && (usedTokens + tokens) > maxTokens;
+    const wouldExceedLines = nonEmptyCount >= maxLines;
+
+    if (!forceLine && (wouldOverflow || wouldExceedLines)) {
+      break;
+    }
+    if (forceLine && wouldExceedLines) {
+      break;
+    }
+
+    selected.push(compressed);
+    seen.add(dedupeKey);
+    usedTokens += tokens;
+    nonEmptyCount += 1;
+  }
+
+  if (selected.length === 0) {
+    return compressMarkdownLines(sourceLines, { maxTokens, maxLines, preserveFirstLine });
+  }
+
+  return selected;
+}
+
+function findMissingAnchors(text, anchorLines) {
+  const normalizedText = String(text ?? '').toLowerCase();
+  return anchorLines.filter((line) => {
+    const compressed = compressLine(line).toLowerCase();
+    return compressed && !normalizedText.includes(compressed);
+  });
+}
+
+function normalizePromptCacheEntry(entry) {
+  if (!entry || typeof entry !== 'object') {
+    return null;
+  }
+
+  const requestKey = typeof entry.requestKey === 'string' ? entry.requestKey.trim() : '';
+  if (!requestKey) {
+    return null;
+  }
+
+  const updatedAt = Number(entry.updatedAt);
+  const hitCount = Number(entry.hitCount);
+  const lastHitAt = Number(entry.lastHitAt);
+
+  return {
+    ...entry,
+    requestKey,
+    updatedAt: Number.isFinite(updatedAt) ? updatedAt : Date.now(),
+    hitCount: Number.isFinite(hitCount) && hitCount >= 0 ? hitCount : 0,
+    lastHitAt: Number.isFinite(lastHitAt) ? lastHitAt : null,
+  };
+}
+
+function normalizePromptCacheDocument(raw, { maxEntries = DEFAULT_PROMPT_CACHE_MAX_ENTRIES } = {}) {
+  const candidates = [];
+  if (Array.isArray(raw?.entries)) {
+    candidates.push(...raw.entries);
+  }
+  if (
+    !Array.isArray(raw?.entries)
+    && raw
+    && typeof raw === 'object'
+    && typeof raw.requestKey === 'string'
+    && raw.requestKey.trim()
+  ) {
+    candidates.push(raw);
+  }
+
+  return {
+    entries: candidates
+      .map((entry) => normalizePromptCacheEntry(entry))
+      .filter(Boolean)
+      .sort((left, right) => (right.updatedAt ?? 0) - (left.updatedAt ?? 0))
+      .filter((entry, index, list) => list.findIndex((candidate) => candidate.requestKey === entry.requestKey) === index)
+      .slice(0, maxEntries),
+  };
+}
+
+export async function readPromptCacheEntry(projectRoot, requestKey, { maxEntries = DEFAULT_PROMPT_CACHE_MAX_ENTRIES, touch = false } = {}) {
+  if (typeof requestKey !== 'string' || !requestKey.trim()) {
+    return null;
+  }
+
+  const runtimePaths = buildRuntimePaths(projectRoot);
+  const cache = normalizePromptCacheDocument(await readJson(runtimePaths.promptCachePath, null), { maxEntries });
+  const entryIndex = cache.entries.findIndex((entry) => entry.requestKey === requestKey);
+  if (entryIndex < 0) {
+    return null;
+  }
+
+  const entry = cache.entries[entryIndex];
+  if (!touch) {
+    return entry;
+  }
+
+  const touchedEntry = {
+    ...entry,
+    updatedAt: Date.now(),
+    lastHitAt: Date.now(),
+    hitCount: (entry.hitCount ?? 0) + 1,
+  };
+  const nextDocument = normalizePromptCacheDocument({
+    entries: [
+      touchedEntry,
+      ...cache.entries.slice(0, entryIndex),
+      ...cache.entries.slice(entryIndex + 1),
+    ],
+  }, { maxEntries });
+  await writeJson(runtimePaths.promptCachePath, nextDocument);
+  return nextDocument.entries[0] ?? touchedEntry;
+}
+
+export async function writePromptCacheEntry(projectRoot, entry, { maxEntries = DEFAULT_PROMPT_CACHE_MAX_ENTRIES } = {}) {
+  const normalizedEntry = normalizePromptCacheEntry(entry);
+  if (!normalizedEntry) {
+    return null;
+  }
+
+  const runtimePaths = buildRuntimePaths(projectRoot);
+  const cache = normalizePromptCacheDocument(await readJson(runtimePaths.promptCachePath, null), { maxEntries });
+  const nextDocument = normalizePromptCacheDocument({
+    entries: [
+      normalizedEntry,
+      ...cache.entries.filter((existingEntry) => existingEntry.requestKey !== normalizedEntry.requestKey),
+    ],
+  }, { maxEntries });
+  await writeJson(runtimePaths.promptCachePath, nextDocument);
+  return nextDocument;
+}
+
+function normalizeCompactHistoryEntry(entry) {
+  if (!entry || typeof entry !== 'object') {
+    return null;
+  }
+
+  const timestamp = Number(entry.timestamp);
+  const tokensBefore = Number(entry.tokensBefore);
+  const tokensAfter = Number(entry.tokensAfter);
+  const savedTokens = Number(entry.savedTokens);
+
+  return {
+    ...entry,
+    timestamp: Number.isFinite(timestamp) ? timestamp : Date.now(),
+    tokensBefore: Number.isFinite(tokensBefore) ? tokensBefore : 0,
+    tokensAfter: Number.isFinite(tokensAfter) ? tokensAfter : 0,
+    savedTokens: Number.isFinite(savedTokens) ? savedTokens : 0,
+  };
+}
+
+function normalizeCompactHistoryDocument(raw, { maxEntries = DEFAULT_COMPACT_HISTORY_MAX_ENTRIES } = {}) {
+  const candidates = Array.isArray(raw?.entries)
+    ? raw.entries
+    : Array.isArray(raw)
+      ? raw
+      : [];
+
+  return {
+    entries: candidates
+      .map((entry) => normalizeCompactHistoryEntry(entry))
+      .filter(Boolean)
+      .sort((left, right) => (right.timestamp ?? 0) - (left.timestamp ?? 0))
+      .slice(0, maxEntries),
+  };
+}
+
+export async function recordCompaction(projectRoot, entry, { maxEntries = DEFAULT_COMPACT_HISTORY_MAX_ENTRIES } = {}) {
+  const normalizedEntry = normalizeCompactHistoryEntry(entry);
+  if (!normalizedEntry || (normalizedEntry.savedTokens ?? 0) <= 0) {
+    return null;
+  }
+
+  const runtimePaths = buildRuntimePaths(projectRoot);
+  const history = normalizeCompactHistoryDocument(await readJson(runtimePaths.compactHistoryPath, null), { maxEntries });
+  const nextDocument = normalizeCompactHistoryDocument({
+    entries: [normalizedEntry, ...history.entries],
+  }, { maxEntries });
+  await writeJson(runtimePaths.compactHistoryPath, nextDocument);
+  return normalizedEntry;
+}

package/templates/.codex/README.md

@@ -0,0 +1,83 @@
+# {{project.name}}
+
+Auto-generated by UKit for OpenAI Codex.
+
+- Packs: {{project.stack}}
+- Package manager: {{runtime.packageManager}}
+
+## Team Workflow (Highest Priority)
+
+- Teammates should only need to remember **`ukit install`**.
+- After `ukit install`, open Claude/Codex and ask naturally.
+- Do not make end users memorize skill names, helper scripts, or routing internals unless they are debugging UKit itself.
+- **Treat helper commands as internal orchestration. Do not ask end users to run them.**
+
+## UKit v1.1.6 Shared Runtime
+
+- Shared runtime state lives in `.ukit/storage/`.
+- Treat `.ukit/storage/config.json` as the source of compact, token-pipeline, router, memory, and validation toggles.
+- Reuse `.ukit/storage/memory/` before asking users to restate prior decisions.
+- For non-trivial work, prefer `ukit memory recall "<current task>"` before widening docs.
+- Reusable cache state lives in `.ukit/storage/cache/prompt-cache.json`, `.ukit/storage/cache/compact-history.json`, `.ukit/storage/cache/compact-pressure.json`, `.ukit/storage/cache/output-history.json`, and preserved raw tool outputs under `.ukit/storage/cache/tee/`.
+- If an older repo still has a visible `ukit/` runtime root, rerun `ukit install`; UKit should migrate the shared runtime into hidden `.ukit/` when safe.
+- Maintainers can inspect runtime state with `ukit status` and `ukit memory export`.
+- Reuse compact `previous-context` / `recent-output` route snapshots before replaying raw history.
+- Threshold-based compact pressure is internal orchestration; Codex users should not need to manage it manually.
+
+## Speed / Reading Contract
+
+- **Trivial**: no docs.
+- **Simple**: `docs/MEMORY.md` only.
+- **Non-trivial**: `docs/MEMORY.md` + `docs/PROJECT.md` + `docs/CODE_MAP.md`.
+- Read `docs/WORKLOG.md` only when recent continuation/debug context matters.
+- Source is truth; if docs are stale, update docs and continue.
+
+## Index-First Loop (for Codex)
+
+1. Refresh/build the index only when needed:
+   - `node .codex/ukit/index/refresh-index.mjs`
+   - fallback: `node .codex/ukit/index/build-index.mjs`
+2. Query likely files:
+   - `node .codex/ukit/index/query-index.mjs "<error|symbol|path>"`
+3. For bug signatures:
+   - `node .codex/ukit/index/triage.mjs "<error signature>"`
+4. Open only the top 1-3 suspect files first, then widen if needed.
+
+For clearly non-code specialist lanes, avoid dragging the source-code index into the route unless the target is actually code.
+
+## Automatic Skill Activation
+
+- End users should not need to know skill names.
+- For every non-trivial task — and again after the first meaningful tool calls — inspect installed project-local skills and **auto-activate the matching skill immediately**.
+- Match from prompt wording plus tool/file evidence.
+- Use the smallest useful set, usually 1-2 skills.
+- If docs work is detected, prefer `.codex/skills/docs-quality/SKILL.md`.
+- If routing is complex or ambiguous, prefer `node .codex/ukit/index/route-task.mjs "<prompt>" [--tool-command <cmd>] [--target <file>]`.
+- That helper should keep shared route memory in `.claude/ukit/skill-router-state.json` so Claude and Codex continue from the same compact route state.
+- When route memory already includes `previous-context` or `recent-output`, reuse it before widening reads or logs.
+- After routing, prefer `node .codex/ukit/index/resolve-context.mjs ...` for context and `node .codex/ukit/index/verify-context.mjs ...` for verification.
+- Follow routed verification policy: targeted first, widen only when scope/risk justifies it, ask before blanket broad runs.
+
+## Selective Subagent Usage (internal only)
+
+- Stay direct for trivial/simple work or tightly coupled code already localized by the index.
+- Delegate only when it reduces context bloat or unlocks meaningful parallel/background progress.
+- Good triggers: noisy side lanes, 3+ independent investigations, long-running background work, or explicit plan/batch execution.
+- End users still only need `ukit install`; subagent choice is internal orchestration.
+
+## Docs Discipline
+
+After significant work, update only what changed:
+- `docs/WORKLOG.md`
+- `docs/MEMORY.md`
+- `docs/CODE_MAP.md`
+- `docs/PROJECT.md`
+
+## Skills
+
+Skills are shared from `.claude/skills/` via symlink at `.codex/skills/`.
+
+## Codex Rulebase
+
+- `.codex/settings.json` — project-shared Codex safe-auto baseline
+- `.codex/settings.local.json` — machine-local overrides (kept on reinstall)