llm-wiki-kit 0.1.0

package/src/redaction.js

@@ -0,0 +1,68 @@
+const SECRET_PATTERNS = [
+  {
+    name: 'private-key',
+    pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
+  },
+  {
+    name: 'env-assignment-secret',
+    pattern: /\b([A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASSWD|API_KEY|PRIVATE_KEY|ACCESS_KEY)[A-Z0-9_]*)\s*=\s*([^\s"'`]{6,}|["'`][^"'`]{6,}["'`])/gi,
+  },
+  {
+    name: 'bearer-token',
+    pattern: /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/g,
+  },
+  {
+    name: 'generic-token',
+    pattern: /\b(?:sk|pk|ghp|gho|github_pat|xoxb|xoxp|AKIA)[A-Za-z0-9._-]{16,}\b/g,
+  },
+  {
+    name: 'email',
+    pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
+  },
+  {
+    name: 'phone',
+    pattern: /(?<![\w])(?!(?:\d{4}-\d{2}-\d{2})(?!\d))(?:\+?\d(?:[\d .()-]*\d){9,})(?![\w])/g,
+  },
+];
+
+const SENSITIVE_PATH_PATTERNS = [
+  /(^|\/)\.env(\.|$)/i,
+  /(^|\/)(?:id_rsa|id_dsa|id_ecdsa|id_ed25519)$/i,
+  /\.(?:pem|key|p12|pfx)$/i,
+  /(^|\/)(?:secrets?|credentials?|raw_private)(\/|$)/i,
+  /(?:token|password|passwd|credential|private[_-]?key)/i,
+];
+
+export function redactText(value, maxLength = 6000) {
+  let text = typeof value === 'string' ? value : JSON.stringify(value ?? '');
+  for (const rule of SECRET_PATTERNS) {
+    text = text.replace(rule.pattern, `[REDACTED:${rule.name}]`);
+  }
+  if (text.length > maxLength) {
+    text = `${text.slice(0, maxLength)}\n[TRUNCATED:${text.length - maxLength}]`;
+  }
+  return text;
+}
+
+export function hasSecretLikeText(value) {
+  const text = typeof value === 'string' ? value : JSON.stringify(value ?? '');
+  return SECRET_PATTERNS.some((rule) => {
+    rule.pattern.lastIndex = 0;
+    return rule.pattern.test(text);
+  });
+}
+
+export function isSensitivePath(path) {
+  if (!path || typeof path !== 'string') return false;
+  return SENSITIVE_PATH_PATTERNS.some((pattern) => pattern.test(path));
+}
+
+export function summarizeForStorage(value, maxLength = 1200) {
+  return redactText(value, maxLength).replace(/\r/g, '').trim();
+}
+
+export function extractPathsFromText(text) {
+  if (!text || typeof text !== 'string') return [];
+  const matches = text.match(/(?:^|\s)([.~\/A-Za-z0-9_@:-][^\s"'`<>|;]{1,220})/g) || [];
+  return matches.map((item) => item.trim()).filter(Boolean);
+}

package/src/state.js

@@ -0,0 +1,73 @@
+import { join } from 'path';
+import { kitDataDir, readJson, sha256, writeJson } from './fs-utils.js';
+import { summarizeForStorage } from './redaction.js';
+
+export function sessionKey(projectRoot, payload) {
+  const sessionId =
+    payload.session_id ||
+    payload.sessionId ||
+    payload.conversation_id ||
+    payload.transcript_path ||
+    'default';
+  return sha256(`${projectRoot}:${sessionId}`).slice(0, 32);
+}
+
+export function statePath(projectRoot, payload) {
+  return join(kitDataDir(), 'state', `${sessionKey(projectRoot, payload)}.json`);
+}
+
+export async function readTurnState(projectRoot, payload) {
+  return (await readJson(statePath(projectRoot, payload), null)) || {
+    projectRoot,
+    session: sessionKey(projectRoot, payload),
+    questions: [],
+    tools: [],
+    files: [],
+    verification: [],
+    updated_at: new Date().toISOString(),
+  };
+}
+
+export async function writeTurnState(projectRoot, payload, state) {
+  state.updated_at = new Date().toISOString();
+  await writeJson(statePath(projectRoot, payload), state);
+}
+
+export async function rememberQuestion(projectRoot, payload, prompt) {
+  const state = await readTurnState(projectRoot, payload);
+  const clean = summarizeForStorage(prompt, 3000);
+  if (clean) state.questions.push({ at: new Date().toISOString(), text: clean });
+  state.questions = state.questions.slice(-5);
+  await writeTurnState(projectRoot, payload, state);
+}
+
+export async function rememberTool(projectRoot, payload, summary) {
+  const state = await readTurnState(projectRoot, payload);
+  const clean = summarizeForStorage(summary, 1500);
+  if (clean) state.tools.push({ at: new Date().toISOString(), text: clean });
+  state.tools = state.tools.slice(-20);
+  const fileMatches = clean.match(/(?:^|\s)([A-Za-z0-9_./@:-]+\.[A-Za-z0-9_./@:-]+)/g) || [];
+  for (const match of fileMatches) state.files.push(match.trim());
+  state.files = [...new Set(state.files)].slice(-30);
+  if (/\b(test|pytest|vitest|jest|playwright|lint|typecheck|build|doctor)\b/i.test(clean)) {
+    state.verification.push(clean);
+    state.verification = state.verification.slice(-10);
+  }
+  await writeTurnState(projectRoot, payload, state);
+}
+
+export async function buildEntryFromState(projectRoot, payload, assistantText) {
+  const state = await readTurnState(projectRoot, payload);
+  const question = state.questions.at(-1)?.text || summarizeForStorage(payload.prompt || payload.user_prompt || '', 2000);
+  const result = summarizeForStorage(assistantText || payload.last_assistant_message || payload.response || '', 4000);
+  const topic = question ? question.replace(/\s+/g, ' ').slice(0, 80) : 'session turn';
+  return {
+    topic,
+    question: question || '(not captured)',
+    work: state.tools.map((item) => `- ${item.text}`).join('\n') || '(not captured)',
+    result: result || '(not captured)',
+    changedFiles: [...new Set(state.files)].map((item) => `- ${item}`).join('\n') || '(not captured)',
+    verification: state.verification.map((item) => `- ${item}`).join('\n') || '(not captured)',
+    followUp: '',
+  };
+}

package/src/templates.js

@@ -0,0 +1,31 @@
+import { runtimeVersion } from './version.js';
+
+export function rootAgentsPolicy() {
+  return `\n<!-- llm-wiki-kit:start -->\n## LLM Wiki Policy\n\nThis repository uses llm-wiki-kit as a hook-first living Markdown wiki for Codex and Claude Code.\n\n- This block supersedes older OMX/OMC/\`omx_wiki/\` LLM Wiki instructions for this repository.\n- Treat chat memory as temporary; durable project knowledge belongs in \`llm-wiki/\`.\n- \`llm-wiki/raw/\` stores immutable or redacted source material. Do not edit raw source files.\n- \`llm-wiki/wiki/\` stores LLM-maintained knowledge pages such as decisions, architecture, debugging, context, concepts, and queries.\n- Before non-trivial work, use the injected LLM Wiki context and consult \`llm-wiki/wiki/index.md\` when present.\n- Capture reusable decisions, debugging findings, verification commands, and open questions into the wiki.\n- Never store credentials, tokens, private keys, \`.env\` contents, customer identifiers, or private personal data.\n- Mark inference explicitly and preserve contradictions instead of silently overwriting them.\n\n<!-- llm-wiki-kit:end -->\n`;
+}
+
+export function llmWikiAgents() {
+  return `# LLM Wiki Agent Rules\n\nGenerated by llm-wiki-kit ${runtimeVersion()}.\n\n## Purpose\nMaintain a living Markdown LLM Wiki from immutable source files and redacted Codex/Claude Code session events.\nThese rules supersede older OMX/OMC/\`omx_wiki/\` LLM Wiki rules for this project.\n\n## Directories\n- \`raw/\`: immutable or redacted source material. Never edit original source captures.\n- \`wiki/\`: AI-maintained knowledge pages.\n- \`outputs/\`: live Q&A summaries, reports, and generated briefs.\n- \`procedures/\`: detailed operating rules for ingest, query, lint, and security.\n\n## Core Rules\n- Never modify \`raw/\` source material except to append redacted session envelopes generated by hooks.\n- Do not state unsupported claims as facts.\n- Mark inference explicitly.\n- Add \`source_ids\` or file references for important claims.\n- Update \`wiki/index.md\` and \`wiki/log.md\` whenever new durable knowledge is created.\n- Preserve contradictions in \`Contradictions\` or \`Open Questions\` instead of overwriting them.\n- Ask before reading private or secret-looking files.\n\n## Page Format\nUse YAML frontmatter when creating wiki pages:\n\n\`\`\`yaml\n---\ntitle: \"\"\ntype: \"source | concept | entity | decision | architecture | debugging | context | query | session-log | convention\"\nsource_ids: []\nstatus: \"draft | reviewed | stale\"\nlast_updated: \"YYYY-MM-DD\"\nconfidence: \"high | medium | low\"\n---\n\`\`\`\n\n## Operations\n- ingest: read new raw files, create or update wiki pages, then update \`wiki/index.md\` and \`wiki/log.md\`.\n- query: start from \`wiki/index.md\`, read relevant wiki pages, answer with source references, and save reusable answers.\n- lint: find stale pages, orphan pages, broken wiki links, missing sources, duplicate concepts, contradictions, and missing links.\n`;
+}
+
+export function indexPage() {
+  return `# LLM Wiki Index\n\nGenerated by llm-wiki-kit.\n\n## Overview\n\nThis is the navigation map for the project living wiki. Keep it short and useful.\n\n## Main Areas\n\n- [Sources](sources/) - source summaries and source IDs.\n- [Concepts](concepts/) - reusable concepts and terminology.\n- [Entities](entities/) - people, systems, modules, vendors, services, and tools.\n- [Decisions](decisions/) - decisions and rationale.\n- [Architecture](architecture/) - system structure and data/control flow.\n- [Debugging](debugging/) - root causes, fixes, verification evidence.\n- [Context](context/) - session continuity and project memory.\n- [Queries](queries/) - durable answers to useful questions.\n\n## Operating Notes\n\n- Start from this index before broad project questions.\n- Read 3-7 relevant pages first; inspect raw sources only when precision matters.\n- Add links using \`[[page-or-topic]]\` when pages become related.\n`;
+}
+
+export function logPage() {
+  return `# LLM Wiki Log\n\nAppend-only operating history for this project wiki.\n`;
+}
+
+export function procedure(name) {
+  const procedures = {
+    'ingest.md': `# Ingest Procedure\n\n1. Read \`wiki/index.md\` first.\n2. Inspect new material under \`raw/inbox/\` or \`raw/sources/\`.\n3. Create or update \`wiki/sources/<slug>.md\` for each source.\n4. Update related concept, entity, decision, architecture, debugging, or context pages.\n5. Prefer updating existing pages over creating duplicates.\n6. Add source references and confidence.\n7. Update \`wiki/index.md\` and append to \`wiki/log.md\`.\n`,
+    'query.md': `# Query Procedure\n\n1. Start from \`wiki/index.md\`.\n2. Search \`wiki/\` for relevant pages.\n3. Read the smallest useful set of pages first.\n4. Use raw sources only when exact evidence matters.\n5. Separate verified facts from inference.\n6. Save reusable answers into \`wiki/queries/\` and link them from related pages.\n`,
+    'lint.md': `# Lint Procedure\n\nCheck for stale pages, orphan pages, broken wiki links, missing sources, duplicate concepts, unsupported claims, and unresolved contradictions. Prefer producing a review report before automatic edits.\n`,
+    'security.md': `# Security Procedure\n\n- Never store credentials, tokens, private keys, \`.env\` contents, customer identifiers, or private personal data.\n- Redact before writing hook payloads or summaries.\n- Full raw transcript capture is disabled by default and must be explicitly enabled by project policy.\n- If a file or prompt looks secret-bearing, do not persist it and ask for confirmation before reading further.\n`,
+  };
+  return procedures[name] || '';
+}
+
+export function gitignore() {
+  return `# llm-wiki-kit safety defaults\nraw/sessions/*.jsonl\nraw/sessions/*.ndjson\nraw/private/\nraw_private/\nsecrets/\n*.key\n*.pem\n*.p12\n*.pfx\n.env\n.env.*\n.cache/\n`;
+}

package/src/update.js

@@ -0,0 +1,133 @@
+import { spawnSync } from 'child_process';
+import { resolve } from 'path';
+import { appendWikiLog, bootstrapProject } from './project.js';
+import { install } from './install.js';
+import { applyProjectTemplateUpdate, inspectProjectState } from './project-state.js';
+import { binPath, detectInstallSource, packageName, runtimeVersion } from './version.js';
+
+function runCommand(command, args, options = {}) {
+  const result = spawnSync(command, args, {
+    encoding: 'utf8',
+    env: options.env || process.env,
+    timeout: options.timeout || 120000,
+  });
+  return {
+    status: result.status,
+    stdout: result.stdout || '',
+    stderr: result.stderr || '',
+    error: result.error || null,
+  };
+}
+
+function npmCommand() {
+  return process.env.LLM_WIKI_KIT_NPM || 'npm';
+}
+
+function binCommand() {
+  return process.env.LLM_WIKI_KIT_BIN || binPath;
+}
+
+function assertCommandOk(result, label) {
+  if (result.status === 0 && !result.error) return;
+  const detail = result.stderr.trim() || result.stdout.trim() || result.error?.message || 'unknown error';
+  throw new Error(`${label} failed: ${detail}`);
+}
+
+export function parseRegistryVersion(output) {
+  return String(output || '').trim().split(/\s+/).at(-1) || '';
+}
+
+export async function checkForUpdate(options = {}) {
+  const target = options.to || 'latest';
+  const installedVersion = runtimeVersion();
+  const result = runCommand(npmCommand(), ['view', `${packageName()}@${target}`, 'version'], {
+    timeout: options.timeout || 30000,
+  });
+  assertCommandOk(result, 'npm view');
+  const latestVersion = parseRegistryVersion(result.stdout);
+  return {
+    installedVersion,
+    latestVersion,
+    target,
+    updateAvailable: latestVersion !== installedVersion,
+  };
+}
+
+export async function postUpdate(options = {}) {
+  const workspace = resolve(options.workspace || process.cwd());
+  const installResult = await install({
+    ...options,
+    workspace,
+    replaceHooks: true,
+  });
+  const projectResult = options.noProject
+    ? null
+    : await applyProjectTemplateUpdate(workspace);
+
+  if (!options.noProject) {
+    await appendWikiLog(workspace, `llm-wiki-kit post-update applied runtime ${runtimeVersion()}; changed templates: ${projectResult.changed.length}`);
+  }
+
+  return {
+    workspace,
+    runtimeVersion: runtimeVersion(),
+    install: installResult,
+    project: projectResult,
+  };
+}
+
+export async function update(options = {}) {
+  const workspace = resolve(options.workspace || process.cwd());
+  if (!options.check && !options.dryRun && detectInstallSource() === 'source' && process.env.LLM_WIKI_KIT_ALLOW_SOURCE_UPDATE !== '1') {
+    throw new Error('llm-wiki update cannot self-update from a source checkout. Install with `npm install -g llm-wiki-kit`, then run `llm-wiki update --workspace <project>`.');
+  }
+
+  const check = await checkForUpdate(options);
+
+  if (options.check) {
+    return {
+      mode: 'check',
+      workspace,
+      ...check,
+      project: options.noProject ? null : await inspectProjectState(workspace),
+    };
+  }
+
+  if (options.dryRun) {
+    return {
+      mode: 'dry-run',
+      workspace,
+      ...check,
+      installSource: detectInstallSource(),
+      project: options.noProject ? null : await applyProjectTemplateUpdate(workspace, { dryRun: true }),
+    };
+  }
+
+  const target = options.to || 'latest';
+  const installResult = runCommand(npmCommand(), ['install', '-g', `${packageName()}@${target}`], {
+    timeout: options.timeout || 120000,
+  });
+  assertCommandOk(installResult, 'npm install -g');
+
+  const postArgs = [binCommand(), 'post-update', '--workspace', workspace];
+  if (options.profile) postArgs.push('--profile', options.profile);
+  if (options.noProject) postArgs.push('--no-project');
+  if (options.codex === false) postArgs.push('--no-codex');
+  if (options.claude === false) postArgs.push('--no-claude');
+  const postResult = runCommand(process.execPath, postArgs, {
+    timeout: options.timeout || 120000,
+  });
+  assertCommandOk(postResult, 'post-update');
+
+  return {
+    mode: 'update',
+    workspace,
+    before: check.installedVersion,
+    target,
+    npmInstall: {
+      stdout: installResult.stdout.trim(),
+      stderr: installResult.stderr.trim(),
+    },
+    postUpdate: postResult.stdout.trim(),
+  };
+}

package/src/version.js

@@ -0,0 +1,27 @@
+import { existsSync, readFileSync } from 'fs';
+import { dirname, join, resolve, sep } from 'path';
+import { fileURLToPath } from 'url';
+
+const here = dirname(fileURLToPath(import.meta.url));
+
+export const packageRoot = resolve(here, '..');
+export const packageJsonPath = join(packageRoot, 'package.json');
+export const binPath = join(packageRoot, 'bin', 'llm-wiki.js');
+
+export function packageMetadata() {
+  return JSON.parse(readFileSync(packageJsonPath, 'utf8'));
+}
+
+export function runtimeVersion() {
+  return packageMetadata().version;
+}
+
+export function packageName() {
+  return packageMetadata().name;
+}
+
+export function detectInstallSource() {
+  if (existsSync(join(packageRoot, '.git'))) return 'source';
+  if (packageRoot.includes(`${sep}node_modules${sep}`)) return 'npm';
+  return 'unknown';
+}