@gitset-dev/cli 2.3.3 → 2.4.0

package/lib/knowledge/map.js

@@ -0,0 +1,239 @@
+'use strict';
+
+const path = require('path');
+
+const JS_IMPORT_RES = [
+  /\brequire\s*\(\s*['"]([^'"]+)['"]\s*\)/g,
+  /\bfrom\s+['"]([^'"]+)['"]/g,
+  /\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g,
+  /^\s*import\s+['"]([^'"]+)['"]/gm,
+];
+
+const PY_IMPORT_RES = [
+  /^\s*from\s+([\w.]+)\s+import\b/gm,
+  /^\s*import\s+([\w.]+(?:\s*,\s*[\w.]+)*)/gm,
+];
+
+const JS_EXTS = ['.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs', '.mts', '.cts'];
+const CONTAINER_DIRS = new Set(['src', 'lib', 'app', 'packages', 'pkg', 'internal', 'cmd', 'api', 'server', 'client']);
+
+function extractImportSpecs(filePath, content) {
+  const ext = path.extname(filePath).toLowerCase();
+  const specs = [];
+  if (JS_EXTS.includes(ext) || ['.astro', '.vue', '.svelte'].includes(ext)) {
+    for (const re of JS_IMPORT_RES) {
+      re.lastIndex = 0;
+      let m;
+      while ((m = re.exec(content)) !== null) specs.push(m[1]);
+    }
+  } else if (ext === '.py') {
+    for (const re of PY_IMPORT_RES) {
+      re.lastIndex = 0;
+      let m;
+      while ((m = re.exec(content)) !== null) {
+        for (const part of m[1].split(',')) specs.push(part.trim());
+      }
+    }
+  }
+  return specs;
+}
+
+function buildFileIndex(files) {
+  const index = new Set(files.map((f) => f.path));
+  return index;
+}
+
+function resolveJsImport(fromFile, spec, fileIndex) {
+  if (!spec.startsWith('./') && !spec.startsWith('../')) return null;
+  const baseDir = path.posix.dirname(fromFile);
+  const joined = path.posix.normalize(path.posix.join(baseDir, spec));
+  const candidates = [joined];
+  for (const ext of JS_EXTS) candidates.push(joined + ext);
+  for (const ext of JS_EXTS) candidates.push(`${joined}/index${ext}`);
+  for (const candidate of candidates) {
+    if (fileIndex.has(candidate)) return candidate;
+  }
+  return null;
+}
+
+function resolvePyImport(spec, fileIndex) {
+  const rel = spec.replace(/\./g, '/');
+  const candidates = [`${rel}.py`, `${rel}/__init__.py`, `src/${rel}.py`, `src/${rel}/__init__.py`];
+  for (const candidate of candidates) {
+    if (fileIndex.has(candidate)) return candidate;
+  }
+  return null;
+}
+
+function moduleKeyFor(filePath) {
+  const segments = filePath.split('/');
+  if (segments.length === 1) return '(root)';
+  if (CONTAINER_DIRS.has(segments[0]) && segments.length > 2) {
+    return `${segments[0]}/${segments[1]}`;
+  }
+  return segments[0];
+}
+
+function parsePackageJson(content) {
+  try {
+    const pkg = JSON.parse(content);
+    return {
+      name: pkg.name || null,
+      version: pkg.version || null,
+      description: pkg.description || null,
+      bin: pkg.bin || null,
+      main: pkg.main || null,
+      type: pkg.type || null,
+      scripts: pkg.scripts || {},
+      dependencies: Object.keys(pkg.dependencies || {}),
+      devDependencies: Object.keys(pkg.devDependencies || {}),
+      engines: pkg.engines || null,
+      packageManager: pkg.packageManager || null,
+    };
+  } catch {
+    return null;
+  }
+}
+
+const COMMAND_REGISTRATION_RES = [
+  /\bcase\s+'([\w:-]+)'/g,
+  /\bcase\s+"([\w:-]+)"/g,
+  /\.command\(\s*['"]([\w:-]+)/g,
+  /add_parser\(\s*['"]([\w-]+)/g,
+];
+
+function extractRegisteredCommands(content) {
+  const names = new Set();
+  for (const re of COMMAND_REGISTRATION_RES) {
+    re.lastIndex = 0;
+    let m;
+    while ((m = re.exec(content)) !== null) {
+      if (!m[1].startsWith('-')) names.add(m[1]);
+    }
+  }
+  return [...names].sort();
+}
+
+function detectEntryPoints(files, manifest) {
+  const fileIndex = new Set(files.map((f) => f.path));
+  const entries = new Set();
+  if (manifest) {
+    if (typeof manifest.main === 'string' && fileIndex.has(manifest.main.replace(/^\.\//, ''))) {
+      entries.add(manifest.main.replace(/^\.\//, ''));
+    }
+    if (manifest.bin) {
+      const bins = typeof manifest.bin === 'string' ? [manifest.bin] : Object.values(manifest.bin);
+      for (const bin of bins) {
+        const norm = String(bin).replace(/^\.\//, '');
+        if (fileIndex.has(norm)) entries.add(norm);
+      }
+    }
+  }
+  for (const candidate of ['index.js', 'index.ts', 'src/index.js', 'src/index.ts', 'server.js', 'main.py', 'app.py', 'main.go', 'src/main.rs']) {
+    if (fileIndex.has(candidate)) entries.add(candidate);
+  }
+  return [...entries].sort();
+}
+
+function buildMap(files, readContentFn) {
+  const fileIndex = buildFileIndex(files);
+  const importsByFile = new Map();
+  const importedBy = new Map();
+  const externalDeps = new Map();
+
+  const contentEligible = files.filter((f) => f.kind === 'source' || f.kind === 'config' || f.kind === 'manifest');
+
+  for (const file of contentEligible) {
+    if (file.kind !== 'source') continue;
+    const content = readContentFn(file);
+    if (!content) continue;
+    const specs = extractImportSpecs(file.path, content);
+    const resolved = [];
+    for (const spec of specs) {
+      const ext = path.extname(file.path).toLowerCase();
+      const target = ext === '.py'
+        ? resolvePyImport(spec, fileIndex)
+        : resolveJsImport(file.path, spec, fileIndex);
+      if (target && target !== file.path) {
+        resolved.push(target);
+        if (!importedBy.has(target)) importedBy.set(target, new Set());
+        importedBy.get(target).add(file.path);
+      } else if (!spec.startsWith('.') && !spec.startsWith('/')) {
+        const root = spec.startsWith('@') ? spec.split('/').slice(0, 2).join('/') : spec.split('/')[0];
+        externalDeps.set(root, (externalDeps.get(root) || 0) + 1);
+      }
+    }
+    importsByFile.set(file.path, [...new Set(resolved)].sort());
+  }
+
+  let manifest = null;
+  const manifestFile = files.find((f) => f.path === 'package.json');
+  if (manifestFile) {
+    const content = readContentFn(manifestFile);
+    if (content) manifest = parsePackageJson(content);
+  }
+
+  const modules = new Map();
+  for (const file of files) {
+    if (file.kind === 'asset' || file.kind === 'sensitive') continue;
+    const key = moduleKeyFor(file.path);
+    if (!modules.has(key)) {
+      modules.set(key, { name: key, files: [], sourceCount: 0, testCount: 0, docCount: 0 });
+    }
+    const mod = modules.get(key);
+    mod.files.push(file);
+    if (file.kind === 'source') mod.sourceCount += 1;
+    if (file.kind === 'test') mod.testCount += 1;
+    if (file.kind === 'doc') mod.docCount += 1;
+  }
+
+  const centrality = new Map();
+  for (const [target, importers] of importedBy) {
+    centrality.set(target, importers.size);
+  }
+
+  const moduleEdges = new Map();
+  for (const [from, targets] of importsByFile) {
+    const fromMod = moduleKeyFor(from);
+    for (const target of targets) {
+      const toMod = moduleKeyFor(target);
+      if (fromMod === toMod) continue;
+      const edgeKey = `${fromMod} -> ${toMod}`;
+      moduleEdges.set(edgeKey, (moduleEdges.get(edgeKey) || 0) + 1);
+    }
+  }
+
+  const entryPoints = detectEntryPoints(files, manifest);
+
+  const registeredCommands = new Set();
+  for (const entry of entryPoints) {
+    const file = files.find((f) => f.path === entry);
+    if (!file) continue;
+    const content = readContentFn(file);
+    if (!content) continue;
+    for (const name of extractRegisteredCommands(content)) registeredCommands.add(name);
+  }
+
+  return {
+    manifest,
+    entryPoints,
+    registeredCommands: [...registeredCommands].sort(),
+    modules: [...modules.values()].sort((a, b) => a.name.localeCompare(b.name)),
+    importsByFile,
+    importedBy,
+    centrality,
+    moduleEdges: [...moduleEdges.entries()].sort((a, b) => a[0].localeCompare(b[0])).map(([edge, count]) => ({ edge, count })),
+    externalDeps: [...externalDeps.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([name, count]) => ({ name, count })),
+  };
+}
+
+module.exports = {
+  buildMap,
+  extractImportSpecs,
+  extractRegisteredCommands,
+  resolveJsImport,
+  resolvePyImport,
+  moduleKeyFor,
+  parsePackageJson,
+  detectEntryPoints,
+};

package/lib/knowledge/secrets.js

@@ -0,0 +1,99 @@
+'use strict';
+
+const KEY_RULES = [
+  { rule: 'aws-access-key', re: /\bAKIA[0-9A-Z]{16}\b/g },
+  { rule: 'github-token', re: /\bgh[pousr]_[A-Za-z0-9]{20,255}\b/g },
+  { rule: 'github-pat', re: /\bgithub_pat_[A-Za-z0-9_]{22,255}\b/g },
+  { rule: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{30,}\b/g },
+  { rule: 'gcp-oauth-key', re: /\bAQ\.[A-Za-z0-9_-]{30,}\b/g },
+  { rule: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g },
+  { rule: 'openai-key', re: /\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b/g },
+  { rule: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g },
+  { rule: 'stripe-key', re: /\b[sr]k_(?:live|test)_[A-Za-z0-9]{16,}\b/g },
+  { rule: 'npm-token', re: /\bnpm_[A-Za-z0-9]{36,}\b/g },
+  { rule: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{5,}\b/g },
+  { rule: 'private-key-block', re: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g },
+];
+
+const ASSIGNMENT_RE = /(\b(?:api[_-]?key|apikey|secret|token|password|passwd|pwd|auth|credential|private[_-]?key)[\w-]*\s*[=:]\s*["']?)([A-Za-z0-9+/_.-]{16,})(["']?)/gi;
+
+const ASSIGNMENT_SAFE_VALUES = /^(?:process\.env|import\.meta|os\.environ|env\(|\$\{|\$[A-Z_]|<[^>]+>|your[_-]|xxx|placeholder|example|changeme|dummy|test)/i;
+
+const PROPERTY_CHAIN_RE = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+$/;
+
+const ENTROPY_TOKEN_RE = /\b[A-Za-z0-9+/_-]{24,}\b/g;
+const ENTROPY_THRESHOLD = 4.2;
+const MAX_FINDINGS_PER_FILE = 5;
+
+function shannonEntropy(str) {
+  const freq = new Map();
+  for (const ch of str) freq.set(ch, (freq.get(ch) || 0) + 1);
+  let entropy = 0;
+  for (const count of freq.values()) {
+    const p = count / str.length;
+    entropy -= p * Math.log2(p);
+  }
+  return entropy;
+}
+
+function looksLikeIdentifier(token) {
+  if (/^[a-z]+(?:[A-Z][a-z0-9]*)+$/.test(token)) return true;
+  if (/^[A-Z0-9_]+$/.test(token)) return true;
+  if (/^[a-z0-9]+(?:[-_][a-z0-9]+)+$/.test(token)) return true;
+  if (/^(?:[0-9a-f]{2}[-:])+[0-9a-f]{2}$/i.test(token)) return true;
+  if (!/[0-9]/.test(token) || !/[a-zA-Z]/.test(token)) return false;
+  return false;
+}
+
+function redactSecrets(content) {
+  if (typeof content !== 'string' || !content) {
+    return { content, findings: [] };
+  }
+
+  const findings = [];
+  let out = content;
+
+  for (const { rule, re } of KEY_RULES) {
+    re.lastIndex = 0;
+    out = out.replace(re, () => {
+      findings.push(rule);
+      return `[REDACTED:${rule}]`;
+    });
+  }
+
+  ASSIGNMENT_RE.lastIndex = 0;
+  out = out.replace(ASSIGNMENT_RE, (full, prefix, value, suffix) => {
+    if (ASSIGNMENT_SAFE_VALUES.test(value)) return full;
+    if (PROPERTY_CHAIN_RE.test(value)) return full;
+    if (value.includes('[REDACTED:')) return full;
+    if (looksLikeIdentifier(value) && shannonEntropy(value) < ENTROPY_THRESHOLD) return full;
+    findings.push('secret-assignment');
+    return `${prefix}[REDACTED:secret-assignment]${suffix}`;
+  });
+
+  ENTROPY_TOKEN_RE.lastIndex = 0;
+  out = out.replace(ENTROPY_TOKEN_RE, (token) => {
+    if (token.includes('REDACTED')) return token;
+    if (looksLikeIdentifier(token)) return token;
+    if (shannonEntropy(token) < ENTROPY_THRESHOLD) return token;
+    findings.push('high-entropy-token');
+    return '[REDACTED:high-entropy-token]';
+  });
+
+  return { content: out, findings };
+}
+
+function sanitizeForPrompt(content) {
+  const { content: redacted, findings } = redactSecrets(content);
+  if (findings.length > MAX_FINDINGS_PER_FILE) {
+    return { content: null, findings, dropped: true };
+  }
+  return { content: redacted, findings, dropped: false };
+}
+
+module.exports = {
+  redactSecrets,
+  sanitizeForPrompt,
+  shannonEntropy,
+  MAX_FINDINGS_PER_FILE,
+};

package/lib/knowledge/state.js

@@ -0,0 +1,81 @@
+'use strict';
+
+const crypto = require('crypto');
+
+const STATE_VERSION = 1;
+const STATE_FILENAME = '.state.json';
+
+function hashContent(content) {
+  return crypto.createHash('sha256').update(content || '').digest('hex').slice(0, 16);
+}
+
+function buildState({ files, moduleSummaries, provider, model, tag, commit }) {
+  const fileHashes = {};
+  for (const file of [...files].sort((a, b) => a.path.localeCompare(b.path))) {
+    if (file.hash) fileHashes[file.path] = file.hash;
+  }
+  const summaries = {};
+  for (const key of Object.keys(moduleSummaries || {}).sort()) {
+    summaries[key] = moduleSummaries[key];
+  }
+  return {
+    version: STATE_VERSION,
+    generatedAt: new Date().toISOString(),
+    provider: provider || null,
+    model: model || null,
+    tag: tag || null,
+    commit: commit || null,
+    fileHashes,
+    moduleSummaries: summaries,
+  };
+}
+
+function parseState(raw) {
+  try {
+    const state = JSON.parse(raw);
+    if (!state || state.version !== STATE_VERSION || typeof state.fileHashes !== 'object') return null;
+    return state;
+  } catch {
+    return null;
+  }
+}
+
+function diffAgainstState(state, currentFiles) {
+  const previous = state && state.fileHashes ? state.fileHashes : {};
+  const changed = [];
+  const added = [];
+  const seen = new Set();
+  for (const file of currentFiles) {
+    if (!file.hash) continue;
+    seen.add(file.path);
+    if (!(file.path in previous)) added.push(file.path);
+    else if (previous[file.path] !== file.hash) changed.push(file.path);
+  }
+  const removed = Object.keys(previous).filter((p) => !seen.has(p));
+  return { changed: changed.sort(), added: added.sort(), removed: removed.sort() };
+}
+
+function affectedModules(diff, moduleKeyFor, importedBy) {
+  const direct = new Set();
+  for (const p of [...diff.changed, ...diff.added, ...diff.removed]) {
+    direct.add(moduleKeyFor(p));
+  }
+  const withImporters = new Set(direct);
+  for (const p of [...diff.changed, ...diff.removed]) {
+    const importers = importedBy && importedBy.get ? importedBy.get(p) : null;
+    if (importers) {
+      for (const importer of importers) withImporters.add(moduleKeyFor(importer));
+    }
+  }
+  return [...withImporters].sort();
+}
+
+module.exports = {
+  STATE_VERSION,
+  STATE_FILENAME,
+  hashContent,
+  buildState,
+  parseState,
+  diffAgainstState,
+  affectedModules,
+};

package/lib/knowledge/validate.js

@@ -0,0 +1,180 @@
+'use strict';
+
+const LINK_RE = /\[[^\]]*\]\(([^)#\s]+)(?:#[^)\s]*)?\)/g;
+const FENCE_RE = /```(?:bash|sh|shell|console|zsh)?\n([\s\S]*?)```/g;
+const CMD_RE = /^\s*\$?\s*(npm|pnpm|yarn|node|git|gitset|npx)\s+(\S+)(?:\s+(\S+))?/;
+
+function extractInternalLinks(markdown) {
+  const links = [];
+  LINK_RE.lastIndex = 0;
+  let m;
+  while ((m = LINK_RE.exec(markdown)) !== null) {
+    const target = m[1];
+    if (/^[a-z]+:\/\//i.test(target)) continue;
+    if (target.startsWith('mailto:')) continue;
+    links.push(target);
+  }
+  return links;
+}
+
+function extractCitedCommands(markdown) {
+  const commands = [];
+  FENCE_RE.lastIndex = 0;
+  let block;
+  while ((block = FENCE_RE.exec(markdown)) !== null) {
+    for (const line of block[1].split('\n')) {
+      const m = CMD_RE.exec(line);
+      if (m) commands.push({ tool: m[1], arg1: m[2] || '', arg2: m[3] || '', line: line.trim() });
+    }
+  }
+  return commands;
+}
+
+function normalizeDocLink(docDir, target) {
+  const clean = target.replace(/^\.\//, '');
+  if (clean.startsWith('../') || clean.startsWith('/')) {
+    return clean.replace(/^(\.\.\/)+/, '').replace(/^\//, '');
+  }
+  return `${docDir}/${clean}`;
+}
+
+function linkResolves(target, { docDir, fileSet, docSet, dirSet }) {
+  const asDocRelative = normalizeDocLink(docDir, target);
+  const asRepoRelative = target.replace(/^\.\//, '');
+  return fileSet.has(asDocRelative)
+    || docSet.has(asDocRelative)
+    || fileSet.has(asRepoRelative)
+    || docSet.has(asRepoRelative)
+    || (dirSet ? dirSet.has(asDocRelative.replace(/\/$/, '')) || dirSet.has(asRepoRelative.replace(/\/$/, '')) : false);
+}
+
+function buildDirSet(repoFiles) {
+  const dirs = new Set();
+  for (const p of repoFiles) {
+    const segments = p.split('/');
+    for (let i = 1; i < segments.length; i += 1) {
+      dirs.add(segments.slice(0, i).join('/'));
+    }
+  }
+  return dirs;
+}
+
+function repairInternalLinks(docs, { docDir, repoFiles }) {
+  const fileSet = new Set(repoFiles);
+  const docSet = new Set(docs.map((d) => `${docDir}/${d.filename}`));
+  const dirSet = buildDirSet(repoFiles);
+  const byBasename = new Map();
+  for (const p of repoFiles) {
+    const base = p.split('/').pop();
+    if (!byBasename.has(base)) byBasename.set(base, []);
+    byBasename.get(base).push(p);
+  }
+
+  const upPrefix = '../'.repeat(docDir.split('/').length);
+  const repairs = [];
+
+  for (const doc of docs) {
+    doc.content = (doc.content || '').replace(
+      /(\]\()([^)#\s]+)((?:#[^)\s]*)?\))/g,
+      (full, open, target, close) => {
+        if (/^[a-z]+:\/\//i.test(target) || target.startsWith('mailto:')) return full;
+        if (linkResolves(target, { docDir, fileSet, docSet, dirSet })) return full;
+
+        const basename = target.split('/').pop();
+        const candidates = new Set([basename]);
+        const deduped = basename.replace(/^(.+?)\.(?=\1\.)/, '');
+        candidates.add(deduped);
+
+        let match = null;
+        for (const candidate of candidates) {
+          const hits = byBasename.get(candidate) || [];
+          if (hits.length === 1) {
+            match = hits[0];
+            break;
+          }
+        }
+        if (!match) {
+          for (const candidate of candidates) {
+            const suffixHits = [];
+            for (const [base, paths] of byBasename) {
+              if (base === candidate || !base.endsWith(candidate)) continue;
+              const before = base[base.length - candidate.length - 1];
+              if (before === '-' || before === '_' || before === '.') suffixHits.push(...paths);
+            }
+            if (suffixHits.length === 1) {
+              match = suffixHits[0];
+              break;
+            }
+          }
+        }
+        if (!match) return full;
+
+        const fixed = `${upPrefix}${match}`;
+        repairs.push({ doc: doc.filename, from: target, to: fixed });
+        return `${open}${fixed}${close}`;
+      },
+    );
+  }
+
+  return repairs;
+}
+
+function missingSections(content, sections) {
+  const markdown = content || '';
+  const missing = [];
+  for (const section of sections || []) {
+    const re = new RegExp(`^##\\s+${section.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*$`, 'm');
+    if (!re.test(markdown)) missing.push(section);
+  }
+  return missing;
+}
+
+function validateDocs({ docs, docDir, repoFiles, packageScripts = {}, cliVerbs = [] }) {
+  const fileSet = new Set(repoFiles);
+  const docSet = new Set(docs.map((d) => `${docDir}/${d.filename}`));
+  const dirSet = buildDirSet(repoFiles);
+  const issues = [];
+
+  for (const doc of docs) {
+    const markdown = doc.content || '';
+
+    if (Array.isArray(doc.sections)) {
+      for (const section of missingSections(markdown, doc.sections)) {
+        issues.push({ doc: doc.filename, type: 'missing-section', detail: section });
+      }
+    }
+
+    for (const target of extractInternalLinks(markdown)) {
+      if (!linkResolves(target, { docDir, fileSet, docSet, dirSet })) {
+        issues.push({ doc: doc.filename, type: 'dead-link', detail: target });
+      }
+    }
+
+    for (const cmd of extractCitedCommands(markdown)) {
+      if ((cmd.tool === 'npm' || cmd.tool === 'pnpm' || cmd.tool === 'yarn')) {
+        const scriptName = cmd.arg1 === 'run' ? cmd.arg2 : cmd.arg1;
+        const builtins = new Set(['install', 'ci', 'test', 'start', 'publish', 'pack', 'link', 'init', 'exec', 'dlx', 'add', 'remove', 'i', 'it', 'update', 'audit', 'sync:ai']);
+        if (scriptName && !builtins.has(scriptName) && !(scriptName in packageScripts)) {
+          if (cmd.arg1 === 'run' || (!builtins.has(cmd.arg1) && cmd.tool === 'pnpm')) {
+            issues.push({ doc: doc.filename, type: 'unknown-script', detail: cmd.line });
+          }
+        }
+      }
+      if (cmd.tool === 'gitset' && cliVerbs.length && cmd.arg1 && !cmd.arg1.startsWith('-')) {
+        if (!cliVerbs.includes(cmd.arg1)) {
+          issues.push({ doc: doc.filename, type: 'unknown-cli-command', detail: cmd.line });
+        }
+      }
+    }
+  }
+
+  return issues;
+}
+
+module.exports = {
+  validateDocs,
+  missingSections,
+  repairInternalLinks,
+  extractInternalLinks,
+  extractCitedCommands,
+};