@gitset-dev/cli 2.3.2 → 2.4.0

package/index.js

@@ -27,6 +27,7 @@ ${theme.bold('AI commands')} (use your own provider key):
   release           Generate release notes from the commit range
   gitignore         Generate a .gitignore for the detected stack
   repo about        AI repo description + topics, applied via gh
+  knowledge         Build an agent-ready knowledge base from your source code
 
 ${theme.bold('Local tools')} (no AI):
   repo license      Generate a LICENSE file (offline)
@@ -121,6 +122,22 @@ Manage & apply a label pack (local pack + gh).
                    without asking (--sync is an alias)`,
   dependabot: `Usage: gitset dependabot [--resolve] [--dry-run] [--save-dev]
 Resolve Dependabot alerts for the current repo (local + gh).`,
+  knowledge: `Usage: gitset knowledge <init|scan|generate|update|automate> [flags]
+Build and maintain docs/gitset-knowledge/ — an agent-ready knowledge base
+derived from your source code, manifests and CI config (never from existing
+prose docs, so it can't inherit their drift). Secrets are redacted locally
+before anything reaches your AI provider.
+  init             scaffold .gitset-knowledge.json (optional scan config)
+  scan             structural pass: zero AI calls, prints plan + cost estimate
+  generate         full run (always shows the estimate and asks first)
+  update           incremental: re-summarizes only changed modules
+  automate         write a GitHub Actions workflow that runs update in CI
+                   and opens a PR when docs change (always asks first)
+  --since <ref>    update selector via git diff instead of content hashes
+  --mode <m>       automate trigger: push | releases | weekly
+  --provider <p>   override the default provider for this run
+  --model <m>      override the model
+  --yes            skip confirmations (CI)`,
   status: `Usage: gitset status
 Show git + configured-provider status.`,
   template: `Usage: gitset template <list|show|edit|path>
@@ -213,6 +230,8 @@ async function main() {
       code = await require('./src/commands/license').runLicenseCommand(rest); break;
     case 'labelspack':
       code = await require('./src/commands/labelspack').runLabelspackCommand(rest); break;
+    case 'knowledge':
+      code = await require('./src/commands/knowledge').runKnowledgeCommand(rest); break;
     case 'dependabot':
       code = (await require('./src/commands/dependabot-resolver')(null, rest)) || 0; break;
     case 'feedback':

package/lib/generate-local.js

@@ -53,6 +53,7 @@ async function generate(opts) {
     provider: cfg.provider,
     model: result.model,
     usage: result.usage,
+    finishReason: result.finishReason,
   };
 }
 

package/lib/knowledge/budget.js

@@ -0,0 +1,85 @@
+'use strict';
+
+const DEFAULTS = {
+  maxFileChars: 6000,
+  signatureChars: 3000,
+  maxBatchChars: 24000,
+  writeInputChars: 40000,
+  summarizeMaxTokens: 2048,
+  writeMaxTokens: 6144,
+};
+
+const DECLARATION_RE = /^[ \t]*(?:export\s+)?(?:default\s+)?(?:async\s+)?(?:function|class|const|let|var|def|interface|type|enum|struct|impl|fn|func|module\.exports|exports\.)[^\n]*$/gm;
+
+function estimateTokens(chars) {
+  return Math.ceil(chars / 4);
+}
+
+function extractSignatures(content, limit) {
+  const head = content.slice(0, Math.floor(limit / 3));
+  const declarations = [];
+  DECLARATION_RE.lastIndex = 0;
+  let m;
+  while ((m = DECLARATION_RE.exec(content)) !== null) {
+    const line = m[0].trim();
+    if (line.length > 3) declarations.push(line.slice(0, 200));
+    if (declarations.length >= 80) break;
+  }
+  const sigBlock = declarations.join('\n');
+  const combined = `${head}\n\n[declarations extracted from the rest of the file]\n${sigBlock}`;
+  return combined.length > limit ? `${combined.slice(0, limit)}\n…(truncated)` : combined;
+}
+
+function budgetFileContent(content, opts = {}) {
+  const { maxFileChars, signatureChars } = { ...DEFAULTS, ...opts };
+  if (content.length <= maxFileChars) {
+    return { content, mode: 'full' };
+  }
+  return { content: extractSignatures(content, signatureChars), mode: 'signatures' };
+}
+
+function batchModules(preparedModules, opts = {}) {
+  const { maxBatchChars } = { ...DEFAULTS, ...opts };
+  const batches = [];
+  for (const mod of preparedModules) {
+    let current = { module: mod.name, part: 1, files: [], chars: 0 };
+    for (const file of mod.files) {
+      const fileChars = file.content ? file.content.length : 0;
+      if (current.files.length > 0 && current.chars + fileChars > maxBatchChars) {
+        batches.push(current);
+        current = { module: mod.name, part: current.part + 1, files: [], chars: 0 };
+      }
+      current.files.push(file);
+      current.chars += fileChars;
+    }
+    if (current.files.length > 0) batches.push(current);
+  }
+  return batches;
+}
+
+function estimateRun({ batches, docCount, systemOverheadChars = 1500 }, opts = {}) {
+  const { summarizeMaxTokens, writeMaxTokens, writeInputChars } = { ...DEFAULTS, ...opts };
+  const summarizeCalls = batches.length;
+  const summarizeInputTokens = batches.reduce(
+    (sum, b) => sum + estimateTokens(b.chars + systemOverheadChars),
+    0,
+  );
+  const writeCalls = docCount;
+  const writeInputTokens = writeCalls * estimateTokens(writeInputChars + systemOverheadChars);
+  return {
+    totalCalls: summarizeCalls + writeCalls,
+    summarizeCalls,
+    writeCalls,
+    estInputTokens: summarizeInputTokens + writeInputTokens,
+    estMaxOutputTokens: summarizeCalls * summarizeMaxTokens + writeCalls * writeMaxTokens,
+  };
+}
+
+module.exports = {
+  DEFAULTS,
+  budgetFileContent,
+  extractSignatures,
+  batchModules,
+  estimateRun,
+  estimateTokens,
+};

package/lib/knowledge/ci.js

@@ -0,0 +1,99 @@
+'use strict';
+
+const WORKFLOW_PATH = '.github/workflows/gitset-knowledge.yml';
+
+const MODES = {
+  push: 'on every push to the default branch (paths-filtered; unchanged code costs nothing)',
+  releases: 'on every published release',
+  weekly: 'weekly (Mondays 06:00 UTC)',
+};
+
+function buildTrigger(mode, defaultBranch) {
+  if (mode === 'push') {
+    return [
+      'on:',
+      '  workflow_dispatch:',
+      '  push:',
+      `    branches: [${defaultBranch}]`,
+      '    paths-ignore:',
+      "      - 'docs/**'",
+      "      - '**.md'",
+      "      - '.gitignore'",
+      "      - 'LICENSE'",
+    ];
+  }
+  if (mode === 'releases') {
+    return [
+      'on:',
+      '  workflow_dispatch:',
+      '  release:',
+      '    types: [published]',
+    ];
+  }
+  if (mode === 'weekly') {
+    return [
+      'on:',
+      '  workflow_dispatch:',
+      '  schedule:',
+      "    - cron: '0 6 * * 1'",
+    ];
+  }
+  throw new Error(`Unknown automation mode "${mode}". One of: ${Object.keys(MODES).join(', ')}`);
+}
+
+function buildKnowledgeWorkflow({ mode, provider, envKey, model, defaultBranch = 'main' }) {
+  if (!provider || !envKey) throw new Error('provider and envKey are required');
+  const updateCmd = `gitset knowledge update --yes --provider ${provider}${model ? ` --model ${model}` : ''}`;
+
+  return [
+    'name: Gitset Knowledge Mapper',
+    `# Incrementally refreshes docs/gitset-knowledge/ with your own AI key.`,
+    `# Requires ONE repository secret: ${envKey} (Settings > Secrets > Actions).`,
+    '# The key is stored encrypted by GitHub and is only sent to your AI',
+    '# provider — never to Gitset. Runs whose mapped source files are',
+    '# unchanged exit without any AI call (content-hash diff), so harmless',
+    '# triggers cost nothing beyond a few CI seconds.',
+    ...buildTrigger(mode, defaultBranch),
+    'permissions:',
+    '  contents: write',
+    '  pull-requests: write',
+    'concurrency:',
+    '  group: gitset-knowledge',
+    '  cancel-in-progress: false',
+    'jobs:',
+    '  update:',
+    '    runs-on: ubuntu-latest',
+    '    steps:',
+    '      - uses: actions/checkout@v4',
+    '      - uses: actions/setup-node@v4',
+    '        with:',
+    "          node-version: '20'",
+    '      - run: npm install -g @gitset-dev/cli',
+    '      - name: Update knowledge base',
+    '        env:',
+    `          ${envKey}: \${{ secrets.${envKey} }}`,
+    `        run: ${updateCmd}`,
+    '      - name: Open a PR when the knowledge base changed',
+    '        env:',
+    '          GH_TOKEN: ${{ github.token }}',
+    '        run: |',
+    '          if git diff --quiet -- docs/gitset-knowledge AGENTS.md; then',
+    '            echo "Knowledge base already up to date."',
+    '            exit 0',
+    '          fi',
+    '          git config user.name "github-actions[bot]"',
+    '          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"',
+    '          git switch -c gitset/knowledge-update',
+    '          git add docs/gitset-knowledge AGENTS.md',
+    '          git commit -m "docs: refresh knowledge base"',
+    '          git push -f origin gitset/knowledge-update',
+    '          gh pr create --title "docs: refresh knowledge base" --body "Automated incremental update by [Gitset](https://gitset.dev) Knowledge Mapper. Only changed modules were re-analyzed; review like any other docs change." || true',
+    '',
+  ].join('\n');
+}
+
+module.exports = {
+  WORKFLOW_PATH,
+  MODES,
+  buildKnowledgeWorkflow,
+};

package/lib/knowledge/discover.js

@@ -0,0 +1,212 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { execFileSync } = require('child_process');
+
+const SOURCE_EXTS = new Set([
+  '.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs', '.mts', '.cts',
+  '.py', '.rb', '.go', '.rs', '.java', '.kt', '.swift', '.c', '.h',
+  '.cpp', '.hpp', '.cc', '.cs', '.php', '.scala', '.ex', '.exs',
+  '.sh', '.bash', '.zsh', '.astro', '.vue', '.svelte',
+]);
+
+const MANIFEST_NAMES = new Set([
+  'package.json', 'pyproject.toml', 'cargo.toml', 'go.mod', 'gemfile',
+  'requirements.txt', 'setup.py', 'setup.cfg', 'composer.json',
+  'pom.xml', 'build.gradle', 'pnpm-workspace.yaml', 'deno.json',
+]);
+
+const CONFIG_PATTERNS = [
+  /^\.?[\w-]+rc(\.\w+)?$/i,
+  /\.config\.(js|cjs|mjs|ts|json)$/i,
+  /^tsconfig(\..+)?\.json$/i,
+  /^dockerfile$/i,
+  /^docker-compose(\..+)?\.ya?ml$/i,
+  /^makefile$/i,
+  /^cloudbuild\.ya?ml$/i,
+  /^\.gitattributes$/i,
+  /^\.editorconfig$/i,
+  /^\.npmrc$/i,
+];
+
+const DOC_EXTS = new Set(['.md', '.mdx', '.rst', '.txt', '.adoc']);
+const DOC_NAMES = new Set(['license', 'notice', 'authors', 'codeowners', 'changelog']);
+
+const LOCKFILE_NAMES = new Set([
+  'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'cargo.lock',
+  'poetry.lock', 'gemfile.lock', 'composer.lock', 'bun.lockb',
+  'deno.lock', 'uv.lock', 'pipfile.lock', 'go.sum',
+]);
+
+const ASSET_EXTS = new Set([
+  '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.webp', '.avif',
+  '.woff', '.woff2', '.ttf', '.otf', '.eot', '.mp4', '.webm', '.mp3',
+  '.wav', '.pdf', '.zip', '.gz', '.tar', '.br', '.wasm', '.map',
+  '.lock', '.lockb', '.min.js', '.min.css',
+]);
+
+const TEST_PATTERNS = [
+  /(^|\/)__tests__\//,
+  /(^|\/)tests?\//,
+  /\.(test|spec)\.[jt]sx?$/,
+  /_test\.(go|py|rb)$/,
+  /(^|\/)conftest\.py$/,
+];
+
+const SENSITIVE_FILE_PATTERNS = [
+  /(^|\/)\.env(\.|$)/i,
+  /\.pem$/i,
+  /\.key$/i,
+  /(^|\/)id_(rsa|ed25519|ecdsa|dsa)(\.|$)/i,
+  /\.p12$/i,
+  /\.pfx$/i,
+  /\.keystore$/i,
+  /(^|\/)credentials?(\.|$)/i,
+  /(^|\/)secrets?\.(json|ya?ml|toml|txt)$/i,
+  /\.tfstate(\.|$)/i,
+  /(^|\/)\.netrc$/i,
+  /(^|\/)\.npmrc$/i,
+];
+
+const ALWAYS_IGNORED_DIRS = new Set([
+  'node_modules', '.git', 'dist', 'build', 'out', 'coverage', '.next',
+  '.astro', '.vercel', '.turbo', '.cache', 'vendor', '__pycache__',
+  '.venv', 'venv', 'target', '.idea', '.vscode', '.DS_Store',
+]);
+
+const OUTPUT_DIR = 'docs/gitset-knowledge';
+const MAX_CONTENT_BYTES = 200 * 1024;
+
+function gitListFiles(rootDir) {
+  try {
+    const out = execFileSync(
+      'git',
+      ['ls-files', '-z', '--cached', '--others', '--exclude-standard'],
+      { cwd: rootDir, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] },
+    );
+    return out.split('\0').filter(Boolean);
+  } catch {
+    return null;
+  }
+}
+
+function walkFiles(rootDir) {
+  const results = [];
+  const stack = [''];
+  while (stack.length) {
+    const rel = stack.pop();
+    const abs = path.join(rootDir, rel);
+    let entries;
+    try {
+      entries = fs.readdirSync(abs, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      if (entry.isSymbolicLink()) continue;
+      const childRel = rel ? `${rel}/${entry.name}` : entry.name;
+      if (entry.isDirectory()) {
+        if (!ALWAYS_IGNORED_DIRS.has(entry.name)) stack.push(childRel);
+      } else if (entry.isFile()) {
+        results.push(childRel);
+      }
+    }
+  }
+  return results;
+}
+
+function isSensitivePath(relPath) {
+  const p = relPath.toLowerCase();
+  return SENSITIVE_FILE_PATTERNS.some((re) => re.test(p));
+}
+
+function classify(relPath) {
+  const base = path.basename(relPath).toLowerCase();
+  const ext = path.extname(base);
+  const stem = base.replace(/\.[^.]*$/, '');
+
+  if (isSensitivePath(relPath)) return 'sensitive';
+  if (LOCKFILE_NAMES.has(base)) return 'asset';
+  if (MANIFEST_NAMES.has(base)) return 'manifest';
+  if (base.endsWith('.gemspec')) return 'manifest';
+  if (TEST_PATTERNS.some((re) => re.test(relPath))) return 'test';
+  if (/^\.github\/workflows\/.+\.ya?ml$/.test(relPath)) return 'config';
+  if (CONFIG_PATTERNS.some((re) => re.test(base))) return 'config';
+  if (DOC_EXTS.has(ext) || ((DOC_NAMES.has(stem) || DOC_NAMES.has(base)) && !SOURCE_EXTS.has(ext))) return 'doc';
+  if (relPath.startsWith('docs/')) return 'doc';
+  if (ASSET_EXTS.has(ext) || base.endsWith('.min.js') || base.endsWith('.min.css')) return 'asset';
+  if (SOURCE_EXTS.has(ext)) return 'source';
+  if (ext === '.json' || ext === '.yaml' || ext === '.yml' || ext === '.toml') return 'config';
+  return 'other';
+}
+
+function looksBinary(buffer) {
+  const slice = buffer.subarray(0, Math.min(buffer.length, 8192));
+  return slice.includes(0);
+}
+
+function discover(rootDir, options = {}) {
+  const { include = [], exclude = [] } = options;
+  let paths = gitListFiles(rootDir);
+  const viaGit = paths !== null;
+  if (!paths) paths = walkFiles(rootDir);
+
+  const excludeRes = exclude.map(globToRegExp);
+  const includeRes = include.map(globToRegExp);
+
+  const files = [];
+  for (const relPath of paths.sort()) {
+    const norm = relPath.split(path.sep).join('/');
+    if (norm.startsWith(`${OUTPUT_DIR}/`)) continue;
+    if (norm.split('/').some((seg) => ALWAYS_IGNORED_DIRS.has(seg))) continue;
+    if (excludeRes.some((re) => re.test(norm))) continue;
+    if (includeRes.length && !includeRes.some((re) => re.test(norm))) continue;
+
+    let size = 0;
+    try {
+      size = fs.statSync(path.join(rootDir, norm)).size;
+    } catch {
+      continue;
+    }
+
+    files.push({ path: norm, kind: classify(norm), size });
+  }
+
+  return { files, viaGit };
+}
+
+function readContent(rootDir, file) {
+  if (file.kind === 'sensitive' || file.kind === 'asset') return null;
+  if (file.size > MAX_CONTENT_BYTES) return null;
+  let buffer;
+  try {
+    buffer = fs.readFileSync(path.join(rootDir, file.path));
+  } catch {
+    return null;
+  }
+  if (looksBinary(buffer)) return null;
+  return buffer.toString('utf8');
+}
+
+function globToRegExp(glob) {
+  const pattern = String(glob)
+    .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+    .replace(/\*\*\/|\*\*|\*|\?/g, (tok) => {
+      if (tok === '**/') return '(?:.*/)?';
+      if (tok === '**') return '.*';
+      if (tok === '*') return '[^/]*';
+      return '[^/]';
+    });
+  return new RegExp(`^${pattern}$`);
+}
+
+module.exports = {
+  discover,
+  classify,
+  readContent,
+  isSensitivePath,
+  globToRegExp,
+  OUTPUT_DIR,
+  MAX_CONTENT_BYTES,
+};

package/lib/knowledge/index.js

@@ -0,0 +1,89 @@
+// VENDORED from gitset-core-v2/lib/knowledge — do not edit here. Run `pnpm sync:ai`.
+'use strict';
+
+const discoverLib = require('./discover');
+const secretsLib = require('./secrets');
+const mapLib = require('./map');
+const budgetLib = require('./budget');
+const stateLib = require('./state');
+const writeLib = require('./write');
+const validateLib = require('./validate');
+const ciLib = require('./ci');
+
+const LLM_ELIGIBLE_KINDS = new Set(['source', 'manifest', 'config']);
+
+function prepareRun(rootDir, options = {}) {
+  const { include = [], exclude = [], budgets = {} } = options;
+
+  const { files, viaGit } = discoverLib.discover(rootDir, { include, exclude });
+
+  const contentCache = new Map();
+  const readCached = (file) => {
+    if (contentCache.has(file.path)) return contentCache.get(file.path);
+    const content = discoverLib.readContent(rootDir, file);
+    contentCache.set(file.path, content);
+    return content;
+  };
+
+  const map = mapLib.buildMap(files, readCached);
+
+  const redactionReport = { totalFindings: 0, droppedFiles: [], findingsByRule: {} };
+  const preparedByModule = new Map();
+
+  for (const file of files) {
+    if (!LLM_ELIGIBLE_KINDS.has(file.kind)) continue;
+    const raw = readCached(file);
+    if (raw === null) continue;
+
+    file.hash = stateLib.hashContent(raw);
+
+    const sanitized = secretsLib.sanitizeForPrompt(raw);
+    for (const rule of sanitized.findings) {
+      redactionReport.totalFindings += 1;
+      redactionReport.findingsByRule[rule] = (redactionReport.findingsByRule[rule] || 0) + 1;
+    }
+    if (sanitized.dropped) {
+      redactionReport.droppedFiles.push(file.path);
+      continue;
+    }
+
+    const budgeted = budgetLib.budgetFileContent(sanitized.content, budgets);
+    const moduleKey = mapLib.moduleKeyFor(file.path);
+    if (!preparedByModule.has(moduleKey)) preparedByModule.set(moduleKey, []);
+    preparedByModule.get(moduleKey).push({
+      path: file.path,
+      kind: file.kind,
+      mode: budgeted.mode,
+      centrality: map.centrality.get(file.path) || 0,
+      content: budgeted.content,
+    });
+  }
+
+  const preparedModules = [...preparedByModule.entries()]
+    .sort((a, b) => a[0].localeCompare(b[0]))
+    .map(([name, prepared]) => ({
+      name,
+      files: prepared.sort((a, b) => b.centrality - a.centrality || a.path.localeCompare(b.path)),
+    }));
+
+  const batches = budgetLib.batchModules(preparedModules, budgets);
+  const estimate = budgetLib.estimateRun(
+    { batches, docCount: writeLib.DOC_SPECS.length },
+    budgets,
+  );
+
+  return { files, map, preparedModules, batches, estimate, redactionReport, viaGit };
+}
+
+module.exports = {
+  prepareRun,
+  LLM_ELIGIBLE_KINDS,
+  ...discoverLib,
+  ...secretsLib,
+  ...mapLib,
+  ...budgetLib,
+  ...stateLib,
+  ...writeLib,
+  ...validateLib,
+  ...ciLib,
+};