nubos-pilot 0.3.0 → 0.4.0

package/lib/workspace-scan.cjs

@@ -0,0 +1,290 @@
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const { NubosPilotError } = require('./core.cjs');
+
+const DEFAULT_IGNORES = Object.freeze(new Set([
+  'node_modules', '.git', '.nubos-pilot', '.planning', '.claude',
+  'vendor', 'target', 'build', 'dist', 'out',
+  '.next', '.nuxt', '.svelte-kit', '.astro',
+  '.venv', 'venv', 'env', '__pycache__', '.pytest_cache', '.mypy_cache', '.ruff_cache',
+  'coverage', '.coverage', '.nyc_output', '.tox',
+  '.idea', '.vscode', '.vs',
+  '.cache', '.turbo', '.parcel-cache', '.gradle',
+  'Pods', 'DerivedData',
+  'tmp', 'temp', '.tmp',
+]));
+
+const MANIFEST_FILES = Object.freeze(new Set([
+  'package.json', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml', 'pnpm-workspace.yaml',
+  'tsconfig.json', 'jsconfig.json',
+  'pyproject.toml', 'setup.py', 'setup.cfg', 'requirements.txt', 'Pipfile', 'Pipfile.lock', 'poetry.lock',
+  'Cargo.toml', 'Cargo.lock',
+  'go.mod', 'go.sum',
+  'composer.json', 'composer.lock',
+  'Gemfile', 'Gemfile.lock', 'Rakefile',
+  'mix.exs', 'rebar.config',
+  'pom.xml', 'build.gradle', 'build.gradle.kts', 'settings.gradle', 'settings.gradle.kts',
+  'Dockerfile', 'docker-compose.yml', 'docker-compose.yaml', 'compose.yml', 'compose.yaml',
+  'Makefile', 'CMakeLists.txt',
+  '.nvmrc', '.tool-versions', '.node-version', '.python-version', '.ruby-version',
+  '.env.example', '.env.sample', '.editorconfig',
+]));
+
+const DOC_FILE_PREFIXES = Object.freeze([
+  'README', 'CHANGELOG', 'LICENSE', 'CONTRIBUTING',
+  'ARCHITECTURE', 'ROADMAP', 'SECURITY', 'CODE_OF_CONDUCT',
+  'AUTHORS', 'NOTICE', 'DESIGN',
+]);
+
+const BINARY_EXTS = Object.freeze(new Set([
+  '.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.ico', '.bmp', '.tiff', '.avif',
+  '.pdf', '.zip', '.tar', '.gz', '.bz2', '.7z', '.rar', '.xz',
+  '.mp3', '.mp4', '.mov', '.avi', '.mkv', '.wav', '.flac', '.ogg', '.webm',
+  '.ttf', '.otf', '.woff', '.woff2', '.eot',
+  '.exe', '.dll', '.so', '.dylib', '.bin', '.o', '.a',
+  '.class', '.jar', '.war', '.pyc', '.pyo',
+  '.db', '.sqlite', '.sqlite3',
+  '.node', '.wasm',
+  '.psd', '.ai', '.sketch', '.fig',
+]));
+
+const MAX_FILE_HASH_BYTES = 512 * 1024;
+const MAX_FILE_CAPTURE_BYTES = 200 * 1024;
+const MAX_FILES_WALKED = 100000;
+const MAX_DEPTH = 12;
+
+function _isDocFile(basename) {
+  const up = basename.toUpperCase();
+  for (const prefix of DOC_FILE_PREFIXES) {
+    if (up === prefix) return true;
+    if (up.startsWith(prefix + '.')) return true;
+  }
+  return false;
+}
+
+function _isDotfileAllowed(name) {
+  if (name === '.nvmrc' || name === '.tool-versions' || name === '.node-version') return true;
+  if (name === '.python-version' || name === '.ruby-version') return true;
+  if (name === '.env.example' || name === '.env.sample') return true;
+  if (name === '.editorconfig' || name === '.gitignore' || name === '.dockerignore') return true;
+  if (name === '.gitattributes') return true;
+  return false;
+}
+
+function _sha256(buffer) {
+  return 'sha256:' + crypto.createHash('sha256').update(buffer).digest('hex');
+}
+
+function _readCapture(absPath) {
+  try {
+    const fd = fs.openSync(absPath, 'r');
+    try {
+      const stat = fs.fstatSync(fd);
+      const toRead = Math.min(stat.size, MAX_FILE_CAPTURE_BYTES);
+      const buf = Buffer.alloc(toRead);
+      if (toRead > 0) fs.readSync(fd, buf, 0, toRead, 0);
+      return {
+        content: buf.toString('utf-8'),
+        size: stat.size,
+        truncated: stat.size > toRead,
+      };
+    } finally {
+      fs.closeSync(fd);
+    }
+  } catch (err) {
+    return { error: err && err.code ? err.code : String(err) };
+  }
+}
+
+function _walk(root, ignores, opts) {
+  const files = [];
+  const skipped = [];
+  let walked = 0;
+
+  function visit(abs, rel, depth) {
+    if (walked >= opts.maxFiles) return;
+    if (depth > opts.maxDepth) return;
+    let entries;
+    try {
+      entries = fs.readdirSync(abs, { withFileTypes: true });
+    } catch (err) {
+      skipped.push({ path: rel || '.', reason: 'readdir-error', detail: err && err.code });
+      return;
+    }
+    for (const entry of entries) {
+      if (walked >= opts.maxFiles) {
+        skipped.push({ path: rel, reason: 'max-files-reached' });
+        return;
+      }
+      const name = entry.name;
+      if (ignores.has(name)) continue;
+      if (name.startsWith('.') && entry.isDirectory()) continue;
+      if (name.startsWith('.') && entry.isFile() && !_isDotfileAllowed(name)) continue;
+
+      const childAbs = path.join(abs, name);
+      const childRel = rel === '' ? name : rel + '/' + name;
+      if (entry.isSymbolicLink()) continue;
+      if (entry.isDirectory()) {
+        visit(childAbs, childRel, depth + 1);
+      } else if (entry.isFile()) {
+        walked++;
+        let stat;
+        try {
+          stat = fs.statSync(childAbs);
+        } catch (err) {
+          skipped.push({ path: childRel, reason: 'stat-error', detail: err && err.code });
+          continue;
+        }
+        files.push({
+          path: childRel,
+          absPath: childAbs,
+          size: stat.size,
+          ext: path.extname(name).toLowerCase(),
+          basename: name,
+        });
+      }
+    }
+  }
+
+  visit(root, '', 0);
+  return { files, skipped };
+}
+
+function _hashFile(file) {
+  if (BINARY_EXTS.has(file.ext)) return null;
+  if (file.size > MAX_FILE_HASH_BYTES) return null;
+  try {
+    return _sha256(fs.readFileSync(file.absPath));
+  } catch {
+    return null;
+  }
+}
+
+function scan(opts) {
+  const options = opts || {};
+  const cwd = path.resolve(options.cwd || process.cwd());
+  const ignores = new Set([...DEFAULT_IGNORES, ...(options.additionalIgnores || [])]);
+  const onProgress = typeof options.onProgress === 'function' ? options.onProgress : () => {};
+  const batchSize = options.batchSize > 0 ? options.batchSize : 500;
+  const maxFiles = options.maxFiles > 0 ? options.maxFiles : MAX_FILES_WALKED;
+  const maxDepth = options.maxDepth > 0 ? options.maxDepth : MAX_DEPTH;
+
+  let rootStat;
+  try {
+    rootStat = fs.statSync(cwd);
+  } catch (err) {
+    throw new NubosPilotError(
+      'scan-cwd-unreadable',
+      `cannot stat cwd: ${cwd}`,
+      { cwd, cause: err && err.code },
+    );
+  }
+  if (!rootStat.isDirectory()) {
+    throw new NubosPilotError(
+      'scan-not-a-directory',
+      `cwd is not a directory: ${cwd}`,
+      { cwd },
+    );
+  }
+
+  onProgress({ phase: 'walk-start', cwd });
+  const { files, skipped } = _walk(cwd, ignores, { maxFiles, maxDepth });
+  onProgress({ phase: 'walk-complete', file_count: files.length, skipped: skipped.length });
+
+  const language_distribution = {};
+  const manifests = {};
+  const docs = {};
+  const fileHashes = [];
+  let totalBytes = 0;
+
+  const totalBatches = Math.ceil(files.length / batchSize);
+  for (let i = 0; i < files.length; i += batchSize) {
+    const batchIndex = Math.floor(i / batchSize);
+    const batch = files.slice(i, i + batchSize);
+    onProgress({
+      phase: 'batch-start',
+      index: batchIndex,
+      total: totalBatches,
+      size: batch.length,
+      files_processed: i,
+      files_total: files.length,
+    });
+
+    for (const f of batch) {
+      totalBytes += f.size;
+      const extKey = f.ext || '<no-ext>';
+      language_distribution[extKey] = (language_distribution[extKey] || 0) + 1;
+
+      const isManifest = MANIFEST_FILES.has(f.basename);
+      const isDoc = _isDocFile(f.basename);
+      if (isManifest || isDoc) {
+        const captured = _readCapture(f.absPath);
+        const entry = { path: f.path, size: f.size, ...captured };
+        if (isManifest) manifests[f.path] = entry;
+        else docs[f.path] = entry;
+      }
+
+      const hash = _hashFile(f);
+      if (hash) {
+        fileHashes.push({
+          path: f.path,
+          size: f.size,
+          sha256: hash,
+          ext: f.ext,
+        });
+      } else {
+        skipped.push({
+          path: f.path,
+          reason: BINARY_EXTS.has(f.ext) ? 'binary' : f.size > MAX_FILE_HASH_BYTES ? 'too-large' : 'hash-error',
+          size: f.size,
+        });
+      }
+    }
+
+    onProgress({
+      phase: 'batch-done',
+      index: batchIndex,
+      total: totalBatches,
+      files_processed: Math.min(i + batch.length, files.length),
+      files_total: files.length,
+    });
+  }
+
+  let git = { is_repo: false };
+  if (typeof options.gitInfo === 'function') {
+    try { git = options.gitInfo(cwd) || { is_repo: false }; }
+    catch { git = { is_repo: false }; }
+  }
+
+  const result = {
+    cwd,
+    scanned_at: new Date().toISOString(),
+    stats: {
+      file_count: files.length,
+      hashed_count: fileHashes.length,
+      manifest_count: Object.keys(manifests).length,
+      doc_count: Object.keys(docs).length,
+      skipped_count: skipped.length,
+      total_bytes: totalBytes,
+    },
+    files: fileHashes,
+    manifests,
+    docs,
+    git,
+    language_distribution,
+    skipped,
+  };
+
+  onProgress({ phase: 'complete', stats: result.stats });
+  return result;
+}
+
+module.exports = {
+  scan,
+  DEFAULT_IGNORES,
+  MANIFEST_FILES,
+  BINARY_EXTS,
+  MAX_FILE_HASH_BYTES,
+  MAX_FILE_CAPTURE_BYTES,
+};

package/lib/workspace-scan.test.cjs

@@ -0,0 +1,212 @@
+const { test, afterEach } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const os = require('node:os');
+const { execFileSync } = require('node:child_process');
+
+const { scan, DEFAULT_IGNORES, MANIFEST_FILES } = require('./workspace-scan.cjs');
+const { workspaceGitInfo } = require('./git.cjs');
+
+const _sandboxes = [];
+
+function makeSandbox() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'np-ws-scan-'));
+  _sandboxes.push(dir);
+  return dir;
+}
+
+function write(root, rel, content) {
+  const abs = path.join(root, rel);
+  fs.mkdirSync(path.dirname(abs), { recursive: true });
+  fs.writeFileSync(abs, content);
+}
+
+afterEach(() => {
+  while (_sandboxes.length) {
+    const dir = _sandboxes.pop();
+    try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
+  }
+});
+
+test('WS-1: empty directory returns zero files', () => {
+  const root = makeSandbox();
+  const result = scan({ cwd: root });
+  assert.equal(result.stats.file_count, 0);
+  assert.equal(result.stats.hashed_count, 0);
+  assert.deepEqual(result.files, []);
+  assert.equal(result.git.is_repo, false);
+});
+
+test('WS-2: walks regular source files and hashes them', () => {
+  const root = makeSandbox();
+  write(root, 'src/index.js', 'console.log("hi");\n');
+  write(root, 'src/util.js', 'module.exports = {};\n');
+  write(root, 'lib/helper.py', 'def f(): pass\n');
+
+  const result = scan({ cwd: root });
+  assert.equal(result.stats.file_count, 3);
+  assert.equal(result.stats.hashed_count, 3);
+  const paths = result.files.map((f) => f.path).sort();
+  assert.deepEqual(paths, ['lib/helper.py', 'src/index.js', 'src/util.js']);
+  for (const f of result.files) {
+    assert.match(f.sha256, /^sha256:[a-f0-9]{64}$/);
+  }
+});
+
+test('WS-3: ignores node_modules / .git / vendor / .nubos-pilot', () => {
+  const root = makeSandbox();
+  write(root, 'src/a.js', 'a');
+  write(root, 'node_modules/foo/index.js', 'foo');
+  write(root, 'vendor/libx/x.php', 'x');
+  write(root, '.git/HEAD', 'ref: refs/heads/main');
+  write(root, '.nubos-pilot/PROJECT.md', '# project');
+
+  const result = scan({ cwd: root });
+  const paths = result.files.map((f) => f.path);
+  assert.deepEqual(paths, ['src/a.js']);
+});
+
+test('WS-4: captures manifest content (package.json) verbatim', () => {
+  const root = makeSandbox();
+  const pkg = { name: 'demo', version: '1.0.0', dependencies: { express: '^4' } };
+  write(root, 'package.json', JSON.stringify(pkg, null, 2));
+
+  const result = scan({ cwd: root });
+  assert.ok(result.manifests['package.json']);
+  const parsed = JSON.parse(result.manifests['package.json'].content);
+  assert.equal(parsed.name, 'demo');
+  assert.equal(parsed.dependencies.express, '^4');
+});
+
+test('WS-5: captures README as a doc', () => {
+  const root = makeSandbox();
+  write(root, 'README.md', '# Demo\n\nHello world.\n');
+
+  const result = scan({ cwd: root });
+  assert.ok(result.docs['README.md']);
+  assert.ok(result.docs['README.md'].content.includes('Demo'));
+});
+
+test('WS-6: language_distribution counts extensions', () => {
+  const root = makeSandbox();
+  write(root, 'a.js', 'a');
+  write(root, 'b.js', 'b');
+  write(root, 'c.py', 'c');
+  write(root, 'Makefile', 'all:\n');
+
+  const result = scan({ cwd: root });
+  assert.equal(result.language_distribution['.js'], 2);
+  assert.equal(result.language_distribution['.py'], 1);
+  assert.equal(result.language_distribution['<no-ext>'], 1);
+});
+
+test('WS-7: onProgress emits batch events', () => {
+  const root = makeSandbox();
+  for (let i = 0; i < 5; i++) write(root, `f${i}.txt`, 'x');
+
+  const events = [];
+  scan({ cwd: root, batchSize: 2, onProgress: (e) => events.push(e.phase) });
+  assert.ok(events.includes('walk-start'));
+  assert.ok(events.includes('walk-complete'));
+  assert.ok(events.includes('batch-start'));
+  assert.ok(events.includes('batch-done'));
+  assert.equal(events[events.length - 1], 'complete');
+});
+
+test('WS-8: binary extensions skip hashing', () => {
+  const root = makeSandbox();
+  write(root, 'logo.png', Buffer.from([0x89, 0x50, 0x4e, 0x47]));
+  write(root, 'code.js', 'x');
+
+  const result = scan({ cwd: root });
+  const hashed = result.files.map((f) => f.path);
+  assert.deepEqual(hashed, ['code.js']);
+  const skippedPaths = result.skipped.map((s) => s.path);
+  assert.ok(skippedPaths.includes('logo.png'));
+});
+
+test('WS-9: additionalIgnores extend the default ignore set', () => {
+  const root = makeSandbox();
+  write(root, 'src/a.js', 'a');
+  write(root, 'generated/big.js', 'g');
+
+  const result = scan({ cwd: root, additionalIgnores: ['generated'] });
+  const paths = result.files.map((f) => f.path);
+  assert.deepEqual(paths, ['src/a.js']);
+});
+
+test('WS-10: detects git repo and captures commits', () => {
+  const root = makeSandbox();
+  write(root, 'a.txt', 'a');
+  try {
+    execFileSync('git', ['init', '-q'], { cwd: root, stdio: 'ignore' });
+    execFileSync('git', ['config', 'user.email', 't@t.t'], { cwd: root, stdio: 'ignore' });
+    execFileSync('git', ['config', 'user.name', 't'], { cwd: root, stdio: 'ignore' });
+    execFileSync('git', ['add', 'a.txt'], { cwd: root, stdio: 'ignore' });
+    execFileSync('git', ['commit', '-q', '-m', 'init'], { cwd: root, stdio: 'ignore' });
+  } catch {
+    return;
+  }
+
+  const result = scan({ cwd: root, gitInfo: workspaceGitInfo });
+  assert.equal(result.git.is_repo, true);
+  assert.ok(Array.isArray(result.git.commits));
+  assert.ok(result.git.commits.length >= 1);
+  assert.equal(result.git.commits[0].subject, 'init');
+});
+
+test('WS-11: throws scan-not-a-directory for file path', () => {
+  const root = makeSandbox();
+  const filePath = path.join(root, 'not-a-dir.txt');
+  fs.writeFileSync(filePath, 'x');
+  assert.throws(
+    () => scan({ cwd: filePath }),
+    (err) => err.code === 'scan-not-a-directory',
+  );
+});
+
+test('WS-12: manifest file triggers capture even in nested dir', () => {
+  const root = makeSandbox();
+  write(root, 'services/api/package.json', JSON.stringify({ name: 'api' }));
+  write(root, 'services/api/src/main.ts', 'export {};');
+
+  const result = scan({ cwd: root });
+  assert.ok(result.manifests['services/api/package.json']);
+});
+
+test('WS-13: stats counts match produced arrays', () => {
+  const root = makeSandbox();
+  write(root, 'a.js', 'a');
+  write(root, 'b.js', 'b');
+  write(root, 'README.md', '# x');
+  write(root, 'package.json', '{}');
+
+  const result = scan({ cwd: root });
+  assert.equal(result.stats.file_count, 4);
+  assert.equal(result.stats.manifest_count, Object.keys(result.manifests).length);
+  assert.equal(result.stats.doc_count, Object.keys(result.docs).length);
+  assert.equal(result.stats.hashed_count, result.files.length);
+});
+
+test('WS-14: dotfiles like .nvmrc are walked but .idea directory is ignored', () => {
+  const root = makeSandbox();
+  write(root, '.nvmrc', '22');
+  write(root, '.idea/workspace.xml', '<x/>');
+  write(root, 'src/a.js', 'a');
+
+  const result = scan({ cwd: root });
+  const paths = result.files.map((f) => f.path).sort();
+  assert.ok(paths.includes('.nvmrc'));
+  assert.ok(paths.includes('src/a.js'));
+  assert.ok(!paths.some((p) => p.startsWith('.idea')));
+});
+
+test('WS-15: MANIFEST_FILES set is frozen and contains known ones', () => {
+  assert.ok(MANIFEST_FILES.has('package.json'));
+  assert.ok(MANIFEST_FILES.has('Cargo.toml'));
+  assert.ok(MANIFEST_FILES.has('go.mod'));
+  assert.ok(MANIFEST_FILES.has('composer.json'));
+  assert.ok(DEFAULT_IGNORES.has('node_modules'));
+  assert.ok(DEFAULT_IGNORES.has('.git'));
+});

package/np-tools.cjs

@@ -15,6 +15,7 @@ const initWorkflows = {
   'discuss-phase-power': require('./bin/np-tools/discuss-phase-power.cjs'),
   'research-phase':      require('./bin/np-tools/research-phase.cjs'),
   'new-project':         require('./bin/np-tools/new-project.cjs'),
+  'discuss-project':     require('./bin/np-tools/discuss-project.cjs'),
   'new-milestone':       require('./bin/np-tools/new-milestone.cjs'),
   'plan-milestone-gaps': require('./bin/np-tools/plan-milestone-gaps.cjs'),
 
@@ -52,6 +53,8 @@ const topLevelCommands = {
   'commit':         require('./bin/np-tools/commit.cjs'),
   'config-get':     require('./bin/np-tools/config.cjs'),
   'dispatch':       require('./bin/np-tools/dispatch.cjs'),
+  'scan-codebase':  require('./bin/np-tools/scan-codebase.cjs'),
+  'update-docs':    require('./bin/np-tools/update-docs.cjs'),
   'doctor':         require('./bin/np-tools/doctor.cjs'),
   'generate-slug':  require('./bin/np-tools/slug.cjs'),
   'metrics':        require('./bin/np-tools/metrics.cjs'),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nubos-pilot",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "AI-driven planning and execution tool for code projects",
   "homepage": "https://github.com/Nubos-AI/nubos-pilot",
   "repository": {

package/templates/PROJECT.md

@@ -1,4 +1,4 @@
-<!-- Placeholders: core_value, created_date, first_milestone_name, first_phase_name, primary_constraints, project_name -->
+<!-- Placeholders: core_value, created_date, domain_text, first_milestone_name, first_phase_name, non_goals_text, primary_constraints, project_description, project_name, strategic_decisions_text, success_criteria_text, target_users_text -->
 # {{project_name}}
 
 ## Project
@@ -7,9 +7,15 @@
 
 ## What This Is
 
-{{project_name}} is an early-stage project. Update this section after the first
-phase ships with a concrete 2-3 sentence description of what the product does
-and who it serves. Use the user's language and framing.
+{{project_description}}
+
+## Domain
+
+{{domain_text}}
+
+## Target Users
+
+{{target_users_text}}
 
 ## Core Value
 
@@ -18,6 +24,18 @@ and who it serves. Use the user's language and framing.
 If everything else fails, this one sentence must remain true. It drives
 prioritization when tradeoffs arise.
 
+## Non-Goals
+
+{{non_goals_text}}
+
+## Success Criteria
+
+{{success_criteria_text}}
+
+## Strategic Decisions
+
+{{strategic_decisions_text}}
+
 ## Constraints
 
 {{primary_constraints}}
@@ -55,6 +73,10 @@ PROJECT.md evolves throughout the project lifecycle.
 2. Core Value check — still the right priority?
 3. Update Current Focus with next milestone/phase
 
+**When scope or positioning shifts:**
+- Run `np:discuss-project` to refresh Domain, Target Users, Non-Goals,
+  Success Criteria, and Strategic Decisions without starting over.
+
 ---
 *Created: {{created_date}}*
 *Last updated: {{created_date}} after np:new-project*