gitnexus 1.6.8-rc.8 → 1.6.8

package/dist/server/api.js

@@ -20,11 +20,16 @@ import { searchFTSFromLbug } from '../core/search/bm25-index.js';
 import { hybridSearch } from '../core/search/hybrid-search.js';
 import { LocalBackend } from '../mcp/local/local-backend.js';
 import { mountMCPEndpoints } from './mcp-http.js';
-import { fork } from 'child_process';
-import { fileURLToPath, pathToFileURL } from 'url';
+import { fileURLToPath } from 'url';
 import { JobManager } from './analyze-job.js';
 import { assertString, escapeRegExp, BadRequestError, createRouteLimiter } from './validation.js';
-import { extractRepoName, getCloneDir, cloneOrPull } from './git-clone.js';
+import { extractRepoName, getCloneDir, cloneOrPull, warnIfInsecureAzureConfig, GITHUB_TOKEN_HOSTS, } from './git-clone.js';
+import { createAnalyzeUploadHandler } from './analyze-upload.js';
+import { createLocalhostOriginGuard, normalizeBoundHost } from './middleware.js';
+import { createLaunchAnalysisWorker } from './analyze-launch.js';
+import { UPLOAD_ROOT } from './upload-paths.js';
+import { sweepStaleUploads } from './upload-sweep.js';
+import { isRfc1918PrivateIpv4 } from './private-ip.js';
 import { logger, flushLoggerSync } from '../core/logger.js';
 const _require = createRequire(import.meta.url);
 const pkg = _require('../../package.json');
@@ -75,21 +80,7 @@ export const isAllowedOrigin = (origin) => {
     // Only allow HTTP(S) origins — reject ftp://, file://, etc.
     if (protocol !== 'http:' && protocol !== 'https:')
         return false;
-    const octets = hostname.split('.').map(Number);
-    if (octets.length !== 4 || octets.some((o) => !Number.isInteger(o) || o < 0 || o > 255)) {
-        return false;
-    }
-    const [a, b] = octets;
-    // 10.0.0.0/8
-    if (a === 10)
-        return true;
-    // 172.16.0.0/12  →  172.16.x.x – 172.31.x.x
-    if (a === 172 && b >= 16 && b <= 31)
-        return true;
-    // 192.168.0.0/16
-    if (a === 192 && b === 168)
-        return true;
-    return false;
+    return isRfc1918PrivateIpv4(hostname);
 };
 export class ClientDisconnectedError extends Error {
     constructor() {
@@ -589,7 +580,45 @@ export const handleQueryRequest = async (req, res, resolveRepo) => {
         res.status(500).json({ error: err.message || 'Query failed' });
     }
 };
+/**
+ * Validate the optional `token` field of POST /api/analyze. Returns an
+ * { status, error } to send, or null when the token is absent or valid.
+ *
+ * The token is a GitHub PAT: charset-restricted (blocks CRLF header
+ * smuggling), length-bounded (1–256), and bound to github.com using the SAME
+ * GITHUB_TOKEN_HOSTS allowlist + hostname parse as resolveGitCredential, so a
+ * token the API accepts is exactly the one buildGitEnv will inject — and one
+ * it rejects is never sent off github.com.
+ *
+ * Exported for unit tests (the route validation is otherwise only reachable
+ * by booting the server).
+ */
+export function validateAnalyzeToken(repoToken, repoUrl) {
+    if (repoToken === undefined)
+        return null;
+    if (typeof repoToken !== 'string')
+        return { status: 400, error: '"token" must be a string' };
+    if (repoToken.length === 0 || repoToken.length > 256)
+        return { status: 400, error: '"token" length must be between 1 and 256' };
+    if (!/^[A-Za-z0-9._~+/=-]+$/.test(repoToken))
+        return { status: 400, error: '"token" contains invalid characters' };
+    if (!repoUrl || typeof repoUrl !== 'string')
+        return { status: 400, error: '"token" requires "url"' };
+    let tokenHost;
+    try {
+        tokenHost = new URL(repoUrl).hostname.toLowerCase();
+    }
+    catch {
+        return { status: 400, error: '"url" must be a valid URL when "token" is provided' };
+    }
+    if (!GITHUB_TOKEN_HOSTS.has(tokenHost))
+        return { status: 400, error: '"token" is only supported for github.com URLs' };
+    return null;
+}
 export const createServer = async (port, host = '127.0.0.1') => {
+    // Surface a cleartext Azure DevOps PAT config at boot (operators rarely
+    // read per-request logs). Warn-only — http:// self-hosted stays supported.
+    warnIfInsecureAzureConfig();
     const app = express();
     app.disable('x-powered-by');
     // Trust X-Forwarded-* headers only when the connection comes from the
@@ -630,6 +659,17 @@ export const createServer = async (port, host = '127.0.0.1') => {
         },
     }));
     app.use(express.json({ limit: '10mb' }));
+    // Same-host origin guard for write routes. Only allows loopback and the
+    // server's own bound host — scoped to prevent CSRF from other LAN devices.
+    const requireLocalhostOrigin = createLocalhostOriginGuard(host);
+    // A wildcard bind (`0.0.0.0`/`::`) has no single host identity for the
+    // same-host check, so browser write routes accept only loopback origins.
+    // Warn the operator so a remote-access deployment isn't silently write-blocked.
+    if (host && normalizeBoundHost(host) === undefined) {
+        logger.warn({ host }, `[gitnexus serve] Bound to a wildcard address (${host}); browser write routes ` +
+            `accept only loopback origins (localhost/127.0.0.1/[::1]). To allow writes from a ` +
+            `specific LAN address, bind --host <that-address> instead of a wildcard.`);
+    }
     // No explicit OPTIONS route is registered. The Chromium Private Network
     // Access header is set by the global middleware above (pre-cors), and
     // `cors()` itself handles OPTIONS preflights for every path. Registering a
@@ -640,6 +680,8 @@ export const createServer = async (port, host = '127.0.0.1') => {
     await backend.init();
     const cleanupMcp = mountMCPEndpoints(app, backend);
     const jobManager = new JobManager();
+    // Backstop: remove any upload staging dirs orphaned by a previous crash.
+    void sweepStaleUploads().catch(() => { });
     // Shared repo lock — prevents concurrent analyze + embed on the same repo path,
     // which would corrupt LadybugDB (analyze calls closeLbug + initLbug while embed has queries in flight).
     const activeRepoPaths = new Set();
@@ -653,6 +695,14 @@ export const createServer = async (port, host = '127.0.0.1') => {
     const releaseRepoLock = (repoPath) => {
         activeRepoPaths.delete(repoPath);
     };
+    // Launch the analyze worker for an already-resolved repo directory. Shared by
+    // the JSON /api/analyze route and the multipart /api/analyze/upload route.
+    const launchAnalysisWorker = createLaunchAnalysisWorker({
+        jobManager,
+        backend,
+        acquireRepoLock,
+        releaseRepoLock,
+    });
     /**
      * Maximum time the hold-queue will wait for an active analysis job to complete.
      * Must stay in sync with the frontend's `fetchRepoInfo({ awaitAnalysis: true })` timeout.
@@ -816,7 +866,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
     // Rate-limited (CodeQL js/missing-rate-limiting): destructive operation
     // doing fs.rm of clone + storage dirs. Default 60 rpm/IP is generous for
     // delete; tighten if abuse is observed.
-    app.delete('/api/repo', createRouteLimiter(), async (req, res) => {
+    app.delete('/api/repo', createRouteLimiter(), requireLocalhostOrigin, async (req, res) => {
         try {
             const repoName = requestedRepo(req);
             if (!repoName) {
@@ -867,6 +917,13 @@ export const createServer = async (port, host = '127.0.0.1') => {
                         /* clone dir may not exist */
                     }
                 }
+                // 2b. Delete the uploaded repo dir if entry.path lives under
+                // UPLOAD_ROOT. Drive this off entry.path (not a name-rederived dir) so
+                // a same-named clone is never affected.
+                const resolvedEntry = path.resolve(entry.path);
+                if (resolvedEntry === UPLOAD_ROOT || resolvedEntry.startsWith(UPLOAD_ROOT + path.sep)) {
+                    await fs.rm(resolvedEntry, { recursive: true, force: true }).catch(() => { });
+                }
                 // 3. Unregister from the global registry
                 const { unregisterRepo } = await import('../storage/repo-manager.js');
                 await unregisterRepo(entry.path);
@@ -1251,9 +1308,9 @@ export const createServer = async (port, host = '127.0.0.1') => {
     });
     // ── Analyze API ──────────────────────────────────────────────────────
     // POST /api/analyze — start a new analysis job
-    app.post('/api/analyze', createRouteLimiter({ limit: 10 }), async (req, res) => {
+    app.post('/api/analyze', createRouteLimiter({ limit: 10 }), requireLocalhostOrigin, async (req, res) => {
         try {
-            const { url: repoUrl, path: repoLocalPath, force, embeddings, dropEmbeddings } = req.body;
+            const { url: repoUrl, path: repoLocalPath, force, embeddings, dropEmbeddings, token: repoToken, } = req.body;
             // Input type validation
             if (repoUrl !== undefined && typeof repoUrl !== 'string') {
                 res.status(400).json({ error: '"url" must be a string' });
@@ -1267,21 +1324,43 @@ export const createServer = async (port, host = '127.0.0.1') => {
                 res.status(400).json({ error: 'Provide "url" (git URL) or "path" (local path)' });
                 return;
             }
-            // Path validation: require absolute path, reject traversal (e.g. /tmp/../etc/passwd)
-            if (repoLocalPath) {
-                if (!path.isAbsolute(repoLocalPath)) {
-                    res.status(400).json({ error: '"path" must be an absolute path' });
-                    return;
-                }
-                if (path.normalize(repoLocalPath) !== path.resolve(repoLocalPath)) {
-                    res.status(400).json({ error: '"path" must not contain traversal sequences' });
-                    return;
-                }
+            // Token: optional, restricted charset to prevent header smuggling
+            // (CRLF), bound length, and bound to github.com (see validateAnalyzeToken).
+            const tokenError = validateAnalyzeToken(repoToken, repoUrl);
+            if (tokenError) {
+                res.status(tokenError.status).json({ error: tokenError.error });
+                return;
+            }
+            // Path validation. The previous `normalize !== resolve` guard was inert
+            // (both collapse `..` identically) and only false-rejected trailing
+            // slashes, so it is dropped. Analyzing a local path the operator names
+            // is the tool's intended capability (same as the CLI); the dangerous
+            // part was cross-origin reach, which is closed by requireLocalhostOrigin
+            // on this route (scoped to the server's own bound host — other LAN
+            // devices are NOT trusted). We only require an absolute path here and
+            // let the analyze worker surface a clear error if it does not exist.
+            // (We do NOT realpath/stat the path in-route: that would be a
+            // user-controlled filesystem read — CodeQL js/path-injection — for no
+            // security gain.)
+            if (repoLocalPath && !path.isAbsolute(repoLocalPath)) {
+                res.status(400).json({ error: '"path" must be an absolute path' });
+                return;
             }
             const job = jobManager.createJob({ repoUrl, repoPath: repoLocalPath });
-            // If job was already running (dedup), just return its id
+            // If job was already running (dedup), just return its id. The token is
+            // not part of the dedup identity and is never stored on the job, so a
+            // token on THIS request had no effect — the existing job already
+            // cloned (or is cloning) with whatever credentials its originating
+            // request supplied. Surface `tokenIgnored` so an authenticated caller
+            // isn't misled into thinking their PAT took effect on a reused job.
             if (job.status !== 'queued') {
-                res.status(202).json({ jobId: job.id, status: job.status });
+                const body = {
+                    jobId: job.id,
+                    status: job.status,
+                };
+                if (repoToken !== undefined)
+                    body.tokenIgnored = true;
+                res.status(202).json(body);
                 return;
             }
             // Mark as active synchronously to prevent race with concurrent requests
@@ -1303,138 +1382,12 @@ export const createServer = async (port, host = '127.0.0.1') => {
                             jobManager.updateJob(job.id, {
                                 progress: { phase: progress.phase, percent: 5, message: progress.message },
                             });
-                        });
+                        }, repoToken ? { token: repoToken } : undefined);
                     }
                     if (!targetPath) {
                         throw new Error('No target path resolved');
                     }
-                    // Acquire shared repo lock (keyed on storagePath to match embed handler)
-                    const analyzeLockKey = getStoragePath(targetPath);
-                    const lockErr = acquireRepoLock(analyzeLockKey);
-                    if (lockErr) {
-                        jobManager.updateJob(job.id, { status: 'failed', error: lockErr });
-                        return;
-                    }
-                    jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' });
-                    // ── Worker fork with auto-retry ──────────────────────────────
-                    //
-                    // Forks a child process with 8GB heap. If the worker crashes
-                    // (OOM, native addon segfault, etc.), it retries up to
-                    // MAX_WORKER_RETRIES times with exponential backoff before
-                    // marking the job as permanently failed.
-                    //
-                    // In dev mode (tsx), registers the tsx ESM hook via a file://
-                    // URL so the child can compile TypeScript on-the-fly.
-                    const MAX_WORKER_RETRIES = 2;
-                    const callerPath = fileURLToPath(import.meta.url);
-                    const isDev = callerPath.endsWith('.ts');
-                    const workerFile = isDev ? 'analyze-worker.ts' : 'analyze-worker.js';
-                    const workerPath = path.join(path.dirname(callerPath), workerFile);
-                    const tsxHookArgs = isDev
-                        ? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href]
-                        : [];
-                    const forkWorker = () => {
-                        const currentJob = jobManager.getJob(job.id);
-                        if (!currentJob || currentJob.status === 'complete' || currentJob.status === 'failed')
-                            return;
-                        const child = fork(workerPath, [], {
-                            execArgv: [...tsxHookArgs, '--max-old-space-size=8192'],
-                            stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
-                        });
-                        // Capture stderr for crash diagnostics
-                        let stderrChunks = '';
-                        child.stderr?.on('data', (chunk) => {
-                            stderrChunks += chunk.toString();
-                            if (stderrChunks.length > 4096)
-                                stderrChunks = stderrChunks.slice(-4096);
-                        });
-                        child.on('message', (msg) => {
-                            if (msg.type === 'progress') {
-                                jobManager.updateJob(job.id, {
-                                    status: 'analyzing',
-                                    progress: { phase: msg.phase, percent: msg.percent, message: msg.message },
-                                });
-                            }
-                            else if (msg.type === 'complete') {
-                                releaseRepoLock(analyzeLockKey);
-                                // Reinitialize backend BEFORE marking complete — ensures the new
-                                // repo is queryable when the client receives the SSE complete event.
-                                backend
-                                    .init()
-                                    .then(() => {
-                                    jobManager.updateJob(job.id, {
-                                        status: 'complete',
-                                        repoName: msg.result.repoName,
-                                    });
-                                })
-                                    .catch((err) => {
-                                    logger.error({ err }, 'backend.init() failed after analyze:');
-                                    jobManager.updateJob(job.id, {
-                                        status: 'failed',
-                                        error: 'Server failed to reload after analysis. Try again.',
-                                    });
-                                });
-                            }
-                            else if (msg.type === 'error') {
-                                releaseRepoLock(analyzeLockKey);
-                                jobManager.updateJob(job.id, {
-                                    status: 'failed',
-                                    error: msg.message,
-                                });
-                            }
-                        });
-                        child.on('error', (err) => {
-                            releaseRepoLock(analyzeLockKey);
-                            jobManager.updateJob(job.id, {
-                                status: 'failed',
-                                error: `Worker process error: ${err.message}`,
-                            });
-                        });
-                        child.on('exit', (code) => {
-                            const j = jobManager.getJob(job.id);
-                            if (!j || j.status === 'complete' || j.status === 'failed')
-                                return;
-                            // Worker crashed — attempt retry if under the limit
-                            if (j.retryCount < MAX_WORKER_RETRIES) {
-                                j.retryCount++;
-                                const delay = 1000 * Math.pow(2, j.retryCount - 1); // 1s, 2s
-                                const lastErr = stderrChunks.trim().split('\n').pop() || '';
-                                logger.warn(`Analyze worker crashed (code ${code}), retry ${j.retryCount}/${MAX_WORKER_RETRIES} in ${delay}ms` +
-                                    (lastErr ? `: ${lastErr}` : ''));
-                                jobManager.updateJob(job.id, {
-                                    status: 'analyzing',
-                                    progress: {
-                                        phase: 'retrying',
-                                        percent: j.progress.percent,
-                                        message: `Worker crashed, retrying (${j.retryCount}/${MAX_WORKER_RETRIES})...`,
-                                    },
-                                });
-                                stderrChunks = '';
-                                setTimeout(forkWorker, delay);
-                            }
-                            else {
-                                // Exhausted retries — permanent failure
-                                releaseRepoLock(analyzeLockKey);
-                                jobManager.updateJob(job.id, {
-                                    status: 'failed',
-                                    error: `Worker crashed ${MAX_WORKER_RETRIES + 1} times (code ${code})${stderrChunks ? ': ' + stderrChunks.trim().split('\n').pop() : ''}`,
-                                });
-                            }
-                        });
-                        // Register child for cancellation + timeout tracking
-                        jobManager.registerChild(job.id, child);
-                        // Send start command to child
-                        child.send({
-                            type: 'start',
-                            repoPath: targetPath,
-                            options: {
-                                force: !!force,
-                                embeddings: !!embeddings,
-                                dropEmbeddings: !!dropEmbeddings,
-                            },
-                        });
-                    };
-                    forkWorker();
+                    launchAnalysisWorker(job, targetPath, { force, embeddings, dropEmbeddings });
                 }
                 catch (err) {
                     if (targetPath)
@@ -1456,6 +1409,15 @@ export const createServer = async (port, host = '127.0.0.1') => {
             }
         }
     });
+    // POST /api/analyze/upload — analyze a browser folder upload.
+    // Securely ingests the multipart upload into a sandbox, promotes it to a
+    // persistent dir, and analyzes it via the shared job/worker machinery.
+    // localhost-only (no cross-origin write reach) + conservative rate limit.
+    app.post('/api/analyze/upload', createRouteLimiter({ limit: 5 }), requireLocalhostOrigin, createAnalyzeUploadHandler({
+        createJob: (params) => jobManager.createJob(params),
+        launch: (job, targetPath, opts) => launchAnalysisWorker(job, targetPath, opts),
+        failJob: (jobId, error) => jobManager.updateJob(jobId, { status: 'failed', error }),
+    }));
     // GET /api/analyze/:jobId — poll job status
     app.get('/api/analyze/:jobId', (req, res) => {
         const job = jobManager.getJob(req.params.jobId);
@@ -1478,8 +1440,9 @@ export const createServer = async (port, host = '127.0.0.1') => {
     // GET /api/analyze/:jobId/progress — SSE stream (shared helper)
     mountSSEProgress(app, '/api/analyze/:jobId/progress', jobManager);
     // DELETE /api/analyze/:jobId — cancel a running analysis job
-    app.delete('/api/analyze/:jobId', (req, res) => {
-        const job = jobManager.getJob(req.params.jobId);
+    app.delete('/api/analyze/:jobId', requireLocalhostOrigin, (req, res) => {
+        const jobId = req.params.jobId;
+        const job = jobManager.getJob(jobId);
         if (!job) {
             res.status(404).json({ error: 'Job not found' });
             return;
@@ -1488,13 +1451,13 @@ export const createServer = async (port, host = '127.0.0.1') => {
             res.status(400).json({ error: `Job already ${job.status}` });
             return;
         }
-        jobManager.cancelJob(req.params.jobId, 'Cancelled by user');
+        jobManager.cancelJob(jobId, 'Cancelled by user');
         res.json({ id: job.id, status: 'failed', error: 'Cancelled by user' });
     });
     // ── Embedding endpoints ────────────────────────────────────────────
     const embedJobManager = new JobManager();
     // POST /api/embed — trigger server-side embedding generation
-    app.post('/api/embed', createRouteLimiter({ limit: 20 }), async (req, res) => {
+    app.post('/api/embed', createRouteLimiter({ limit: 20 }), requireLocalhostOrigin, async (req, res) => {
         try {
             const entry = await resolveRepo(requestedRepo(req));
             if (!entry) {
@@ -1616,8 +1579,9 @@ export const createServer = async (port, host = '127.0.0.1') => {
     // GET /api/embed/:jobId/progress — SSE stream (shared helper)
     mountSSEProgress(app, '/api/embed/:jobId/progress', embedJobManager);
     // DELETE /api/embed/:jobId — cancel embedding job
-    app.delete('/api/embed/:jobId', (req, res) => {
-        const job = embedJobManager.getJob(req.params.jobId);
+    app.delete('/api/embed/:jobId', requireLocalhostOrigin, (req, res) => {
+        const jobId = req.params.jobId;
+        const job = embedJobManager.getJob(jobId);
         if (!job) {
             res.status(404).json({ error: 'Job not found' });
             return;
@@ -1626,7 +1590,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
             res.status(400).json({ error: `Job already ${job.status}` });
             return;
         }
-        embedJobManager.cancelJob(req.params.jobId, 'Cancelled by user');
+        embedJobManager.cancelJob(jobId, 'Cancelled by user');
         res.json({ id: job.id, status: 'failed', error: 'Cancelled by user' });
     });
     // ── Web UI (served at root) ───────────────────────────────────────

package/dist/server/git-clone.d.ts

@@ -1,9 +1,10 @@
 /**
  * Git Clone Utility
  *
- * Shallow-clones repositories into ~/.gitnexus/repos/{name}/.
+ * Shallow-clones repositories into the clone root (getGlobalDir()/repos/{name}/).
  * If already cloned, does git pull instead.
  */
+export declare const REPO_NAME_PATTERN: RegExp;
 /**
  * Extract the repository name from a git URL (HTTPS or SSH).
  *
@@ -34,6 +35,25 @@ export interface CloneProgress {
  *
  * Exported so the separator placement is testable without mocking spawn.
  */
+/**
+ * Detect Azure DevOps URLs — both self-hosted (via AZURE_DEVOPS_URL env)
+ * and cloud (dev.azure.com / *.visualstudio.com).
+ *
+ * Self-hosted Azure DevOps Server instances use arbitrary hostnames
+ * (e.g. `http://tfs.corp.example/Collection/Project/_git/Repo`), so the
+ * function checks `AZURE_DEVOPS_URL` first. Cloud addresses are a
+ * hardcoded fallback so PAT injection works out-of-the-box for
+ * dev.azure.com without extra configuration.
+ */
+export declare function isAzureDevOpsUrl(url: string): boolean;
+/**
+ * One-time startup warning when AZURE_DEVOPS_URL is configured over cleartext
+ * http:// — the Azure DevOps PAT would then be sent unencrypted on every
+ * clone. Self-hosted instances that only serve http are still supported (we
+ * do not refuse), but operators rarely read request-time logs, so surface it
+ * at boot too. Call once from server startup.
+ */
+export declare function warnIfInsecureAzureConfig(): void;
 export declare function buildCloneArgs(url: string, targetDir: string): string[];
 /**
  * Normalize a git URL into a comparable form.
@@ -96,4 +116,25 @@ export declare function assertRemoteMatchesRequestedUrl(targetDir: string, reque
  *     `--` (e.g. `--upload-pack=evil`) cannot be interpreted as a git option
  *     (CodeQL js/second-order-command-line-injection).
  */
-export declare function cloneOrPull(url: string, targetDir: string, onProgress?: (progress: CloneProgress) => void): Promise<string>;
+export declare function cloneOrPull(url: string, targetDir: string, onProgress?: (progress: CloneProgress) => void, options?: {
+    token?: string;
+}): Promise<string>;
+/**
+ * Hosts the per-request GitHub PAT may be sent to. Exported so the
+ * /api/analyze boundary check and this injection-site check share one
+ * allowlist (they must agree, or a token accepted by the API could be
+ * silently dropped — or worse — at injection).
+ */
+export declare const GITHUB_TOKEN_HOSTS: ReadonlySet<string>;
+/**
+ * Build the spawn env for `git`. Suppresses credential prompts and, when a
+ * credential resolves (see resolveGitCredential), injects a single
+ * host-scoped Authorization header via the `GIT_CONFIG_*` env protocol
+ * (git ≥2.31) so credentials never appear in argv or the URL. Appends after
+ * any existing `GIT_CONFIG_COUNT` rather than overwriting it. Exported for
+ * unit tests.
+ */
+export declare function buildGitEnv(baseEnv: NodeJS.ProcessEnv, options?: {
+    token?: string;
+    url?: string;
+}): NodeJS.ProcessEnv;