gitnexus 1.6.8-rc.13 → 1.6.8-rc.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/analyze-launch.d.ts +29 -0
- package/dist/server/analyze-launch.js +137 -0
- package/dist/server/analyze-upload.d.ts +33 -0
- package/dist/server/analyze-upload.js +123 -0
- package/dist/server/analyze-worker.d.ts +17 -1
- package/dist/server/api.js +46 -140
- package/dist/server/git-clone.d.ts +1 -0
- package/dist/server/git-clone.js +1 -1
- package/dist/server/middleware.d.ts +11 -0
- package/dist/server/middleware.js +27 -0
- package/dist/server/upload-ingest.d.ts +56 -0
- package/dist/server/upload-ingest.js +276 -0
- package/dist/server/upload-paths.d.ts +31 -0
- package/dist/server/upload-paths.js +51 -0
- package/dist/server/upload-sweep.d.ts +20 -0
- package/dist/server/upload-sweep.js +57 -0
- package/package.json +3 -1
- package/web/assets/{agent-CKPMqImC.js → agent-B0-_Ihgi.js} +1 -1
- package/web/assets/{architectureDiagram-UL44E2DR-D1EXI0zA.js → architectureDiagram-UL44E2DR-PdDPNXTO.js} +1 -1
- package/web/assets/{chunk-LCXTWHL2-Dfmux4m1.js → chunk-LCXTWHL2-u4-0IIru.js} +1 -1
- package/web/assets/{chunk-RG4AUYOV-CAkzcoRj.js → chunk-RG4AUYOV--R3bV0KH.js} +1 -1
- package/web/assets/{classDiagram-KGZ6W3CR-Bv93af_b.js → classDiagram-KGZ6W3CR-CLDcI2Tc.js} +1 -1
- package/web/assets/{classDiagram-v2-72OJOZXJ-CLfEqHUa.js → classDiagram-v2-72OJOZXJ-dwefssU_.js} +1 -1
- package/web/assets/{diagram-3NCE3AQN-BqAtKUpW.js → diagram-3NCE3AQN-DMYhnaiQ.js} +1 -1
- package/web/assets/{diagram-GF46GFSD-CaBG6n6o.js → diagram-GF46GFSD-CgWD3Go0.js} +1 -1
- package/web/assets/{diagram-QXG6HAR7-CZ-O3rcV.js → diagram-QXG6HAR7-Bwu9Wi_f.js} +1 -1
- package/web/assets/{diagram-WEQXMOUZ-P4lSL4GH.js → diagram-WEQXMOUZ-CaxeST-3.js} +1 -1
- package/web/assets/{erDiagram-L5TCEMPS-QN2eEP1e.js → erDiagram-L5TCEMPS-WlblWt40.js} +1 -1
- package/web/assets/{flowDiagram-H6V6AXG4-CXbXImlN.js → flowDiagram-H6V6AXG4-CRrYkD90.js} +1 -1
- package/web/assets/index-COMMmbxW.css +2 -0
- package/web/assets/index-D_6rUFGW.js +626 -0
- package/web/assets/{infoDiagram-3YFTVSEB-DmLICZx1.js → infoDiagram-3YFTVSEB-pvT6ArHz.js} +1 -1
- package/web/assets/{ishikawaDiagram-BNXS4ZKH-6LKOvBfp.js → ishikawaDiagram-BNXS4ZKH-D_0fBKRi.js} +1 -1
- package/web/assets/{kanban-definition-75IXJCU3-DgDi9oJT.js → kanban-definition-75IXJCU3-DGABptDC.js} +1 -1
- package/web/assets/{mindmap-definition-2TDM6QVE-CwR5sBB-.js → mindmap-definition-2TDM6QVE-r-bFdhPw.js} +1 -1
- package/web/assets/{pieDiagram-CU6KROY3-By8g6f6B.js → pieDiagram-CU6KROY3-CS3o1Ug4.js} +1 -1
- package/web/assets/{requirementDiagram-JXO7QTGE-7oDcJ1_J.js → requirementDiagram-JXO7QTGE-BMYGXwjH.js} +1 -1
- package/web/assets/{sequenceDiagram-VS2MUI6T-FySeKCUy.js → sequenceDiagram-VS2MUI6T-BJnTiLwO.js} +1 -1
- package/web/assets/{stateDiagram-7D4R322I-CB2nABwH.js → stateDiagram-7D4R322I-G35bS8CQ.js} +1 -1
- package/web/assets/{stateDiagram-v2-36443NZ5-COBGd2RL.js → stateDiagram-v2-36443NZ5-BUaZpLs6.js} +1 -1
- package/web/assets/{timeline-definition-O6YCAMPW-Ds2CnVZK.js → timeline-definition-O6YCAMPW-B9o1j771.js} +1 -1
- package/web/assets/{vennDiagram-MWXL3ELB-DUIEwXWp.js → vennDiagram-MWXL3ELB-Vz5Sin3H.js} +1 -1
- package/web/assets/{wardleyDiagram-CUQ6CDDI-DEiFPQih.js → wardleyDiagram-CUQ6CDDI-D42Nu4nI.js} +1 -1
- package/web/assets/{xychartDiagram-N2JHSOCM-BUbayhST.js → xychartDiagram-N2JHSOCM-CSImZ1y9.js} +1 -1
- package/web/index.html +2 -2
- package/web/assets/index-BKWA-m7o.css +0 -2
- package/web/assets/index-CG6q8eTs.js +0 -626
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared analyze-worker launcher.
|
|
3
|
+
*
|
|
4
|
+
* Forks the analyze worker for an already-resolved repo directory and owns the
|
|
5
|
+
* lock + auto-retry + IPC machinery. Used by both the JSON `/api/analyze` route
|
|
6
|
+
* and the multipart `/api/analyze/upload` route. Dependency-injected (like
|
|
7
|
+
* createAnalyzeUploadHandler) so the seam is testable and api.ts stays smaller.
|
|
8
|
+
*
|
|
9
|
+
* NOTE: this module must live alongside analyze-worker.{ts,js} — the worker
|
|
10
|
+
* path is resolved relative to `import.meta.url`.
|
|
11
|
+
*/
|
|
12
|
+
import type { JobManager } from './analyze-job.js';
|
|
13
|
+
export interface LaunchDeps {
|
|
14
|
+
jobManager: JobManager;
|
|
15
|
+
backend: {
|
|
16
|
+
init: () => Promise<unknown>;
|
|
17
|
+
};
|
|
18
|
+
acquireRepoLock: (key: string) => string | null;
|
|
19
|
+
releaseRepoLock: (key: string) => void;
|
|
20
|
+
}
|
|
21
|
+
export interface LaunchOptions {
|
|
22
|
+
force?: boolean;
|
|
23
|
+
embeddings?: boolean;
|
|
24
|
+
dropEmbeddings?: boolean;
|
|
25
|
+
registryName?: string;
|
|
26
|
+
}
|
|
27
|
+
export declare function createLaunchAnalysisWorker(deps: LaunchDeps): (job: {
|
|
28
|
+
id: string;
|
|
29
|
+
}, targetPath: string, opts: LaunchOptions) => void;
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared analyze-worker launcher.
|
|
3
|
+
*
|
|
4
|
+
* Forks the analyze worker for an already-resolved repo directory and owns the
|
|
5
|
+
* lock + auto-retry + IPC machinery. Used by both the JSON `/api/analyze` route
|
|
6
|
+
* and the multipart `/api/analyze/upload` route. Dependency-injected (like
|
|
7
|
+
* createAnalyzeUploadHandler) so the seam is testable and api.ts stays smaller.
|
|
8
|
+
*
|
|
9
|
+
* NOTE: this module must live alongside analyze-worker.{ts,js} — the worker
|
|
10
|
+
* path is resolved relative to `import.meta.url`.
|
|
11
|
+
*/
|
|
12
|
+
import path from 'path';
|
|
13
|
+
import { fork } from 'child_process';
|
|
14
|
+
import { fileURLToPath, pathToFileURL } from 'url';
|
|
15
|
+
import { createRequire } from 'node:module';
|
|
16
|
+
import { getStoragePath } from '../storage/repo-manager.js';
|
|
17
|
+
import { logger } from '../core/logger.js';
|
|
18
|
+
const _require = createRequire(import.meta.url);
|
|
19
|
+
const MAX_WORKER_RETRIES = 2;
|
|
20
|
+
export function createLaunchAnalysisWorker(deps) {
|
|
21
|
+
const { jobManager, backend, acquireRepoLock, releaseRepoLock } = deps;
|
|
22
|
+
return function launchAnalysisWorker(job, targetPath, opts) {
|
|
23
|
+
// Acquire shared repo lock (keyed on storagePath to match embed handler)
|
|
24
|
+
const analyzeLockKey = getStoragePath(targetPath);
|
|
25
|
+
const lockErr = acquireRepoLock(analyzeLockKey);
|
|
26
|
+
if (lockErr) {
|
|
27
|
+
jobManager.updateJob(job.id, { status: 'failed', error: lockErr });
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' });
|
|
31
|
+
// ── Worker fork with auto-retry ──────────────────────────────
|
|
32
|
+
const callerPath = fileURLToPath(import.meta.url);
|
|
33
|
+
const isDev = callerPath.endsWith('.ts');
|
|
34
|
+
const workerFile = isDev ? 'analyze-worker.ts' : 'analyze-worker.js';
|
|
35
|
+
const workerPath = path.join(path.dirname(callerPath), workerFile);
|
|
36
|
+
const tsxHookArgs = isDev
|
|
37
|
+
? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href]
|
|
38
|
+
: [];
|
|
39
|
+
const forkWorker = () => {
|
|
40
|
+
const currentJob = jobManager.getJob(job.id);
|
|
41
|
+
if (!currentJob || currentJob.status === 'complete' || currentJob.status === 'failed')
|
|
42
|
+
return;
|
|
43
|
+
const child = fork(workerPath, [], {
|
|
44
|
+
execArgv: [...tsxHookArgs, '--max-old-space-size=8192'],
|
|
45
|
+
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
|
|
46
|
+
});
|
|
47
|
+
// Capture stderr for crash diagnostics
|
|
48
|
+
let stderrChunks = '';
|
|
49
|
+
child.stderr?.on('data', (chunk) => {
|
|
50
|
+
stderrChunks += chunk.toString();
|
|
51
|
+
if (stderrChunks.length > 4096)
|
|
52
|
+
stderrChunks = stderrChunks.slice(-4096);
|
|
53
|
+
});
|
|
54
|
+
child.on('message', (msg) => {
|
|
55
|
+
if (msg.type === 'progress') {
|
|
56
|
+
jobManager.updateJob(job.id, {
|
|
57
|
+
status: 'analyzing',
|
|
58
|
+
progress: { phase: msg.phase, percent: msg.percent, message: msg.message },
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
else if (msg.type === 'complete') {
|
|
62
|
+
releaseRepoLock(analyzeLockKey);
|
|
63
|
+
// Reinitialize backend BEFORE marking complete — ensures the new repo
|
|
64
|
+
// is queryable when the client receives the SSE complete event.
|
|
65
|
+
backend
|
|
66
|
+
.init()
|
|
67
|
+
.then(() => {
|
|
68
|
+
jobManager.updateJob(job.id, { status: 'complete', repoName: msg.result.repoName });
|
|
69
|
+
})
|
|
70
|
+
.catch((err) => {
|
|
71
|
+
logger.error({ err }, 'backend.init() failed after analyze:');
|
|
72
|
+
jobManager.updateJob(job.id, {
|
|
73
|
+
status: 'failed',
|
|
74
|
+
error: 'Server failed to reload after analysis. Try again.',
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
else if (msg.type === 'error') {
|
|
79
|
+
releaseRepoLock(analyzeLockKey);
|
|
80
|
+
jobManager.updateJob(job.id, { status: 'failed', error: msg.message });
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
child.on('error', (err) => {
|
|
84
|
+
releaseRepoLock(analyzeLockKey);
|
|
85
|
+
jobManager.updateJob(job.id, {
|
|
86
|
+
status: 'failed',
|
|
87
|
+
error: `Worker process error: ${err.message}`,
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
child.on('exit', (code) => {
|
|
91
|
+
const j = jobManager.getJob(job.id);
|
|
92
|
+
if (!j || j.status === 'complete' || j.status === 'failed')
|
|
93
|
+
return;
|
|
94
|
+
// Worker crashed — attempt retry if under the limit
|
|
95
|
+
if (j.retryCount < MAX_WORKER_RETRIES) {
|
|
96
|
+
j.retryCount++;
|
|
97
|
+
const delay = 1000 * Math.pow(2, j.retryCount - 1); // 1s, 2s
|
|
98
|
+
const lastErr = stderrChunks.trim().split('\n').pop() || '';
|
|
99
|
+
logger.warn(`Analyze worker crashed (code ${code}), retry ${j.retryCount}/${MAX_WORKER_RETRIES} in ${delay}ms` +
|
|
100
|
+
(lastErr ? `: ${lastErr}` : ''));
|
|
101
|
+
jobManager.updateJob(job.id, {
|
|
102
|
+
status: 'analyzing',
|
|
103
|
+
progress: {
|
|
104
|
+
phase: 'retrying',
|
|
105
|
+
percent: j.progress.percent,
|
|
106
|
+
message: `Worker crashed, retrying (${j.retryCount}/${MAX_WORKER_RETRIES})...`,
|
|
107
|
+
},
|
|
108
|
+
});
|
|
109
|
+
stderrChunks = '';
|
|
110
|
+
setTimeout(forkWorker, delay);
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
// Exhausted retries — permanent failure
|
|
114
|
+
releaseRepoLock(analyzeLockKey);
|
|
115
|
+
jobManager.updateJob(job.id, {
|
|
116
|
+
status: 'failed',
|
|
117
|
+
error: `Worker crashed ${MAX_WORKER_RETRIES + 1} times (code ${code})${stderrChunks ? ': ' + stderrChunks.trim().split('\n').pop() : ''}`,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
// Register child for cancellation + timeout tracking
|
|
122
|
+
jobManager.registerChild(job.id, child);
|
|
123
|
+
// Send start command to child
|
|
124
|
+
child.send({
|
|
125
|
+
type: 'start',
|
|
126
|
+
repoPath: targetPath,
|
|
127
|
+
options: {
|
|
128
|
+
force: !!opts.force,
|
|
129
|
+
embeddings: !!opts.embeddings,
|
|
130
|
+
dropEmbeddings: !!opts.dropEmbeddings,
|
|
131
|
+
...(opts.registryName ? { registryName: opts.registryName } : {}),
|
|
132
|
+
},
|
|
133
|
+
});
|
|
134
|
+
};
|
|
135
|
+
forkWorker();
|
|
136
|
+
};
|
|
137
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* POST /api/analyze/upload — analyze a browser folder upload.
|
|
3
|
+
*
|
|
4
|
+
* Securely ingests the multipart upload into a sandbox (upload-ingest.ts),
|
|
5
|
+
* promotes it to a persistent app-controlled directory, and analyzes it via
|
|
6
|
+
* the same job/worker machinery as a git clone — never returning a server
|
|
7
|
+
* path to the client. Factored as a dependency-injected handler so the job
|
|
8
|
+
* machinery (createJob + the worker launcher) can be mocked in unit tests.
|
|
9
|
+
*/
|
|
10
|
+
import type { Request, Response } from 'express';
|
|
11
|
+
import { ingestUpload } from './upload-ingest.js';
|
|
12
|
+
import type { AnalyzeJob } from './analyze-job.js';
|
|
13
|
+
/** Minimal job shape the handler needs (a subset of the real AnalyzeJob). */
|
|
14
|
+
export type UploadJobRef = Pick<AnalyzeJob, 'id' | 'status'>;
|
|
15
|
+
export interface AnalyzeUploadDeps {
|
|
16
|
+
/** Create (or throw on busy) an analysis job for the given upload dir. */
|
|
17
|
+
createJob: (params: {
|
|
18
|
+
repoPath: string;
|
|
19
|
+
}) => UploadJobRef;
|
|
20
|
+
/** Launch the analyze worker against an already-resolved repo directory. */
|
|
21
|
+
launch: (job: UploadJobRef, targetPath: string, opts: {
|
|
22
|
+
registryName: string;
|
|
23
|
+
}) => void;
|
|
24
|
+
/**
|
|
25
|
+
* Mark a created job failed. The job occupies the single analysis slot from
|
|
26
|
+
* createJob onward, so ANY error before launch must release it — otherwise a
|
|
27
|
+
* leaked non-terminal job wedges all future analyses until restart.
|
|
28
|
+
*/
|
|
29
|
+
failJob: (jobId: string, error: string) => void;
|
|
30
|
+
/** Injectable for tests (defaults to the real ingestUpload). */
|
|
31
|
+
ingest?: typeof ingestUpload;
|
|
32
|
+
}
|
|
33
|
+
export declare function createAnalyzeUploadHandler(deps: AnalyzeUploadDeps): (req: Request, res: Response) => Promise<void>;
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* POST /api/analyze/upload — analyze a browser folder upload.
|
|
3
|
+
*
|
|
4
|
+
* Securely ingests the multipart upload into a sandbox (upload-ingest.ts),
|
|
5
|
+
* promotes it to a persistent app-controlled directory, and analyzes it via
|
|
6
|
+
* the same job/worker machinery as a git clone — never returning a server
|
|
7
|
+
* path to the client. Factored as a dependency-injected handler so the job
|
|
8
|
+
* machinery (createJob + the worker launcher) can be mocked in unit tests.
|
|
9
|
+
*/
|
|
10
|
+
import path from 'path';
|
|
11
|
+
import fsp from 'fs/promises';
|
|
12
|
+
import { ingestUpload } from './upload-ingest.js';
|
|
13
|
+
import { UPLOAD_ROOT, getUploadDir, deriveUploadName } from './upload-paths.js';
|
|
14
|
+
import { BadRequestError } from './validation.js';
|
|
15
|
+
/** Cap on collision-suffix attempts when allocating an upload dir name. */
|
|
16
|
+
const MAX_NAME_COLLISION_TRIES = 100;
|
|
17
|
+
/**
|
|
18
|
+
* Find an available upload directory name, appending `-2`, `-3`, … on
|
|
19
|
+
* collision with an existing upload. Bounded to avoid an unbounded scan.
|
|
20
|
+
*/
|
|
21
|
+
async function pickAvailableName(base) {
|
|
22
|
+
for (let i = 0; i < MAX_NAME_COLLISION_TRIES; i++) {
|
|
23
|
+
const name = i === 0 ? base : `${base}-${i + 1}`;
|
|
24
|
+
let dir;
|
|
25
|
+
try {
|
|
26
|
+
dir = getUploadDir(name);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
try {
|
|
32
|
+
await fsp.access(dir);
|
|
33
|
+
// exists → try the next suffix
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return name; // ENOENT → available
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
throw new BadRequestError(`Could not allocate an upload directory after ${MAX_NAME_COLLISION_TRIES} attempts`, 409);
|
|
40
|
+
}
|
|
41
|
+
export function createAnalyzeUploadHandler(deps) {
|
|
42
|
+
const ingest = deps.ingest ?? ingestUpload;
|
|
43
|
+
return async function handleAnalyzeUploadRequest(req, res) {
|
|
44
|
+
let stageRoot;
|
|
45
|
+
let promotedDir;
|
|
46
|
+
let createdJobId;
|
|
47
|
+
let launched = false;
|
|
48
|
+
try {
|
|
49
|
+
const result = await ingest(req);
|
|
50
|
+
stageRoot = result.stageRoot;
|
|
51
|
+
const baseName = deriveUploadName(result.topLevelName);
|
|
52
|
+
if (!baseName) {
|
|
53
|
+
throw new BadRequestError('Uploaded folder has no usable name');
|
|
54
|
+
}
|
|
55
|
+
// webkitRelativePath prefixes every entry with the picked folder, so the
|
|
56
|
+
// real repo root is stageRoot/<topLevelName>. Validate it is a directory
|
|
57
|
+
// BEFORE taking the single analysis slot — a malformed (non-folder)
|
|
58
|
+
// upload must not be able to occupy the slot.
|
|
59
|
+
const innerRoot = path.join(result.stageRoot, result.topLevelName);
|
|
60
|
+
let innerIsDir = false;
|
|
61
|
+
try {
|
|
62
|
+
innerIsDir = (await fsp.stat(innerRoot)).isDirectory();
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
innerIsDir = false;
|
|
66
|
+
}
|
|
67
|
+
if (!innerIsDir) {
|
|
68
|
+
throw new BadRequestError('Upload must be a folder');
|
|
69
|
+
}
|
|
70
|
+
const finalName = await pickAvailableName(baseName);
|
|
71
|
+
const finalDir = getUploadDir(finalName);
|
|
72
|
+
// createJob occupies the single analysis slot (throws 'already in
|
|
73
|
+
// progress' → 409). From here on, ANY error before launch MUST release
|
|
74
|
+
// the slot via failJob in the catch, or the server wedges all analyses.
|
|
75
|
+
let job;
|
|
76
|
+
try {
|
|
77
|
+
job = deps.createJob({ repoPath: finalDir });
|
|
78
|
+
}
|
|
79
|
+
catch (err) {
|
|
80
|
+
const msg = err instanceof Error ? err.message : '';
|
|
81
|
+
if (msg.includes('already in progress')) {
|
|
82
|
+
throw new BadRequestError(msg, 409);
|
|
83
|
+
}
|
|
84
|
+
throw err;
|
|
85
|
+
}
|
|
86
|
+
createdJobId = job.id;
|
|
87
|
+
// Promote staging → persistent upload dir. Both live under UPLOAD_ROOT's
|
|
88
|
+
// filesystem, so this rename stays atomic (no EXDEV).
|
|
89
|
+
await fsp.mkdir(UPLOAD_ROOT, { recursive: true });
|
|
90
|
+
await fsp.rename(innerRoot, finalDir);
|
|
91
|
+
promotedDir = finalDir;
|
|
92
|
+
const oldStage = stageRoot;
|
|
93
|
+
stageRoot = undefined;
|
|
94
|
+
await fsp.rm(oldStage, { recursive: true, force: true }).catch(() => { });
|
|
95
|
+
// Drop any crafted index the upload may have carried (a `.gitnexus`
|
|
96
|
+
// segment passes containment); the worker will build a fresh one.
|
|
97
|
+
await fsp
|
|
98
|
+
.rm(path.join(finalDir, '.gitnexus'), { recursive: true, force: true })
|
|
99
|
+
.catch(() => { });
|
|
100
|
+
deps.launch(job, finalDir, { registryName: finalName });
|
|
101
|
+
launched = true;
|
|
102
|
+
res.status(202).json({ jobId: job.id, status: job.status });
|
|
103
|
+
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
// Release the single analysis slot if a job was created but never
|
|
106
|
+
// launched — otherwise the leaked queued job blocks all future analyses.
|
|
107
|
+
if (createdJobId && !launched) {
|
|
108
|
+
deps.failJob(createdJobId, err instanceof Error ? err.message : 'Upload failed');
|
|
109
|
+
}
|
|
110
|
+
if (stageRoot) {
|
|
111
|
+
await fsp.rm(stageRoot, { recursive: true, force: true }).catch(() => { });
|
|
112
|
+
}
|
|
113
|
+
if (promotedDir && !launched) {
|
|
114
|
+
await fsp.rm(promotedDir, { recursive: true, force: true }).catch(() => { });
|
|
115
|
+
}
|
|
116
|
+
if (err instanceof BadRequestError) {
|
|
117
|
+
res.status(err.status).json({ error: err.message });
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
res.status(500).json({ error: 'Upload failed' });
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
}
|
|
@@ -10,4 +10,20 @@
|
|
|
10
10
|
* Child -> Parent: { type: 'complete', result: AnalyzeResult }
|
|
11
11
|
* Child -> Parent: { type: 'error', message: string }
|
|
12
12
|
*/
|
|
13
|
-
|
|
13
|
+
import { type AnalyzeResultIpc } from './analyze-worker-ipc.js';
|
|
14
|
+
export interface ProgressMessage {
|
|
15
|
+
type: 'progress';
|
|
16
|
+
phase: string;
|
|
17
|
+
percent: number;
|
|
18
|
+
message: string;
|
|
19
|
+
}
|
|
20
|
+
export interface CompleteMessage {
|
|
21
|
+
type: 'complete';
|
|
22
|
+
result: AnalyzeResultIpc;
|
|
23
|
+
}
|
|
24
|
+
export interface ErrorMessage {
|
|
25
|
+
type: 'error';
|
|
26
|
+
message: string;
|
|
27
|
+
}
|
|
28
|
+
/** Child → parent IPC messages. Shared with the parent-side launcher. */
|
|
29
|
+
export type WorkerMessage = ProgressMessage | CompleteMessage | ErrorMessage;
|
package/dist/server/api.js
CHANGED
|
@@ -20,11 +20,15 @@ import { searchFTSFromLbug } from '../core/search/bm25-index.js';
|
|
|
20
20
|
import { hybridSearch } from '../core/search/hybrid-search.js';
|
|
21
21
|
import { LocalBackend } from '../mcp/local/local-backend.js';
|
|
22
22
|
import { mountMCPEndpoints } from './mcp-http.js';
|
|
23
|
-
import {
|
|
24
|
-
import { fileURLToPath, pathToFileURL } from 'url';
|
|
23
|
+
import { fileURLToPath } from 'url';
|
|
25
24
|
import { JobManager } from './analyze-job.js';
|
|
26
25
|
import { assertString, escapeRegExp, BadRequestError, createRouteLimiter } from './validation.js';
|
|
27
26
|
import { extractRepoName, getCloneDir, cloneOrPull } from './git-clone.js';
|
|
27
|
+
import { createAnalyzeUploadHandler } from './analyze-upload.js';
|
|
28
|
+
import { requireLocalhostOrigin } from './middleware.js';
|
|
29
|
+
import { createLaunchAnalysisWorker } from './analyze-launch.js';
|
|
30
|
+
import { UPLOAD_ROOT } from './upload-paths.js';
|
|
31
|
+
import { sweepStaleUploads } from './upload-sweep.js';
|
|
28
32
|
import { logger, flushLoggerSync } from '../core/logger.js';
|
|
29
33
|
const _require = createRequire(import.meta.url);
|
|
30
34
|
const pkg = _require('../../package.json');
|
|
@@ -640,6 +644,8 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
640
644
|
await backend.init();
|
|
641
645
|
const cleanupMcp = mountMCPEndpoints(app, backend);
|
|
642
646
|
const jobManager = new JobManager();
|
|
647
|
+
// Backstop: remove any upload staging dirs orphaned by a previous crash.
|
|
648
|
+
void sweepStaleUploads().catch(() => { });
|
|
643
649
|
// Shared repo lock — prevents concurrent analyze + embed on the same repo path,
|
|
644
650
|
// which would corrupt LadybugDB (analyze calls closeLbug + initLbug while embed has queries in flight).
|
|
645
651
|
const activeRepoPaths = new Set();
|
|
@@ -653,6 +659,14 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
653
659
|
const releaseRepoLock = (repoPath) => {
|
|
654
660
|
activeRepoPaths.delete(repoPath);
|
|
655
661
|
};
|
|
662
|
+
// Launch the analyze worker for an already-resolved repo directory. Shared by
|
|
663
|
+
// the JSON /api/analyze route and the multipart /api/analyze/upload route.
|
|
664
|
+
const launchAnalysisWorker = createLaunchAnalysisWorker({
|
|
665
|
+
jobManager,
|
|
666
|
+
backend,
|
|
667
|
+
acquireRepoLock,
|
|
668
|
+
releaseRepoLock,
|
|
669
|
+
});
|
|
656
670
|
/**
|
|
657
671
|
* Maximum time the hold-queue will wait for an active analysis job to complete.
|
|
658
672
|
* Must stay in sync with the frontend's `fetchRepoInfo({ awaitAnalysis: true })` timeout.
|
|
@@ -867,6 +881,13 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
867
881
|
/* clone dir may not exist */
|
|
868
882
|
}
|
|
869
883
|
}
|
|
884
|
+
// 2b. Delete the uploaded repo dir if entry.path lives under
|
|
885
|
+
// UPLOAD_ROOT. Drive this off entry.path (not a name-rederived dir) so
|
|
886
|
+
// a same-named clone is never affected.
|
|
887
|
+
const resolvedEntry = path.resolve(entry.path);
|
|
888
|
+
if (resolvedEntry === UPLOAD_ROOT || resolvedEntry.startsWith(UPLOAD_ROOT + path.sep)) {
|
|
889
|
+
await fs.rm(resolvedEntry, { recursive: true, force: true }).catch(() => { });
|
|
890
|
+
}
|
|
870
891
|
// 3. Unregister from the global registry
|
|
871
892
|
const { unregisterRepo } = await import('../storage/repo-manager.js');
|
|
872
893
|
await unregisterRepo(entry.path);
|
|
@@ -1251,7 +1272,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1251
1272
|
});
|
|
1252
1273
|
// ── Analyze API ──────────────────────────────────────────────────────
|
|
1253
1274
|
// POST /api/analyze — start a new analysis job
|
|
1254
|
-
app.post('/api/analyze', createRouteLimiter({ limit: 10 }), async (req, res) => {
|
|
1275
|
+
app.post('/api/analyze', createRouteLimiter({ limit: 10 }), requireLocalhostOrigin, async (req, res) => {
|
|
1255
1276
|
try {
|
|
1256
1277
|
const { url: repoUrl, path: repoLocalPath, force, embeddings, dropEmbeddings } = req.body;
|
|
1257
1278
|
// Input type validation
|
|
@@ -1267,16 +1288,18 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1267
1288
|
res.status(400).json({ error: 'Provide "url" (git URL) or "path" (local path)' });
|
|
1268
1289
|
return;
|
|
1269
1290
|
}
|
|
1270
|
-
// Path validation
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1291
|
+
// Path validation. The previous `normalize !== resolve` guard was inert
|
|
1292
|
+
// (both collapse `..` identically) and only false-rejected trailing
|
|
1293
|
+
// slashes, so it is dropped. Analyzing a local path the operator names
|
|
1294
|
+
// is the tool's intended capability (same as the CLI); the dangerous
|
|
1295
|
+
// part was cross-origin reach, which is closed by requireLocalhostOrigin
|
|
1296
|
+
// on this route. We only require an absolute path here and let the
|
|
1297
|
+
// analyze worker surface a clear error if it does not exist. (We do NOT
|
|
1298
|
+
// realpath/stat the path in-route: that would be a user-controlled
|
|
1299
|
+
// filesystem read — CodeQL js/path-injection — for no security gain.)
|
|
1300
|
+
if (repoLocalPath && !path.isAbsolute(repoLocalPath)) {
|
|
1301
|
+
res.status(400).json({ error: '"path" must be an absolute path' });
|
|
1302
|
+
return;
|
|
1280
1303
|
}
|
|
1281
1304
|
const job = jobManager.createJob({ repoUrl, repoPath: repoLocalPath });
|
|
1282
1305
|
// If job was already running (dedup), just return its id
|
|
@@ -1308,133 +1331,7 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1308
1331
|
if (!targetPath) {
|
|
1309
1332
|
throw new Error('No target path resolved');
|
|
1310
1333
|
}
|
|
1311
|
-
|
|
1312
|
-
const analyzeLockKey = getStoragePath(targetPath);
|
|
1313
|
-
const lockErr = acquireRepoLock(analyzeLockKey);
|
|
1314
|
-
if (lockErr) {
|
|
1315
|
-
jobManager.updateJob(job.id, { status: 'failed', error: lockErr });
|
|
1316
|
-
return;
|
|
1317
|
-
}
|
|
1318
|
-
jobManager.updateJob(job.id, { repoPath: targetPath, status: 'analyzing' });
|
|
1319
|
-
// ── Worker fork with auto-retry ──────────────────────────────
|
|
1320
|
-
//
|
|
1321
|
-
// Forks a child process with 8GB heap. If the worker crashes
|
|
1322
|
-
// (OOM, native addon segfault, etc.), it retries up to
|
|
1323
|
-
// MAX_WORKER_RETRIES times with exponential backoff before
|
|
1324
|
-
// marking the job as permanently failed.
|
|
1325
|
-
//
|
|
1326
|
-
// In dev mode (tsx), registers the tsx ESM hook via a file://
|
|
1327
|
-
// URL so the child can compile TypeScript on-the-fly.
|
|
1328
|
-
const MAX_WORKER_RETRIES = 2;
|
|
1329
|
-
const callerPath = fileURLToPath(import.meta.url);
|
|
1330
|
-
const isDev = callerPath.endsWith('.ts');
|
|
1331
|
-
const workerFile = isDev ? 'analyze-worker.ts' : 'analyze-worker.js';
|
|
1332
|
-
const workerPath = path.join(path.dirname(callerPath), workerFile);
|
|
1333
|
-
const tsxHookArgs = isDev
|
|
1334
|
-
? ['--import', pathToFileURL(_require.resolve('tsx/esm')).href]
|
|
1335
|
-
: [];
|
|
1336
|
-
const forkWorker = () => {
|
|
1337
|
-
const currentJob = jobManager.getJob(job.id);
|
|
1338
|
-
if (!currentJob || currentJob.status === 'complete' || currentJob.status === 'failed')
|
|
1339
|
-
return;
|
|
1340
|
-
const child = fork(workerPath, [], {
|
|
1341
|
-
execArgv: [...tsxHookArgs, '--max-old-space-size=8192'],
|
|
1342
|
-
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
|
|
1343
|
-
});
|
|
1344
|
-
// Capture stderr for crash diagnostics
|
|
1345
|
-
let stderrChunks = '';
|
|
1346
|
-
child.stderr?.on('data', (chunk) => {
|
|
1347
|
-
stderrChunks += chunk.toString();
|
|
1348
|
-
if (stderrChunks.length > 4096)
|
|
1349
|
-
stderrChunks = stderrChunks.slice(-4096);
|
|
1350
|
-
});
|
|
1351
|
-
child.on('message', (msg) => {
|
|
1352
|
-
if (msg.type === 'progress') {
|
|
1353
|
-
jobManager.updateJob(job.id, {
|
|
1354
|
-
status: 'analyzing',
|
|
1355
|
-
progress: { phase: msg.phase, percent: msg.percent, message: msg.message },
|
|
1356
|
-
});
|
|
1357
|
-
}
|
|
1358
|
-
else if (msg.type === 'complete') {
|
|
1359
|
-
releaseRepoLock(analyzeLockKey);
|
|
1360
|
-
// Reinitialize backend BEFORE marking complete — ensures the new
|
|
1361
|
-
// repo is queryable when the client receives the SSE complete event.
|
|
1362
|
-
backend
|
|
1363
|
-
.init()
|
|
1364
|
-
.then(() => {
|
|
1365
|
-
jobManager.updateJob(job.id, {
|
|
1366
|
-
status: 'complete',
|
|
1367
|
-
repoName: msg.result.repoName,
|
|
1368
|
-
});
|
|
1369
|
-
})
|
|
1370
|
-
.catch((err) => {
|
|
1371
|
-
logger.error({ err }, 'backend.init() failed after analyze:');
|
|
1372
|
-
jobManager.updateJob(job.id, {
|
|
1373
|
-
status: 'failed',
|
|
1374
|
-
error: 'Server failed to reload after analysis. Try again.',
|
|
1375
|
-
});
|
|
1376
|
-
});
|
|
1377
|
-
}
|
|
1378
|
-
else if (msg.type === 'error') {
|
|
1379
|
-
releaseRepoLock(analyzeLockKey);
|
|
1380
|
-
jobManager.updateJob(job.id, {
|
|
1381
|
-
status: 'failed',
|
|
1382
|
-
error: msg.message,
|
|
1383
|
-
});
|
|
1384
|
-
}
|
|
1385
|
-
});
|
|
1386
|
-
child.on('error', (err) => {
|
|
1387
|
-
releaseRepoLock(analyzeLockKey);
|
|
1388
|
-
jobManager.updateJob(job.id, {
|
|
1389
|
-
status: 'failed',
|
|
1390
|
-
error: `Worker process error: ${err.message}`,
|
|
1391
|
-
});
|
|
1392
|
-
});
|
|
1393
|
-
child.on('exit', (code) => {
|
|
1394
|
-
const j = jobManager.getJob(job.id);
|
|
1395
|
-
if (!j || j.status === 'complete' || j.status === 'failed')
|
|
1396
|
-
return;
|
|
1397
|
-
// Worker crashed — attempt retry if under the limit
|
|
1398
|
-
if (j.retryCount < MAX_WORKER_RETRIES) {
|
|
1399
|
-
j.retryCount++;
|
|
1400
|
-
const delay = 1000 * Math.pow(2, j.retryCount - 1); // 1s, 2s
|
|
1401
|
-
const lastErr = stderrChunks.trim().split('\n').pop() || '';
|
|
1402
|
-
logger.warn(`Analyze worker crashed (code ${code}), retry ${j.retryCount}/${MAX_WORKER_RETRIES} in ${delay}ms` +
|
|
1403
|
-
(lastErr ? `: ${lastErr}` : ''));
|
|
1404
|
-
jobManager.updateJob(job.id, {
|
|
1405
|
-
status: 'analyzing',
|
|
1406
|
-
progress: {
|
|
1407
|
-
phase: 'retrying',
|
|
1408
|
-
percent: j.progress.percent,
|
|
1409
|
-
message: `Worker crashed, retrying (${j.retryCount}/${MAX_WORKER_RETRIES})...`,
|
|
1410
|
-
},
|
|
1411
|
-
});
|
|
1412
|
-
stderrChunks = '';
|
|
1413
|
-
setTimeout(forkWorker, delay);
|
|
1414
|
-
}
|
|
1415
|
-
else {
|
|
1416
|
-
// Exhausted retries — permanent failure
|
|
1417
|
-
releaseRepoLock(analyzeLockKey);
|
|
1418
|
-
jobManager.updateJob(job.id, {
|
|
1419
|
-
status: 'failed',
|
|
1420
|
-
error: `Worker crashed ${MAX_WORKER_RETRIES + 1} times (code ${code})${stderrChunks ? ': ' + stderrChunks.trim().split('\n').pop() : ''}`,
|
|
1421
|
-
});
|
|
1422
|
-
}
|
|
1423
|
-
});
|
|
1424
|
-
// Register child for cancellation + timeout tracking
|
|
1425
|
-
jobManager.registerChild(job.id, child);
|
|
1426
|
-
// Send start command to child
|
|
1427
|
-
child.send({
|
|
1428
|
-
type: 'start',
|
|
1429
|
-
repoPath: targetPath,
|
|
1430
|
-
options: {
|
|
1431
|
-
force: !!force,
|
|
1432
|
-
embeddings: !!embeddings,
|
|
1433
|
-
dropEmbeddings: !!dropEmbeddings,
|
|
1434
|
-
},
|
|
1435
|
-
});
|
|
1436
|
-
};
|
|
1437
|
-
forkWorker();
|
|
1334
|
+
launchAnalysisWorker(job, targetPath, { force, embeddings, dropEmbeddings });
|
|
1438
1335
|
}
|
|
1439
1336
|
catch (err) {
|
|
1440
1337
|
if (targetPath)
|
|
@@ -1456,6 +1353,15 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1456
1353
|
}
|
|
1457
1354
|
}
|
|
1458
1355
|
});
|
|
1356
|
+
// POST /api/analyze/upload — analyze a browser folder upload.
|
|
1357
|
+
// Securely ingests the multipart upload into a sandbox, promotes it to a
|
|
1358
|
+
// persistent dir, and analyzes it via the shared job/worker machinery.
|
|
1359
|
+
// localhost-only (no cross-origin write reach) + conservative rate limit.
|
|
1360
|
+
app.post('/api/analyze/upload', createRouteLimiter({ limit: 5 }), requireLocalhostOrigin, createAnalyzeUploadHandler({
|
|
1361
|
+
createJob: (params) => jobManager.createJob(params),
|
|
1362
|
+
launch: (job, targetPath, opts) => launchAnalysisWorker(job, targetPath, opts),
|
|
1363
|
+
failJob: (jobId, error) => jobManager.updateJob(jobId, { status: 'failed', error }),
|
|
1364
|
+
}));
|
|
1459
1365
|
// GET /api/analyze/:jobId — poll job status
|
|
1460
1366
|
app.get('/api/analyze/:jobId', (req, res) => {
|
|
1461
1367
|
const job = jobManager.getJob(req.params.jobId);
|
package/dist/server/git-clone.js
CHANGED
|
@@ -17,7 +17,7 @@ const CLONE_ROOT = path.resolve(path.join(os.homedir(), '.gitnexus', 'repos'));
|
|
|
17
17
|
// Rejecting anything else (including `..`, `/`, `\`, shell metacharacters)
|
|
18
18
|
// guarantees getCloneDir(repoName) cannot escape CLONE_ROOT regardless of
|
|
19
19
|
// how the caller derived repoName.
|
|
20
|
-
const REPO_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/;
|
|
20
|
+
export const REPO_NAME_PATTERN = /^[a-zA-Z0-9._-]+$/;
|
|
21
21
|
/**
|
|
22
22
|
* Extract the repository name from a git URL (HTTPS or SSH).
|
|
23
23
|
*
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared Express route guards (alongside createRouteLimiter in validation.ts).
|
|
3
|
+
*/
|
|
4
|
+
import type { Request, Response } from 'express';
|
|
5
|
+
/**
|
|
6
|
+
* Restrict a route to localhost browser origins. Non-browser requests (no
|
|
7
|
+
* Origin header, e.g. curl / the CLI) pass through. This closes cross-origin
|
|
8
|
+
* reach (the allow-listed public deploy + Private Network Access) to write
|
|
9
|
+
* routes without affecting read routes.
|
|
10
|
+
*/
|
|
11
|
+
export declare function requireLocalhostOrigin(req: Request, res: Response, next: () => void): void;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared Express route guards (alongside createRouteLimiter in validation.ts).
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Restrict a route to localhost browser origins. Non-browser requests (no
|
|
6
|
+
* Origin header, e.g. curl / the CLI) pass through. This closes cross-origin
|
|
7
|
+
* reach (the allow-listed public deploy + Private Network Access) to write
|
|
8
|
+
* routes without affecting read routes.
|
|
9
|
+
*/
|
|
10
|
+
export function requireLocalhostOrigin(req, res, next) {
|
|
11
|
+
const origin = req.headers.origin;
|
|
12
|
+
if (origin === undefined) {
|
|
13
|
+
next();
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
try {
|
|
17
|
+
const hostname = new URL(origin).hostname;
|
|
18
|
+
if (hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1') {
|
|
19
|
+
next();
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
/* malformed origin → reject */
|
|
25
|
+
}
|
|
26
|
+
res.status(403).json({ error: 'This endpoint is restricted to localhost origins' });
|
|
27
|
+
}
|