archicore 0.2.4 → 0.2.5

package/dist/cli/utils/upload-utils.js

@@ -0,0 +1,495 @@
+/**
+ * ArchiCore CLI - Upload Utilities
+ *
+ * Утилиты для загрузки больших проектов:
+ * - Chunked uploads для больших файлов
+ * - Retry логика
+ * - Детальная обработка ошибок
+ */
+import { loadConfig } from './config.js';
+// Лимиты для chunked upload
+const MAX_PAYLOAD_SIZE = 10 * 1024 * 1024; // 10MB per chunk
+const MAX_SYMBOLS_PER_CHUNK = 5000;
+const MAX_FILES_PER_CHUNK = 100;
+const UPLOAD_TIMEOUT = 60000; // 60 секунд на chunk
+const MAX_RETRIES = 3;
+/**
+ * Определение размера JSON в байтах
+ */
+function estimateJsonSize(data) {
+    return new Blob([JSON.stringify(data)]).size;
+}
+/**
+ * Детальный анализ ошибки fetch
+ */
+export function analyzeNetworkError(error) {
+    const errorStr = String(error);
+    const errorName = error instanceof Error ? error.name : 'Unknown';
+    const errorMessage = error instanceof Error ? error.message : errorStr;
+    // Анализ различных типов ошибок
+    if (errorStr.includes('fetch failed') || errorStr.includes('ECONNREFUSED')) {
+        return {
+            code: 'CONNECTION_FAILED',
+            message: 'Cannot connect to ArchiCore server',
+            suggestion: 'Check that the server is running and accessible. Verify your internet connection.',
+            technicalDetails: `Network error: ${errorMessage}`,
+        };
+    }
+    if (errorStr.includes('ETIMEDOUT') || errorStr.includes('timeout')) {
+        return {
+            code: 'TIMEOUT',
+            message: 'Request timed out',
+            suggestion: 'The project may be too large. Try indexing a smaller subset first.',
+            technicalDetails: `Timeout after request: ${errorMessage}`,
+        };
+    }
+    if (errorStr.includes('ECONNRESET') || errorStr.includes('socket hang up')) {
+        return {
+            code: 'CONNECTION_RESET',
+            message: 'Connection was reset by server',
+            suggestion: 'The payload may be too large. ArchiCore will try chunked upload.',
+            technicalDetails: `Connection reset: ${errorMessage}`,
+        };
+    }
+    if (errorStr.includes('PayloadTooLargeError') || errorStr.includes('request entity too large')) {
+        return {
+            code: 'PAYLOAD_TOO_LARGE',
+            message: 'Data too large for single upload',
+            suggestion: 'Using chunked upload mode for large projects.',
+            technicalDetails: `Payload size exceeded: ${errorMessage}`,
+        };
+    }
+    if (errorStr.includes('CERT') || errorStr.includes('SSL') || errorStr.includes('certificate')) {
+        return {
+            code: 'SSL_ERROR',
+            message: 'SSL/TLS certificate error',
+            suggestion: 'Check server SSL configuration or try HTTP instead of HTTPS.',
+            technicalDetails: `SSL error: ${errorMessage}`,
+        };
+    }
+    if (errorStr.includes('ENOTFOUND') || errorStr.includes('getaddrinfo')) {
+        return {
+            code: 'DNS_ERROR',
+            message: 'Cannot resolve server hostname',
+            suggestion: 'Check your internet connection and server URL configuration.',
+            technicalDetails: `DNS resolution failed: ${errorMessage}`,
+        };
+    }
+    // Generic network error
+    return {
+        code: 'NETWORK_ERROR',
+        message: 'Network request failed',
+        suggestion: 'Check your internet connection and try again.',
+        technicalDetails: `${errorName}: ${errorMessage}`,
+    };
+}
+/**
+ * Детальный анализ HTTP ошибки
+ */
+export function analyzeHttpError(status, responseBody) {
+    const body = responseBody || {};
+    const serverError = body.error || body.message || '';
+    switch (status) {
+        case 400:
+            return {
+                code: 'BAD_REQUEST',
+                message: 'Invalid request data',
+                suggestion: 'The index data format may be corrupted. Try re-indexing.',
+                technicalDetails: serverError || 'Server rejected the request as invalid',
+            };
+        case 401:
+            return {
+                code: 'UNAUTHORIZED',
+                message: 'Authentication required',
+                suggestion: 'Your session may have expired. Run /logout and log in again.',
+                technicalDetails: serverError || 'Access token is invalid or expired',
+            };
+        case 403:
+            return {
+                code: 'FORBIDDEN',
+                message: 'Access denied to this project',
+                suggestion: 'You may not have permission to this project. Try re-initializing with /index.',
+                technicalDetails: serverError || 'Server denied access to the requested resource',
+            };
+        case 404:
+            return {
+                code: 'NOT_FOUND',
+                message: 'Project not found on server',
+                suggestion: 'The project may have been deleted. Try running /index again.',
+                technicalDetails: serverError || 'Project ID does not exist on server',
+            };
+        case 413:
+            return {
+                code: 'PAYLOAD_TOO_LARGE',
+                message: 'Project data too large for single upload',
+                suggestion: 'ArchiCore will automatically use chunked upload.',
+                technicalDetails: serverError || 'Request payload exceeded server limit',
+            };
+        case 429:
+            return {
+                code: 'RATE_LIMITED',
+                message: 'Too many requests',
+                suggestion: 'Wait a few minutes and try again.',
+                technicalDetails: serverError || 'Rate limit exceeded',
+            };
+        case 500:
+            return {
+                code: 'SERVER_ERROR',
+                message: 'Server internal error',
+                suggestion: 'The server encountered an error processing your project. This may be due to project size.',
+                technicalDetails: serverError || 'Internal server error - check server logs for details',
+            };
+        case 502:
+        case 503:
+        case 504:
+            return {
+                code: 'SERVER_UNAVAILABLE',
+                message: 'Server is temporarily unavailable',
+                suggestion: 'Wait a few minutes and try again. The server may be overloaded.',
+                technicalDetails: serverError || `Server returned ${status}`,
+            };
+        default:
+            return {
+                code: 'HTTP_ERROR',
+                message: `Request failed with status ${status}`,
+                suggestion: 'Check server status and try again.',
+                technicalDetails: serverError || `HTTP ${status} error`,
+            };
+    }
+}
+/**
+ * Выполнение fetch с timeout и retry
+ */
+async function fetchWithRetry(url, options, timeout = UPLOAD_TIMEOUT, maxRetries = MAX_RETRIES) {
+    let lastError = null;
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), timeout);
+            const response = await fetch(url, {
+                ...options,
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            return response;
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+            // Не повторяем для определённых ошибок
+            const errorStr = String(error);
+            if (errorStr.includes('AbortError') ||
+                errorStr.includes('ENOTFOUND') ||
+                errorStr.includes('CERT')) {
+                throw error;
+            }
+            // Экспоненциальная задержка перед повтором
+            if (attempt < maxRetries) {
+                const delay = Math.min(1000 * Math.pow(2, attempt - 1), 10000);
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
+        }
+    }
+    throw lastError || new Error('Request failed after retries');
+}
+/**
+ * Загрузка индекса с автоматическим chunked upload для больших проектов
+ */
+export async function uploadIndexData(projectId, data, onProgress) {
+    const config = await loadConfig();
+    const url = `${config.serverUrl}/api/projects/${projectId}/upload-index`;
+    // Оценка размера данных
+    const estimatedSize = estimateJsonSize(data);
+    const isLargeProject = estimatedSize > MAX_PAYLOAD_SIZE ||
+        data.symbols.length > MAX_SYMBOLS_PER_CHUNK * 2;
+    onProgress?.({
+        phase: 'preparing',
+        current: 0,
+        total: 100,
+        message: `Preparing upload (${(estimatedSize / 1024 / 1024).toFixed(1)} MB)...`,
+    });
+    // Для небольших проектов - обычная загрузка
+    if (!isLargeProject) {
+        console.log(`[DEBUG] Using single request upload for ${data.symbols.length} symbols`);
+        return uploadSingleRequest(url, projectId, data, config.accessToken || '', onProgress);
+    }
+    // Для больших проектов - chunked upload
+    console.log(`[DEBUG] Using chunked upload for large project (${data.symbols.length} symbols)`);
+    return uploadChunked(url, projectId, data, config.accessToken || '', onProgress);
+}
+/**
+ * Обычная загрузка одним запросом
+ */
+async function uploadSingleRequest(url, projectId, data, accessToken, onProgress) {
+    onProgress?.({
+        phase: 'uploading',
+        current: 50,
+        total: 100,
+        message: 'Uploading index data...',
+    });
+    try {
+        const response = await fetchWithRetry(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${accessToken}`,
+            },
+            body: JSON.stringify(data),
+        });
+        if (!response.ok) {
+            const errorBody = await response.json().catch(() => ({}));
+            console.log(`[DEBUG] HTTP error ${response.status}: ${JSON.stringify(errorBody)}`);
+            const errorDetails = analyzeHttpError(response.status, errorBody);
+            // Если payload слишком большой - пробуем chunked
+            if (response.status === 413) {
+                console.log(`[DEBUG] Payload too large, trying chunked upload`);
+                return uploadChunked(url, projectId, data, accessToken, onProgress);
+            }
+            return {
+                success: false,
+                error: errorDetails.message,
+                errorDetails,
+            };
+        }
+        const result = await response.json();
+        onProgress?.({
+            phase: 'done',
+            current: 100,
+            total: 100,
+            message: 'Upload complete!',
+        });
+        return {
+            success: true,
+            statistics: result.statistics,
+        };
+    }
+    catch (error) {
+        console.log(`[DEBUG] uploadSingleRequest caught error: ${error}`);
+        const errorDetails = analyzeNetworkError(error);
+        console.log(`[DEBUG] Analyzed error: ${JSON.stringify(errorDetails)}`);
+        // Если ошибка связана с размером - пробуем chunked
+        if (errorDetails.code === 'CONNECTION_RESET' ||
+            errorDetails.code === 'PAYLOAD_TOO_LARGE') {
+            console.log(`[DEBUG] Trying chunked upload due to ${errorDetails.code}`);
+            return uploadChunked(url, projectId, data, accessToken, onProgress);
+        }
+        return {
+            success: false,
+            error: errorDetails.message,
+            errorDetails,
+        };
+    }
+}
+/**
+ * Chunked upload для больших проектов
+ */
+async function uploadChunked(baseUrl, projectId, data, accessToken, onProgress) {
+    const config = await loadConfig();
+    // Разбиваем данные на chunks
+    const symbolChunks = chunkArray(data.symbols, MAX_SYMBOLS_PER_CHUNK);
+    const astChunks = chunkArray(data.asts, MAX_FILES_PER_CHUNK);
+    const fileChunks = chunkArray(data.fileContents, MAX_FILES_PER_CHUNK);
+    const totalChunks = symbolChunks.length + astChunks.length + fileChunks.length + 1; // +1 for graph
+    let completedChunks = 0;
+    onProgress?.({
+        phase: 'uploading',
+        current: 0,
+        total: totalChunks,
+        message: `Uploading in ${totalChunks} chunks...`,
+    });
+    try {
+        // 1. Инициализируем chunked upload
+        const initResponse = await fetchWithRetry(`${config.serverUrl}/api/projects/${projectId}/upload-index/init`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${accessToken}`,
+            },
+            body: JSON.stringify({
+                statistics: data.statistics,
+                totalChunks,
+            }),
+        });
+        if (!initResponse.ok) {
+            // Fallback: если сервер не поддерживает chunked upload,
+            // отправляем только минимальные данные
+            return uploadMinimalData(baseUrl, projectId, data, accessToken, onProgress);
+        }
+        const initResult = await initResponse.json();
+        const uploadId = initResult.uploadId;
+        // 2. Загружаем ASTs по частям
+        for (let i = 0; i < astChunks.length; i++) {
+            await uploadChunk(config.serverUrl, projectId, uploadId, 'asts', i, astChunks[i], accessToken);
+            completedChunks++;
+            onProgress?.({
+                phase: 'uploading',
+                current: completedChunks,
+                total: totalChunks,
+                message: `Uploading ASTs (${i + 1}/${astChunks.length})...`,
+            });
+        }
+        // 3. Загружаем символы по частям
+        for (let i = 0; i < symbolChunks.length; i++) {
+            await uploadChunk(config.serverUrl, projectId, uploadId, 'symbols', i, symbolChunks[i], accessToken);
+            completedChunks++;
+            onProgress?.({
+                phase: 'uploading',
+                current: completedChunks,
+                total: totalChunks,
+                message: `Uploading symbols (${i + 1}/${symbolChunks.length})...`,
+            });
+        }
+        // 4. Загружаем граф
+        await uploadChunk(config.serverUrl, projectId, uploadId, 'graph', 0, data.graph, accessToken);
+        completedChunks++;
+        onProgress?.({
+            phase: 'uploading',
+            current: completedChunks,
+            total: totalChunks,
+            message: 'Uploading dependency graph...',
+        });
+        // 5. Загружаем содержимое файлов (опционально)
+        for (let i = 0; i < fileChunks.length; i++) {
+            await uploadChunk(config.serverUrl, projectId, uploadId, 'fileContents', i, fileChunks[i], accessToken);
+            completedChunks++;
+            onProgress?.({
+                phase: 'uploading',
+                current: completedChunks,
+                total: totalChunks,
+                message: `Uploading file contents (${i + 1}/${fileChunks.length})...`,
+            });
+        }
+        // 6. Финализируем upload
+        onProgress?.({
+            phase: 'processing',
+            current: totalChunks,
+            total: totalChunks,
+            message: 'Finalizing index...',
+        });
+        const finalizeResponse = await fetchWithRetry(`${config.serverUrl}/api/projects/${projectId}/upload-index/finalize`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${accessToken}`,
+            },
+            body: JSON.stringify({ uploadId }),
+        }, 120000 // 2 минуты на финализацию
+        );
+        if (!finalizeResponse.ok) {
+            const errorBody = await finalizeResponse.json().catch(() => ({}));
+            const errorDetails = analyzeHttpError(finalizeResponse.status, errorBody);
+            return {
+                success: false,
+                error: errorDetails.message,
+                errorDetails,
+            };
+        }
+        const result = await finalizeResponse.json();
+        onProgress?.({
+            phase: 'done',
+            current: totalChunks,
+            total: totalChunks,
+            message: 'Upload complete!',
+        });
+        return {
+            success: true,
+            statistics: result.statistics,
+        };
+    }
+    catch (error) {
+        const errorDetails = analyzeNetworkError(error);
+        return {
+            success: false,
+            error: errorDetails.message,
+            errorDetails,
+        };
+    }
+}
+/**
+ * Загрузка минимальных данных (fallback для серверов без chunked upload)
+ */
+async function uploadMinimalData(url, _projectId, data, accessToken, onProgress) {
+    onProgress?.({
+        phase: 'uploading',
+        current: 50,
+        total: 100,
+        message: 'Uploading minimal index (large project mode)...',
+    });
+    // Отправляем только граф и статистику (без содержимого файлов и AST)
+    const minimalData = {
+        asts: data.asts.slice(0, MAX_FILES_PER_CHUNK), // Только первые N файлов
+        symbols: data.symbols.slice(0, MAX_SYMBOLS_PER_CHUNK), // Только первые N символов
+        graph: data.graph,
+        statistics: data.statistics,
+        // Без fileContents - это самая большая часть
+    };
+    try {
+        const response = await fetchWithRetry(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${accessToken}`,
+            },
+            body: JSON.stringify(minimalData),
+        }, 120000); // 2 минуты
+        if (!response.ok) {
+            const errorBody = await response.json().catch(() => ({}));
+            const errorDetails = analyzeHttpError(response.status, errorBody);
+            return {
+                success: false,
+                error: errorDetails.message,
+                errorDetails,
+            };
+        }
+        const result = await response.json();
+        onProgress?.({
+            phase: 'done',
+            current: 100,
+            total: 100,
+            message: 'Upload complete (minimal mode)!',
+        });
+        return {
+            success: true,
+            statistics: result.statistics,
+        };
+    }
+    catch (error) {
+        const errorDetails = analyzeNetworkError(error);
+        return {
+            success: false,
+            error: errorDetails.message,
+            errorDetails,
+        };
+    }
+}
+/**
+ * Загрузка одного chunk
+ */
+async function uploadChunk(serverUrl, projectId, uploadId, chunkType, chunkIndex, chunkData, accessToken) {
+    const response = await fetchWithRetry(`${serverUrl}/api/projects/${projectId}/upload-index/chunk`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${accessToken}`,
+        },
+        body: JSON.stringify({
+            uploadId,
+            chunkType,
+            chunkIndex,
+            data: chunkData,
+        }),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to upload chunk ${chunkType}[${chunkIndex}]`);
+    }
+}
+/**
+ * Разбиение массива на chunks
+ */
+function chunkArray(array, chunkSize) {
+    const chunks = [];
+    for (let i = 0; i < array.length; i += chunkSize) {
+        chunks.push(array.slice(i, i + chunkSize));
+    }
+    return chunks.length > 0 ? chunks : [[]];
+}
+//# sourceMappingURL=upload-utils.js.map

package/dist/server/routes/api.js

@@ -165,7 +165,146 @@ apiRouter.post('/projects/:id/upload-index', authMiddleware, checkProjectAccess,
     }
     catch (error) {
         Logger.error('Failed to upload indexed data:', error);
-        res.status(500).json({ error: 'Failed to upload indexed data' });
+        // Provide more specific error messages
+        const errorStr = String(error);
+        let errorMessage = 'Failed to upload indexed data';
+        let errorCode = 'UPLOAD_FAILED';
+        if (errorStr.includes('ENOMEM') || errorStr.includes('heap')) {
+            errorMessage = 'Server ran out of memory processing the project';
+            errorCode = 'OUT_OF_MEMORY';
+        }
+        else if (errorStr.includes('ENOSPC')) {
+            errorMessage = 'Server ran out of disk space';
+            errorCode = 'DISK_FULL';
+        }
+        else if (errorStr.includes('timeout')) {
+            errorMessage = 'Processing timed out for large project';
+            errorCode = 'TIMEOUT';
+        }
+        res.status(500).json({
+            error: errorMessage,
+            code: errorCode,
+            details: process.env.NODE_ENV === 'development' ? errorStr : undefined
+        });
+    }
+});
+// Хранилище для chunked uploads
+const chunkedUploads = new Map();
+// Очистка старых uploads (старше 1 часа)
+setInterval(() => {
+    const oneHourAgo = Date.now() - 60 * 60 * 1000;
+    for (const [uploadId, data] of chunkedUploads.entries()) {
+        if (data.createdAt < oneHourAgo) {
+            chunkedUploads.delete(uploadId);
+        }
+    }
+}, 5 * 60 * 1000); // Проверка каждые 5 минут
+/**
+ * POST /api/projects/:id/upload-index/init
+ * Инициализация chunked upload
+ */
+apiRouter.post('/projects/:id/upload-index/init', authMiddleware, checkProjectAccess, async (req, res) => {
+    try {
+        const { id } = req.params;
+        const { statistics, totalChunks } = req.body;
+        const uploadId = `upload_${id}_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`;
+        chunkedUploads.set(uploadId, {
+            projectId: id,
+            statistics: statistics || { totalFiles: 0, totalSymbols: 0 },
+            asts: [],
+            symbols: [],
+            graph: null,
+            fileContents: [],
+            createdAt: Date.now(),
+        });
+        Logger.info(`Chunked upload initialized: ${uploadId} (${totalChunks} chunks expected)`);
+        res.json({ uploadId, message: 'Chunked upload initialized' });
+    }
+    catch (error) {
+        Logger.error('Failed to init chunked upload:', error);
+        res.status(500).json({ error: 'Failed to initialize chunked upload' });
+    }
+});
+/**
+ * POST /api/projects/:id/upload-index/chunk
+ * Загрузка одного chunk
+ */
+apiRouter.post('/projects/:id/upload-index/chunk', authMiddleware, checkProjectAccess, async (req, res) => {
+    try {
+        const { uploadId, chunkType, chunkIndex, data } = req.body;
+        const upload = chunkedUploads.get(uploadId);
+        if (!upload) {
+            res.status(404).json({ error: 'Upload session not found or expired' });
+            return;
+        }
+        // Добавляем данные в соответствующий массив
+        switch (chunkType) {
+            case 'asts':
+                upload.asts.push(...data);
+                break;
+            case 'symbols':
+                upload.symbols.push(...data);
+                break;
+            case 'graph':
+                upload.graph = data;
+                break;
+            case 'fileContents':
+                upload.fileContents.push(...data);
+                break;
+            default:
+                res.status(400).json({ error: `Unknown chunk type: ${chunkType}` });
+                return;
+        }
+        Logger.debug(`Chunk received: ${chunkType}[${chunkIndex}] for upload ${uploadId}`);
+        res.json({ success: true, received: chunkType, index: chunkIndex });
+    }
+    catch (error) {
+        Logger.error('Failed to process chunk:', error);
+        res.status(500).json({ error: 'Failed to process chunk' });
+    }
+});
+/**
+ * POST /api/projects/:id/upload-index/finalize
+ * Финализация chunked upload
+ */
+apiRouter.post('/projects/:id/upload-index/finalize', authMiddleware, checkProjectAccess, async (req, res) => {
+    try {
+        const { id } = req.params;
+        const { uploadId } = req.body;
+        const upload = chunkedUploads.get(uploadId);
+        if (!upload) {
+            res.status(404).json({ error: 'Upload session not found or expired' });
+            return;
+        }
+        if (upload.projectId !== id) {
+            res.status(403).json({ error: 'Upload session does not match project' });
+            return;
+        }
+        Logger.info(`Finalizing chunked upload ${uploadId}: ${upload.asts.length} ASTs, ${upload.symbols.length} symbols`);
+        // Собираем все данные и отправляем в projectService
+        // Type cast because chunked data comes as unknown from JSON
+        const result = await projectService.uploadIndexedData(id, {
+            asts: upload.asts,
+            symbols: upload.symbols,
+            graph: (upload.graph || { nodes: [], edges: [] }),
+            fileContents: upload.fileContents,
+            statistics: {
+                totalFiles: upload.statistics.totalFiles || upload.asts.length,
+                totalSymbols: upload.statistics.totalSymbols || upload.symbols.length,
+            },
+        });
+        // Удаляем upload session
+        chunkedUploads.delete(uploadId);
+        res.json(result);
+    }
+    catch (error) {
+        Logger.error('Failed to finalize chunked upload:', error);
+        const errorStr = String(error);
+        let errorMessage = 'Failed to finalize upload';
+        if (errorStr.includes('ENOMEM') || errorStr.includes('heap')) {
+            errorMessage = 'Server ran out of memory processing the project';
+        }
+        res.status(500).json({ error: errorMessage });
     }
 });
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archicore",
-  "version": "0.2.4",
+  "version": "0.2.5",
   "description": "AI Software Architect - code analysis, impact prediction, semantic search",
   "main": "dist/index.js",
   "type": "module",