archicore 0.1.5 → 0.1.7

package/dist/cli/commands/interactive.js

@@ -17,6 +17,57 @@ const state = {
     history: [],
     user: null,
 };
+// Track token usage for session
+let sessionTokensUsed = 0;
+let lastOperationTokens = 0;
+/**
+ * Estimate tokens from text (roughly 1 token = 4 chars)
+ */
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Helper for authenticated API calls with token tracking
+ */
+async function apiFetch(url, options = {}) {
+    const config = await loadConfig();
+    const headers = {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${config.accessToken}`,
+        ...options.headers,
+    };
+    // Estimate input tokens from request body
+    const inputTokens = options.body ? estimateTokens(String(options.body)) : 0;
+    const response = await fetch(url, { ...options, headers });
+    // Try to get tokens from header first
+    const tokensHeader = response.headers.get('X-Tokens-Used');
+    if (tokensHeader) {
+        lastOperationTokens = parseInt(tokensHeader, 10);
+    }
+    else {
+        // Estimate based on response size (will be updated when body is read)
+        lastOperationTokens = inputTokens;
+    }
+    return response;
+}
+/**
+ * Update token count from response data
+ */
+function updateTokensFromResponse(data) {
+    const responseText = JSON.stringify(data);
+    const outputTokens = estimateTokens(responseText);
+    lastOperationTokens += outputTokens;
+    sessionTokensUsed += lastOperationTokens;
+}
+/**
+ * Display token usage after operation
+ */
+function showTokenUsage() {
+    if (lastOperationTokens > 0) {
+        console.log(colors.dim(`  [${lastOperationTokens.toLocaleString()} tokens]`));
+        lastOperationTokens = 0; // Reset for next operation
+    }
+}
 export async function startInteractiveMode() {
     // Prevent process from exiting on unhandled errors
     process.on('uncaughtException', (error) => {
@@ -212,6 +263,10 @@ async function handleCommand(input) {
         case 'export':
             await handleExportCommand(args);
             break;
+        case 'docs':
+        case 'documentation':
+            await handleDocsCommand(args);
+            break;
         case 'status':
             await handleStatusCommand();
             break;
@@ -312,7 +367,64 @@ async function handleIndexCommand() {
         });
         if (!response.ok) {
             const errorData = await response.json().catch(() => ({}));
-            throw new Error(errorData.error || 'Upload failed');
+            // If access denied, re-register project with current user
+            if (response.status === 403 && errorData.error?.includes('Access denied')) {
+                indexSpinner.update('Re-registering project with current user...');
+                // Clear old project ID and re-register
+                const reRegisterResponse = await fetch(`${config.serverUrl}/api/projects`, {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'Authorization': `Bearer ${config.accessToken}`,
+                    },
+                    body: JSON.stringify({ name: state.projectName, path: state.projectPath }),
+                });
+                if (reRegisterResponse.ok) {
+                    const reData = await reRegisterResponse.json();
+                    state.projectId = reData.id || reData.project?.id;
+                    // Retry upload with new project ID
+                    indexSpinner.update('Uploading to server...');
+                    const retryResponse = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/upload-index`, {
+                        method: 'POST',
+                        headers: {
+                            'Content-Type': 'application/json',
+                            'Authorization': `Bearer ${config.accessToken}`,
+                        },
+                        body: JSON.stringify({
+                            asts: astsArray,
+                            symbols: symbolsArray,
+                            graph: graphData,
+                            fileContents,
+                            statistics: {
+                                totalFiles: asts.size,
+                                totalSymbols: symbols.size,
+                            },
+                        }),
+                    });
+                    if (!retryResponse.ok) {
+                        throw new Error('Upload failed after re-registration');
+                    }
+                    // Use retry response for data
+                    const retryData = await retryResponse.json();
+                    indexSpinner.succeed('Project re-registered and indexed');
+                    printKeyValue('Files', String(retryData.statistics?.filesCount || asts.size));
+                    printKeyValue('Symbols', String(retryData.statistics?.symbolsCount || symbols.size));
+                    // Update local config with new project ID
+                    const localProjectRetry = await getLocalProject(state.projectPath);
+                    if (localProjectRetry && state.projectId) {
+                        localProjectRetry.id = state.projectId;
+                        localProjectRetry.indexed = true;
+                        await fs.writeFile(pathModule.default.join(state.projectPath, '.archicore', 'project.json'), JSON.stringify(localProjectRetry, null, 2));
+                    }
+                    return; // Exit early - already handled
+                }
+                else {
+                    throw new Error('Failed to re-register project');
+                }
+            }
+            else {
+                throw new Error(errorData.error || 'Upload failed');
+            }
         }
         const data = await response.json();
         indexSpinner.succeed('Project indexed and uploaded');
@@ -341,9 +453,8 @@ async function handleAnalyzeCommand(args) {
     const spinner = createSpinner('Analyzing impact...').start();
     try {
         const config = await loadConfig();
-        const response = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/analyze`, {
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/analyze`, {
             method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({
                 description,
                 files: [],
@@ -394,6 +505,7 @@ async function handleAnalyzeCommand(args) {
                 console.log(`  ${colors.info(icons.info)} ${rec.description}`);
             }
         }
+        showTokenUsage();
     }
     catch (error) {
         spinner.fail('Analysis failed');
@@ -413,9 +525,8 @@ async function handleSearchCommand(query) {
     const spinner = createSpinner('Searching...').start();
     try {
         const config = await loadConfig();
-        const response = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/search`, {
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/search`, {
             method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({ query, limit: 10 }),
         });
         if (!response.ok)
@@ -440,8 +551,8 @@ async function handleSearchCommand(query) {
                     .replace(/\n/g, ' ');
                 console.log(`     ${colors.dim(preview)}${preview.length >= 100 ? '...' : ''}`);
             }
-            console.log();
         }
+        showTokenUsage();
     }
     catch (error) {
         spinner.fail('Search failed');
@@ -457,14 +568,14 @@ async function handleQuery(query) {
     const spinner = createSpinner('Thinking...').start();
     try {
         const config = await loadConfig();
-        const response = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/ask`, {
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/ask`, {
             method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({ question: query }),
         });
         if (!response.ok)
             throw new Error('Query failed');
         const data = await response.json();
+        updateTokensFromResponse(data);
         spinner.stop();
         console.log();
         console.log(colors.primary('  ' + icons.lightning + ' ArchiCore'));
@@ -475,6 +586,7 @@ async function handleQuery(query) {
         for (const line of lines) {
             console.log('  ' + line);
         }
+        showTokenUsage();
     }
     catch (error) {
         spinner.fail('Query failed');
@@ -489,7 +601,7 @@ async function handleDeadCodeCommand() {
     const spinner = createSpinner('Analyzing dead code...').start();
     try {
         const config = await loadConfig();
-        const response = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/dead-code`);
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/dead-code`);
         if (!response.ok)
             throw new Error('Analysis failed');
         const data = await response.json();
@@ -520,7 +632,7 @@ async function handleSecurityCommand() {
     const spinner = createSpinner('Analyzing security...').start();
     try {
         const config = await loadConfig();
-        const response = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/security`);
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/security`);
         if (!response.ok)
             throw new Error('Analysis failed');
         const data = await response.json();
@@ -562,7 +674,7 @@ async function handleMetricsCommand() {
     const spinner = createSpinner('Calculating metrics...').start();
     try {
         const config = await loadConfig();
-        const response = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/metrics`);
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/metrics`);
         if (!response.ok)
             throw new Error('Analysis failed');
         const data = await response.json();
@@ -591,9 +703,8 @@ async function handleExportCommand(args) {
     const spinner = createSpinner(`Exporting as ${format}...`).start();
     try {
         const config = await loadConfig();
-        const response = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/export`, {
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/export`, {
             method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({ format }),
         });
         if (!response.ok)
@@ -647,6 +758,10 @@ async function handleStatusCommand() {
         catch {
             // Ignore subscription fetch errors
         }
+        // Show session token usage
+        if (sessionTokensUsed > 0) {
+            console.log(`  Session tokens: ${colors.highlight(sessionTokensUsed.toLocaleString())}`);
+        }
     }
     if (state.projectId) {
         console.log(`  Project: ${colors.primary(state.projectName || state.projectId)}`);
@@ -664,7 +779,7 @@ async function handleDuplicationCommand() {
     const spinner = createSpinner('Analyzing code duplication...').start();
     try {
         const config = await loadConfig();
-        const response = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/duplication`);
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/duplication`);
         if (!response.ok)
             throw new Error('Analysis failed');
         const data = await response.json();
@@ -701,7 +816,7 @@ async function handleRefactoringCommand() {
     const spinner = createSpinner('Generating refactoring suggestions...').start();
     try {
         const config = await loadConfig();
-        const response = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/refactoring`);
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/refactoring`);
         if (!response.ok)
             throw new Error('Analysis failed');
         const data = await response.json();
@@ -746,7 +861,7 @@ async function handleRulesCommand() {
     const spinner = createSpinner('Checking architectural rules...').start();
     try {
         const config = await loadConfig();
-        const response = await fetch(`${config.serverUrl}/api/projects/${state.projectId}/rules`);
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/rules`);
         if (!response.ok)
             throw new Error('Analysis failed');
         const data = await response.json();
@@ -773,4 +888,35 @@ async function handleRulesCommand() {
         throw error;
     }
 }
+async function handleDocsCommand(args) {
+    if (!state.projectId) {
+        printError('No project indexed');
+        printInfo('Use /index first');
+        return;
+    }
+    const format = args[0] || 'markdown';
+    const output = args[1] || `documentation.${format === 'markdown' ? 'md' : format}`;
+    const spinner = createSpinner('Generating documentation...').start();
+    try {
+        const config = await loadConfig();
+        const response = await apiFetch(`${config.serverUrl}/api/projects/${state.projectId}/documentation`, {
+            method: 'POST',
+            body: JSON.stringify({ format }),
+        });
+        if (!response.ok)
+            throw new Error('Documentation generation failed');
+        const data = await response.json();
+        updateTokensFromResponse(data);
+        const { writeFile } = await import('fs/promises');
+        await writeFile(output, data.documentation);
+        spinner.succeed('Documentation generated');
+        printKeyValue('Output', output);
+        printKeyValue('Format', format);
+        showTokenUsage();
+    }
+    catch (error) {
+        spinner.fail('Documentation generation failed');
+        throw error;
+    }
+}
 //# sourceMappingURL=interactive.js.map

package/dist/code-index/ast-parser.js

@@ -144,9 +144,41 @@ export class ASTParser {
                 /^(?:\w+\s+)*(\w+)\s*\([^)]*\)\s*{/,
                 /^struct\s+(\w+)/,
                 /^typedef\s+.*\s+(\w+)\s*;/
-            ]
+            ],
+            html: [
+                /<(\w+)\s+id="(\w+)"/, // Elements with id
+                /<script[^>]*>/, // Script tags
+                /<style[^>]*>/ // Style tags
+            ],
+            css: [
+                /^\.(\w[\w-]*)\s*\{/, // Class selectors
+                /^#(\w[\w-]*)\s*\{/, // ID selectors
+                /^@media\s+/, // Media queries
+                /^@keyframes\s+(\w+)/ // Animations
+            ],
+            scss: [
+                /^\.(\w[\w-]*)\s*\{/,
+                /^\$(\w[\w-]*)\s*:/, // Variables
+                /^@mixin\s+(\w+)/, // Mixins
+                /^@include\s+(\w+)/ // Include
+            ],
+            json: [], // JSON doesn't have functions/classes
+            yaml: [], // YAML doesn't have functions/classes
+            xml: [
+                /<(\w+)\s+.*?>/ // XML elements
+            ],
+            sql: [
+                /^CREATE\s+(?:TABLE|VIEW|FUNCTION|PROCEDURE)\s+(\w+)/i,
+                /^ALTER\s+TABLE\s+(\w+)/i,
+                /^INSERT\s+INTO\s+(\w+)/i
+            ],
+            shell: [
+                /^(\w+)\s*\(\)\s*\{/, // Function definitions
+                /^function\s+(\w+)/ // Function keyword
+            ],
+            markdown: [] // Markdown doesn't have functions/classes
         };
-        const langPatterns = patterns[language] || patterns.javascript;
+        const langPatterns = patterns[language] || [];
         lines.forEach((line, index) => {
             const trimmedLine = line.trim();
             for (const pattern of langPatterns) {

package/dist/github/github-service.d.ts

@@ -71,6 +71,14 @@ export declare class GitHubService {
      * Disconnect repository
      */
     disconnectRepository(userId: string, repoId: string): Promise<boolean>;
+    /**
+     * Disconnect repository by project ID
+     */
+    disconnectRepositoryByProjectId(projectId: string): Promise<boolean>;
+    /**
+     * Update repository's project ID
+     */
+    updateRepositoryProjectId(repoId: string, projectId: string): Promise<void>;
     /**
      * Get connected repositories for user
      */

package/dist/github/github-service.js

@@ -12,6 +12,8 @@ const CONNECTIONS_FILE = 'github-connections.json';
 const REPOSITORIES_FILE = 'github-repositories.json';
 // Encryption key (in production, use env variable)
 const ENCRYPTION_KEY = process.env.GITHUB_ENCRYPTION_KEY || 'archicore-github-key-32bytes!!';
+// Singleton instance
+let instance = null;
 export class GitHubService {
     dataDir;
     connections = [];
@@ -22,11 +24,16 @@ export class GitHubService {
     redirectUri;
     webhookBaseUrl;
     constructor(dataDir = DATA_DIR) {
+        // Singleton pattern - return existing instance
+        if (instance) {
+            return instance;
+        }
         this.dataDir = dataDir;
         this.clientId = process.env.GITHUB_CLIENT_ID || '';
         this.clientSecret = process.env.GITHUB_CLIENT_SECRET || '';
         this.redirectUri = process.env.GITHUB_REDIRECT_URI || 'http://localhost:3000/api/github/callback';
         this.webhookBaseUrl = process.env.ARCHICORE_WEBHOOK_URL || 'http://localhost:3000/api/github/webhook';
+        instance = this;
     }
     async ensureInitialized() {
         if (this.initialized)
@@ -222,7 +229,8 @@ export class GitHubService {
         let page = 1;
         const perPage = 100;
         while (true) {
-            const response = await fetch(`https://api.github.com/user/repos?per_page=${perPage}&page=${page}&sort=updated`, {
+            // visibility=all includes private repos, affiliation includes all repo types
+            const response = await fetch(`https://api.github.com/user/repos?per_page=${perPage}&page=${page}&sort=updated&visibility=all&affiliation=owner,collaborator,organization_member`, {
                 headers: {
                     'Authorization': `Bearer ${accessToken}`,
                     'Accept': 'application/vnd.github.v3+json',
@@ -230,7 +238,16 @@ export class GitHubService {
                 }
             });
             if (!response.ok) {
-                throw new Error('Failed to list repositories');
+                const errorBody = await response.text();
+                Logger.error(`GitHub API error ${response.status}: ${errorBody}`);
+                // Check for specific errors
+                if (response.status === 401) {
+                    throw new Error('GitHub token expired or invalid. Please reconnect GitHub.');
+                }
+                else if (response.status === 403) {
+                    throw new Error('GitHub rate limit exceeded or insufficient permissions.');
+                }
+                throw new Error(`Failed to list repositories: ${response.status}`);
             }
             const data = await response.json();
             repos.push(...data);
@@ -359,6 +376,40 @@ export class GitHubService {
         Logger.info(`Repository disconnected: ${repo.fullName}`);
         return true;
     }
+    /**
+     * Disconnect repository by project ID
+     */
+    async disconnectRepositoryByProjectId(projectId) {
+        await this.ensureInitialized();
+        const repo = this.repositories.find(r => r.projectId === projectId);
+        if (!repo)
+            return false;
+        // Delete webhook
+        if (repo.webhookId) {
+            try {
+                await this.deleteWebhook(repo.userId, repo.fullName, repo.webhookId);
+            }
+            catch (e) {
+                Logger.warn(`Failed to delete webhook: ${e}`);
+            }
+        }
+        this.repositories = this.repositories.filter(r => r.projectId !== projectId);
+        await this.saveRepositories();
+        Logger.info(`Repository disconnected by projectId: ${repo.fullName}`);
+        return true;
+    }
+    /**
+     * Update repository's project ID
+     */
+    async updateRepositoryProjectId(repoId, projectId) {
+        await this.ensureInitialized();
+        const repo = this.repositories.find(r => r.id === repoId);
+        if (repo) {
+            repo.projectId = projectId;
+            repo.updatedAt = new Date();
+            await this.saveRepositories();
+        }
+    }
     /**
      * Get connected repositories for user
      */
@@ -560,18 +611,36 @@ export class GitHubService {
             throw new Error('GitHub not connected');
         }
         const url = `https://api.github.com/repos/${fullName}/zipball/${ref || 'HEAD'}`;
+        Logger.info(`Downloading from: ${url}`);
         const response = await fetch(url, {
             headers: {
                 'Authorization': `Bearer ${accessToken}`,
-                'Accept': 'application/vnd.github.v3+json',
-                'User-Agent': 'ArchiCore'
-            }
+                'Accept': 'application/vnd.github+json',
+                'User-Agent': 'ArchiCore',
+                'X-GitHub-Api-Version': '2022-11-28'
+            },
+            redirect: 'follow'
         });
+        Logger.info(`GitHub response status: ${response.status}, content-type: ${response.headers.get('content-type')}`);
         if (!response.ok) {
-            throw new Error('Failed to download repository');
+            const errorText = await response.text();
+            Logger.error(`GitHub download error: ${response.status} - ${errorText}`);
+            throw new Error(`Failed to download repository: ${response.status}`);
         }
         const arrayBuffer = await response.arrayBuffer();
-        return Buffer.from(arrayBuffer);
+        const buffer = Buffer.from(arrayBuffer);
+        // Validate that we got a ZIP file (should start with PK\x03\x04)
+        if (buffer.length < 100) {
+            Logger.error(`Downloaded content too small (${buffer.length} bytes), content: ${buffer.toString('utf8').substring(0, 200)}`);
+            throw new Error(`Downloaded file too small - possibly an error response`);
+        }
+        // Check ZIP magic bytes
+        if (buffer[0] !== 0x50 || buffer[1] !== 0x4B) {
+            Logger.error(`Invalid ZIP file, first bytes: ${buffer[0]}, ${buffer[1]}, content: ${buffer.toString('utf8').substring(0, 200)}`);
+            throw new Error('Downloaded file is not a valid ZIP archive');
+        }
+        Logger.info(`Valid ZIP downloaded: ${buffer.length} bytes`);
+        return buffer;
     }
     // ===== UTILITY =====
     /**