@arcteninc/core 0.0.133 → 0.0.135

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcteninc/core",
-  "version": "0.0.133",
+  "version": "0.0.135",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.mjs",

package/scripts/cli-extract-types-auto.ts

@@ -1566,4 +1566,12 @@ if (import.meta.main) {
   autoDiscoverAndExtract(projectRoot, outputPath);
 }
 
-export { autoDiscoverAndExtract };
+export {
+  autoDiscoverAndExtract,
+  findToolUsageFiles,
+  extractToolNamesFromFile,
+  findFunctionDefinition,
+  extractFunctionMetadata,
+};
+
+export type { FunctionMetadata, ToolUsage };

package/scripts/cli-init-wizard.ts

@@ -14,6 +14,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { glob } from 'glob';
 import * as readline from 'readline';
+import { autoDiscoverAndExtract } from './cli-extract-types-auto.ts';
 
 // Types
 interface FunctionMetadata {
@@ -197,6 +198,19 @@ async function scanCodebase(): Promise<CodebaseContext> {
   // Build file tree
   const fileTree = buildFileTree(projectRoot);
 
+  // Debug: log context size
+  const contextStr = JSON.stringify({
+    fileTree,
+    framework,
+    dependencies,
+    functions,
+    stats: {
+      totalFunctions: functions.length,
+      skippedPaths: EXCLUDE_PATHS,
+    },
+  });
+  console.log(`📊 Context size: ${(contextStr.length / 1024).toFixed(2)} KB (~${Math.round(contextStr.length / 4)} tokens)`);
+
   return {
     fileTree,
     framework,
@@ -252,65 +266,58 @@ function getDependencies(projectRoot: string): Record<string, string> {
   return { ...packageJson.dependencies };
 }
 
-// Helper: Scan for functions
+// Helper: Scan for functions (reuses existing autoDiscoverAndExtract)
 async function scanFunctions(projectRoot: string): Promise<FunctionMetadata[]> {
-  const pattern = path.join(projectRoot, '**/*.{ts,tsx,js,jsx}').replace(/\\/g, '/');
-  const files = await glob(pattern, {
-    ignore: EXCLUDE_PATHS.map(p => `**/${p}/**`),
-  });
+  console.log('🔍 Using TypeScript AST to scan for exported functions...');
 
-  const functions: FunctionMetadata[] = [];
+  // Generate metadata file using existing extraction logic
+  const outputPath = path.join(projectRoot, '.arcten', 'tool-metadata.ts');
 
-  // Simple regex-based extraction (can be enhanced with AST parsing)
-  for (const file of files) {
-    const content = fs.readFileSync(file, 'utf-8');
-
-    // Match exported functions
-    const functionRegex = /export\s+(async\s+)?function\s+(\w+)\s*\(([^)]*)\)/g;
-    let match;
-
-    while ((match = functionRegex.exec(content)) !== null) {
-      const [, isAsync, name, params] = match;
-      functions.push({
-        name,
-        file: path.relative(projectRoot, file),
-        params: parseParams(params),
-        jsDoc: extractJsDoc(content, match.index),
-        sourceCode: extractFunctionSource(content, match.index),
-      });
-    }
+  // Ensure .arcten directory exists
+  const arctenDir = path.dirname(outputPath);
+  if (!fs.existsSync(arctenDir)) {
+    fs.mkdirSync(arctenDir, { recursive: true });
   }
 
-  return functions;
-}
+  // Run the auto-discovery
+  await autoDiscoverAndExtract(projectRoot, outputPath);
 
-// Helper: Parse function parameters
-function parseParams(paramsStr: string): any {
-  if (!paramsStr.trim()) return [];
+  // Read the generated metadata
+  const content = fs.readFileSync(outputPath, 'utf-8');
+  const metadataMatch = content.match(/export const toolMetadata = ([\s\S]+) as const;/);
 
-  return paramsStr.split(',').map(p => {
-    const [name, type] = p.trim().split(':');
-    return { name: name?.trim(), type: type?.trim() };
-  });
-}
+  if (!metadataMatch) {
+    return [];
+  }
 
-// Helper: Extract JSDoc comment
-function extractJsDoc(content: string, index: number): string | undefined {
-  const beforeFunction = content.substring(0, index);
-  const jsDocMatch = beforeFunction.match(/\/\*\*([\s\S]*?)\*\/\s*$/);
-  return jsDocMatch ? jsDocMatch[1].trim() : undefined;
-}
+  const metadata = JSON.parse(metadataMatch[1]);
+  const functions: FunctionMetadata[] = [];
+
+  // Convert to wizard format
+  for (const [name, fn] of Object.entries(metadata.functions as any)) {
+    // Find which file this function was discovered from
+    const fileIndex = metadata.discoveredFrom.findIndex((f: string) =>
+      content.includes(name)
+    );
+
+    functions.push({
+      name,
+      file: metadata.discoveredFrom[fileIndex] || metadata.discoveredFrom[0] || 'unknown',
+      params: Object.entries(fn.parameters?.properties || {}).map(([paramName, prop]: [string, any]) => ({
+        name: paramName,
+        type: prop.type || 'any',
+      })),
+      returnType: fn.returnType,
+      jsDoc: fn.description,
+      sourceCode: `${name}(${Object.keys(fn.parameters?.properties || {}).join(', ')})`,
+    });
+  }
 
-// Helper: Extract function source (first 1000 chars)
-function extractFunctionSource(content: string, index: number): string {
-  const remaining = content.substring(index);
-  const endMatch = remaining.match(/\n\}/);
-  const endIndex = endMatch ? endMatch.index! + 2 : Math.min(1000, remaining.length);
-  return remaining.substring(0, endIndex);
+  return functions;
 }
 
-// Helper: Build file tree
-function buildFileTree(projectRoot: string, depth: number = 0, maxDepth: number = 4): string {
+// Helper: Build file tree (limited depth to reduce tokens)
+function buildFileTree(projectRoot: string, depth: number = 0, maxDepth: number = 2): string {
   if (depth >= maxDepth) return '';
 
   const entries = fs.readdirSync(projectRoot, { withFileTypes: true });
@@ -332,6 +339,53 @@ function buildFileTree(projectRoot: string, depth: number = 0, maxDepth: number
   return tree;
 }
 
+// Step 2.5: Handle missing descriptions
+async function handleMissingDescriptions(
+  rl: readline.Interface,
+  functions: FunctionMetadata[]
+): Promise<FunctionMetadata[]> {
+  // Find functions with generic "Execute X" descriptions
+  const missingDescriptions = functions.filter(fn =>
+    !fn.jsDoc || fn.jsDoc.startsWith('Execute ')
+  );
+
+  if (missingDescriptions.length === 0) {
+    return functions;
+  }
+
+  console.log(`\n⚠️  ${missingDescriptions.length} function${missingDescriptions.length > 1 ? 's are' : ' is'} missing JSDoc descriptions\n`);
+  console.log('How would you like to handle these?');
+  console.log('  1. Edit descriptions now (one by one)');
+  console.log('  2. Skip - I\'ll add JSDoc comments manually');
+  console.log('  3. Skip - I\'ll edit in dashboard later');
+
+  const choice = await ask(rl, '\n? Choose option (1-3): ');
+
+  if (choice === '1') {
+    // Edit descriptions one by one
+    console.log('');
+    for (const fn of missingDescriptions) {
+      console.log(`📝 ${fn.name} (${fn.file})`);
+      console.log(`   Current: ${fn.jsDoc || `Execute ${fn.name}`}\n`);
+
+      const newDesc = await ask(rl, '   Enter description (or press Enter to skip): ');
+
+      if (newDesc.trim()) {
+        fn.jsDoc = newDesc.trim();
+        console.log(`   ✓ Updated description for ${fn.name}\n`);
+      } else {
+        console.log(`   ⏭️  Skipped ${fn.name}\n`);
+      }
+    }
+    console.log('✅ Description editing complete\n');
+  } else {
+    console.log('\n⏭️  Skipping description editing\n');
+    console.log('💡 You can add JSDoc comments to your code or edit descriptions in the dashboard.');
+  }
+
+  return functions;
+}
+
 // Step 3: Get user intent
 async function getUserIntent(rl: readline.Interface): Promise<string> {
   console.log('\n💭 Step 3: Define your agent\'s purpose\n');
@@ -352,6 +406,8 @@ async function getUserIntent(rl: readline.Interface): Promise<string> {
 
 // Helper: Get JWT token from API key
 async function getAuthToken(apiKey: string, serverUrl: string): Promise<string> {
+  console.log(`🔑 Getting auth token from ${serverUrl}/token...`);
+
   try {
     const response = await fetch(`${serverUrl}/token`, {
       method: 'POST',
@@ -361,13 +417,23 @@ async function getAuthToken(apiKey: string, serverUrl: string): Promise<string>
       body: JSON.stringify({ apiKey }),
     });
 
+    console.log(`🔑 Token response status: ${response.status}`);
+
     if (!response.ok) {
       const error = await response.json();
+      console.error(`🔑 Token error:`, error);
       throw new Error(error.error || 'Failed to authenticate');
     }
 
     const data = await response.json();
-    return data.token;
+    const token = data.clientToken || data.token; // Server returns "clientToken"
+    console.log(`🔑 Got token: ${token ? token.substring(0, 20) + '...' : 'NO TOKEN'}`);
+
+    if (!token) {
+      throw new Error('No token received from server');
+    }
+
+    return token;
   } catch (error: any) {
     if (error.cause?.code === 'ENOTFOUND' || error.cause?.code === 'ECONNREFUSED' || error.message.includes('fetch')) {
       console.error(`\n❌ Unable to connect to: ${serverUrl}`);
@@ -453,6 +519,8 @@ async function runAnalysis(
 
   // Get JWT token from API key
   const token = await getAuthToken(apiKey, serverUrl);
+  console.log(`📡 Using token for analysis: ${token ? token.substring(0, 20) + '...' : 'NO TOKEN'}`);
+  console.log(`📡 Calling: ${serverUrl}/tools/analyze`);
 
   const response = await fetch(`${serverUrl}/tools/analyze`, {
     method: 'POST',
@@ -463,15 +531,26 @@ async function runAnalysis(
     body: JSON.stringify({ context, userIntent }),
   });
 
+  console.log(`📡 Analysis response status: ${response.status}`);
+
   if (!response.ok) {
     const error = await response.json();
+    console.error(`📡 Analysis error response:`, error);
     throw new Error(error.error || 'Analysis failed');
   }
 
   const result = await response.json();
+  console.log(`📡 Analysis result:`, JSON.stringify(result).substring(0, 200));
+
+  if (!result || !result.success) {
+    throw new Error(result?.error || 'Analysis failed - invalid response');
+  }
+
+  const tokensUsed = result.tokensUsed || 0;
+  const actualCost = result.stats?.actualCost || 0;
 
-  console.log(`✓ Analysis complete - ${result.tokensUsed.toLocaleString()} tokens used`);
-  console.log(`  Actual cost: $${result.stats.actualCost.toFixed(2)}`);
+  console.log(`✓ Analysis complete - ${tokensUsed.toLocaleString()} tokens used`);
+  console.log(`  Actual cost: $${actualCost.toFixed(2)}`);
 
   return result;
 }
@@ -479,9 +558,10 @@ async function runAnalysis(
 // Step 7: Sync to dashboard
 async function syncToDashboard(
   apiKey: string,
-  convexUrl: string,
+  serverUrl: string,
   projectId: string,
-  tools: ToolRecommendation[]
+  tools: ToolRecommendation[],
+  agents?: AgentConfig[]
 ): Promise<void> {
   console.log('\n☁️  Step 7: Syncing to dashboard...\n');
 
@@ -498,41 +578,58 @@ async function syncToDashboard(
     sensitiveParams: tool.sensitiveParams || [],
   }));
 
+  // Convert agents to the format expected by bulkUpsert
+  const agentsData = agents?.map(agent => ({
+    name: agent.name,
+    systemPrompt: agent.systemPrompt,
+    enabledTools: agent.enabledTools,
+    description: agent.description || `Agent: ${agent.name}`,
+  })) || [];
+
   try {
-    // Call Convex bulkUpsert mutation via HTTP API
-    const response = await fetch(`${convexUrl}/api/mutation`, {
+    // Get JWT token
+    const token = await getAuthToken(apiKey, serverUrl);
+
+    // Call server endpoint which will handle Convex mutation
+    const response = await fetch(`${serverUrl}/tools/sync`, {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
-        'Authorization': `Bearer ${apiKey}`,
+        'Authorization': `Bearer ${token}`,
       },
       body: JSON.stringify({
-        path: 'tools:bulkUpsert',
-        args: { projectId, tools: toolsData },
-        format: 'json',
+        projectId,
+        tools: toolsData,
+        agents: agentsData,
       }),
     });
 
     if (!response.ok) {
       const errorText = await response.text().catch(() => 'Unknown error');
-      throw new Error(`Failed to sync tools: ${response.status} ${errorText}`);
+      throw new Error(`Failed to sync: ${response.status} ${errorText}`);
     }
 
     const result = await response.json();
 
     if (!result.success) {
-      throw new Error(result.error || 'Failed to sync tools to dashboard');
+      throw new Error(result.error || 'Failed to sync to dashboard');
     }
 
-    console.log(`✅ Synced ${tools.length} tools to dashboard`);
+    console.log(`✅ Synced ${tools.length} tools${agents && agents.length > 0 ? ` and ${agents.length} agents` : ''} to dashboard`);
 
     if (result.upserted) {
       const created = result.upserted.filter((t: any) => t.created).length;
       const updated = result.upserted.filter((t: any) => !t.created).length;
-      console.log(`   📝 Created: ${created} | Updated: ${updated}`);
+      console.log(`   📝 Tools - Created: ${created} | Updated: ${updated}`);
+    }
+
+    if (result.agents && agents && agents.length > 0) {
+      const agentCreated = result.agents.filter((a: any) => a.created).length;
+      const agentUpdated = result.agents.filter((a: any) => !a.created).length;
+      console.log(`   📝 Agents - Created: ${agentCreated} | Updated: ${agentUpdated}`);
     }
   } catch (error: any) {
-    console.error(`❌ Failed to sync tools: ${error.message}`);
+    console.error(`❌ Failed to sync: ${error.message}`);
     throw error;
   }
 }
@@ -699,7 +796,6 @@ async function main() {
 
   const rl = createPrompt();
   const serverUrl = process.env.ARCTEN_SERVER_URL || 'https://api.arcten.com';
-  const convexUrl = process.env.CONVEX_URL || 'https://convex.arcten.com';
 
   try {
     // Check for existing setup
@@ -731,6 +827,11 @@ async function main() {
     // Step 2: Scan codebase
     const context = await scanCodebase();
 
+    // Step 2.5: Handle missing descriptions (only in full wizard mode, not sync/allow-all)
+    if (!skipAnalysis && !allowAll) {
+      context.functions = await handleMissingDescriptions(rl, context.functions);
+    }
+
     let analysis: AnalysisResult | null = null;
 
     if (allowAll) {
@@ -749,7 +850,7 @@ async function main() {
         sensitiveParams: [],
       }));
 
-      await syncToDashboard(apiKey, convexUrl, projectId, allTools);
+      await syncToDashboard(apiKey, serverUrl, projectId, allTools);
       await generateLocalConfig(projectId, 'default');
 
       console.log('\n✅ Setup complete!\n');
@@ -770,7 +871,7 @@ async function main() {
         sensitiveParams: [],
       }));
 
-      await syncToDashboard(apiKey, convexUrl, projectId, basicTools);
+      await syncToDashboard(apiKey, serverUrl, projectId, basicTools);
       await generateLocalConfig(projectId, 'default');
 
       console.log('\n✅ Sync complete!\n');
@@ -798,7 +899,7 @@ async function main() {
         console.log(`  Placements suggested: ${analysis.suggestedPlacements.length}`);
 
         // Step 7: Sync to dashboard
-        await syncToDashboard(apiKey, convexUrl, projectId, analysis.recommendations);
+        await syncToDashboard(apiKey, serverUrl, projectId, analysis.recommendations, analysis.suggestedAgents);
 
         // Step 7.5: Display placement suggestions
         await displayPlacementSuggestions(analysis.suggestedPlacements);
@@ -816,7 +917,7 @@ async function main() {
           sensitiveParams: [],
         }));
 
-        await syncToDashboard(apiKey, convexUrl, projectId, basicTools);
+        await syncToDashboard(apiKey, serverUrl, projectId, basicTools);
         await generateLocalConfig(projectId, 'default');
       }