@arcteninc/core 0.0.138 → 0.0.140

package/scripts/cli-sync-wrapper.cjs

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+/**
+ * Wrapper for cli-sync.ts
+ * Handles TypeScript execution via tsx
+ */
+
+const { spawn, execSync } = require('child_process');
+const path = require('path');
+const fs = require('fs');
+
+const scriptPath = path.join(__dirname, 'cli-sync.ts');
+
+// Check if script exists
+if (!fs.existsSync(scriptPath)) {
+    console.error('❌ cli-sync.ts not found');
+    process.exit(1);
+}
+
+// Try tsx first (most reliable for TypeScript)
+function tryTsx() {
+    try {
+        execSync('npx tsx --version', { stdio: 'ignore' });
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+// Try bun
+function tryBun() {
+    try {
+        execSync('bun --version', { stdio: 'ignore' });
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+let runner, args;
+
+if (tryTsx()) {
+    runner = 'npx';
+    args = ['tsx', scriptPath, ...process.argv.slice(2)];
+} else if (tryBun()) {
+    runner = 'bun';
+    args = [scriptPath, ...process.argv.slice(2)];
+} else {
+    console.error('❌ No TypeScript runner found.');
+    console.error('');
+    console.error('Install one of:');
+    console.error('  npm install -g tsx');
+    console.error('  OR install Bun: https://bun.sh');
+    process.exit(1);
+}
+
+const child = spawn(runner, args, {
+    stdio: 'inherit',
+    cwd: process.cwd(),
+    shell: true
+});
+
+child.on('error', (error) => {
+    console.error('❌ Failed to run sync:', error.message);
+    process.exit(1);
+});
+
+child.on('exit', (code) => {
+    process.exit(code || 0);
+});

package/scripts/cli-sync.ts

@@ -0,0 +1,596 @@
+#!/usr/bin/env node
+/**
+ * Arcten Sync - Sync tools from codebase to dashboard
+ *
+ * Runs:
+ * 1. arcten-extract-types (generates .arcten/tool-metadata.ts)
+ * 2. Generates .arcten/arcten.tools.ts with imports and arrays
+ * 3. Updates .arcten/sync-state.json for diff tracking
+ * 4. Syncs to dashboard (functions are READONLY)
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as readline from 'readline';
+import { execSync } from 'child_process';
+
+interface ToolMetadata {
+  generated: string;
+  discoveredFrom: string[];
+  functions: Record<string, {
+    name: string;
+    description?: string;
+    parameters: unknown;
+    returnType: string;
+    isAsync: boolean;
+  }>;
+  toolOrder: string[];
+}
+
+interface SyncState {
+  lastSync: string;
+  toolsFile: string;
+  functions: Record<string, {
+    classification: 'safe' | 'sensitive';
+  }>;
+}
+
+interface ClassificationResult {
+  source: string;
+  classifications: Record<string, 'safe' | 'sensitive'>;
+  safe: string[];
+  sensitive: string[];
+}
+
+// State
+let rl: readline.Interface;
+let apiKey: string;
+let projectId: string;
+let serverUrl: string;
+
+/**
+ * Read tool-metadata.ts to get current functions
+ */
+function readToolMetadata(projectRoot: string): ToolMetadata | null {
+  const metadataPath = path.join(projectRoot, '.arcten', 'tool-metadata.ts');
+
+  if (!fs.existsSync(metadataPath)) {
+    return null;
+  }
+
+  const content = fs.readFileSync(metadataPath, 'utf-8');
+
+  // Extract the object from "export const toolMetadata = { ... } as const;"
+  const match = content.match(/export const toolMetadata = ({[\s\S]*}) as const;/);
+  if (!match) {
+    return null;
+  }
+
+  try {
+    // Use eval to parse the object (it's generated code, safe to eval)
+    const metadata = eval(`(${match[1]})`);
+    return metadata as ToolMetadata;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Read sync-state.json to get previous state
+ */
+function readSyncState(projectRoot: string): SyncState | null {
+  const statePath = path.join(projectRoot, '.arcten', 'sync-state.json');
+
+  if (!fs.existsSync(statePath)) {
+    return null;
+  }
+
+  try {
+    const content = fs.readFileSync(statePath, 'utf-8');
+    const parsed = JSON.parse(content) as SyncState;
+    // Validate the structure
+    if (!parsed.functions || typeof parsed.functions !== 'object') {
+      return null;
+    }
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Write sync-state.json
+ */
+function writeSyncState(projectRoot: string, state: SyncState): void {
+  const arctenDir = path.join(projectRoot, '.arcten');
+  if (!fs.existsSync(arctenDir)) {
+    fs.mkdirSync(arctenDir, { recursive: true });
+  }
+
+  const statePath = path.join(arctenDir, 'sync-state.json');
+  fs.writeFileSync(statePath, JSON.stringify(state, null, 2));
+}
+
+/**
+ * Generate arcten.tools.ts with imports and arrays
+ */
+function generateToolsFile(
+  projectRoot: string,
+  toolsFilePath: string,
+  safeTools: string[],
+  sensitiveTools: string[]
+): void {
+  const arctenDir = path.join(projectRoot, '.arcten');
+  if (!fs.existsSync(arctenDir)) {
+    fs.mkdirSync(arctenDir, { recursive: true });
+  }
+
+  // Calculate relative import path from .arcten to tools file
+  const toolsFileRelative = path.relative(arctenDir, path.join(projectRoot, toolsFilePath));
+  const importPath = toolsFileRelative.replace(/\.ts$/, '').replace(/\\/g, '/');
+
+  const allTools = [...safeTools, ...sensitiveTools];
+
+  const content = `// .arcten/arcten.tools.ts - AUTO-GENERATED by arcten sync
+// DO NOT EDIT - run \`arcten sync\` to regenerate
+
+import {
+${allTools.map(t => `  ${t},`).join('\n')}
+} from '${importPath}';
+
+// Safe tools - auto-execute without user approval
+export const safeTools = [
+${safeTools.map(t => `  ${t},`).join('\n')}
+] as const;
+
+// Sensitive tools - require user approval before execution
+export const sensitiveTools = [
+${sensitiveTools.map(t => `  ${t},`).join('\n')}
+] as const;
+
+// All tools combined
+export const allTools = [...safeTools, ...sensitiveTools];
+
+// Tool names for the safeToolNames prop
+export const safeToolNames = safeTools.map(fn => fn.name);
+
+// Type exports
+export type SafeToolName = typeof safeTools[number]['name'];
+export type SensitiveToolName = typeof sensitiveTools[number]['name'];
+export type AllToolName = SafeToolName | SensitiveToolName;
+`;
+
+  const outputPath = path.join(arctenDir, 'arcten.tools.ts');
+  fs.writeFileSync(outputPath, content);
+}
+
+/**
+ * Classify tools using AI or pattern fallback
+ */
+async function classifyTools(
+  tools: Array<{ name: string; description?: string }>,
+  existingClassifications: Record<string, { classification: 'safe' | 'sensitive' }>
+): Promise<ClassificationResult> {
+  // Filter to only new tools (not already classified)
+  const newTools = tools.filter(t => !existingClassifications[t.name]);
+
+  if (newTools.length === 0) {
+    // All tools already classified
+    const classifications: Record<string, 'safe' | 'sensitive'> = {};
+    for (const tool of tools) {
+      classifications[tool.name] = existingClassifications[tool.name]?.classification || 'sensitive';
+    }
+    return {
+      source: 'existing',
+      classifications,
+      safe: tools.filter(t => classifications[t.name] === 'safe').map(t => t.name),
+      sensitive: tools.filter(t => classifications[t.name] === 'sensitive').map(t => t.name),
+    };
+  }
+
+  // Try AI classification for new tools
+  try {
+    const response = await fetch(`${serverUrl}/ai-init/classify`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'x-arcten-api-key': apiKey,
+      },
+      body: JSON.stringify({
+        tools: newTools.map(t => ({ name: t.name, jsDoc: t.description })),
+      }),
+    });
+
+    if (response.ok) {
+      const aiResult = await response.json() as ClassificationResult;
+
+      // Merge with existing classifications
+      const mergedClassifications: Record<string, 'safe' | 'sensitive'> = {};
+      for (const tool of tools) {
+        if (existingClassifications[tool.name]) {
+          mergedClassifications[tool.name] = existingClassifications[tool.name].classification;
+        } else {
+          mergedClassifications[tool.name] = aiResult.classifications[tool.name] || 'sensitive';
+        }
+      }
+
+      return {
+        source: 'ai',
+        classifications: mergedClassifications,
+        safe: tools.filter(t => mergedClassifications[t.name] === 'safe').map(t => t.name),
+        sensitive: tools.filter(t => mergedClassifications[t.name] === 'sensitive').map(t => t.name),
+      };
+    }
+  } catch {
+    // Fall through to pattern fallback
+  }
+
+  // Pattern-based fallback
+  const safePatterns = /^(get|list|search|find|fetch|load|calculate|count)/i;
+  const sensitivePatterns = /^(create|update|delete|set|add|remove|regenerate)/i;
+  const sensitiveReads = /^(get|fetch).*(api.?key|credential|secret|password|token)/i;
+
+  const classifications: Record<string, 'safe' | 'sensitive'> = {};
+
+  for (const tool of tools) {
+    if (existingClassifications[tool.name]) {
+      classifications[tool.name] = existingClassifications[tool.name].classification;
+    } else if (sensitiveReads.test(tool.name)) {
+      classifications[tool.name] = 'sensitive';
+    } else if (sensitivePatterns.test(tool.name)) {
+      classifications[tool.name] = 'sensitive';
+    } else if (safePatterns.test(tool.name)) {
+      classifications[tool.name] = 'safe';
+    } else {
+      classifications[tool.name] = 'sensitive'; // Default to sensitive
+    }
+  }
+
+  return {
+    source: 'pattern-fallback',
+    classifications,
+    safe: tools.filter(t => classifications[t.name] === 'safe').map(t => t.name),
+    sensitive: tools.filter(t => classifications[t.name] === 'sensitive').map(t => t.name),
+  };
+}
+
+/**
+ * Sync to dashboard
+ */
+async function syncToDashboard(
+  tools: Array<{ name: string; description?: string; requiresApproval: boolean }>
+): Promise<{ success: boolean; error?: string }> {
+  try {
+    // Get JWT token
+    const tokenResponse = await fetch(`${serverUrl}/token`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ apiKey }),
+    });
+
+    if (!tokenResponse.ok) {
+      return { success: false, error: 'Failed to authenticate' };
+    }
+
+    const { clientToken: token } = await tokenResponse.json();
+
+    // Format tools for sync
+    const toolsData = tools.map(tool => ({
+      name: tool.name,
+      description: tool.description || `Function: ${tool.name}`,
+      signature: JSON.stringify({}),
+      isEnabled: true,
+      isOverridable: true,
+      requiresApproval: tool.requiresApproval,
+      sensitiveParams: [],
+      readonly: true, // Functions are readonly in dashboard
+      syncedAt: Date.now(),
+    }));
+
+    // Sync to dashboard
+    const syncResponse = await fetch(`${serverUrl}/tools/sync`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${token}`,
+      },
+      body: JSON.stringify({
+        projectId,
+        tools: toolsData,
+      }),
+    });
+
+    if (!syncResponse.ok) {
+      const error = await syncResponse.text();
+      return { success: false, error: `Sync failed: ${error}` };
+    }
+
+    return { success: true };
+  } catch (error: any) {
+    return { success: false, error: error.message };
+  }
+}
+
+/**
+ * Check for API key
+ */
+function checkApiKey(): { apiKey: string; projectId: string } | null {
+  const envFiles = ['.env.local', '.env'];
+
+  for (const envFile of envFiles) {
+    const envPath = path.join(process.cwd(), envFile);
+    if (fs.existsSync(envPath)) {
+      const envContent = fs.readFileSync(envPath, 'utf-8');
+      const match = envContent.match(/ARCTEN_API_KEY=([^\s\n]+)/);
+      if (match) {
+        const key = match[1];
+        const projectIdMatch = key.match(/sk_(proj_[a-zA-Z0-9]+)_/);
+        if (projectIdMatch) {
+          return { apiKey: key, projectId: projectIdMatch[1] };
+        }
+      }
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Ask user a yes/no question
+ */
+function askYesNo(question: string): Promise<boolean> {
+  return new Promise((resolve) => {
+    rl.question(`${question} (y/n) `, (answer) => {
+      resolve(answer.trim().toLowerCase().startsWith('y'));
+    });
+  });
+}
+
+/**
+ * Main sync function
+ */
+async function main() {
+  const projectRoot = process.cwd();
+  const args = process.argv.slice(2);
+  const autoYes = args.includes('--yes') || args.includes('-y');
+  const quiet = autoYes; // Quiet mode when auto-yes
+
+  if (!quiet) {
+    console.log('');
+    console.log('Arcten Sync');
+    console.log('');
+  }
+
+  // Check for API key
+  const auth = checkApiKey();
+  if (!auth) {
+    if (!quiet) {
+      console.log('No API key found. Run `arcten init` first.');
+    }
+    process.exit(1);
+  }
+
+  apiKey = auth.apiKey;
+  projectId = auth.projectId;
+  serverUrl = process.env.ARCTEN_SERVER_URL || 'https://api.arcten.com';
+
+  // Create readline interface (only if interactive)
+  if (!autoYes) {
+    rl = readline.createInterface({
+      input: process.stdin,
+      output: process.stdout,
+    });
+  }
+
+  try {
+    // Step 1: Run arcten-extract-types to update tool-metadata.ts
+    if (!quiet) console.log('Extracting tool types...');
+    // Get script directory (ESM compatible)
+    const scriptDir = path.dirname(new URL(import.meta.url).pathname).replace(/^\/([A-Z]:)/, '$1');
+
+    try {
+      // Try running the wrapper script directly first
+      const extractScript = path.join(scriptDir, 'cli-extract-types-auto-wrapper.js');
+      if (fs.existsSync(extractScript)) {
+        execSync(`node "${extractScript}"`, {
+          cwd: projectRoot,
+          stdio: quiet ? 'pipe' : ['inherit', 'pipe', 'pipe'],
+        });
+        if (!quiet) console.log('  Done');
+      } else {
+        // Fall back to npx
+        execSync('npx arcten-extract-types', {
+          cwd: projectRoot,
+          stdio: quiet ? 'pipe' : ['inherit', 'pipe', 'pipe'],
+        });
+        if (!quiet) console.log('  Done');
+      }
+    } catch (error: any) {
+      if (!quiet) console.log('  Warning: Could not run extract-types, continuing with existing metadata...');
+    }
+
+    // Step 2: Read tool-metadata.ts
+    if (!quiet) console.log('Reading tool metadata...');
+    const metadata = readToolMetadata(projectRoot);
+
+    if (!metadata) {
+      if (!quiet) {
+        console.log('');
+        console.log('No tool-metadata.ts found.');
+        console.log('Make sure you have a tools file (e.g., app/tools.ts) with exported functions.');
+        console.log('');
+      }
+      if (rl) rl.close();
+      process.exit(1);
+    }
+
+    const currentFunctions = Object.keys(metadata.functions);
+    const toolsFile = metadata.discoveredFrom[0] || 'app/tools.ts';
+
+    if (!quiet) console.log(`Found ${currentFunctions.length} functions from ${toolsFile}`);
+
+    // Step 2: Read previous sync state
+    const previousState = readSyncState(projectRoot);
+    const previousFunctions = previousState
+      ? Object.keys(previousState.functions)
+      : [];
+
+    // Step 3: Compute diff
+    const newFunctions = currentFunctions.filter(f => !previousFunctions.includes(f));
+    const removedFunctions = previousFunctions.filter(f => !currentFunctions.includes(f));
+
+    // If no changes and in auto mode, exit silently
+    if (newFunctions.length === 0 && removedFunctions.length === 0 && previousState) {
+      if (autoYes) {
+        // Silent exit - no changes needed
+        if (rl) rl.close();
+        process.exit(0);
+      }
+
+      console.log('');
+      console.log('No changes detected since last sync.');
+
+      const resync = await askYesNo('Re-sync anyway?');
+      if (!resync) {
+        console.log('Sync cancelled.');
+        if (rl) rl.close();
+        process.exit(0);
+      }
+    }
+
+    // Show changes if any
+    if (newFunctions.length > 0) {
+      console.log('');
+      console.log(`New functions since last sync:`);
+      for (const fn of newFunctions) {
+        console.log(`  + ${fn}`);
+      }
+    }
+
+    if (removedFunctions.length > 0) {
+      console.log('');
+      console.log(`Removed functions:`);
+      for (const fn of removedFunctions) {
+        console.log(`  - ${fn}`);
+      }
+    }
+
+    // Step 4: Classify tools (AI for new, preserve existing)
+    if (!quiet) {
+      console.log('');
+      console.log('Classifying tools...');
+    }
+
+    const existingClassifications = previousState?.functions || {};
+    const toolsToClassify = currentFunctions.map(name => ({
+      name,
+      description: metadata.functions[name]?.description,
+    }));
+
+    const classification = await classifyTools(toolsToClassify, existingClassifications);
+
+    if (!quiet) {
+      console.log('');
+      console.log(`Classification (${classification.source}):`);
+      console.log(`  Safe (auto-execute): ${classification.safe.length} tools`);
+      if (classification.safe.length > 0 && classification.safe.length <= 10) {
+        console.log(`    ${classification.safe.join(', ')}`);
+      }
+      console.log(`  Sensitive (needs approval): ${classification.sensitive.length} tools`);
+      if (classification.sensitive.length > 0 && classification.sensitive.length <= 10) {
+        console.log(`    ${classification.sensitive.join(', ')}`);
+      }
+    }
+
+    // Step 5: Confirm with user (skip if auto-yes)
+    if (!autoYes) {
+      console.log('');
+      const proceed = await askYesNo('Does this look right?');
+
+      if (!proceed) {
+        console.log('');
+        console.log('You can adjust classifications in .arcten/sync-state.json and re-run sync.');
+        if (rl) rl.close();
+        process.exit(0);
+      }
+    }
+
+    // Step 6: Generate arcten.tools.ts
+    if (!quiet) {
+      console.log('');
+      console.log('Generating .arcten/arcten.tools.ts...');
+    }
+    generateToolsFile(projectRoot, toolsFile.replace(/\\/g, '/'), classification.safe, classification.sensitive);
+    if (!quiet) console.log('  Created .arcten/arcten.tools.ts');
+
+    // Step 7: Update sync-state.json
+    const newState: SyncState = {
+      lastSync: new Date().toISOString(),
+      toolsFile,
+      functions: {},
+    };
+    for (const name of currentFunctions) {
+      newState.functions[name] = {
+        classification: classification.classifications[name],
+      };
+    }
+    writeSyncState(projectRoot, newState);
+    if (!quiet) console.log('  Updated .arcten/sync-state.json');
+
+    // Step 8: Sync to dashboard (skip logging in quiet mode unless there's an error)
+    if (!quiet) {
+      console.log('');
+      console.log('Syncing to dashboard...');
+    }
+
+    const toolsForDashboard = currentFunctions.map(name => ({
+      name,
+      description: metadata.functions[name]?.description,
+      requiresApproval: classification.classifications[name] === 'sensitive',
+    }));
+
+    const syncResult = await syncToDashboard(toolsForDashboard);
+
+    if (syncResult.success) {
+      if (!quiet) console.log(`  Synced ${toolsForDashboard.length} tools to dashboard`);
+    } else {
+      // Always show dashboard errors
+      console.log(`  Warning: Dashboard sync failed: ${syncResult.error}`);
+      if (!quiet) console.log('  Local files were still updated.');
+    }
+
+    if (!quiet) {
+      console.log('');
+      console.log('Sync complete!');
+      console.log('');
+      console.log('Usage in your components:');
+      console.log('');
+      console.log('  import { allTools, safeToolNames } from "./.arcten/arcten.tools";');
+      console.log('');
+      console.log('  <ArctenAgent');
+      console.log('    tools={allTools}');
+      console.log('    safeToolNames={safeToolNames}');
+      console.log('  />');
+      console.log('');
+    } else {
+      // Brief message when there were changes in quiet mode
+      if (newFunctions.length > 0 || removedFunctions.length > 0) {
+        console.log(`✓ Synced ${newFunctions.length} new, ${removedFunctions.length} removed tools`);
+      }
+    }
+
+  } catch (error: any) {
+    console.error('Error:', error.message);
+    process.exit(1);
+  } finally {
+    if (rl) rl.close();
+  }
+}
+
+// Run
+main().catch((error) => {
+  console.error('Fatal error:', error);
+  process.exit(1);
+});
+
+export { main };