devtopia 1.8.3 → 2.0.0

package/dist/commands/submit.js

@@ -3,6 +3,7 @@ import { extname, resolve, dirname, join } from 'path';
 import { createInterface } from 'readline';
 import { API_BASE } from '../config.js';
 import { loadIdentity, hasIdentity } from '../identity.js';
+import { validateExecution } from '../executor.js';
 const LANG_MAP = {
     '.ts': 'typescript',
     '.js': 'javascript',
@@ -10,74 +11,26 @@ const LANG_MAP = {
     '.mjs': 'javascript',
     '.cjs': 'javascript',
 };
-// Valid categories - grouped by type
-const CATEGORIES = [
-    // Data & Parsing
-    { id: 'data', name: 'Data Processing' },
-    { id: 'json', name: 'JSON & Objects' },
-    { id: 'csv', name: 'CSV & Tables' },
-    { id: 'xml', name: 'XML & Markup' },
-    { id: 'yaml', name: 'YAML & Config' },
-    // Text & Strings
-    { id: 'text', name: 'Text & NLP' },
-    { id: 'string', name: 'String Utils' },
-    { id: 'regex', name: 'Regex & Patterns' },
-    { id: 'format', name: 'Formatting' },
-    // Web & Network
-    { id: 'web', name: 'Web & HTTP' },
-    { id: 'api', name: 'API Tools' },
-    { id: 'url', name: 'URL & Links' },
-    { id: 'html', name: 'HTML & DOM' },
-    // Security & Crypto
-    { id: 'crypto', name: 'Crypto & Security' },
-    { id: 'hash', name: 'Hashing' },
-    { id: 'encode', name: 'Encoding' },
-    { id: 'auth', name: 'Authentication' },
-    // Math & Numbers
-    { id: 'math', name: 'Math & Numbers' },
-    { id: 'stats', name: 'Statistics' },
-    { id: 'convert', name: 'Conversion' },
-    { id: 'random', name: 'Random & Generate' },
-    // Date & Time
-    { id: 'time', name: 'Date & Time' },
-    { id: 'timezone', name: 'Timezones' },
-    // Files & System
-    { id: 'file', name: 'File & I/O' },
-    { id: 'path', name: 'Paths' },
-    { id: 'compress', name: 'Compression' },
-    // Arrays & Collections
-    { id: 'array', name: 'Arrays & Lists' },
-    { id: 'sort', name: 'Sort & Search' },
-    { id: 'set', name: 'Sets & Maps' },
-    // Validation
-    { id: 'validate', name: 'Validation' },
-    { id: 'sanitize', name: 'Sanitization' },
-    // Generation
-    { id: 'generate', name: 'Generation' },
-    { id: 'template', name: 'Templates' },
-    // AI & ML
-    { id: 'ai', name: 'AI & ML' },
-    { id: 'nlp', name: 'NLP' },
-    // Media
-    { id: 'image', name: 'Images' },
-    { id: 'color', name: 'Colors' },
-    { id: 'qr', name: 'QR & Barcodes' },
-    // Dev Tools
-    { id: 'cli', name: 'CLI & Terminal' },
-    { id: 'debug', name: 'Debug & Test' },
-    { id: 'diff', name: 'Diff & Compare' },
-    // General
-    { id: 'util', name: 'Utilities' },
-    { id: 'other', name: 'Other' },
-];
+/**
+ * Fetch categories from the API (single source of truth)
+ */
+async function fetchCategories() {
+    try {
+        const res = await fetch(`${API_BASE}/api/categories`);
+        const data = await res.json();
+        return (data.categories || []).map((c) => ({
+            id: c.id,
+            name: c.name,
+            description: c.description,
+        }));
+    }
+    catch {
+        // Fallback: return empty array — submit will still work, server validates
+        return [];
+    }
+}
 /**
  * Extract description from source code comments
- * Supports multiple formats:
- * - // tool-name - description
- * - # tool-name - description
- * - JSDoc: * tool-name - description
- * - JSDoc: First sentence after title in /** block
- * - JSDoc @description tag
  */
 function extractDescription(source) {
     const lines = source.split('\n');
@@ -85,7 +38,6 @@ function extractDescription(source) {
     let jsdocTitle = false;
     for (const line of lines) {
         const trimmed = line.trim();
-        // Track JSDoc block
         if (trimmed === '/**') {
             inJSDoc = true;
             jsdocTitle = false;
@@ -95,7 +47,7 @@ function extractDescription(source) {
             inJSDoc = false;
             continue;
         }
-        // TypeScript/JavaScript single line: // tool-name - description
+        // TypeScript/JavaScript: // tool-name - description
         const tsMatch = line.match(/^\/\/\s*[\w-]+\s*[-–:]\s*(.+)$/);
         if (tsMatch)
             return tsMatch[1].trim();
@@ -103,55 +55,35 @@ function extractDescription(source) {
         const pyMatch = line.match(/^#\s*[\w-]+\s*[-–:]\s*(.+)$/);
         if (pyMatch)
             return pyMatch[1].trim();
-        // JSDoc block comment: * tool-name - description
-        const jsdocToolMatch = line.match(/^\s*\*\s*[\w-]+\s*[-–:]\s*(.+)$/);
-        if (jsdocToolMatch)
-            return jsdocToolMatch[1].trim();
-        // JSDoc @description tag
+        // JSDoc: * @description
         const jsdocDescMatch = line.match(/^\s*\*\s*@description\s+(.+)$/);
         if (jsdocDescMatch)
             return jsdocDescMatch[1].trim();
-        // Inside JSDoc: look for description after title
+        // JSDoc: first content line after /** that describes the tool
         if (inJSDoc) {
-            // Skip @ tags
             if (trimmed.match(/^\*\s*@/))
                 continue;
-            // Extract content after *
             const jsdocContent = trimmed.match(/^\*\s*(.+)$/);
             if (jsdocContent) {
                 const content = jsdocContent[1].trim();
-                // Skip if it looks like a title (short, capitalized, no punctuation)
                 if (!jsdocTitle && content.length < 50 && !content.includes('.') && /^[A-Z]/.test(content)) {
                     jsdocTitle = true;
                     continue;
                 }
-                // After title, next non-empty content line is likely the description
                 if (jsdocTitle && content.length > 10) {
-                    // Return first sentence
                     const firstSentence = content.match(/^[^.!?]+[.!?]?/);
                     if (firstSentence)
                         return firstSentence[0].trim();
                     return content;
                 }
             }
-            // Skip empty JSDoc lines
             if (trimmed === '*')
                 continue;
         }
-        // Skip empty lines, shebang, opening markers, and usage lines
-        if (trimmed === '' ||
-            line.startsWith('#!') ||
-            trimmed === '/**' ||
-            trimmed === '*/' ||
-            trimmed === '*' ||
-            trimmed.toLowerCase().includes('usage:')) {
+        if (trimmed === '' || line.startsWith('#!') || trimmed === '/**' || trimmed === '*/' || trimmed === '*' || trimmed.toLowerCase().includes('usage:')) {
             continue;
         }
-        // Stop if we hit non-comment code
-        if (!trimmed.startsWith('//') &&
-            !trimmed.startsWith('#') &&
-            !trimmed.startsWith('*') &&
-            !trimmed.startsWith('/*')) {
+        if (!trimmed.startsWith('//') && !trimmed.startsWith('#') && !trimmed.startsWith('*') && !trimmed.startsWith('/*')) {
             break;
         }
     }
@@ -164,17 +96,13 @@ function extractDescriptionFromReadme(readme) {
     const lines = readme.split('\n');
     let foundTitle = false;
     for (const line of lines) {
-        // Skip title lines (# Title)
         if (line.startsWith('#')) {
             foundTitle = true;
             continue;
         }
-        // Skip empty lines after title
         if (foundTitle && line.trim() === '')
             continue;
-        // First non-empty line after title is the description
         if (foundTitle && line.trim()) {
-            // Take first sentence or up to 100 chars
             const desc = line.trim();
             const firstSentence = desc.match(/^[^.!?]+[.!?]?/);
             if (firstSentence && firstSentence[0].length <= 100) {
@@ -190,7 +118,20 @@ function extractDescriptionFromReadme(readme) {
  */
 function detectCategory(description, source) {
     const text = `${description || ''} ${source}`.toLowerCase();
-    // Specific categories first (more specific matches)
+    // Use-case categories first (higher-level, more meaningful)
+    if ((text.includes('scrape') || text.includes('crawl') || text.includes('extract from page') || text.includes('web scraping')) && text.includes('html'))
+        return 'scraping';
+    if (text.includes('health check') || text.includes('uptime') || text.includes('monitor') || text.includes('observ'))
+        return 'monitoring';
+    if ((text.includes('pipeline') || text.includes('batch') || text.includes('schedule') || text.includes('automat')) && !text.includes('json'))
+        return 'automation';
+    if ((text.includes('api') && (text.includes('retry') || text.includes('integration') || text.includes('client'))) || text.includes('api integration'))
+        return 'integration';
+    if ((text.includes('analys') || text.includes('aggregate') || text.includes('report') || text.includes('insight')) && (text.includes('data') || text.includes('text')))
+        return 'analysis';
+    if (text.includes('content') && (text.includes('process') || text.includes('generat') || text.includes('transform')))
+        return 'content';
+    // Technical categories
     if (text.includes('jwt') || text.includes('oauth') || text.includes('token') || text.includes('auth'))
         return 'auth';
     if (text.includes('base64') || text.includes('hex') || text.includes('encode') || text.includes('decode'))
@@ -215,9 +156,21 @@ function detectCategory(description, source) {
         return 'validate';
     if (text.includes('sanitize') || text.includes('clean') || text.includes('escape'))
         return 'sanitize';
+    if (text.includes('polymarket'))
+        return 'polymarket';
+    if (text.includes('database') || text.includes('sql') || text.includes('query builder'))
+        return 'database';
+    if (text.includes('email'))
+        return 'email';
+    if (text.includes('graph') || text.includes('dfs') || text.includes('bfs') || text.includes('traversal'))
+        return 'graph';
+    if (text.includes('log') && (text.includes('structured') || text.includes('logging')))
+        return 'logging';
+    if (text.includes('geo') || text.includes('haversine') || text.includes('latitude') || text.includes('longitude'))
+        return 'geo';
     if (text.includes('qr') || text.includes('barcode'))
         return 'qr';
-    if (text.includes('color') || text.includes('rgb') || text.includes('hex') || text.includes('hsl'))
+    if (text.includes('color') || text.includes('rgb') || text.includes('hsl'))
         return 'color';
     if (text.includes('image') || text.includes('resize') || text.includes('thumbnail'))
         return 'image';
@@ -241,7 +194,7 @@ function detectCategory(description, source) {
         return 'nlp';
     if (text.includes('cli') || text.includes('terminal') || text.includes('command'))
         return 'cli';
-    if (text.includes('debug') || text.includes('test') || text.includes('log'))
+    if (text.includes('debug') || text.includes('test'))
         return 'debug';
     if (text.includes('html') || text.includes('dom') || text.includes('scrape'))
         return 'html';
@@ -249,7 +202,6 @@ function detectCategory(description, source) {
         return 'url';
     if (text.includes('api') || text.includes('rest') || text.includes('graphql'))
         return 'api';
-    // Broader categories
     if (text.includes('fetch') || text.includes('http') || text.includes('request'))
         return 'web';
     if (text.includes('encrypt') || text.includes('crypto') || text.includes('secure'))
@@ -286,6 +238,46 @@ function findReadme(filePath, toolName) {
     }
     return null;
 }
+/**
+ * Validate tool source: check it defines proper I/O
+ */
+function validateToolSource(source, language) {
+    const warnings = [];
+    if (language === 'javascript' || language === 'typescript') {
+        const hasArgvParse = /process\.argv\[2\]/.test(source);
+        const definesMain = /\bfunction\s+main\s*\(/.test(source);
+        const hasConsoleLog = /console\.log\s*\(/.test(source);
+        const hasJSONStringify = /JSON\.stringify/.test(source);
+        if (!hasArgvParse && !definesMain) {
+            warnings.push('Tool does not read input (process.argv[2]) or define main(). It may not work when run.');
+        }
+        if (!hasConsoleLog && !definesMain) {
+            warnings.push('Tool does not output results (console.log). It may produce no output when run.');
+        }
+        if (!hasJSONStringify && !definesMain) {
+            warnings.push('Tool does not use JSON.stringify. Output may not be valid JSON.');
+        }
+        if (definesMain && !hasArgvParse) {
+            // This will be auto-wrapped by executor — that's fine
+        }
+    }
+    if (language === 'python') {
+        const hasSysArgv = /sys\.argv/.test(source);
+        const definesMain = /\bdef\s+main\s*\(/.test(source);
+        const hasPrint = /\bprint\s*\(/.test(source);
+        const hasJsonDumps = /json\.dumps/.test(source);
+        if (!hasSysArgv && !definesMain) {
+            warnings.push('Tool does not read input (sys.argv) or define main(). It may not work when run.');
+        }
+        if (!hasPrint && !definesMain) {
+            warnings.push('Tool does not print output. It may produce nothing when run.');
+        }
+        if (!hasJsonDumps && !definesMain) {
+            warnings.push('Tool does not use json.dumps. Output may not be valid JSON.');
+        }
+    }
+    return { valid: true, warnings };
+}
 export async function submit(name, file, options) {
     // Check identity
     if (!hasIdentity()) {
@@ -320,6 +312,15 @@ export async function submit(name, file, options) {
     }
     // Read source
     const source = readFileSync(filePath, 'utf-8');
+    // ── Validate tool source format ──
+    const { warnings } = validateToolSource(source, language);
+    if (warnings.length > 0) {
+        console.log(`\n⚠️  Source validation warnings:`);
+        for (const w of warnings) {
+            console.log(`   - ${w}`);
+        }
+        console.log(`   (tool may still work if executor can auto-wrap it)\n`);
+    }
     // Find and read README
     let readmePath = options.readme ? resolve(options.readme) : findReadme(filePath, name);
     let readme = null;
@@ -339,11 +340,9 @@ export async function submit(name, file, options) {
     }
     // Extract or use provided description
     let description = options.description || extractDescription(source);
-    // Try to get description from README if not found in source
     if (!description && readme) {
         description = extractDescriptionFromReadme(readme);
     }
-    // Description is required
     if (!description) {
         console.log(`\n❌ Description required. Please provide a description for your tool.`);
         console.log(`\n   Option 1: Use -d "Your description here"`);
@@ -352,13 +351,15 @@ export async function submit(name, file, options) {
         console.log(`   Option 3: Add a description after the title in your README\n`);
         process.exit(1);
     }
+    // ── Fetch categories from API (single source of truth) ──
+    const CATEGORIES = await fetchCategories();
     // Determine category
     let category = options.category;
-    if (category && !CATEGORIES.find(c => c.id === category)) {
+    if (category && CATEGORIES.length > 0 && !CATEGORIES.find(c => c.id === category)) {
         console.log(`\n❌ Invalid category: ${category}`);
         console.log(`\n   Valid categories:`);
         for (const cat of CATEGORIES) {
-            console.log(`   ${cat.id.padEnd(10)} ${cat.name}`);
+            console.log(`   ${cat.id.padEnd(12)} ${cat.name}`);
         }
         console.log();
         process.exit(1);
@@ -367,35 +368,25 @@ export async function submit(name, file, options) {
     if (!category) {
         category = detectCategory(description, source);
     }
-    // Prompt for category confirmation/selection if not provided via CLI
-    if (!options.category && process.stdin.isTTY) {
-        const rl = createInterface({
-            input: process.stdin,
-            output: process.stdout
-        });
+    // Prompt for category confirmation if TTY
+    if (!options.category && process.stdin.isTTY && CATEGORIES.length > 0) {
+        const rl = createInterface({ input: process.stdin, output: process.stdout });
         const detectedCat = CATEGORIES.find(c => c.id === category);
         console.log(`\n📁 Category Selection`);
         console.log(`   Auto-detected: ${detectedCat?.name || category} (${category})`);
         console.log(`\n   Common categories:`);
-        // Show most common/relevant categories
-        const commonCategories = [
-            'api', 'json', 'data', 'text', 'web', 'crypto', 'file',
-            'array', 'validate', 'util', 'other'
-        ];
+        const commonCategories = ['api', 'json', 'data', 'text', 'web', 'crypto', 'file', 'array', 'validate', 'util', 'other'];
         for (const catId of commonCategories) {
             const cat = CATEGORIES.find(c => c.id === catId);
             if (cat) {
-                const marker = cat.id === category ? ' ← detected' : '';
+                const marker = cat.id === category ? ' <- detected' : '';
                 console.log(`   ${cat.id.padEnd(12)} ${cat.name}${marker}`);
             }
         }
         console.log(`\n   (Use 'devtopia categories' to see all categories)`);
         console.log(`\n   Press Enter to use detected category, or type a category ID:`);
         const answer = await new Promise((resolve) => {
-            rl.question(`   Category [${category}]: `, (ans) => {
-                rl.close();
-                resolve(ans.trim());
-            });
+            rl.question(`   Category [${category}]: `, (ans) => { rl.close(); resolve(ans.trim()); });
         });
         if (answer) {
             const selectedCat = CATEGORIES.find(c => c.id === answer.toLowerCase());
@@ -415,16 +406,58 @@ export async function submit(name, file, options) {
     const dependencies = options.deps
         ? options.deps.split(',').map(d => d.trim()).filter(Boolean)
         : [];
-    // Parse builds_on (parent tools this tool extends/composes)
+    // Parse builds_on
     const buildsOn = options.buildsOn
         ? options.buildsOn.split(',').map(d => d.trim()).filter(Boolean)
         : [];
-    // Encourage lineage tracking
     if (!options.buildsOn && buildsOn.length === 0) {
         console.log(`\n💡 Tip: Use --builds-on to show lineage!`);
         console.log(`   Example: --builds-on api-request,json-validate`);
         console.log(`   This helps others see how tools build on each other.\n`);
     }
+    // ── Extract schema from JSDoc @param tags ──
+    let inputSchema = null;
+    let outputSchema = null;
+    if (options.schema) {
+        try {
+            const schemaContent = JSON.parse(readFileSync(resolve(options.schema), 'utf-8'));
+            inputSchema = schemaContent.inputs || null;
+            outputSchema = schemaContent.outputs || null;
+        }
+        catch {
+            console.log(`\n⚠️  Could not parse schema file: ${options.schema}`);
+        }
+    }
+    if (!inputSchema) {
+        // Auto-extract from JSDoc @param tags
+        const paramMatches = source.matchAll(/@param\s+\{(\w+)\}\s+([\w.]+)\s*[-–]?\s*(.*)/g);
+        const extracted = {};
+        for (const match of paramMatches) {
+            const type = match[1].toLowerCase();
+            const paramName = match[2].replace(/^params\./, '');
+            if (paramName !== 'params') {
+                extracted[paramName] = type;
+            }
+        }
+        if (Object.keys(extracted).length > 0) {
+            inputSchema = extracted;
+        }
+    }
+    // ── Pre-submit execution validation ──
+    if (!options.skipValidation) {
+        console.log(`\n   Validating tool execution...`);
+        const validation = await validateExecution(source, language);
+        if (!validation.valid) {
+            const reason = validation.stderr
+                ? validation.stderr.split('\n')[0]
+                : (validation.stdout ? 'Tool produced non-JSON output.' : 'Tool produced no output.');
+            console.log(`\n❌ Tool failed validation: ${reason}`);
+            console.log(`\n   Your tool must produce JSON output (even error JSON like {"error":"..."}).`);
+            console.log(`   Use --skip-validation to bypass this check.\n`);
+            process.exit(1);
+        }
+        console.log(`   Validation passed (exit ${validation.exitCode}, ${validation.stdout.length} bytes output)`);
+    }
     const catInfo = CATEGORIES.find(c => c.id === category);
     console.log(`\n📦 Submitting ${name}...`);
     console.log(`   File:     ${file}`);
@@ -433,6 +466,8 @@ export async function submit(name, file, options) {
     console.log(`   README:   ${readmePath}`);
     if (description)
         console.log(`   Desc:     ${description}`);
+    if (inputSchema)
+        console.log(`   Schema:   ${Object.keys(inputSchema).join(', ')}`);
     if (dependencies.length)
         console.log(`   Deps:     ${dependencies.join(', ')}`);
     if (buildsOn.length)
@@ -451,6 +486,8 @@ export async function submit(name, file, options) {
                 source,
                 dependencies,
                 builds_on: buildsOn.length > 0 ? buildsOn : undefined,
+                input_schema: inputSchema ? JSON.stringify(inputSchema) : undefined,
+                output_schema: outputSchema ? JSON.stringify(outputSchema) : undefined,
             }),
         });
         const data = await res.json();
@@ -465,6 +502,14 @@ export async function submit(name, file, options) {
         if (buildsOn.length > 0) {
             console.log(`   Builds on: ${buildsOn.join(', ')}`);
         }
+        // Show similar tools warning (duplicate detection)
+        if (data.similar_tools && data.similar_tools.length > 0) {
+            console.log(`\n⚠️  Similar tools already exist:`);
+            for (const sim of data.similar_tools) {
+                console.log(`   - ${sim.name}: "${sim.description || 'No description'}"`);
+            }
+            console.log(`   Consider composing with existing tools instead of creating new ones.`);
+        }
         console.log(`\n   Others can now:`);
         console.log(`   $ devtopia cat ${name}`);
         console.log(`   $ devtopia run ${name} '{...}'\n`);

package/dist/executor.d.ts

@@ -4,8 +4,34 @@ interface ExecutionResult {
     error?: string;
     durationMs: number;
 }
+interface ExecuteOptions {
+    strictJson?: boolean;
+}
+interface ValidationResult {
+    valid: boolean;
+    stdout: string;
+    stderr: string;
+    exitCode: number | null;
+}
+/**
+ * Get the interpreter command for a language
+ */
+export declare function getInterpreter(language: string): {
+    cmd: string;
+    args: string[];
+    ext: string;
+};
+/**
+ * Detect if source defines a `function main(...)` without calling it,
+ * and wrap it with the standard argv boilerplate so it actually executes.
+ */
+export declare function wrapSourceIfNeeded(source: string, language: string): string;
 /**
- * Execute a tool locally
+ * Validate that a tool source actually runs and produces valid JSON output.
+ * Runs with {} as input, 10s timeout.
+ * Pass: process produces valid JSON stdout (even error JSON like {"error":"..."}).
+ * Fail: no stdout, non-JSON stdout with exit 0, or process hangs.
  */
-export declare function executeTool(toolName: string, input: any): Promise<ExecutionResult>;
+export declare function validateExecution(source: string, language: string): Promise<ValidationResult>;
+export declare function executeTool(toolName: string, input: any, options?: ExecuteOptions): Promise<ExecutionResult>;
 export {};