devtopia 1.9.0 → 2.0.0

package/dist/executor.js

@@ -18,7 +18,7 @@ async function fetchTool(name) {
 /**
  * Get the interpreter command for a language
  */
-function getInterpreter(language) {
+export function getInterpreter(language) {
     switch (language) {
         case 'typescript':
             return { cmd: 'npx', args: ['tsx'], ext: '.ts' };
@@ -31,102 +31,272 @@ function getInterpreter(language) {
     }
 }
 /**
- * Execute a tool locally
+ * Detect if source defines a `function main(...)` without calling it,
+ * and wrap it with the standard argv boilerplate so it actually executes.
  */
-export async function executeTool(toolName, input) {
+export function wrapSourceIfNeeded(source, language) {
+    if (language === 'javascript' || language === 'typescript') {
+        const definesMain = /\bfunction\s+main\s*\(/.test(source);
+        const callsMain = /\bmain\s*\(/.test(source.replace(/function\s+main\s*\(/, ''));
+        const hasArgvParse = /process\.argv\[2\]/.test(source);
+        if (definesMain && !callsMain && !hasArgvParse) {
+            return `${source}
+
+// --- Devtopia auto-wrapper: call main() with argv input ---
+const __input = JSON.parse(process.argv[2] || '{}');
+const __result = main(__input);
+if (__result !== undefined) {
+  if (__result instanceof Promise) {
+    __result.then(r => console.log(JSON.stringify(r))).catch(e => {
+      console.error(e.message || e);
+      process.exit(1);
+    });
+  } else {
+    console.log(JSON.stringify(__result));
+  }
+}
+`;
+        }
+    }
+    if (language === 'python') {
+        const definesMain = /\bdef\s+main\s*\(/.test(source);
+        const callsMain = /\bmain\s*\(/.test(source.replace(/def\s+main\s*\(/, ''));
+        const hasArgvParse = /sys\.argv/.test(source);
+        if (definesMain && !callsMain && !hasArgvParse) {
+            return `${source}
+
+# --- Devtopia auto-wrapper: call main() with argv input ---
+import json, sys
+__input = json.loads(sys.argv[1] if len(sys.argv) > 1 else '{}')
+__result = main(__input)
+if __result is not None:
+    print(json.dumps(__result))
+`;
+        }
+    }
+    return source;
+}
+// ─── Composition Runtime Templates ───────────────────────────────────────────
+const JS_RUNTIME = `// devtopia-runtime.js — Composition runtime for Devtopia tools
+// Usage: const { devtopiaRun } = require('./devtopia-runtime');
+//        const result = devtopiaRun('tool-name', { input: 'data' });
+
+const { execSync } = require('child_process');
+
+function devtopiaRun(toolName, input) {
+  const inputStr = JSON.stringify(input || {});
+  const cliCmd = process.env.DEVTOPIA_CLI || 'npx devtopia';
+  try {
+    const stdout = execSync(
+      \`\${cliCmd} run \${toolName} '\${inputStr.replace(/'/g, "'\\\\''")}' --json --quiet\`,
+      { encoding: 'utf-8', timeout: 30000, stdio: ['pipe', 'pipe', 'pipe'] }
+    );
+    const text = (stdout || '').trim();
+    const parsed = text ? JSON.parse(text) : null;
+    if (parsed && typeof parsed === 'object' && (parsed.error || parsed.ok === false)) {
+      throw new Error(parsed.error || 'Unknown error');
+    }
+    return parsed;
+  } catch (err) {
+    const stderr = err && err.stderr ? String(err.stderr).trim() : '';
+    const message = stderr || (err && err.message ? err.message : 'Unknown error');
+    throw new Error(\`devtopiaRun(\${toolName}) failed: \${message}\`);
+  }
+}
+
+module.exports = { devtopiaRun };
+`;
+const PY_RUNTIME = `# devtopia_runtime.py — Composition runtime for Devtopia tools
+# Usage: from devtopia_runtime import devtopia_run
+#        result = devtopia_run('tool-name', {'input': 'data'})
+
+import subprocess, json, os
+
+def devtopia_run(tool_name, input_data=None):
+    input_str = json.dumps(input_data or {})
+    cli_cmd = os.environ.get('DEVTOPIA_CLI', 'npx devtopia')
+    try:
+        result = subprocess.run(
+            (cli_cmd.split() + ['run', tool_name, input_str, '--json', '--quiet']),
+            capture_output=True, text=True, timeout=30
+        )
+        if result.returncode != 0:
+            raise RuntimeError(f"devtopia_run({tool_name}) failed: {result.stderr}")
+        output = (result.stdout or '').strip()
+        parsed = json.loads(output) if output else None
+        if isinstance(parsed, dict) and (parsed.get('error') is not None or parsed.get('ok') is False):
+            raise RuntimeError(parsed.get('error') or 'Unknown error')
+        return parsed
+    except subprocess.TimeoutExpired:
+        raise RuntimeError(f"devtopia_run({tool_name}) timed out")
+`;
+/**
+ * Write composition runtime helper into the working directory
+ */
+function writeRuntimeHelper(workDir, language) {
+    if (language === 'javascript' || language === 'typescript') {
+        writeFileSync(join(workDir, 'devtopia-runtime.js'), JS_RUNTIME);
+    }
+    if (language === 'python') {
+        writeFileSync(join(workDir, 'devtopia_runtime.py'), PY_RUNTIME);
+    }
+}
+// ─── Validate tool execution (pre-submit) ────────────────────────────────────
+/**
+ * Validate that a tool source actually runs and produces valid JSON output.
+ * Runs with {} as input, 10s timeout.
+ * Pass: process produces valid JSON stdout (even error JSON like {"error":"..."}).
+ * Fail: no stdout, non-JSON stdout with exit 0, or process hangs.
+ */
+export async function validateExecution(source, language) {
+    const workDir = join(tmpdir(), `devtopia-validate-${randomUUID()}`);
+    mkdirSync(workDir, { recursive: true });
+    const { cmd, args, ext } = getInterpreter(language);
+    const toolFile = join(workDir, `tool${ext}`);
+    const wrappedSource = wrapSourceIfNeeded(source, language);
+    writeFileSync(toolFile, wrappedSource);
+    // Also write runtime helper in case tool uses it
+    writeRuntimeHelper(workDir, language);
+    return new Promise((resolve) => {
+        const proc = spawn(cmd, [...args, toolFile, '{}'], {
+            cwd: workDir,
+            env: { ...process.env },
+            timeout: 10000,
+        });
+        let stdout = '';
+        let stderr = '';
+        proc.stdout.on('data', (data) => { stdout += data.toString(); });
+        proc.stderr.on('data', (data) => { stderr += data.toString(); });
+        proc.on('close', (code) => {
+            try {
+                if (existsSync(workDir))
+                    rmSync(workDir, { recursive: true, force: true });
+            }
+            catch { }
+            const trimmedOut = stdout.trim();
+            // No output at all = broken
+            if (trimmedOut.length === 0) {
+                resolve({ valid: false, stdout: '', stderr: stderr.trim(), exitCode: code });
+                return;
+            }
+            // Check if output is valid JSON
+            let isJson = false;
+            try {
+                JSON.parse(trimmedOut);
+                isJson = true;
+            }
+            catch { }
+            // Non-zero exit with JSON stderr/stdout = valid (error handling works)
+            if (code !== 0 && isJson) {
+                resolve({ valid: true, stdout: trimmedOut, stderr: stderr.trim(), exitCode: code });
+                return;
+            }
+            // Exit 0 with non-JSON output = invalid (must produce JSON)
+            if (code === 0 && !isJson) {
+                resolve({
+                    valid: false,
+                    stdout: trimmedOut,
+                    stderr: 'Tool produced non-JSON output. All Devtopia tools must output valid JSON.',
+                    exitCode: code,
+                });
+                return;
+            }
+            // Exit 0 with JSON = valid, non-zero with non-JSON stderr = broken
+            resolve({ valid: isJson, stdout: trimmedOut, stderr: stderr.trim(), exitCode: code });
+        });
+        proc.on('error', (err) => {
+            try {
+                if (existsSync(workDir))
+                    rmSync(workDir, { recursive: true, force: true });
+            }
+            catch { }
+            resolve({ valid: false, stdout: '', stderr: err.message, exitCode: null });
+        });
+    });
+}
+// ─── Execute a tool locally ──────────────────────────────────────────────────
+export async function executeTool(toolName, input, options = {}) {
     const startTime = Date.now();
+    const strictJson = options.strictJson === true;
     try {
-        // Fetch the tool
         const tool = await fetchTool(toolName);
-        // Fetch dependencies
-        const deps = new Map();
-        for (const depName of tool.dependencies) {
-            const dep = await fetchTool(depName);
-            deps.set(depName, dep);
-        }
-        // Create temp directory
         const workDir = join(tmpdir(), `devtopia-${randomUUID()}`);
         mkdirSync(workDir, { recursive: true });
-        // Write tool source
         const { cmd, args, ext } = getInterpreter(tool.language);
         const toolFile = join(workDir, `tool${ext}`);
-        // Inject a simple devtopia runtime for dependencies
-        let source = tool.source;
-        if (deps.size > 0) {
-            const depComment = `// Dependencies: ${[...deps.keys()].join(', ')}\n`;
-            source = depComment + source;
-        }
+        const source = wrapSourceIfNeeded(tool.source, tool.language);
         writeFileSync(toolFile, source);
-        // Execute and wait for completion
+        // Always write runtime helper so tools can compose
+        writeRuntimeHelper(workDir, tool.language);
         const result = await new Promise((resolve) => {
             const inputStr = JSON.stringify(input);
             const proc = spawn(cmd, [...args, toolFile, inputStr], {
                 cwd: workDir,
-                env: {
-                    ...process.env,
-                    INPUT: inputStr,
-                },
-                timeout: 30000, // 30 second timeout
+                env: { ...process.env, INPUT: inputStr },
+                timeout: 30000,
             });
             let stdout = '';
             let stderr = '';
-            proc.stdout.on('data', (data) => {
-                stdout += data.toString();
-            });
-            proc.stderr.on('data', (data) => {
-                stderr += data.toString();
-            });
+            proc.stdout.on('data', (data) => { stdout += data.toString(); });
+            proc.stderr.on('data', (data) => { stderr += data.toString(); });
             proc.on('close', (code) => {
                 const durationMs = Date.now() - startTime;
+                const trimmedOut = stdout.trim();
+                const trimmedErr = stderr.trim();
+                const tryParseJson = (text) => {
+                    if (!text)
+                        return null;
+                    try {
+                        return JSON.parse(text);
+                    }
+                    catch {
+                        return null;
+                    }
+                };
                 if (code !== 0) {
-                    resolve({
-                        success: false,
-                        output: null,
-                        error: stderr || `Process exited with code ${code}`,
-                        durationMs,
-                    });
+                    if (strictJson) {
+                        const parsed = tryParseJson(trimmedOut) ?? tryParseJson(trimmedErr);
+                        resolve({
+                            success: false,
+                            output: parsed,
+                            error: (parsed && typeof parsed === 'object' && parsed.error) ? String(parsed.error) : (trimmedErr || `Process exited with code ${code}`),
+                            durationMs,
+                        });
+                        return;
+                    }
+                    resolve({ success: false, output: null, error: trimmedErr || `Process exited with code ${code}`, durationMs });
                     return;
                 }
-                try {
-                    // Parse JSON output
-                    const output = JSON.parse(stdout.trim());
-                    resolve({
-                        success: true,
-                        output,
-                        durationMs,
-                    });
+                if (trimmedOut.length === 0) {
+                    if (strictJson) {
+                        resolve({ success: false, output: null, error: 'Tool produced no output', durationMs });
+                    }
+                    else {
+                        resolve({ success: true, output: '', durationMs });
+                    }
+                    return;
                 }
-                catch {
-                    // Return raw output if not JSON
-                    resolve({
-                        success: true,
-                        output: stdout.trim(),
-                        durationMs,
-                    });
+                const parsed = tryParseJson(trimmedOut);
+                if (parsed === null) {
+                    if (strictJson) {
+                        resolve({ success: false, output: null, error: 'Tool produced non-JSON output', durationMs });
+                    }
+                    else {
+                        resolve({ success: true, output: trimmedOut, durationMs });
+                    }
+                    return;
                 }
+                resolve({ success: true, output: parsed, durationMs });
             });
             proc.on('error', (err) => {
-                resolve({
-                    success: false,
-                    output: null,
-                    error: err.message,
-                    durationMs: Date.now() - startTime,
-                });
+                resolve({ success: false, output: null, error: err.message, durationMs: Date.now() - startTime });
             });
         });
-        // Cleanup temp directory after execution completes
-        if (existsSync(workDir)) {
+        if (existsSync(workDir))
             rmSync(workDir, { recursive: true, force: true });
-        }
         return result;
     }
     catch (err) {
-        const durationMs = Date.now() - startTime;
-        return {
-            success: false,
-            output: null,
-            error: err.message,
-            durationMs,
-        };
+        return { success: false, output: null, error: err.message, durationMs: Date.now() - startTime };
     }
 }

package/dist/index.js

@@ -10,11 +10,13 @@ import { run } from './commands/run.js';
 import { categories } from './commands/categories.js';
 import { updateLineage } from './commands/lineage.js';
 import { showDocs } from './commands/docs.js';
+import { search } from './commands/search.js';
+import { compose } from './commands/compose.js';
 const program = new Command();
 program
     .name('devtopia')
     .description('CLI for Devtopia - AI agent tool registry')
-    .version('1.9.0')
+    .version('2.0.0')
     .addHelpText('before', `
 🐝 Devtopia — AI Agent Tool Registry
 
@@ -53,7 +55,7 @@ KEY PRINCIPLES
 program
     .command('start')
     .description('Learn about Devtopia - start here if you\'re new!')
-    .action(start);
+    .action(async () => { await start(); });
 program
     .command('docs [name]')
     .description('View Devtopia documentation (agents, contributing, cli, tool-format, faq)')
@@ -66,7 +68,7 @@ Examples:
   $ devtopia docs tool-format   # Show tool format specification
   $ devtopia docs faq           # Show FAQ
 
-These docs match the GitHub repository: https://github.com/Devtopia/Devtopia
+These docs match the GitHub repository: https://github.com/DevtopiaHub/Devtopia
   `)
     .action((name) => showDocs(name));
 // =============================================================================
@@ -97,6 +99,11 @@ program
     .command('categories')
     .description('List all available categories')
     .action(categories);
+program
+    .command('search <query>')
+    .description('Search tools by name or description')
+    .option('-l, --limit <n>', 'Max results (default: 20)', '20')
+    .action((query, options) => search(query, options));
 program
     .command('cat <tool>')
     .description('View tool README and source code')
@@ -114,6 +121,8 @@ program
     .option('-c, --category <id>', 'Category (auto-detected if not specified)')
     .option('--deps <deps>', 'Comma-separated dependencies')
     .option('--builds-on <tools>', 'Comma-separated parent tools this extends/composes (RECOMMENDED: shows lineage)')
+    .option('--skip-validation', 'Skip pre-submit execution validation')
+    .option('--schema <path>', 'Path to JSON file with input/output schemas')
     .addHelpText('after', `
 Examples:
   $ devtopia submit my-tool ./my-tool.js -d "Does something useful"
@@ -124,6 +133,18 @@ Examples:
    This creates visible dependency chains and helps the ecosystem grow.
   `)
     .action(submit);
+program
+    .command('compose <name>')
+    .description('Generate a composed tool scaffold pre-wired to call parent tools')
+    .requiredOption('--uses <tools>', 'Comma-separated parent tools to compose')
+    .addHelpText('after', `
+Example:
+  $ devtopia compose my-pipeline --uses json-validate,json-flatten,csv-writer
+
+  Generates my-pipeline.js and my-pipeline.md with devtopiaRun() calls
+  pre-wired for each parent tool. Edit the TODOs, test, then submit.
+  `)
+    .action((name, options) => compose(name, options));
 program
     .command('lineage <tool> [builds-on]')
     .description('Update tool lineage - specify which tools this builds on')
@@ -143,7 +164,9 @@ Examples:
 program
     .command('run <tool> [input]')
     .description('Run a tool locally (fetches source, executes on your machine)')
-    .action(run);
+    .option('--json', 'Output JSON only (no logs). Errors return {"error":"..."}')
+    .option('--quiet', 'Suppress status logs (prints only tool output)')
+    .action((tool, input, options) => run(tool, input, options));
 // =============================================================================
 // Parse
 // =============================================================================

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "devtopia",
-  "version": "1.9.0",
+  "version": "2.0.0",
   "description": "CLI for Devtopia - AI agent tool registry",
   "type": "module",
   "bin": {