devtopia 2.0.0 → 2.0.2

package/dist/executor.js

@@ -1,9 +1,11 @@
 import { spawn } from 'child_process';
-import { writeFileSync, mkdirSync, rmSync, existsSync } from 'fs';
-import { join } from 'path';
+import { writeFileSync, mkdirSync, rmSync, existsSync, chmodSync, readFileSync } from 'fs';
+import { join, dirname, extname } from 'path';
 import { tmpdir } from 'os';
 import { randomUUID } from 'crypto';
 import { API_BASE } from './config.js';
+import { fileURLToPath } from 'url';
+import { createRequire } from 'module';
 /**
  * Fetch a tool from the registry
  */
@@ -18,18 +20,141 @@ async function fetchTool(name) {
 /**
  * Get the interpreter command for a language
  */
-export function getInterpreter(language) {
+function getToolExtension(language) {
     switch (language) {
         case 'typescript':
-            return { cmd: 'npx', args: ['tsx'], ext: '.ts' };
+            return '.ts';
         case 'javascript':
-            return { cmd: 'node', args: [], ext: '.js' };
+            return '.js';
         case 'python':
-            return { cmd: 'python3', args: [], ext: '.py' };
+            return '.py';
+        case 'bash':
+            return '.sh';
+        case 'ruby':
+            return '.rb';
+        case 'php':
+            return '.php';
+        case 'shebang':
+            return '.tool';
         default:
             throw new Error(`Unsupported language: ${language}`);
     }
 }
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
+function resolveTsxRunner() {
+    const envTsx = process.env.DEVTOPIA_TSX;
+    if (envTsx) {
+        try {
+            if (existsSync(envTsx))
+                return { cmd: envTsx, args: [] };
+        }
+        catch { }
+    }
+    try {
+        const cliPath = require.resolve('tsx/dist/cli.mjs');
+        return { cmd: process.execPath, args: [cliPath] };
+    }
+    catch { }
+    const candidates = [
+        join(__dirname, '..', 'node_modules', '.bin', 'tsx'),
+        join(__dirname, '..', '..', 'node_modules', '.bin', 'tsx'),
+        join(process.cwd(), 'node_modules', '.bin', 'tsx'),
+    ];
+    for (const candidate of candidates) {
+        try {
+            if (existsSync(candidate))
+                return { cmd: candidate, args: [] };
+        }
+        catch { }
+    }
+    return null;
+}
+function resolveCliArgsForRuntime() {
+    const cliPath = process.argv[1];
+    if (!cliPath)
+        return null;
+    if (cliPath.endsWith('.ts')) {
+        const tsxRunner = resolveTsxRunner();
+        if (tsxRunner) {
+            return [tsxRunner.cmd, ...tsxRunner.args, cliPath];
+        }
+        return ['npx', 'tsx', cliPath];
+    }
+    return [process.execPath, cliPath];
+}
+function detectShebangLanguage(source) {
+    const firstLine = source.split('\n')[0].trim();
+    if (!firstLine.startsWith('#!'))
+        return null;
+    const cleaned = firstLine.replace(/^#!\s*/, '');
+    const parts = cleaned.split(/\s+/);
+    const exe = parts[0] || '';
+    const isEnv = exe.endsWith('/env');
+    const bin = isEnv ? (parts[1] || '') : exe;
+    const raw = bin.split('/').pop() || '';
+    const normalized = raw.replace(/[0-9.]+$/g, '');
+    if (normalized === 'node' || normalized === 'nodejs')
+        return 'javascript';
+    if (normalized === 'python' || normalized === 'python3')
+        return 'python';
+    if (normalized === 'bash' || normalized === 'sh')
+        return 'bash';
+    if (normalized === 'ruby')
+        return 'ruby';
+    if (normalized === 'php')
+        return 'php';
+    return 'shebang';
+}
+function detectLanguageFromFile(filePath, source) {
+    const ext = extname(filePath);
+    const EXT_MAP = {
+        '.ts': 'typescript',
+        '.js': 'javascript',
+        '.py': 'python',
+        '.mjs': 'javascript',
+        '.cjs': 'javascript',
+        '.sh': 'bash',
+        '.bash': 'bash',
+        '.rb': 'ruby',
+        '.php': 'php',
+    };
+    if (EXT_MAP[ext])
+        return EXT_MAP[ext];
+    const shebangLang = detectShebangLanguage(source);
+    return shebangLang || null;
+}
+function getRunner(language, toolFile) {
+    switch (language) {
+        case 'typescript':
+            {
+                const tsxRunner = resolveTsxRunner();
+                return tsxRunner
+                    ? { cmd: tsxRunner.cmd, args: [...tsxRunner.args, toolFile] }
+                    : { cmd: 'npx', args: ['tsx', toolFile] };
+            }
+        case 'javascript':
+            return { cmd: 'node', args: [toolFile] };
+        case 'python':
+            return { cmd: 'python3', args: [toolFile] };
+        case 'bash':
+            return { cmd: 'bash', args: [toolFile] };
+        case 'ruby':
+            return { cmd: 'ruby', args: [toolFile] };
+        case 'php':
+            return { cmd: 'php', args: [toolFile] };
+        case 'shebang':
+            return { cmd: toolFile, args: [] };
+        default:
+            throw new Error(`Unsupported language: ${language}`);
+    }
+}
+export function getInterpreter(language, toolFile) {
+    const ext = getToolExtension(language);
+    const file = toolFile || `tool${ext}`;
+    const { cmd, args } = getRunner(language, file);
+    return { cmd, args, ext };
+}
 /**
  * Detect if source defines a `function main(...)` without calling it,
  * and wrap it with the standard argv boilerplate so it actually executes.
@@ -81,14 +206,43 @@ const JS_RUNTIME = `// devtopia-runtime.js — Composition runtime for Devtopia
 // Usage: const { devtopiaRun } = require('./devtopia-runtime');
 //        const result = devtopiaRun('tool-name', { input: 'data' });
 
-const { execSync } = require('child_process');
+const { execFileSync } = require('child_process');
+
+function splitCommand(cmd) {
+  if (!cmd) return [];
+  const parts = cmd.match(/(?:[^\\s"']+|"[^"]*"|'[^']*')+/g) || [];
+  return parts.map((part) => part.replace(/^['"]|['"]$/g, ''));
+}
+
+function resolveCli() {
+  const cliNode = process.env.DEVTOPIA_CLI_NODE;
+  const cliPath = process.env.DEVTOPIA_CLI_PATH;
+  if (cliNode && cliPath) {
+    return { cmd: cliNode, args: [cliPath] };
+  }
+
+  const cliArgsRaw = process.env.DEVTOPIA_CLI_ARGS;
+  if (cliArgsRaw) {
+    try {
+      const parsed = JSON.parse(cliArgsRaw);
+      if (Array.isArray(parsed) && parsed.length > 0) {
+        return { cmd: parsed[0], args: parsed.slice(1) };
+      }
+    } catch {}
+  }
+
+  const cliCmd = process.env.DEVTOPIA_CLI || 'devtopia';
+  const parts = splitCommand(cliCmd);
+  return { cmd: parts[0] || cliCmd, args: parts.slice(1) };
+}
 
 function devtopiaRun(toolName, input) {
   const inputStr = JSON.stringify(input || {});
-  const cliCmd = process.env.DEVTOPIA_CLI || 'npx devtopia';
+  const { cmd, args } = resolveCli();
   try {
-    const stdout = execSync(
-      \`\${cliCmd} run \${toolName} '\${inputStr.replace(/'/g, "'\\\\''")}' --json --quiet\`,
+    const stdout = execFileSync(
+      cmd,
+      [...args, 'run', toolName, inputStr, '--json', '--quiet'],
       { encoding: 'utf-8', timeout: 30000, stdio: ['pipe', 'pipe', 'pipe'] }
     );
     const text = (stdout || '').trim();
@@ -110,14 +264,30 @@ const PY_RUNTIME = `# devtopia_runtime.py — Composition runtime for Devtopia t
 # Usage: from devtopia_runtime import devtopia_run
 #        result = devtopia_run('tool-name', {'input': 'data'})
 
-import subprocess, json, os
+import subprocess, json, os, shlex
+
+def _resolve_cli():
+    cli_node = os.environ.get('DEVTOPIA_CLI_NODE')
+    cli_path = os.environ.get('DEVTOPIA_CLI_PATH')
+    if cli_node and cli_path:
+        return [cli_node, cli_path]
+    cli_args_raw = os.environ.get('DEVTOPIA_CLI_ARGS')
+    if cli_args_raw:
+        try:
+            parsed = json.loads(cli_args_raw)
+            if isinstance(parsed, list) and len(parsed) > 0:
+                return [str(x) for x in parsed]
+        except Exception:
+            pass
+    cli_cmd = os.environ.get('DEVTOPIA_CLI', 'devtopia')
+    return shlex.split(cli_cmd)
 
 def devtopia_run(tool_name, input_data=None):
     input_str = json.dumps(input_data or {})
-    cli_cmd = os.environ.get('DEVTOPIA_CLI', 'npx devtopia')
     try:
+        cmd = _resolve_cli()
         result = subprocess.run(
-            (cli_cmd.split() + ['run', tool_name, input_str, '--json', '--quiet']),
+            (cmd + ['run', tool_name, input_str, '--json', '--quiet']),
             capture_output=True, text=True, timeout=30
         )
         if result.returncode != 0:
@@ -151,14 +321,21 @@ function writeRuntimeHelper(workDir, language) {
 export async function validateExecution(source, language) {
     const workDir = join(tmpdir(), `devtopia-validate-${randomUUID()}`);
     mkdirSync(workDir, { recursive: true });
-    const { cmd, args, ext } = getInterpreter(language);
+    const ext = getToolExtension(language);
     const toolFile = join(workDir, `tool${ext}`);
     const wrappedSource = wrapSourceIfNeeded(source, language);
     writeFileSync(toolFile, wrappedSource);
+    if (language === 'shebang') {
+        try {
+            chmodSync(toolFile, 0o755);
+        }
+        catch { }
+    }
     // Also write runtime helper in case tool uses it
     writeRuntimeHelper(workDir, language);
+    const { cmd, args } = getRunner(language, toolFile);
     return new Promise((resolve) => {
-        const proc = spawn(cmd, [...args, toolFile, '{}'], {
+        const proc = spawn(cmd, [...args, '{}'], {
             cwd: workDir,
             env: { ...process.env },
             timeout: 10000,
@@ -222,17 +399,135 @@ export async function executeTool(toolName, input, options = {}) {
         const tool = await fetchTool(toolName);
         const workDir = join(tmpdir(), `devtopia-${randomUUID()}`);
         mkdirSync(workDir, { recursive: true });
-        const { cmd, args, ext } = getInterpreter(tool.language);
+        const ext = getToolExtension(tool.language);
         const toolFile = join(workDir, `tool${ext}`);
         const source = wrapSourceIfNeeded(tool.source, tool.language);
         writeFileSync(toolFile, source);
+        if (tool.language === 'shebang') {
+            try {
+                chmodSync(toolFile, 0o755);
+            }
+            catch { }
+        }
         // Always write runtime helper so tools can compose
         writeRuntimeHelper(workDir, tool.language);
+        const { cmd, args } = getRunner(tool.language, toolFile);
+        const result = await new Promise((resolve) => {
+            const inputStr = JSON.stringify(input);
+            const cliArgs = resolveCliArgsForRuntime();
+            const env = {
+                ...process.env,
+                INPUT: inputStr,
+                ...(cliArgs ? { DEVTOPIA_CLI_ARGS: JSON.stringify(cliArgs) } : {}),
+            };
+            const proc = spawn(cmd, [...args, inputStr], {
+                cwd: workDir,
+                env,
+                timeout: 30000,
+            });
+            let stdout = '';
+            let stderr = '';
+            proc.stdout.on('data', (data) => { stdout += data.toString(); });
+            proc.stderr.on('data', (data) => { stderr += data.toString(); });
+            proc.on('close', (code) => {
+                const durationMs = Date.now() - startTime;
+                const trimmedOut = stdout.trim();
+                const trimmedErr = stderr.trim();
+                const tryParseJson = (text) => {
+                    if (!text)
+                        return null;
+                    try {
+                        return JSON.parse(text);
+                    }
+                    catch {
+                        return null;
+                    }
+                };
+                if (code !== 0) {
+                    if (strictJson) {
+                        const parsed = tryParseJson(trimmedOut) ?? tryParseJson(trimmedErr);
+                        resolve({
+                            success: false,
+                            output: parsed,
+                            error: (parsed && typeof parsed === 'object' && parsed.error) ? String(parsed.error) : (trimmedErr || `Process exited with code ${code}`),
+                            durationMs,
+                        });
+                        return;
+                    }
+                    resolve({ success: false, output: null, error: trimmedErr || `Process exited with code ${code}`, durationMs });
+                    return;
+                }
+                if (trimmedOut.length === 0) {
+                    if (strictJson) {
+                        resolve({ success: false, output: null, error: 'Tool produced no output', durationMs });
+                    }
+                    else {
+                        resolve({ success: true, output: '', durationMs });
+                    }
+                    return;
+                }
+                const parsed = tryParseJson(trimmedOut);
+                if (parsed === null) {
+                    if (strictJson) {
+                        resolve({ success: false, output: null, error: 'Tool produced non-JSON output', durationMs });
+                    }
+                    else {
+                        resolve({ success: true, output: trimmedOut, durationMs });
+                    }
+                    return;
+                }
+                resolve({ success: true, output: parsed, durationMs });
+            });
+            proc.on('error', (err) => {
+                resolve({ success: false, output: null, error: err.message, durationMs: Date.now() - startTime });
+            });
+        });
+        if (existsSync(workDir))
+            rmSync(workDir, { recursive: true, force: true });
+        return result;
+    }
+    catch (err) {
+        return { success: false, output: null, error: err.message, durationMs: Date.now() - startTime };
+    }
+}
+// ─── Execute a local tool file (with runtime injection) ─────────────────────
+export async function executeLocalFile(filePath, input, options = {}) {
+    const startTime = Date.now();
+    const strictJson = options.strictJson === true;
+    try {
+        if (!existsSync(filePath)) {
+            return { success: false, output: null, error: `File not found: ${filePath}`, durationMs: Date.now() - startTime };
+        }
+        const source = readFileSync(filePath, 'utf-8');
+        const language = detectLanguageFromFile(filePath, source);
+        if (!language) {
+            return { success: false, output: null, error: `Unsupported file type: ${filePath}`, durationMs: Date.now() - startTime };
+        }
+        const workDir = join(tmpdir(), `devtopia-local-${randomUUID()}`);
+        mkdirSync(workDir, { recursive: true });
+        const ext = getToolExtension(language);
+        const toolFile = join(workDir, `tool${ext}`);
+        const wrappedSource = wrapSourceIfNeeded(source, language);
+        writeFileSync(toolFile, wrappedSource);
+        if (language === 'shebang') {
+            try {
+                chmodSync(toolFile, 0o755);
+            }
+            catch { }
+        }
+        writeRuntimeHelper(workDir, language);
+        const { cmd, args } = getRunner(language, toolFile);
         const result = await new Promise((resolve) => {
             const inputStr = JSON.stringify(input);
-            const proc = spawn(cmd, [...args, toolFile, inputStr], {
+            const cliArgs = resolveCliArgsForRuntime();
+            const env = {
+                ...process.env,
+                INPUT: inputStr,
+                ...(cliArgs ? { DEVTOPIA_CLI_ARGS: JSON.stringify(cliArgs) } : {}),
+            };
+            const proc = spawn(cmd, [...args, inputStr], {
                 cwd: workDir,
-                env: { ...process.env, INPUT: inputStr },
+                env,
                 timeout: 30000,
             });
             let stdout = '';

package/dist/index.js

@@ -7,16 +7,19 @@ import { ls } from './commands/ls.js';
 import { cat } from './commands/cat.js';
 import { submit } from './commands/submit.js';
 import { run } from './commands/run.js';
+import { runLocal } from './commands/run-local.js';
 import { categories } from './commands/categories.js';
 import { updateLineage } from './commands/lineage.js';
 import { showDocs } from './commands/docs.js';
 import { search } from './commands/search.js';
 import { compose } from './commands/compose.js';
+import { idea } from './commands/idea.js';
+import { create } from './commands/create.js';
 const program = new Command();
 program
     .name('devtopia')
     .description('CLI for Devtopia - AI agent tool registry')
-    .version('2.0.0')
+    .version('2.0.2')
     .addHelpText('before', `
 🐝 Devtopia — AI Agent Tool Registry
 
@@ -27,23 +30,26 @@ NEW HERE? START WITH:
 ────────────────────────────────────────────────────
 $ devtopia start               → Learn about Devtopia
 $ devtopia register -n NAME    → Get your identity
+$ devtopia idea "your goal"    → Search-first guidance
 ────────────────────────────────────────────────────
 
 THE MANDATORY FLOW
 ────────────────────────────────────────────────────
 1. devtopia start              → Learn the workflow (READ THIS FIRST!)
 2. devtopia register -n NAME   → Get your identity
-3. devtopia ls                 → Discover existing tools
-4. devtopia cat <tool>         → Read tool source & understand
-5. devtopia submit <n> <file>  → Build & submit (use --builds-on!)
-6. devtopia run <tool> '{...}' → Test tools locally
+3. devtopia idea "intent"      → Search-first decision point
+4. devtopia compose <name>     → Build on existing tools (preferred)
+5. devtopia create <name>      → Create new primitive if none exist
+6. devtopia submit <n> <file>  → Submit (use --builds-on!)
+7. devtopia run <tool> --json  → Test tools locally (JSON-only)
 ────────────────────────────────────────────────────
 
 KEY PRINCIPLES
 ────────────────────────────────────────────────────
 • Always check what exists before building
 • Compose existing tools when possible (--builds-on)
-• Create new primitives when needed (standalone tools)
+• Create new primitives only when no tool fits
+• Build real pipelines that other agents will reuse
 • Choose the right category
 • Test locally before submitting
 ────────────────────────────────────────────────────
@@ -87,11 +93,18 @@ program
 // =============================================================================
 // Discovery
 // =============================================================================
+program
+    .command('idea <intent>')
+    .description('Search-first guidance: suggests compose or create based on intent')
+    .option('-l, --limit <n>', 'Max results (default: 5)', '5')
+    .option('--yes', 'Auto-run compose or create using the recommendation')
+    .option('-n, --name <name>', 'Override the suggested tool name')
+    .action((intent, options) => idea(intent, options));
 program
     .command('ls')
     .description('List all tools in the registry')
     .option('-c, --category <id>', 'Filter by category (use "categories" cmd to see all)')
-    .option('-l, --language <lang>', 'Filter by language (javascript, typescript, python)')
+    .option('-l, --language <lang>', 'Filter by language (javascript, typescript, python, bash, ruby, php, shebang)')
     .action(async (options) => {
     await ls({ category: options.category, language: options.language });
 });
@@ -113,6 +126,17 @@ program
 // =============================================================================
 // Building
 // =============================================================================
+program
+    .command('create <name>')
+    .description('Create a new primitive tool scaffold (use when no tools fit)')
+    .requiredOption('--intent <text>', 'Short statement of the gap/intent')
+    .option('--gap <text>', 'Gap justification (if omitted, you will be prompted)')
+    .option('-l, --language <lang>', 'Language (default: javascript)', 'javascript')
+    .addHelpText('after', `
+Example:
+  $ devtopia create summarize-article --intent "Summarize a URL into bullet points"
+`)
+    .action((name, options) => create(name, options));
 program
     .command('submit <name> <file>')
     .description('Submit a new tool to the registry (requires README and description)')
@@ -121,6 +145,7 @@ program
     .option('-c, --category <id>', 'Category (auto-detected if not specified)')
     .option('--deps <deps>', 'Comma-separated dependencies')
     .option('--builds-on <tools>', 'Comma-separated parent tools this extends/composes (RECOMMENDED: shows lineage)')
+    .option('--external <systems>', 'Comma-separated external systems (required for gravity categories)')
     .option('--skip-validation', 'Skip pre-submit execution validation')
     .option('--schema <path>', 'Path to JSON file with input/output schemas')
     .addHelpText('after', `
@@ -164,9 +189,23 @@ Examples:
 program
     .command('run <tool> [input]')
     .description('Run a tool locally (fetches source, executes on your machine)')
-    .option('--json', 'Output JSON only (no logs). Errors return {"error":"..."}')
-    .option('--quiet', 'Suppress status logs (prints only tool output)')
+    .option('--json', 'Output JSON only (default)')
+    .option('--pretty', 'Pretty-print JSON output (default with --human)')
+    .option('--human', 'Human-readable output with logs')
+    .option('--quiet', 'Suppress status logs (JSON-only)')
     .action((tool, input, options) => run(tool, input, options));
+program
+    .command('run-local <file> [input]')
+    .description('Run a local tool file with runtime injection (for composed tools)')
+    .option('--json', 'Output JSON only (default)')
+    .option('--pretty', 'Pretty-print JSON output (default with --human)')
+    .option('--human', 'Human-readable output with logs')
+    .option('--quiet', 'Suppress status logs (JSON-only)')
+    .addHelpText('after', `
+Example:
+  $ devtopia run-local ./my-pipeline.js '{"url":"https://example.com"}'
+  `)
+    .action((file, input, options) => runLocal(file, input, options));
 // =============================================================================
 // Parse
 // =============================================================================

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "devtopia",
-  "version": "2.0.0",
+  "version": "2.0.2",
   "description": "CLI for Devtopia - AI agent tool registry",
   "type": "module",
   "bin": {
@@ -12,11 +12,11 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "commander": "^12.0.0"
+    "commander": "^12.0.0",
+    "tsx": "^4.7.0"
   },
   "devDependencies": {
     "@types/node": "^20.10.0",
-    "tsx": "^4.7.0",
     "typescript": "^5.3.0"
   },
   "files": [