devtopia 1.9.0 → 2.0.1

package/dist/executor.js

@@ -1,9 +1,11 @@
 import { spawn } from 'child_process';
-import { writeFileSync, mkdirSync, rmSync, existsSync } from 'fs';
-import { join } from 'path';
+import { writeFileSync, mkdirSync, rmSync, existsSync, chmodSync, readFileSync } from 'fs';
+import { join, dirname, extname } from 'path';
 import { tmpdir } from 'os';
 import { randomUUID } from 'crypto';
 import { API_BASE } from './config.js';
+import { fileURLToPath } from 'url';
+import { createRequire } from 'module';
 /**
  * Fetch a tool from the registry
  */
@@ -18,115 +20,578 @@ async function fetchTool(name) {
 /**
  * Get the interpreter command for a language
  */
-function getInterpreter(language) {
+function getToolExtension(language) {
     switch (language) {
         case 'typescript':
-            return { cmd: 'npx', args: ['tsx'], ext: '.ts' };
+            return '.ts';
         case 'javascript':
-            return { cmd: 'node', args: [], ext: '.js' };
+            return '.js';
         case 'python':
-            return { cmd: 'python3', args: [], ext: '.py' };
+            return '.py';
+        case 'bash':
+            return '.sh';
+        case 'ruby':
+            return '.rb';
+        case 'php':
+            return '.php';
+        case 'shebang':
+            return '.tool';
         default:
             throw new Error(`Unsupported language: ${language}`);
     }
 }
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
+function resolveTsxRunner() {
+    const envTsx = process.env.DEVTOPIA_TSX;
+    if (envTsx) {
+        try {
+            if (existsSync(envTsx))
+                return { cmd: envTsx, args: [] };
+        }
+        catch { }
+    }
+    try {
+        const cliPath = require.resolve('tsx/dist/cli.mjs');
+        return { cmd: process.execPath, args: [cliPath] };
+    }
+    catch { }
+    const candidates = [
+        join(__dirname, '..', 'node_modules', '.bin', 'tsx'),
+        join(__dirname, '..', '..', 'node_modules', '.bin', 'tsx'),
+        join(process.cwd(), 'node_modules', '.bin', 'tsx'),
+    ];
+    for (const candidate of candidates) {
+        try {
+            if (existsSync(candidate))
+                return { cmd: candidate, args: [] };
+        }
+        catch { }
+    }
+    return null;
+}
+function resolveCliArgsForRuntime() {
+    const cliPath = process.argv[1];
+    if (!cliPath)
+        return null;
+    if (cliPath.endsWith('.ts')) {
+        const tsxRunner = resolveTsxRunner();
+        if (tsxRunner) {
+            return [tsxRunner.cmd, ...tsxRunner.args, cliPath];
+        }
+        return ['npx', 'tsx', cliPath];
+    }
+    return [process.execPath, cliPath];
+}
+function detectShebangLanguage(source) {
+    const firstLine = source.split('\n')[0].trim();
+    if (!firstLine.startsWith('#!'))
+        return null;
+    const cleaned = firstLine.replace(/^#!\s*/, '');
+    const parts = cleaned.split(/\s+/);
+    const exe = parts[0] || '';
+    const isEnv = exe.endsWith('/env');
+    const bin = isEnv ? (parts[1] || '') : exe;
+    const raw = bin.split('/').pop() || '';
+    const normalized = raw.replace(/[0-9.]+$/g, '');
+    if (normalized === 'node' || normalized === 'nodejs')
+        return 'javascript';
+    if (normalized === 'python' || normalized === 'python3')
+        return 'python';
+    if (normalized === 'bash' || normalized === 'sh')
+        return 'bash';
+    if (normalized === 'ruby')
+        return 'ruby';
+    if (normalized === 'php')
+        return 'php';
+    return 'shebang';
+}
+function detectLanguageFromFile(filePath, source) {
+    const ext = extname(filePath);
+    const EXT_MAP = {
+        '.ts': 'typescript',
+        '.js': 'javascript',
+        '.py': 'python',
+        '.mjs': 'javascript',
+        '.cjs': 'javascript',
+        '.sh': 'bash',
+        '.bash': 'bash',
+        '.rb': 'ruby',
+        '.php': 'php',
+    };
+    if (EXT_MAP[ext])
+        return EXT_MAP[ext];
+    const shebangLang = detectShebangLanguage(source);
+    return shebangLang || null;
+}
+function getRunner(language, toolFile) {
+    switch (language) {
+        case 'typescript':
+            {
+                const tsxRunner = resolveTsxRunner();
+                return tsxRunner
+                    ? { cmd: tsxRunner.cmd, args: [...tsxRunner.args, toolFile] }
+                    : { cmd: 'npx', args: ['tsx', toolFile] };
+            }
+        case 'javascript':
+            return { cmd: 'node', args: [toolFile] };
+        case 'python':
+            return { cmd: 'python3', args: [toolFile] };
+        case 'bash':
+            return { cmd: 'bash', args: [toolFile] };
+        case 'ruby':
+            return { cmd: 'ruby', args: [toolFile] };
+        case 'php':
+            return { cmd: 'php', args: [toolFile] };
+        case 'shebang':
+            return { cmd: toolFile, args: [] };
+        default:
+            throw new Error(`Unsupported language: ${language}`);
+    }
+}
+export function getInterpreter(language, toolFile) {
+    const ext = getToolExtension(language);
+    const file = toolFile || `tool${ext}`;
+    const { cmd, args } = getRunner(language, file);
+    return { cmd, args, ext };
+}
 /**
- * Execute a tool locally
+ * Detect if source defines a `function main(...)` without calling it,
+ * and wrap it with the standard argv boilerplate so it actually executes.
  */
-export async function executeTool(toolName, input) {
+export function wrapSourceIfNeeded(source, language) {
+    if (language === 'javascript' || language === 'typescript') {
+        const definesMain = /\bfunction\s+main\s*\(/.test(source);
+        const callsMain = /\bmain\s*\(/.test(source.replace(/function\s+main\s*\(/, ''));
+        const hasArgvParse = /process\.argv\[2\]/.test(source);
+        if (definesMain && !callsMain && !hasArgvParse) {
+            return `${source}
+
+// --- Devtopia auto-wrapper: call main() with argv input ---
+const __input = JSON.parse(process.argv[2] || '{}');
+const __result = main(__input);
+if (__result !== undefined) {
+  if (__result instanceof Promise) {
+    __result.then(r => console.log(JSON.stringify(r))).catch(e => {
+      console.error(e.message || e);
+      process.exit(1);
+    });
+  } else {
+    console.log(JSON.stringify(__result));
+  }
+}
+`;
+        }
+    }
+    if (language === 'python') {
+        const definesMain = /\bdef\s+main\s*\(/.test(source);
+        const callsMain = /\bmain\s*\(/.test(source.replace(/def\s+main\s*\(/, ''));
+        const hasArgvParse = /sys\.argv/.test(source);
+        if (definesMain && !callsMain && !hasArgvParse) {
+            return `${source}
+
+# --- Devtopia auto-wrapper: call main() with argv input ---
+import json, sys
+__input = json.loads(sys.argv[1] if len(sys.argv) > 1 else '{}')
+__result = main(__input)
+if __result is not None:
+    print(json.dumps(__result))
+`;
+        }
+    }
+    return source;
+}
+// ─── Composition Runtime Templates ───────────────────────────────────────────
+const JS_RUNTIME = `// devtopia-runtime.js — Composition runtime for Devtopia tools
+// Usage: const { devtopiaRun } = require('./devtopia-runtime');
+//        const result = devtopiaRun('tool-name', { input: 'data' });
+
+const { execFileSync } = require('child_process');
+
+function splitCommand(cmd) {
+  if (!cmd) return [];
+  const parts = cmd.match(/(?:[^\\s"']+|"[^"]*"|'[^']*')+/g) || [];
+  return parts.map((part) => part.replace(/^['"]|['"]$/g, ''));
+}
+
+function resolveCli() {
+  const cliNode = process.env.DEVTOPIA_CLI_NODE;
+  const cliPath = process.env.DEVTOPIA_CLI_PATH;
+  if (cliNode && cliPath) {
+    return { cmd: cliNode, args: [cliPath] };
+  }
+
+  const cliArgsRaw = process.env.DEVTOPIA_CLI_ARGS;
+  if (cliArgsRaw) {
+    try {
+      const parsed = JSON.parse(cliArgsRaw);
+      if (Array.isArray(parsed) && parsed.length > 0) {
+        return { cmd: parsed[0], args: parsed.slice(1) };
+      }
+    } catch {}
+  }
+
+  const cliCmd = process.env.DEVTOPIA_CLI || 'devtopia';
+  const parts = splitCommand(cliCmd);
+  return { cmd: parts[0] || cliCmd, args: parts.slice(1) };
+}
+
+function devtopiaRun(toolName, input) {
+  const inputStr = JSON.stringify(input || {});
+  const { cmd, args } = resolveCli();
+  try {
+    const stdout = execFileSync(
+      cmd,
+      [...args, 'run', toolName, inputStr, '--json', '--quiet'],
+      { encoding: 'utf-8', timeout: 30000, stdio: ['pipe', 'pipe', 'pipe'] }
+    );
+    const text = (stdout || '').trim();
+    const parsed = text ? JSON.parse(text) : null;
+    if (parsed && typeof parsed === 'object' && (parsed.error || parsed.ok === false)) {
+      throw new Error(parsed.error || 'Unknown error');
+    }
+    return parsed;
+  } catch (err) {
+    const stderr = err && err.stderr ? String(err.stderr).trim() : '';
+    const message = stderr || (err && err.message ? err.message : 'Unknown error');
+    throw new Error(\`devtopiaRun(\${toolName}) failed: \${message}\`);
+  }
+}
+
+module.exports = { devtopiaRun };
+`;
+const PY_RUNTIME = `# devtopia_runtime.py — Composition runtime for Devtopia tools
+# Usage: from devtopia_runtime import devtopia_run
+#        result = devtopia_run('tool-name', {'input': 'data'})
+
+import subprocess, json, os, shlex
+
+def _resolve_cli():
+    cli_node = os.environ.get('DEVTOPIA_CLI_NODE')
+    cli_path = os.environ.get('DEVTOPIA_CLI_PATH')
+    if cli_node and cli_path:
+        return [cli_node, cli_path]
+    cli_args_raw = os.environ.get('DEVTOPIA_CLI_ARGS')
+    if cli_args_raw:
+        try:
+            parsed = json.loads(cli_args_raw)
+            if isinstance(parsed, list) and len(parsed) > 0:
+                return [str(x) for x in parsed]
+        except Exception:
+            pass
+    cli_cmd = os.environ.get('DEVTOPIA_CLI', 'devtopia')
+    return shlex.split(cli_cmd)
+
+def devtopia_run(tool_name, input_data=None):
+    input_str = json.dumps(input_data or {})
+    try:
+        cmd = _resolve_cli()
+        result = subprocess.run(
+            (cmd + ['run', tool_name, input_str, '--json', '--quiet']),
+            capture_output=True, text=True, timeout=30
+        )
+        if result.returncode != 0:
+            raise RuntimeError(f"devtopia_run({tool_name}) failed: {result.stderr}")
+        output = (result.stdout or '').strip()
+        parsed = json.loads(output) if output else None
+        if isinstance(parsed, dict) and (parsed.get('error') is not None or parsed.get('ok') is False):
+            raise RuntimeError(parsed.get('error') or 'Unknown error')
+        return parsed
+    except subprocess.TimeoutExpired:
+        raise RuntimeError(f"devtopia_run({tool_name}) timed out")
+`;
+/**
+ * Write composition runtime helper into the working directory
+ */
+function writeRuntimeHelper(workDir, language) {
+    if (language === 'javascript' || language === 'typescript') {
+        writeFileSync(join(workDir, 'devtopia-runtime.js'), JS_RUNTIME);
+    }
+    if (language === 'python') {
+        writeFileSync(join(workDir, 'devtopia_runtime.py'), PY_RUNTIME);
+    }
+}
+// ─── Validate tool execution (pre-submit) ────────────────────────────────────
+/**
+ * Validate that a tool source actually runs and produces valid JSON output.
+ * Runs with {} as input, 10s timeout.
+ * Pass: process produces valid JSON stdout (even error JSON like {"error":"..."}).
+ * Fail: no stdout, non-JSON stdout with exit 0, or process hangs.
+ */
+export async function validateExecution(source, language) {
+    const workDir = join(tmpdir(), `devtopia-validate-${randomUUID()}`);
+    mkdirSync(workDir, { recursive: true });
+    const ext = getToolExtension(language);
+    const toolFile = join(workDir, `tool${ext}`);
+    const wrappedSource = wrapSourceIfNeeded(source, language);
+    writeFileSync(toolFile, wrappedSource);
+    if (language === 'shebang') {
+        try {
+            chmodSync(toolFile, 0o755);
+        }
+        catch { }
+    }
+    // Also write runtime helper in case tool uses it
+    writeRuntimeHelper(workDir, language);
+    const { cmd, args } = getRunner(language, toolFile);
+    return new Promise((resolve) => {
+        const proc = spawn(cmd, [...args, '{}'], {
+            cwd: workDir,
+            env: { ...process.env },
+            timeout: 10000,
+        });
+        let stdout = '';
+        let stderr = '';
+        proc.stdout.on('data', (data) => { stdout += data.toString(); });
+        proc.stderr.on('data', (data) => { stderr += data.toString(); });
+        proc.on('close', (code) => {
+            try {
+                if (existsSync(workDir))
+                    rmSync(workDir, { recursive: true, force: true });
+            }
+            catch { }
+            const trimmedOut = stdout.trim();
+            // No output at all = broken
+            if (trimmedOut.length === 0) {
+                resolve({ valid: false, stdout: '', stderr: stderr.trim(), exitCode: code });
+                return;
+            }
+            // Check if output is valid JSON
+            let isJson = false;
+            try {
+                JSON.parse(trimmedOut);
+                isJson = true;
+            }
+            catch { }
+            // Non-zero exit with JSON stderr/stdout = valid (error handling works)
+            if (code !== 0 && isJson) {
+                resolve({ valid: true, stdout: trimmedOut, stderr: stderr.trim(), exitCode: code });
+                return;
+            }
+            // Exit 0 with non-JSON output = invalid (must produce JSON)
+            if (code === 0 && !isJson) {
+                resolve({
+                    valid: false,
+                    stdout: trimmedOut,
+                    stderr: 'Tool produced non-JSON output. All Devtopia tools must output valid JSON.',
+                    exitCode: code,
+                });
+                return;
+            }
+            // Exit 0 with JSON = valid, non-zero with non-JSON stderr = broken
+            resolve({ valid: isJson, stdout: trimmedOut, stderr: stderr.trim(), exitCode: code });
+        });
+        proc.on('error', (err) => {
+            try {
+                if (existsSync(workDir))
+                    rmSync(workDir, { recursive: true, force: true });
+            }
+            catch { }
+            resolve({ valid: false, stdout: '', stderr: err.message, exitCode: null });
+        });
+    });
+}
+// ─── Execute a tool locally ──────────────────────────────────────────────────
+export async function executeTool(toolName, input, options = {}) {
     const startTime = Date.now();
+    const strictJson = options.strictJson === true;
     try {
-        // Fetch the tool
         const tool = await fetchTool(toolName);
-        // Fetch dependencies
-        const deps = new Map();
-        for (const depName of tool.dependencies) {
-            const dep = await fetchTool(depName);
-            deps.set(depName, dep);
-        }
-        // Create temp directory
         const workDir = join(tmpdir(), `devtopia-${randomUUID()}`);
         mkdirSync(workDir, { recursive: true });
-        // Write tool source
-        const { cmd, args, ext } = getInterpreter(tool.language);
+        const ext = getToolExtension(tool.language);
         const toolFile = join(workDir, `tool${ext}`);
-        // Inject a simple devtopia runtime for dependencies
-        let source = tool.source;
-        if (deps.size > 0) {
-            const depComment = `// Dependencies: ${[...deps.keys()].join(', ')}\n`;
-            source = depComment + source;
-        }
+        const source = wrapSourceIfNeeded(tool.source, tool.language);
         writeFileSync(toolFile, source);
-        // Execute and wait for completion
+        if (tool.language === 'shebang') {
+            try {
+                chmodSync(toolFile, 0o755);
+            }
+            catch { }
+        }
+        // Always write runtime helper so tools can compose
+        writeRuntimeHelper(workDir, tool.language);
+        const { cmd, args } = getRunner(tool.language, toolFile);
         const result = await new Promise((resolve) => {
             const inputStr = JSON.stringify(input);
-            const proc = spawn(cmd, [...args, toolFile, inputStr], {
+            const cliArgs = resolveCliArgsForRuntime();
+            const env = {
+                ...process.env,
+                INPUT: inputStr,
+                ...(cliArgs ? { DEVTOPIA_CLI_ARGS: JSON.stringify(cliArgs) } : {}),
+            };
+            const proc = spawn(cmd, [...args, inputStr], {
                 cwd: workDir,
-                env: {
-                    ...process.env,
-                    INPUT: inputStr,
-                },
-                timeout: 30000, // 30 second timeout
+                env,
+                timeout: 30000,
             });
             let stdout = '';
             let stderr = '';
-            proc.stdout.on('data', (data) => {
-                stdout += data.toString();
+            proc.stdout.on('data', (data) => { stdout += data.toString(); });
+            proc.stderr.on('data', (data) => { stderr += data.toString(); });
+            proc.on('close', (code) => {
+                const durationMs = Date.now() - startTime;
+                const trimmedOut = stdout.trim();
+                const trimmedErr = stderr.trim();
+                const tryParseJson = (text) => {
+                    if (!text)
+                        return null;
+                    try {
+                        return JSON.parse(text);
+                    }
+                    catch {
+                        return null;
+                    }
+                };
+                if (code !== 0) {
+                    if (strictJson) {
+                        const parsed = tryParseJson(trimmedOut) ?? tryParseJson(trimmedErr);
+                        resolve({
+                            success: false,
+                            output: parsed,
+                            error: (parsed && typeof parsed === 'object' && parsed.error) ? String(parsed.error) : (trimmedErr || `Process exited with code ${code}`),
+                            durationMs,
+                        });
+                        return;
+                    }
+                    resolve({ success: false, output: null, error: trimmedErr || `Process exited with code ${code}`, durationMs });
+                    return;
+                }
+                if (trimmedOut.length === 0) {
+                    if (strictJson) {
+                        resolve({ success: false, output: null, error: 'Tool produced no output', durationMs });
+                    }
+                    else {
+                        resolve({ success: true, output: '', durationMs });
+                    }
+                    return;
+                }
+                const parsed = tryParseJson(trimmedOut);
+                if (parsed === null) {
+                    if (strictJson) {
+                        resolve({ success: false, output: null, error: 'Tool produced non-JSON output', durationMs });
+                    }
+                    else {
+                        resolve({ success: true, output: trimmedOut, durationMs });
+                    }
+                    return;
+                }
+                resolve({ success: true, output: parsed, durationMs });
             });
-            proc.stderr.on('data', (data) => {
-                stderr += data.toString();
+            proc.on('error', (err) => {
+                resolve({ success: false, output: null, error: err.message, durationMs: Date.now() - startTime });
+            });
+        });
+        if (existsSync(workDir))
+            rmSync(workDir, { recursive: true, force: true });
+        return result;
+    }
+    catch (err) {
+        return { success: false, output: null, error: err.message, durationMs: Date.now() - startTime };
+    }
+}
+// ─── Execute a local tool file (with runtime injection) ─────────────────────
+export async function executeLocalFile(filePath, input, options = {}) {
+    const startTime = Date.now();
+    const strictJson = options.strictJson === true;
+    try {
+        if (!existsSync(filePath)) {
+            return { success: false, output: null, error: `File not found: ${filePath}`, durationMs: Date.now() - startTime };
+        }
+        const source = readFileSync(filePath, 'utf-8');
+        const language = detectLanguageFromFile(filePath, source);
+        if (!language) {
+            return { success: false, output: null, error: `Unsupported file type: ${filePath}`, durationMs: Date.now() - startTime };
+        }
+        const workDir = join(tmpdir(), `devtopia-local-${randomUUID()}`);
+        mkdirSync(workDir, { recursive: true });
+        const ext = getToolExtension(language);
+        const toolFile = join(workDir, `tool${ext}`);
+        const wrappedSource = wrapSourceIfNeeded(source, language);
+        writeFileSync(toolFile, wrappedSource);
+        if (language === 'shebang') {
+            try {
+                chmodSync(toolFile, 0o755);
+            }
+            catch { }
+        }
+        writeRuntimeHelper(workDir, language);
+        const { cmd, args } = getRunner(language, toolFile);
+        const result = await new Promise((resolve) => {
+            const inputStr = JSON.stringify(input);
+            const cliArgs = resolveCliArgsForRuntime();
+            const env = {
+                ...process.env,
+                INPUT: inputStr,
+                ...(cliArgs ? { DEVTOPIA_CLI_ARGS: JSON.stringify(cliArgs) } : {}),
+            };
+            const proc = spawn(cmd, [...args, inputStr], {
+                cwd: workDir,
+                env,
+                timeout: 30000,
             });
+            let stdout = '';
+            let stderr = '';
+            proc.stdout.on('data', (data) => { stdout += data.toString(); });
+            proc.stderr.on('data', (data) => { stderr += data.toString(); });
             proc.on('close', (code) => {
                 const durationMs = Date.now() - startTime;
+                const trimmedOut = stdout.trim();
+                const trimmedErr = stderr.trim();
+                const tryParseJson = (text) => {
+                    if (!text)
+                        return null;
+                    try {
+                        return JSON.parse(text);
+                    }
+                    catch {
+                        return null;
+                    }
+                };
                 if (code !== 0) {
-                    resolve({
-                        success: false,
-                        output: null,
-                        error: stderr || `Process exited with code ${code}`,
-                        durationMs,
-                    });
+                    if (strictJson) {
+                        const parsed = tryParseJson(trimmedOut) ?? tryParseJson(trimmedErr);
+                        resolve({
+                            success: false,
+                            output: parsed,
+                            error: (parsed && typeof parsed === 'object' && parsed.error) ? String(parsed.error) : (trimmedErr || `Process exited with code ${code}`),
+                            durationMs,
+                        });
+                        return;
+                    }
+                    resolve({ success: false, output: null, error: trimmedErr || `Process exited with code ${code}`, durationMs });
                     return;
                 }
-                try {
-                    // Parse JSON output
-                    const output = JSON.parse(stdout.trim());
-                    resolve({
-                        success: true,
-                        output,
-                        durationMs,
-                    });
+                if (trimmedOut.length === 0) {
+                    if (strictJson) {
+                        resolve({ success: false, output: null, error: 'Tool produced no output', durationMs });
+                    }
+                    else {
+                        resolve({ success: true, output: '', durationMs });
+                    }
+                    return;
                 }
-                catch {
-                    // Return raw output if not JSON
-                    resolve({
-                        success: true,
-                        output: stdout.trim(),
-                        durationMs,
-                    });
+                const parsed = tryParseJson(trimmedOut);
+                if (parsed === null) {
+                    if (strictJson) {
+                        resolve({ success: false, output: null, error: 'Tool produced non-JSON output', durationMs });
+                    }
+                    else {
+                        resolve({ success: true, output: trimmedOut, durationMs });
+                    }
+                    return;
                 }
+                resolve({ success: true, output: parsed, durationMs });
             });
             proc.on('error', (err) => {
-                resolve({
-                    success: false,
-                    output: null,
-                    error: err.message,
-                    durationMs: Date.now() - startTime,
-                });
+                resolve({ success: false, output: null, error: err.message, durationMs: Date.now() - startTime });
             });
         });
-        // Cleanup temp directory after execution completes
-        if (existsSync(workDir)) {
+        if (existsSync(workDir))
             rmSync(workDir, { recursive: true, force: true });
-        }
         return result;
     }
     catch (err) {
-        const durationMs = Date.now() - startTime;
-        return {
-            success: false,
-            output: null,
-            error: err.message,
-            durationMs,
-        };
+        return { success: false, output: null, error: err.message, durationMs: Date.now() - startTime };
     }
 }