@tenex-chat/backend 0.9.1

package/src/commands/daemon.ts

@@ -0,0 +1,198 @@
+import { getDaemon } from "@/daemon";
+import { getNDK, initNDK } from "@/nostr/ndkClient";
+import { config } from "@/services/ConfigService";
+import { llmOpsRegistry } from "@/services/LLMOperationsRegistry";
+import { SchedulerService } from "@/services/scheduling";
+import { eventLoopMonitor } from "@/telemetry/EventLoopMonitor";
+import { logger } from "@/utils/logger";
+import { runInteractiveSetup } from "./setup/interactive";
+import chalk from "chalk";
+import { Command } from "commander";
+
+/**
+ * Daemon command - runs all projects in a single process
+ */
+export const daemonCommand = new Command("daemon")
+    .description("Start the TENEX daemon to manage all projects")
+    .option("-w, --whitelist <pubkeys>", "Comma-separated list of whitelisted pubkeys")
+    .option("-c, --config <path>", "Path to config file")
+    .option("-v, --verbose", "Enable verbose logging")
+    .option("-b, --boot <pattern>", "Auto-boot projects whose d-tag contains this pattern (can be used multiple times)", (value: string, prev: string[]) => {
+        return prev ? [...prev, value] : [value];
+    }, [])
+    .option("--supervised", "Run in supervised mode (enables graceful restart via SIGHUP)")
+    .action(async (options) => {
+        // Enable verbose logging if requested
+        if (options.verbose) {
+            process.env.LOG_LEVEL = "debug";
+        }
+
+        // Load configuration (MCP config will be loaded later per-project with metadataPath)
+        const { config: globalConfig, llms: globalLLMs } = await config.loadConfig();
+
+        // Initialize daemon logging
+        await logger.initDaemonLogging();
+
+        // Get whitelisted pubkeys
+        const whitelistedPubkeys = config.getWhitelistedPubkeys(
+            options.whitelist,
+            globalConfig
+        );
+
+        // Check for required configurations
+        const needsSetup =
+            whitelistedPubkeys.length === 0 ||
+            !globalLLMs.configurations ||
+            Object.keys(globalLLMs.configurations).length === 0;
+
+        if (needsSetup) {
+            if (whitelistedPubkeys.length === 0) {
+                logger.info("No whitelisted pubkeys found. Starting interactive setup...");
+            }
+            if (!globalLLMs.configurations || Object.keys(globalLLMs.configurations).length === 0) {
+                logger.info("No LLM configurations found. Starting interactive setup...");
+            }
+
+            // Run interactive setup
+            const setupConfig = await runInteractiveSetup();
+
+            // Save the setup configuration and reload
+            await config.saveGlobalConfig(setupConfig);
+        }
+
+        console.log(chalk.cyan("╔════════════════════════════════════════╗"));
+        console.log(chalk.cyan("║       TENEX Daemon Starting            ║"));
+        console.log(chalk.cyan("╚════════════════════════════════════════╝"));
+        console.log();
+
+        // Initialize NDK for Nostr communication
+        console.log(chalk.gray("🔌 Initializing Nostr connection..."));
+        await initNDK();
+        console.log(chalk.gray("✓ Nostr connected"));
+
+        // Get scheduler service instance (initialization deferred until callbacks are registered)
+        const schedulerService = SchedulerService.getInstance();
+
+        // DIAGNOSTIC: Start event loop monitoring for concurrent streaming bottleneck analysis
+        eventLoopMonitor.start(
+            () => llmOpsRegistry.getActiveOperationsCount(),
+            100, // Sample every 100ms
+            50   // Consider >50ms lag as blocked
+        );
+        console.log(chalk.gray("📊 Event loop monitoring enabled"));
+
+        // Get the daemon instance
+        console.log(chalk.gray("🚀 Starting daemon..."));
+        const daemon = getDaemon();
+
+        // Set supervised mode if enabled
+        if (options.supervised) {
+            daemon.setSupervisedMode(true);
+            console.log(chalk.yellow("🔄 Supervised mode enabled (SIGHUP triggers graceful restart)"));
+        }
+
+        // Set boot patterns if provided
+        const bootPatterns: string[] = options.boot || [];
+        if (bootPatterns.length > 0) {
+            daemon.setAutoBootPatterns(bootPatterns);
+            console.log(chalk.yellow(`🚀 Auto-boot patterns: ${bootPatterns.join(", ")}`));
+        }
+
+        // Register scheduler shutdown with daemon's shutdown handlers
+        daemon.addShutdownHandler(async () => {
+            schedulerService.shutdown();
+            // Stop event loop monitor and log final stats
+            const stats = eventLoopMonitor.getStats();
+            logger.info("[EventLoopMonitor] Final stats on shutdown", {
+                peakLagMs: stats.peakLagMs,
+                avgLagMs: Math.round(stats.avgLagMs * 100) / 100,
+                sampleCount: stats.sampleCount,
+                blockedCount: stats.blockedCount,
+                blockedPercentage: stats.sampleCount > 0
+                    ? Math.round((stats.blockedCount / stats.sampleCount) * 10000) / 100
+                    : 0,
+            });
+            eventLoopMonitor.stop();
+        });
+
+        try {
+            // Start the daemon
+            await daemon.start();
+            console.log(chalk.gray("✓ Daemon core started"));
+
+            // Load restart state if this is a restart (auto-boot previously booted projects)
+            if (options.supervised) {
+                await daemon.loadRestartState();
+            }
+
+            // Register project callbacks with scheduler service (dependency injection)
+            // This enables Layer 3 (SchedulerService) to use Layer 4 (Daemon) functionality
+            // without direct imports, maintaining architectural separation
+            schedulerService.setProjectCallbacks(
+                // Boot handler: called when a scheduled task needs to boot a project
+                async (projectId: string) => {
+                    await daemon.startRuntime(projectId);
+                },
+                // State resolver: called to check if a project is already running
+                (projectId: string) => {
+                    return daemon.getActiveRuntimes().has(projectId);
+                },
+                // Target resolver: called to resolve the target pubkey (may reroute to PM)
+                (projectId: string, originalTargetPubkey: string) => {
+                    const runtime = daemon.getActiveRuntimes().get(projectId);
+                    if (!runtime) {
+                        // Project not running, use original target
+                        return originalTargetPubkey;
+                    }
+
+                    const context = runtime.getContext();
+                    if (!context) {
+                        // No context available, use original target
+                        return originalTargetPubkey;
+                    }
+
+                    // Check if the target agent is in this project
+                    const targetAgent = context.getAgentByPubkey(originalTargetPubkey);
+                    if (targetAgent) {
+                        // Target agent exists in project, use original target
+                        return originalTargetPubkey;
+                    }
+
+                    // Target agent is NOT in this project - route to PM instead
+                    const pm = context.projectManager;
+                    if (!pm) {
+                        // No PM available, use original target
+                        return originalTargetPubkey;
+                    }
+
+                    return pm.pubkey;
+                }
+            );
+            console.log(chalk.gray("✓ Scheduler callbacks registered"));
+
+            // Initialize scheduler AFTER callbacks are registered
+            // This ensures catch-up tasks can use ensureProjectRunning() for auto-boot
+            console.log(chalk.gray("⚙️  Initializing scheduler service..."));
+            await schedulerService.initialize(getNDK(), ".tenex");
+            console.log(chalk.gray("✓ Scheduler initialized"));
+
+            console.log(chalk.green("✅ Daemon started successfully"));
+
+            // Log initial status
+            const status = daemon.getStatus();
+            console.log(chalk.blue("📊 Initial Status:"));
+            console.log(chalk.gray(`   Known Projects: ${status.knownProjects}`));
+            console.log(chalk.gray(`   Active Projects: ${status.activeProjects}`));
+            console.log(chalk.gray(`   Total Agents: ${status.totalAgents}`));
+            console.log();
+
+            // Keep the process alive
+            await new Promise(() => {
+                // This promise never resolves, keeping the daemon running
+            });
+        } catch (error) {
+            logger.error("Failed to start daemon", { error });
+            console.error(chalk.red("❌ Failed to start daemon:"), error);
+            process.exit(1);
+        }
+    });

package/src/commands/doctor.ts

@@ -0,0 +1,134 @@
+import { Command } from "commander";
+import chalk from "chalk";
+import { agentStorage, type StoredAgent } from "@/agents/AgentStorage";
+import { NDKAgentDefinition } from "@/events/NDKAgentDefinition";
+import { initNDK, getNDK } from "@/nostr/ndkClient";
+import { NDKPrivateKeySigner } from "@nostr-dev-kit/ndk";
+
+const refetchCommand = new Command("refetch")
+    .description("Refetch and update all agent definitions from Nostr")
+    .action(repairAgents);
+
+const orphansCommand = new Command("orphans")
+    .description("List agents not assigned to any project")
+    .option("--purge", "Delete orphaned agents")
+    .action(async (options) => {
+        await findOrphanedAgents(!!options.purge);
+    });
+
+const agentsCommand = new Command("agents")
+    .description("Agent diagnostics and repair")
+    .addCommand(refetchCommand)
+    .addCommand(orphansCommand);
+
+export const doctorCommand = new Command("doctor")
+    .description("Diagnose and repair TENEX state")
+    .addCommand(agentsCommand);
+
+function agentChanged(before: StoredAgent, after: StoredAgent): boolean {
+    if (before.name !== after.name) return true;
+    if (before.role !== after.role) return true;
+    if (before.description !== after.description) return true;
+    if (before.instructions !== after.instructions) return true;
+    if (before.useCriteria !== after.useCriteria) return true;
+    const oldTools = before.default?.tools?.slice().sort().join("\0") ?? "";
+    const newTools = after.default?.tools?.slice().sort().join("\0") ?? "";
+    return oldTools !== newTools;
+}
+
+async function repairAgents(): Promise<void> {
+    await agentStorage.initialize();
+    await initNDK();
+    const ndk = getNDK();
+
+    const agents = await agentStorage.getAllAgents();
+    const nostrAgents = agents.filter((a) => a.eventId);
+    const skipped = agents.length - nostrAgents.length;
+
+    console.log(chalk.blue(`Checking ${nostrAgents.length} Nostr agent(s)...`));
+
+    let updated = 0;
+    let failed = 0;
+
+    for (const agent of nostrAgents) {
+        const pubkey = new NDKPrivateKeySigner(agent.nsec).pubkey;
+        const label = `${agent.slug} (${pubkey.substring(0, 8)}...)`;
+
+        const event = await ndk.fetchEvent(agent.eventId!, { groupable: false });
+        if (!event) {
+            console.log(chalk.yellow(`  ⚠ ${label}: event not found on relays, skipping`));
+            failed++;
+            continue;
+        }
+
+        const agentDef = NDKAgentDefinition.from(event);
+        const toolTags = event.tags
+            .filter((t) => t[0] === "tool" && t[1])
+            .map((t) => t[1] as string);
+
+        const newDefault = {
+            ...agent.default,
+            tools: toolTags.length > 0 ? toolTags : undefined,
+        };
+        const updatedAgent = {
+            ...agent,
+            name: agentDef.title || agent.name,
+            role: agentDef.role || agent.role,
+            description: agentDef.description ?? agent.description,
+            instructions: agentDef.instructions ?? agent.instructions,
+            useCriteria: agentDef.useCriteria ?? agent.useCriteria,
+            default: Object.values(newDefault).some((v) => v !== undefined) ? newDefault : undefined,
+        };
+
+        const toolsDisplay = toolTags.length > 0 ? toolTags.join(", ") : "(none)";
+        const suffix = chalk.gray(`  [tools: ${toolsDisplay}]`);
+
+        if (agentChanged(agent, updatedAgent)) {
+            await agentStorage.saveAgent(updatedAgent);
+            console.log(chalk.green(`  ✓ ${label}: updated`) + suffix);
+            updated++;
+        } else {
+            console.log(chalk.gray(`  ${label}: ok`) + suffix);
+        }
+    }
+
+    console.log(
+        chalk.blue(
+            `\nDone: ${updated} updated, ${skipped} skipped (no eventId), ${failed} failed`
+        )
+    );
+}
+
+async function findOrphanedAgents(purge: boolean): Promise<void> {
+    await agentStorage.initialize();
+    const agents = await agentStorage.getAllAgents();
+
+    const orphans: Array<{ agent: StoredAgent; pubkey: string }> = [];
+    for (const agent of agents) {
+        const pubkey = new NDKPrivateKeySigner(agent.nsec).pubkey;
+        const projects = await agentStorage.getAgentProjects(pubkey);
+        if (projects.length === 0) {
+            orphans.push({ agent, pubkey });
+        }
+    }
+
+    if (orphans.length === 0) {
+        console.log(chalk.green("No orphaned agents found."));
+        return;
+    }
+
+    console.log(chalk.yellow(`Found ${orphans.length} orphaned agent(s):`));
+    for (const { agent, pubkey } of orphans) {
+        const source = agent.eventId ? `nostr:${agent.eventId.substring(0, 8)}...` : "local";
+        console.log(chalk.gray(`  ${agent.slug} (${pubkey.substring(0, 8)}...)  [${source}]`));
+    }
+
+    if (!purge) return;
+
+    console.log(chalk.blue(`\nPurging ${orphans.length} orphaned agent(s)...`));
+    for (const { agent, pubkey } of orphans) {
+        await agentStorage.deleteAgent(pubkey);
+        console.log(chalk.green(`  ✓ deleted ${agent.slug}`));
+    }
+    console.log(chalk.blue(`Done: ${orphans.length} deleted`));
+}

package/src/commands/setup/embed.ts

@@ -0,0 +1,228 @@
+import * as fileSystem from "@/lib/fs";
+import { PROVIDER_IDS } from "@/llm/providers/provider-ids";
+import {
+    type EmbeddingConfig,
+    EmbeddingProviderFactory,
+} from "@/services/rag/EmbeddingProviderFactory";
+import { config as configService } from "@/services/ConfigService";
+import { logger } from "@/utils/logger";
+import { Command } from "commander";
+import inquirer from "inquirer";
+
+/**
+ * Providers that support embeddings (OpenAI-compatible API)
+ */
+const EMBEDDING_CAPABLE_PROVIDERS = [
+    PROVIDER_IDS.OPENAI,
+    PROVIDER_IDS.OPENROUTER,
+] as const;
+
+/**
+ * Display names for providers
+ */
+const PROVIDER_DISPLAY_NAMES: Record<string, string> = {
+    [PROVIDER_IDS.OPENAI]: "OpenAI",
+    [PROVIDER_IDS.OPENROUTER]: "OpenRouter",
+};
+
+/**
+ * Common embedding models by provider
+ */
+const EMBEDDING_MODELS: Record<string, Array<{ name: string; value: string }>> = {
+    [PROVIDER_IDS.OPENAI]: [
+        { name: "text-embedding-3-small (fast, good quality)", value: "text-embedding-3-small" },
+        { name: "text-embedding-3-large (slower, best quality)", value: "text-embedding-3-large" },
+        { name: "text-embedding-ada-002 (legacy)", value: "text-embedding-ada-002" },
+    ],
+    [PROVIDER_IDS.OPENROUTER]: [
+        { name: "openai/text-embedding-3-small", value: "openai/text-embedding-3-small" },
+        { name: "openai/text-embedding-3-large", value: "openai/text-embedding-3-large" },
+        { name: "openai/text-embedding-ada-002", value: "openai/text-embedding-ada-002" },
+    ],
+};
+
+/**
+ * Command for configuring embedding provider
+ */
+export const embedCommand = new Command("embed")
+    .description(
+        "Configure embedding model for RAG (global by default, --project for current project)"
+    )
+    .option("--project", "Use project-specific configuration instead of global")
+    .action(async (options) => {
+        try {
+            const scope: "global" | "project" = options.project ? "project" : "global";
+            const projectPath = process.cwd();
+            const baseDir =
+                scope === "project"
+                    ? configService.getProjectPath(projectPath)
+                    : configService.getGlobalPath();
+
+            if (scope === "project") {
+                // Check if we're in a TENEX project
+                if (!(await fileSystem.directoryExists(baseDir))) {
+                    logger.error(
+                        "No .tenex directory found. Make sure you're in a TENEX project directory."
+                    );
+                    process.exitCode = 1;
+                    return;
+                }
+            } else {
+                // Ensure global config directory exists
+                await fileSystem.ensureDirectory(baseDir);
+            }
+
+            // Load existing configuration
+            const existing = await EmbeddingProviderFactory.loadConfiguration({
+                scope,
+                projectPath: scope === "project" ? projectPath : undefined,
+            });
+
+            // Load configured providers from providers.json
+            const providersConfig = await configService.loadTenexProviders(configService.getGlobalPath());
+            if (Object.keys(providersConfig.providers).length === 0) {
+                logger.error(
+                    "No providers configured. Run `tenex setup providers` before configuring embeddings."
+                );
+                process.exitCode = 1;
+                return;
+            }
+            const configuredProviders = Object.keys(providersConfig.providers);
+
+            // Build provider choices: local + any configured embedding-capable providers
+            const providerChoices: Array<{ name: string; value: string }> = [
+                { name: "Local Transformers (runs on your machine)", value: "local" },
+            ];
+
+            for (const providerId of EMBEDDING_CAPABLE_PROVIDERS) {
+                if (configuredProviders.includes(providerId)) {
+                    const displayName = PROVIDER_DISPLAY_NAMES[providerId] || providerId;
+                    providerChoices.push({
+                        name: `${displayName} (configured)`,
+                        value: providerId,
+                    });
+                }
+            }
+
+            // Prompt for provider selection
+            const { provider } = await inquirer.prompt([
+                {
+                    type: "select",
+                    name: "provider",
+                    message: "Select embedding provider:",
+                    choices: providerChoices,
+                    default: existing?.provider || "local",
+                },
+            ]);
+
+            let model: string;
+
+            if (provider !== "local") {
+                // OpenAI-compatible provider configuration
+                const displayName = PROVIDER_DISPLAY_NAMES[provider] || provider;
+                const modelChoices = EMBEDDING_MODELS[provider] || [
+                    { name: "Enter custom model ID", value: "custom" },
+                ];
+
+                // Add custom model option if not already present
+                if (!modelChoices.some((c) => c.value === "custom")) {
+                    modelChoices.push({ name: "Enter custom model ID", value: "custom" });
+                }
+
+                const providerAnswers = await inquirer.prompt([
+                    {
+                        type: "select",
+                        name: "model",
+                        message: `Select ${displayName} embedding model:`,
+                        choices: modelChoices,
+                        default: existing?.provider === provider ? existing?.model : modelChoices[0]?.value,
+                    },
+                ]);
+
+                if (providerAnswers.model === "custom") {
+                    const customAnswer = await inquirer.prompt([
+                        {
+                            type: "input",
+                            name: "customModel",
+                            message: "Enter model ID:",
+                            validate: (input: string) =>
+                                input.trim().length > 0 || "Model ID cannot be empty",
+                        },
+                    ]);
+                    model = customAnswer.customModel;
+                } else {
+                    model = providerAnswers.model;
+                }
+            } else {
+                // Local transformer configuration
+                const localAnswers = await inquirer.prompt([
+                    {
+                        type: "select",
+                        name: "model",
+                        message: "Select local embedding model:",
+                        choices: [
+                            {
+                                name: "all-MiniLM-L6-v2 (default, fast, good for general use)",
+                                value: "Xenova/all-MiniLM-L6-v2",
+                            },
+                            {
+                                name: "all-mpnet-base-v2 (larger, better quality)",
+                                value: "Xenova/all-mpnet-base-v2",
+                            },
+                            {
+                                name: "paraphrase-multilingual-MiniLM-L12-v2 (multilingual support)",
+                                value: "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
+                            },
+                            {
+                                name: "Custom model (enter HuggingFace model ID)",
+                                value: "custom",
+                            },
+                        ],
+                        default: existing?.model || "Xenova/all-MiniLM-L6-v2",
+                    },
+                ]);
+
+                if (localAnswers.model === "custom") {
+                    const customAnswer = await inquirer.prompt([
+                        {
+                            type: "input",
+                            name: "customModel",
+                            message:
+                                "Enter HuggingFace model ID (e.g., sentence-transformers/all-MiniLM-L6-v2):",
+                            validate: (input: string) =>
+                                input.trim().length > 0 || "Model ID cannot be empty",
+                        },
+                    ]);
+                    model = customAnswer.customModel;
+                } else {
+                    model = localAnswers.model;
+                }
+            }
+
+            // Save configuration (API key comes from providers.json, not stored here)
+            const embeddingConfig: EmbeddingConfig = {
+                provider,
+                model,
+            };
+
+            // Save model selection to embed.json
+            await EmbeddingProviderFactory.saveConfiguration(embeddingConfig, scope, {
+                projectPath: scope === "project" ? projectPath : undefined,
+            });
+
+            logger.info(
+                `✅ Embedding configuration saved to ${scope} config\n` +
+                    `   Provider: ${provider}\n` +
+                    `   Model: ${model}`
+            );
+        } catch (error: unknown) {
+            // Handle SIGINT (Ctrl+C) gracefully
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            if (errorMessage?.includes("SIGINT") || errorMessage?.includes("force closed")) {
+                return;
+            }
+
+            logger.error(`Failed to configure embedding model: ${error}`);
+            process.exitCode = 1;
+        }
+    });