@elizaos/agent 2.0.0-alpha.144 → 2.0.0-alpha.152

package/packages/agent/src/evals/coordinator-preflight.js

@@ -1,296 +1 @@
-import { execFileSync } from "node:child_process";
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { isConnectorConfigured } from "../config/plugin-auto-enable.js";
-import { CoordinatorEvalClient, resolveCoordinatorEvalBaseUrl, } from "./coordinator-eval-client.js";
-const SUPPORTED_CONNECTOR_CHANNELS = [
-    "discord",
-    "telegram",
-    "slack",
-    "whatsapp",
-    "signal",
-    "matrix",
-    "wechat",
-];
-const CHANNEL_CONNECTOR_KEYS = {
-    discord: ["discord"],
-    telegram: ["telegram", "telegramAccount"],
-    slack: ["slack"],
-    whatsapp: ["whatsapp"],
-    signal: ["signal"],
-    matrix: ["matrix"],
-    wechat: ["wechat"],
-};
-function getHomeDir() {
-    return (process.env.HOME?.trim() || process.env.USERPROFILE?.trim() || os.homedir());
-}
-function readJsonFile(filePath) {
-    try {
-        const parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
-        return parsed && typeof parsed === "object" && !Array.isArray(parsed)
-            ? parsed
-            : null;
-    }
-    catch {
-        return null;
-    }
-}
-function resolveElizaConfigPath() {
-    const explicit = process.env.ELIZA_CONFIG_PATH?.trim() ||
-        process.env.ELIZA_CONFIG_PATH?.trim();
-    if (explicit)
-        return explicit;
-    const stateDir = process.env.ELIZA_STATE_DIR?.trim() ||
-        process.env.ELIZA_STATE_DIR?.trim() ||
-        path.join(getHomeDir(), ".eliza");
-    const namespace = process.env.ELIZA_NAMESPACE?.trim();
-    const filename = !namespace || namespace === "eliza" ? "eliza.json" : `${namespace}.json`;
-    return path.join(stateDir, filename);
-}
-function commandExists(command) {
-    try {
-        execFileSync(process.platform === "win32" ? "where" : "which", [command], {
-            stdio: "ignore",
-            timeout: 3_000,
-        });
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
-function detectShareCapabilities(config) {
-    const capabilities = [];
-    const gateway = config && typeof config.gateway === "object" && config.gateway
-        ? config.gateway
-        : null;
-    const gatewayTailscale = gateway && typeof gateway.tailscale === "object" && gateway.tailscale
-        ? gateway.tailscale
-        : null;
-    const gatewayRemote = gateway && typeof gateway.remote === "object" && gateway.remote
-        ? gateway.remote
-        : null;
-    const tailscaleMode = typeof gatewayTailscale?.mode === "string" ? gatewayTailscale.mode : null;
-    if (tailscaleMode && tailscaleMode !== "off") {
-        capabilities.push(`tailscale:${tailscaleMode}`);
-    }
-    if (typeof gatewayRemote?.url === "string" && gatewayRemote.url.trim()) {
-        capabilities.push("gateway-remote-url");
-    }
-    if (typeof gatewayRemote?.sshTarget === "string" &&
-        gatewayRemote.sshTarget.trim()) {
-        capabilities.push("gateway-remote-ssh");
-    }
-    if (typeof gateway?.mode === "string" && gateway.mode === "remote") {
-        capabilities.push("gateway-remote-mode");
-    }
-    return capabilities;
-}
-function normalizeConnectorConfig(connectors) {
-    const normalized = {};
-    for (const [key, value] of Object.entries(connectors)) {
-        if (!value || typeof value !== "object" || Array.isArray(value))
-            continue;
-        normalized[key] = value;
-    }
-    return normalized;
-}
-function normalizeConnectorHealth(connectors) {
-    const normalized = {};
-    if (!connectors)
-        return normalized;
-    for (const [key, value] of Object.entries(connectors)) {
-        if (typeof value !== "string" || !value.trim())
-            continue;
-        normalized[key] = value.trim();
-    }
-    return normalized;
-}
-function connectorKeysForChannel(channel) {
-    return CHANNEL_CONNECTOR_KEYS[channel];
-}
-function buildChannelReadiness(params) {
-    const readiness = [
-        {
-            channel: "app_chat",
-            connectorKeys: [],
-            configured: true,
-            configReady: true,
-            healthStatuses: {},
-            available: true,
-            reason: "App chat is always available when the Eliza API is reachable.",
-        },
-    ];
-    for (const channel of SUPPORTED_CONNECTOR_CHANNELS) {
-        const connectorKeys = connectorKeysForChannel(channel);
-        const configuredKeys = connectorKeys.filter((key) => params.connectorConfig[key] !== undefined);
-        const healthStatuses = Object.fromEntries(connectorKeys.flatMap((key) => typeof params.connectorHealth[key] === "string"
-            ? [[key, params.connectorHealth[key]]]
-            : []));
-        const configReady = configuredKeys.some((key) => isConnectorConfigured(key, params.connectorConfig[key]));
-        const available = connectorKeys.some((key) => typeof params.connectorHealth[key] === "string" &&
-            params.connectorHealth[key] === "ok" &&
-            params.connectorConfig[key] !== undefined &&
-            isConnectorConfigured(key, params.connectorConfig[key]));
-        let reason;
-        if (available) {
-            reason = "Eliza reported a live connector runtime for this channel.";
-        }
-        else if (configuredKeys.length === 0) {
-            reason = "This channel is not configured in Eliza.";
-        }
-        else if (!configReady) {
-            reason =
-                "This channel is present in config but is missing required credentials or auth state.";
-        }
-        else if (Object.values(healthStatuses).includes("missing")) {
-            reason =
-                "This channel is configured, but the runtime did not load its connector plugin.";
-        }
-        else if (Object.values(healthStatuses).includes("configured")) {
-            reason =
-                "This channel is configured, but Eliza has not confirmed a live connector runtime yet.";
-        }
-        else if (Object.values(healthStatuses).includes("unknown")) {
-            reason =
-                "This channel is configured, but the connector health monitor could not classify its runtime state.";
-        }
-        else if (Object.keys(healthStatuses).length === 0) {
-            reason =
-                "This channel is configured, but Eliza did not report runtime health for it.";
-        }
-        else {
-            reason = "This channel is not currently available for live evaluation.";
-        }
-        readiness.push({
-            channel,
-            connectorKeys,
-            configured: configuredKeys.length > 0,
-            configReady,
-            healthStatuses,
-            available,
-            reason,
-        });
-    }
-    return readiness;
-}
-export async function runCoordinatorPreflight(options) {
-    const baseUrl = resolveCoordinatorEvalBaseUrl(options?.baseUrl);
-    const client = new CoordinatorEvalClient(baseUrl);
-    const configPath = resolveElizaConfigPath();
-    const config = readJsonFile(configPath);
-    const checks = [];
-    const addCheck = (id, status, summary, details) => {
-        checks.push({ id, status, summary, ...(details ? { details } : {}) });
-    };
-    addCheck("local-cli-codex", commandExists("codex") ? "pass" : "fail", commandExists("codex")
-        ? "Codex CLI is installed."
-        : "Codex CLI is not installed.");
-    addCheck("local-cli-claude", commandExists("claude") ? "pass" : "fail", commandExists("claude")
-        ? "Claude Code CLI is installed."
-        : "Claude Code CLI is not installed.");
-    addCheck("local-auth-files", fs.existsSync(path.join(getHomeDir(), ".codex", "auth.json")) ||
-        fs.existsSync(path.join(getHomeDir(), ".claude", ".credentials.json"))
-        ? "pass"
-        : "warn", "Local Codex/Claude auth files were inspected.", {
-        codexAuthFile: fs.existsSync(path.join(getHomeDir(), ".codex", "auth.json")),
-        claudeCredentialsFile: fs.existsSync(path.join(getHomeDir(), ".claude", ".credentials.json")),
-    });
-    const shareCapabilities = detectShareCapabilities(config);
-    addCheck("share-capabilities", shareCapabilities.length > 0 ? "pass" : "warn", shareCapabilities.length > 0
-        ? "Share or remote-preview capabilities were discovered in config."
-        : "No explicit remote share capability was discovered in config.", { shareCapabilities, configPath });
-    let coordinatorStatusResponse;
-    try {
-        coordinatorStatusResponse = await client.requestJson("/api/coding-agents/coordinator/status");
-    }
-    catch (error) {
-        addCheck("eliza-api", "fail", "Eliza API is not reachable at the configured base URL.", {
-            baseUrl,
-            error: error instanceof Error ? error.message : String(error),
-        });
-        return {
-            ok: false,
-            baseUrl,
-            configPath,
-            availableChannels: [],
-            supportedConnectors: SUPPORTED_CONNECTOR_CHANNELS,
-            channelReadiness: [],
-            shareCapabilities,
-            checks,
-        };
-    }
-    const subscriptionStatus = await client.requestJson("/api/subscription/status");
-    const providerMap = new Map((subscriptionStatus.providers ?? []).flatMap((provider) => provider.provider ? [[provider.provider, provider]] : []));
-    const codexProvider = providerMap.get("openai-codex");
-    const claudeProvider = providerMap.get("anthropic-subscription");
-    addCheck("subscription-openai-codex", codexProvider?.configured && codexProvider.valid ? "pass" : "fail", codexProvider?.configured && codexProvider.valid
-        ? "OpenAI Codex subscription is configured and valid."
-        : "OpenAI Codex subscription is missing or invalid.", codexProvider);
-    addCheck("subscription-anthropic", claudeProvider?.configured && claudeProvider.valid ? "pass" : "fail", claudeProvider?.configured && claudeProvider.valid
-        ? "Claude subscription is configured and valid for task-agent use."
-        : "Claude subscription is missing or invalid for task-agent use.", claudeProvider);
-    const frameworks = Array.isArray(coordinatorStatusResponse?.frameworks)
-        ? coordinatorStatusResponse.frameworks
-        : [];
-    const frameworkMap = new Map(frameworks.flatMap((framework) => framework.id ? [[framework.id, framework]] : []));
-    for (const id of ["codex", "claude"]) {
-        const framework = frameworkMap.get(id);
-        const ready = framework?.installed === true &&
-            (framework.authReady === true || framework.subscriptionReady === true);
-        addCheck(`framework-${id}`, ready ? "pass" : "fail", ready
-            ? `${id} is installed and ready for coordinator task execution.`
-            : `${id} is not ready for coordinator task execution.`, framework);
-    }
-    const trajectoryConfig = await client.requestJson("/api/trajectories/config");
-    addCheck("trajectory-logging", trajectoryConfig.enabled === true ? "pass" : "fail", trajectoryConfig.enabled === true
-        ? "Trajectory logging is enabled."
-        : "Trajectory logging is disabled.", trajectoryConfig);
-    const connectorsResponse = await client.requestJson("/api/connectors");
-    const healthResponse = await client.requestJson("/api/health");
-    const connectorConfig = normalizeConnectorConfig(connectorsResponse.connectors ?? {});
-    const connectorHealth = normalizeConnectorHealth(healthResponse.connectors);
-    const channelReadiness = buildChannelReadiness({
-        connectorConfig,
-        connectorHealth,
-    });
-    const availableChannels = channelReadiness
-        .filter((channel) => channel.available)
-        .map((channel) => channel.channel);
-    const readyConnectorChannels = channelReadiness
-        .filter((channel) => channel.channel !== "app_chat" && channel.available)
-        .map((channel) => channel.channel);
-    const configuredConnectorChannels = channelReadiness
-        .filter((channel) => channel.channel !== "app_chat" && channel.configured)
-        .map((channel) => channel.channel);
-    addCheck("connectors", readyConnectorChannels.length > 0
-        ? "pass"
-        : configuredConnectorChannels.length > 0
-            ? "fail"
-            : "warn", readyConnectorChannels.length > 0
-        ? "At least one external connector is live and ready for evaluation."
-        : configuredConnectorChannels.length > 0
-            ? "External connectors are configured, but none are currently live."
-            : "No external connectors are configured; live eval coverage is limited to app chat.", {
-        readyChannels: readyConnectorChannels,
-        configuredChannels: configuredConnectorChannels,
-        supportedConnectorChannels: SUPPORTED_CONNECTOR_CHANNELS,
-        connectorHealth,
-    });
-    for (const readiness of channelReadiness) {
-        if (readiness.channel === "app_chat")
-            continue;
-        addCheck(`channel-${readiness.channel}`, readiness.available ? "pass" : readiness.configured ? "fail" : "warn", readiness.reason, readiness);
-    }
-    return {
-        ok: checks.every((check) => check.status !== "fail"),
-        baseUrl,
-        configPath,
-        availableChannels,
-        supportedConnectors: SUPPORTED_CONNECTOR_CHANNELS,
-        channelReadiness,
-        shareCapabilities,
-        checks,
-    };
-}
+export * from "@elizaos/app-task-coordinator/evals/coordinator-preflight";

package/packages/agent/src/evals/coordinator-scenarios.d.ts

@@ -1,24 +1,2 @@
-export type CoordinatorEvalChannel = "app_chat" | "discord" | "telegram" | "slack" | "whatsapp" | "signal" | "matrix" | "wechat";
-export type CoordinatorScenarioFamily = "build_and_edit" | "continuation" | "preview_and_share" | "pause_resume_stop" | "history_and_reporting" | "research_and_planning" | "connector_behavior" | "recovery_and_failover" | "task_management" | "visibility_and_audit";
-export type CoordinatorScenarioProfile = "smoke" | "core" | "full";
-export type CoordinatorScenarioTurn = {
-    speaker: "user";
-    text: string;
-};
-export interface CoordinatorScenario {
-    id: string;
-    family: CoordinatorScenarioFamily;
-    profile: CoordinatorScenarioProfile;
-    title: string;
-    summary: string;
-    channels: CoordinatorEvalChannel[];
-    requiredCapabilities: string[];
-    turns: CoordinatorScenarioTurn[];
-    doneWhen: string[];
-    evidence: string[];
-}
-export declare const coordinatorScenarios: CoordinatorScenario[];
-export declare const coordinatorScenarioById: Map<string, CoordinatorScenario>;
-export declare function listCoordinatorScenarios(profile?: CoordinatorScenarioProfile): CoordinatorScenario[];
-export declare function countCoordinatorScenariosByFamily(): Record<CoordinatorScenarioFamily, number>;
+export * from "@elizaos/app-task-coordinator/evals/coordinator-scenarios";
 //# sourceMappingURL=coordinator-scenarios.d.ts.map

package/packages/agent/src/evals/coordinator-scenarios.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"coordinator-scenarios.d.ts","sourceRoot":"","sources":["../../../../../src/evals/coordinator-scenarios.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,sBAAsB,GAC9B,UAAU,GACV,SAAS,GACT,UAAU,GACV,OAAO,GACP,UAAU,GACV,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;AAEb,MAAM,MAAM,yBAAyB,GACjC,gBAAgB,GAChB,cAAc,GACd,mBAAmB,GACnB,mBAAmB,GACnB,uBAAuB,GACvB,uBAAuB,GACvB,oBAAoB,GACpB,uBAAuB,GACvB,iBAAiB,GACjB,sBAAsB,CAAC;AAE3B,MAAM,MAAM,0BAA0B,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;AAEnE,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,yBAAyB,CAAC;IAClC,OAAO,EAAE,0BAA0B,CAAC;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,sBAAsB,EAAE,CAAC;IACnC,oBAAoB,EAAE,MAAM,EAAE,CAAC;IAC/B,KAAK,EAAE,uBAAuB,EAAE,CAAC;IACjC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAiBD,eAAO,MAAM,oBAAoB,EAAE,mBAAmB,EAunCrD,CAAC;AAEF,eAAO,MAAM,uBAAuB,kCAEnC,CAAC;AAEF,wBAAgB,wBAAwB,CACtC,OAAO,GAAE,0BAAmC,GAC3C,mBAAmB,EAAE,CAYvB;AAED,wBAAgB,iCAAiC,IAAI,MAAM,CACzD,yBAAyB,EACzB,MAAM,CACP,CAmBA"}
+{"version":3,"file":"coordinator-scenarios.d.ts","sourceRoot":"","sources":["../../../../../src/evals/coordinator-scenarios.ts"],"names":[],"mappings":"AAAA,cAAc,2DAA2D,CAAC"}