@mcoda/core 0.1.8 → 0.1.11

package/dist/services/agents/GatewayAgentService.js

@@ -1,16 +1,21 @@
 import path from "node:path";
 import fs from "node:fs";
+import { fileURLToPath } from "node:url";
 import { AgentService } from "@mcoda/agents";
 import { DocdexClient } from "@mcoda/integrations";
 import { GlobalRepository, WorkspaceRepository } from "@mcoda/db";
-import { canonicalizeCommandName, getCommandRequiredCapabilities } from "@mcoda/shared";
+import { READY_TO_CODE_REVIEW, canonicalizeCommandName, getCommandRequiredCapabilities } from "@mcoda/shared";
 import { JobService } from "../jobs/JobService.js";
 import { TaskSelectionService } from "../execution/TaskSelectionService.js";
 import { RoutingService } from "./RoutingService.js";
+import { isDocContextExcluded, normalizeDocType } from "../shared/ProjectGuidance.js";
 const DEFAULT_GATEWAY_PROMPT = [
     "You are the gateway agent. Read the task context and docdex snippets, digest the task, decide what is done vs. remaining, and plan the work.",
     "You must identify concrete file paths to modify or create before offloading.",
+    "If new directories are required for planned files, list them explicitly in dirsToCreate.",
     "Do not use placeholders like (unknown), TBD, or glob patterns in file paths.",
+    "Do not assume repository structure; only name paths grounded in provided file content or docdex context.",
+    "If you add or modify tests, ensure tests/all.js is updated (or state that it already covers the new tests).",
     "If docdex returns no results, say so in docdexNotes.",
     "Do not leave currentState, todo, or understanding blank.",
     "Put reasoningSummary near the top of the JSON object so it appears early in the stream.",
@@ -28,12 +33,15 @@ const DEFAULT_GATEWAY_PROMPT = [
     '  "discipline": "backend|frontend|uiux|docs|architecture|qa|planning|ops|other",',
     '  "filesLikelyTouched": ["path/to/file.ext"],',
     '  "filesToCreate": ["path/to/new_file.ext"],',
+    '  "dirsToCreate": ["path/to/new_dir"],',
     '  "assumptions": ["assumption 1"],',
     '  "risks": ["risk 1"],',
     '  "docdexNotes": ["notes about docdex coverage/gaps"]',
     "}",
     "If information is missing, keep arrays empty and mention the gap in assumptions or docdexNotes.",
 ].join("\n");
+const REPO_PROMPTS_DIR = fileURLToPath(new URL("../../../../../prompts/", import.meta.url));
+const resolveRepoPromptPath = (filename) => path.join(REPO_PROMPTS_DIR, filename);
 const REQUIRED_PROMPT_MARKERS = [
     '"summary"',
     '"reasoningSummary"',
@@ -42,27 +50,63 @@ const REQUIRED_PROMPT_MARKERS = [
     '"understanding"',
     '"filesLikelyTouched"',
     '"filesToCreate"',
+    '"dirsToCreate"',
 ];
 const hasRequiredPromptMarkers = (content) => REQUIRED_PROMPT_MARKERS.every((marker) => content.includes(marker));
 const DEFAULT_JOB_PROMPT = "You are an mcoda agent that follows workspace runbooks and responds with actionable, concise output.";
 const DEFAULT_CHARACTER_PROMPT = "Write clearly, avoid hallucinations, cite assumptions, and prioritize risk mitigation for the user.";
-const extractJson = (raw) => {
-    const fenced = raw.match(/```json([\s\S]*?)```/);
-    const candidate = fenced ? fenced[1] : raw;
-    const start = candidate.indexOf("{");
-    const end = candidate.lastIndexOf("}");
-    if (start === -1 || end === -1 || end <= start)
-        return undefined;
-    const body = candidate.slice(start, end + 1);
+const ROUTING_PROMPT_MARKERS = [
+    "routing gateway",
+    "choose a route",
+    "devstral-local",
+    "glm-worker",
+    "codex-architect",
+    "complexity from 1 to 5",
+];
+const sanitizeAgentOutput = (output) => {
+    if (!output.includes("[agent-io]"))
+        return output;
+    const lines = output.split(/\r?\n/);
+    const cleaned = [];
+    for (const line of lines) {
+        if (!line.includes("[agent-io]")) {
+            cleaned.push(line);
+            continue;
+        }
+        if (/^\s*\[agent-io\]\s*(?:begin|input|meta)/i.test(line)) {
+            continue;
+        }
+        const stripped = line.replace(/^\s*\[agent-io\]\s*(?:output\s*)?/i, "");
+        if (stripped.trim())
+            cleaned.push(stripped);
+    }
+    return cleaned.join("\n");
+};
+const extractJsonOnly = (raw) => {
+    const trimmed = raw.trim();
+    if (!trimmed)
+        return { jsonOnly: false };
+    let candidate = trimmed;
+    if (trimmed.startsWith("```")) {
+        const match = trimmed.match(/^```(?:json)?\s*([\s\S]*?)```$/i);
+        if (!match)
+            return { jsonOnly: false };
+        candidate = match[1].trim();
+    }
+    else if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
+        return { jsonOnly: false };
+    }
     try {
-        return JSON.parse(body);
+        return { payload: JSON.parse(candidate), jsonOnly: true };
     }
     catch {
-        return undefined;
+        return { jsonOnly: true };
     }
 };
 const estimateTokens = (text) => Math.max(1, Math.ceil((text ?? "").length / 4));
 const clamp = (value, min, max) => Math.min(max, Math.max(min, value));
+const STRONG_TIER_MIN_COMPLEXITY = 5;
+const SPECIALIST_TIER_MIN_COMPLEXITY = 8;
 const normalizeList = (value) => {
     if (Array.isArray(value))
         return value.map((item) => String(item)).filter(Boolean);
@@ -81,6 +125,20 @@ const normalizeTextField = (value) => {
     }
     return undefined;
 };
+const isRoutingPrompt = (value) => {
+    const lower = value.toLowerCase();
+    return ROUTING_PROMPT_MARKERS.some((marker) => lower.includes(marker));
+};
+const sanitizeGatewayPrompt = (value) => {
+    if (!value)
+        return undefined;
+    const trimmed = value.trim();
+    if (!trimmed)
+        return undefined;
+    if (isRoutingPrompt(trimmed))
+        return undefined;
+    return trimmed;
+};
 const isPlaceholderPath = (value) => {
     const lower = value.trim().toLowerCase();
     if (!lower)
@@ -93,7 +151,77 @@ const isPlaceholderPath = (value) => {
     return false;
 };
 const normalizeFileList = (value) => normalizeList(value).map((item) => item.trim()).filter((item) => item.length > 0 && !isPlaceholderPath(item));
-const listMissingFields = (raw) => {
+const normalizeDirList = (value) => normalizeList(value)
+    .map((item) => item.trim().replace(/[/\\]+$/, ""))
+    .filter((item) => item.length > 0 && !isPlaceholderPath(item));
+const isDirMarker = (value) => {
+    const trimmed = value.trim();
+    if (!trimmed)
+        return false;
+    if (/[/\\]$/.test(trimmed))
+        return true;
+    if (/^\s*(?:dir|directory)\s*:/i.test(trimmed))
+        return true;
+    if (/\(\s*dir\s*\)\s*$/i.test(trimmed))
+        return true;
+    return false;
+};
+const stripDirMarker = (value) => {
+    let trimmed = value.trim();
+    trimmed = trimmed.replace(/^\s*(?:dir|directory)\s*:/i, "");
+    trimmed = trimmed.replace(/\(\s*dir\s*\)\s*$/i, "");
+    trimmed = trimmed.replace(/[/\\]+$/, "");
+    return trimmed.trim();
+};
+const splitFileAndDirEntries = (values) => {
+    const files = [];
+    const dirs = [];
+    for (const value of values) {
+        const trimmed = value.trim();
+        if (!trimmed)
+            continue;
+        if (isDirMarker(trimmed)) {
+            const cleaned = stripDirMarker(trimmed);
+            if (cleaned)
+                dirs.push(cleaned);
+            continue;
+        }
+        files.push(trimmed);
+    }
+    return { files, dirs };
+};
+const hasFileContextJustification = (raw) => {
+    const notes = [...normalizeList(raw?.assumptions), ...normalizeList(raw?.docdexNotes)]
+        .map((item) => item.toLowerCase())
+        .join(" ");
+    if (!notes.trim())
+        return false;
+    const patterns = [
+        /\bno file(?:s)?\b/,
+        /\bmissing file(?:s)?\b/,
+        /\bunknown file(?:s)?\b/,
+        /\bfile context\b/,
+        /\binsufficient context\b/,
+        /\bnot enough context\b/,
+        /\bnot provided\b/,
+        /\bdocdex unavailable\b/,
+        /\bdocdex missing\b/,
+        /\bno matching docs?\b/,
+        /\bno matching documents?\b/,
+        /\bno results\b/,
+    ];
+    return patterns.some((pattern) => pattern.test(notes));
+};
+const assessFileListCoverage = (raw) => {
+    const filesLikelyTouched = normalizeFileList(raw?.filesLikelyTouched);
+    const rawCreate = normalizeList(raw?.filesToCreate);
+    const splitCreate = splitFileAndDirEntries(rawCreate);
+    const filesToCreate = normalizeFileList(splitCreate.files);
+    const dirsToCreate = [...normalizeDirList(raw?.dirsToCreate), ...normalizeDirList(splitCreate.dirs)];
+    const empty = filesLikelyTouched.length === 0 && filesToCreate.length === 0 && dirsToCreate.length === 0;
+    return { empty, justified: empty && hasFileContextJustification(raw) };
+};
+const listMissingFields = (raw, options) => {
     const missing = [];
     const summary = normalizeTextField(raw?.summary);
     const reasoningSummary = normalizeTextField(raw?.reasoningSummary);
@@ -102,7 +230,11 @@ const listMissingFields = (raw) => {
     const understanding = normalizeTextField(raw?.understanding);
     const plan = normalizeList(raw?.plan);
     const filesLikelyTouched = normalizeFileList(raw?.filesLikelyTouched);
-    const filesToCreate = normalizeFileList(raw?.filesToCreate);
+    const rawCreate = normalizeList(raw?.filesToCreate);
+    const splitCreate = splitFileAndDirEntries(rawCreate);
+    const filesToCreate = normalizeFileList(splitCreate.files);
+    const dirsToCreate = [...normalizeDirList(raw?.dirsToCreate), ...normalizeDirList(splitCreate.dirs)];
+    const allowEmptyFiles = options?.allowEmptyFiles ?? false;
     if (!summary)
         missing.push("summary");
     if (!reasoningSummary)
@@ -115,8 +247,9 @@ const listMissingFields = (raw) => {
         missing.push("understanding");
     if (plan.length === 0)
         missing.push("plan");
-    if (filesLikelyTouched.length === 0 && filesToCreate.length === 0)
-        missing.push("files");
+    if (!allowEmptyFiles && filesLikelyTouched.length === 0 && filesToCreate.length === 0 && dirsToCreate.length === 0) {
+        missing.push("filesLikelyTouched", "filesToCreate", "dirsToCreate");
+    }
     return missing;
 };
 const normalizeDiscipline = (value) => {
@@ -174,24 +307,33 @@ const scoreUsage = (discipline, bestUsage, capabilities) => {
         score += 0.5;
     return score;
 };
+const EXPLORATION_RATE = 0.1;
 const DEFAULT_STATUS_FILTER = [
     "not_started",
     "in_progress",
-    "blocked",
-    "ready_to_review",
+    READY_TO_CODE_REVIEW,
     "ready_to_qa",
     "completed",
     "cancelled",
     "failed",
     "skipped",
 ];
-const summarizeDoc = (doc, index) => {
+const summarizeDoc = (doc, index, warnings) => {
     const title = doc.title ?? doc.path ?? doc.id ?? `doc-${index + 1}`;
     const excerptSource = doc.segments?.[0]?.content ?? doc.content ?? "";
     const excerpt = excerptSource ? (excerptSource.length > 480 ? `${excerptSource.slice(0, 480)}...` : excerptSource) : undefined;
+    const normalized = normalizeDocType({
+        docType: doc.docType,
+        path: doc.path,
+        title: doc.title,
+        content: excerptSource,
+    });
+    if (normalized.downgraded && warnings) {
+        warnings.push(`Docdex docType downgraded from SDS to DOC for ${doc.path ?? doc.title ?? doc.id}: ${normalized.reason ?? "not_sds"}`);
+    }
     return {
         id: doc.id ?? `doc-${index + 1}`,
-        docType: doc.docType,
+        docType: normalized.docType,
         title,
         path: doc.path,
         excerpt,
@@ -210,6 +352,25 @@ const buildDocContext = (docs) => {
         }),
     ].join("\n");
 };
+const PLACEHOLDER_DEPENDENCY_PATTERN = /^(?:t|task-?)\d+$/i;
+const normalizeDependencies = (deps) => {
+    if (!deps?.length)
+        return undefined;
+    const seen = new Set();
+    const filtered = [];
+    for (const dep of deps) {
+        const trimmed = dep.trim();
+        if (!trimmed)
+            continue;
+        if (PLACEHOLDER_DEPENDENCY_PATTERN.test(trimmed))
+            continue;
+        if (seen.has(trimmed))
+            continue;
+        seen.add(trimmed);
+        filtered.push(trimmed);
+    }
+    return filtered.length ? filtered : undefined;
+};
 const buildTaskContext = (tasks) => {
     if (tasks.length === 0)
         return "Task context: (no task records found)";
@@ -240,9 +401,11 @@ export class GatewayAgentService {
         const globalRepo = await GlobalRepository.create();
         const agentService = new AgentService(globalRepo);
         const routingService = await RoutingService.create();
+        const docdexRepoId = workspace.config?.docdexRepoId ?? process.env.MCODA_DOCDEX_REPO_ID ?? process.env.DOCDEX_REPO_ID;
         const docdex = new DocdexClient({
             workspaceRoot: workspace.workspaceRoot,
             baseUrl: workspace.config?.docdexUrl ?? process.env.MCODA_DOCDEX_URL,
+            repoId: docdexRepoId,
         });
         const workspaceRepo = await WorkspaceRepository.create(workspace.workspaceRoot);
         const jobService = new JobService(workspace, workspaceRepo);
@@ -291,10 +454,31 @@ export class GatewayAgentService {
         await maybeClose(this.deps.workspaceRepo);
         await maybeClose(this.deps.routingService);
     }
+    setDocdexAvailability(available, reason) {
+        if (available)
+            return;
+        const docdex = this.deps.docdex;
+        if (docdex && typeof docdex.disable === "function") {
+            docdex.disable(reason);
+        }
+    }
+    async saveRepoMemory(text) {
+        const docdex = this.deps.docdex;
+        if (!docdex || typeof docdex.memorySave !== "function")
+            return;
+        await docdex.memorySave(text);
+    }
+    async savePreference(category, content, agentId = "default") {
+        const docdex = this.deps.docdex;
+        if (!docdex || typeof docdex.savePreference !== "function")
+            return;
+        await docdex.savePreference(agentId, category, content);
+    }
     async loadGatewayPrompts(agentId) {
         const agentPrompts = "getPrompts" in this.deps.agentService ? await this.deps.agentService.getPrompts(agentId) : undefined;
-        const mcodaPromptPath = path.join(this.workspace.workspaceRoot, ".mcoda", "prompts", "gateway-agent.md");
+        const mcodaPromptPath = path.join(this.workspace.mcodaDir, "prompts", "gateway-agent.md");
         const workspacePromptPath = path.join(this.workspace.workspaceRoot, "prompts", "gateway-agent.md");
+        const repoPromptPath = resolveRepoPromptPath("gateway-agent.md");
         try {
             await fs.promises.mkdir(path.dirname(mcodaPromptPath), { recursive: true });
             await fs.promises.access(mcodaPromptPath);
@@ -305,7 +489,13 @@ export class GatewayAgentService {
                 await fs.promises.copyFile(workspacePromptPath, mcodaPromptPath);
             }
             catch {
-                await fs.promises.writeFile(mcodaPromptPath, DEFAULT_GATEWAY_PROMPT, "utf8");
+                try {
+                    await fs.promises.access(repoPromptPath);
+                    await fs.promises.copyFile(repoPromptPath, mcodaPromptPath);
+                }
+                catch {
+                    await fs.promises.writeFile(mcodaPromptPath, DEFAULT_GATEWAY_PROMPT, "utf8");
+                }
             }
         }
         try {
@@ -319,7 +509,15 @@ export class GatewayAgentService {
                     }
                 }
                 catch {
-                    /* ignore */
+                    try {
+                        const repoPrompt = await fs.promises.readFile(repoPromptPath, "utf8");
+                        if (hasRequiredPromptMarkers(repoPrompt)) {
+                            nextPrompt = repoPrompt.trim();
+                        }
+                    }
+                    catch {
+                        /* ignore */
+                    }
                 }
                 await fs.promises.writeFile(mcodaPromptPath, nextPrompt, "utf8");
             }
@@ -327,7 +525,7 @@ export class GatewayAgentService {
         catch {
             /* ignore */
         }
-        const commandPromptFiles = (await this.readPromptFiles([mcodaPromptPath, workspacePromptPath])).filter(hasRequiredPromptMarkers);
+        const commandPromptFiles = (await this.readPromptFiles([mcodaPromptPath, workspacePromptPath, repoPromptPath])).filter(hasRequiredPromptMarkers);
         const mergedCommandPrompt = (() => {
             const parts = [...commandPromptFiles];
             const agentCommandPrompt = agentPrompts?.commandPrompts?.["gateway-agent"];
@@ -338,9 +536,11 @@ export class GatewayAgentService {
                 parts.push(DEFAULT_GATEWAY_PROMPT);
             return parts.filter(Boolean).join("\n\n");
         })();
+        const sanitizedJobPrompt = sanitizeGatewayPrompt(agentPrompts?.jobPrompt);
+        const sanitizedCharacterPrompt = sanitizeGatewayPrompt(agentPrompts?.characterPrompt);
         return {
-            jobPrompt: agentPrompts?.jobPrompt ?? DEFAULT_JOB_PROMPT,
-            characterPrompt: agentPrompts?.characterPrompt ?? DEFAULT_CHARACTER_PROMPT,
+            jobPrompt: sanitizedJobPrompt ?? DEFAULT_JOB_PROMPT,
+            characterPrompt: sanitizedCharacterPrompt ?? DEFAULT_CHARACTER_PROMPT,
             commandPrompt: mergedCommandPrompt,
         };
     }
@@ -362,14 +562,15 @@ export class GatewayAgentService {
                     const caps = await this.deps.globalRepo.getAgentCapabilities(overrideAgent.id);
                     const missing = requiredCaps.filter((cap) => !caps.includes(cap));
                     const health = await this.deps.globalRepo.getAgentHealth(overrideAgent.id);
-                    if (missing.length === 0 && health?.status !== "unreachable") {
-                        return overrideAgent;
+                    if (health?.status === "unreachable") {
+                        warnings.push(`Override agent ${overrideAgent.slug} is unreachable; ignoring override.`);
                     }
-                    if (missing.length) {
-                        warnings.push(`Override agent ${overrideAgent.slug} is missing gateway capabilities (${missing.join(", ")}); ignoring override.`);
+                    else if (missing.length === 0) {
+                        return overrideAgent;
                     }
-                    else if (health?.status === "unreachable") {
-                        warnings.push(`Override agent ${overrideAgent.slug} is unreachable; ignoring override.`);
+                    else {
+                        warnings.push(`Override agent ${overrideAgent.slug} is missing gateway capabilities (${missing.join(", ")}); proceeding with override as requested.`);
+                        return overrideAgent;
                     }
                 }
                 catch (overrideError) {
@@ -411,7 +612,7 @@ export class GatewayAgentService {
                     if (text && onChunk)
                         onChunk(text);
                 }
-                return { output, durationSeconds: (Date.now() - startedAt) / 1000 };
+                return { output: sanitizeAgentOutput(output), durationSeconds: (Date.now() - startedAt) / 1000 };
             }
         }
         catch (error) {
@@ -424,7 +625,7 @@ export class GatewayAgentService {
             input: prompt,
             metadata: { command: "gateway-agent", job },
         });
-        const output = response.output ?? "";
+        const output = sanitizeAgentOutput(response.output ?? "");
         if (output && onChunk)
             onChunk(output);
         return { output, durationSeconds: (Date.now() - startedAt) / 1000 };
@@ -445,27 +646,81 @@ export class GatewayAgentService {
                 taskKeys: request.taskKeys,
                 statusFilter: request.statusFilter?.length ? request.statusFilter : DEFAULT_STATUS_FILTER,
                 limit,
+                ignoreDependencies: request.taskKeys && request.taskKeys.length > 0 ? true : request.ignoreDependencies,
             };
             const selection = await this.taskSelectionService.selectTasks(filters);
             if (selection.warnings.length)
                 warnings.push(...selection.warnings);
-            const combined = [...selection.ordered, ...selection.blocked];
-            return combined.slice(0, limit).map((entry) => ({
-                key: entry.task.key,
-                title: entry.task.title,
-                description: entry.task.description ?? undefined,
-                status: entry.task.status,
-                storyPoints: entry.task.storyPoints ?? undefined,
-                storyKey: entry.task.storyKey,
-                storyTitle: entry.task.storyTitle,
-                epicKey: entry.task.epicKey,
-                epicTitle: entry.task.epicTitle,
-                acceptanceCriteria: entry.task.acceptanceCriteria,
-                dependencies: entry.dependencies.keys,
+            const combined = [...selection.ordered];
+            if (combined.length) {
+                return combined.slice(0, limit).map((entry) => ({
+                    key: entry.task.key,
+                    title: entry.task.title,
+                    description: entry.task.description ?? undefined,
+                    status: entry.task.status,
+                    storyPoints: entry.task.storyPoints ?? undefined,
+                    storyKey: entry.task.storyKey,
+                    storyTitle: entry.task.storyTitle,
+                    epicKey: entry.task.epicKey,
+                    epicTitle: entry.task.epicTitle,
+                    acceptanceCriteria: entry.task.acceptanceCriteria,
+                    dependencies: normalizeDependencies(entry.dependencies.keys),
+                }));
+            }
+            if (!request.taskKeys?.length)
+                return [];
+            warnings.push("Gateway task selection returned no tasks; falling back to direct task lookup.");
+            const tasks = await Promise.all(request.taskKeys.map((key) => this.deps.workspaceRepo.getTaskByKey(key)));
+            const found = tasks.filter((task) => Boolean(task));
+            const missing = request.taskKeys.filter((key, index) => !tasks[index]);
+            if (missing.length) {
+                warnings.push(`Fallback task lookup missing keys: ${missing.join(", ")}`);
+            }
+            if (!found.length)
+                return [];
+            let withRelations = [];
+            try {
+                const related = await this.deps.workspaceRepo.getTasksWithRelations(found.map((task) => task.id));
+                const relatedById = new Map(related.map((task) => [task.id, task]));
+                withRelations = found.map((task) => relatedById.get(task.id) ?? task);
+            }
+            catch {
+                withRelations = found;
+            }
+            return withRelations.slice(0, limit).map((task) => ({
+                key: task.key,
+                title: task.title,
+                description: task.description ?? undefined,
+                status: task.status,
+                storyPoints: task.storyPoints ?? undefined,
+                storyKey: task.storyKey,
+                storyTitle: task.storyTitle,
+                epicKey: task.epicKey,
+                epicTitle: task.epicTitle,
+                acceptanceCriteria: task.acceptanceCriteria,
+                dependencies: undefined,
             }));
         }
         catch (error) {
             warnings.push(`Task lookup failed: ${error.message}`);
+            if (request.taskKeys?.length) {
+                warnings.push("Task lookup failed; attempting direct task fallback.");
+                try {
+                    const tasks = await Promise.all(request.taskKeys.map((key) => this.deps.workspaceRepo.getTaskByKey(key)));
+                    const found = tasks.filter((task) => Boolean(task));
+                    return found.map((task) => ({
+                        key: task.key,
+                        title: task.title,
+                        description: task.description ?? undefined,
+                        status: task.status,
+                        storyPoints: task.storyPoints ?? undefined,
+                        dependencies: undefined,
+                    }));
+                }
+                catch {
+                    return [];
+                }
+            }
             return [];
         }
     }
@@ -509,22 +764,59 @@ export class GatewayAgentService {
         const maxDocs = request.maxDocs ?? 4;
         if (maxDocs <= 0)
             return [];
+        if (typeof this.deps.docdex?.ensureRepoScope === "function") {
+            try {
+                await this.deps.docdex.ensureRepoScope();
+            }
+            catch (error) {
+                warnings.push(`Docdex scope missing: ${error.message}`);
+                return [];
+            }
+        }
         const docTypes = this.pickDocTypes(request.job, request.inputText);
         const query = this.buildQuerySeed(tasks, request.inputText);
         const summaries = [];
+        let openApiIncluded = false;
+        let reindexed = false;
         for (const docType of docTypes) {
             if (summaries.length >= maxDocs)
                 break;
             try {
-                const docs = await this.deps.docdex.search({
+                let docs = await this.deps.docdex.search({
                     projectKey: request.projectKey,
                     docType,
                     query,
                 });
+                if (!docs.length && !reindexed && typeof this.deps.docdex.reindex === "function") {
+                    reindexed = true;
+                    try {
+                        warnings.push("Docdex search returned no results; reindexing and retrying.");
+                        await this.deps.docdex.reindex();
+                        docs = await this.deps.docdex.search({
+                            projectKey: request.projectKey,
+                            docType,
+                            query,
+                        });
+                    }
+                    catch (error) {
+                        warnings.push(`Docdex reindex failed: ${error.message}`);
+                    }
+                }
                 for (const doc of docs) {
                     if (summaries.length >= maxDocs)
                         break;
-                    summaries.push(summarizeDoc(doc, summaries.length));
+                    const ref = doc.path ?? doc.title ?? doc.id;
+                    if (isDocContextExcluded(ref, false)) {
+                        warnings.push(`Docdex doc filtered from gateway context: ${ref}`);
+                        continue;
+                    }
+                    const summary = summarizeDoc(doc, summaries.length, warnings);
+                    if (summary.docType === "OPENAPI") {
+                        if (openApiIncluded)
+                            continue;
+                        openApiIncluded = true;
+                    }
+                    summaries.push(summary);
                 }
             }
             catch (error) {
@@ -547,7 +839,10 @@ export class GatewayAgentService {
         const understanding = normalizeTextField(raw?.understanding) ?? "";
         const plan = normalizeList(raw?.plan);
         const filesLikelyTouched = normalizeFileList(raw?.filesLikelyTouched);
-        const filesToCreate = normalizeFileList(raw?.filesToCreate);
+        const rawCreate = normalizeList(raw?.filesToCreate);
+        const splitCreate = splitFileAndDirEntries(rawCreate);
+        const filesToCreate = normalizeFileList(splitCreate.files);
+        const dirsToCreate = [...normalizeDirList(raw?.dirsToCreate), ...normalizeDirList(splitCreate.dirs)];
         const complexityRaw = Number(raw?.complexity);
         const complexity = Number.isFinite(complexityRaw) ? clamp(Math.round(complexityRaw), 1, 10) : 5;
         const discipline = normalizeDiscipline(typeof raw?.discipline === "string" ? raw.discipline : undefined) ??
@@ -577,6 +872,7 @@ export class GatewayAgentService {
             discipline,
             filesLikelyTouched,
             filesToCreate,
+            dirsToCreate,
             assumptions: normalizeList(raw?.assumptions),
             risks: normalizeList(raw?.risks),
             docdexNotes: normalizeList(raw?.docdexNotes),
@@ -592,6 +888,36 @@ export class GatewayAgentService {
         const isInside = (relative) => !relative.startsWith("..") && !path.isAbsolute(relative);
         const touched = [];
         const created = [];
+        const dirs = [];
+        const dirSet = new Set();
+        for (const dir of analysis.dirsToCreate ?? []) {
+            const { relative, resolved } = normalize(dir);
+            if (!isInside(relative)) {
+                warnings.push(`Gateway directory path outside workspace ignored: ${dir}`);
+                continue;
+            }
+            try {
+                const stat = await fs.promises.stat(resolved);
+                if (stat.isDirectory()) {
+                    const normalized = relative.replace(/\\/g, "/");
+                    if (!dirSet.has(normalized)) {
+                        dirSet.add(normalized);
+                        dirs.push(normalized);
+                    }
+                    continue;
+                }
+                warnings.push(`Gateway directory path is not a directory: ${dir}`);
+                continue;
+            }
+            catch {
+                const normalized = relative.replace(/\\/g, "/");
+                if (!dirSet.has(normalized)) {
+                    dirSet.add(normalized);
+                    dirs.push(normalized);
+                }
+            }
+        }
+        const dirPrefixes = Array.from(dirSet).map((dir) => (dir.endsWith("/") ? dir : `${dir}/`));
         for (const file of analysis.filesLikelyTouched) {
             const { relative, resolved } = normalize(file);
             if (!isInside(relative)) {
@@ -626,8 +952,14 @@ export class GatewayAgentService {
                 }
             }
             catch {
-                warnings.push(`Gateway create path parent does not exist: ${file}`);
-                continue;
+                const parentRelative = path.relative(root, parent).replace(/\\/g, "/");
+                const parentAllowed = parentRelative === "" ||
+                    dirSet.has(parentRelative) ||
+                    dirPrefixes.some((prefix) => parentRelative.startsWith(prefix));
+                if (!parentAllowed) {
+                    warnings.push(`Gateway create path parent does not exist: ${file}`);
+                    continue;
+                }
             }
             try {
                 const stat = await fs.promises.stat(resolved);
@@ -646,17 +978,22 @@ export class GatewayAgentService {
             ...analysis,
             filesLikelyTouched: touched,
             filesToCreate: created,
+            dirsToCreate: dirs,
         };
     }
-    async listCandidates(requiredCaps, discipline) {
+    async listCandidates(requiredCaps, discipline, avoidAgents = []) {
         const agents = await this.deps.globalRepo.listAgents();
         if (agents.length === 0) {
             throw new Error("No agents available; register one with mcoda agent add");
         }
+        const avoidSet = new Set(avoidAgents.map((value) => value.toLowerCase()));
         const health = await this.deps.globalRepo.listAgentHealthSummary();
         const healthById = new Map(health.map((row) => [row.agentId, row]));
         const candidates = [];
         for (const agent of agents) {
+            const slug = agent.slug ?? agent.id;
+            if (avoidSet.has(agent.id.toLowerCase()) || avoidSet.has(slug.toLowerCase()))
+                continue;
             const capabilities = await this.deps.globalRepo.getAgentCapabilities(agent.id);
             const missing = requiredCaps.filter((cap) => !capabilities.includes(cap));
             if (missing.length)
@@ -672,6 +1009,9 @@ export class GatewayAgentService {
             const usageScore = scoreUsage(discipline, agent.bestUsage, capabilities);
             const cost = agent.costPerMillion ?? Number.POSITIVE_INFINITY;
             const adjustedQuality = healthEntry?.status === "degraded" ? quality - 0.5 : quality;
+            const maxComplexity = typeof agent.maxComplexity === "number" && Number.isFinite(agent.maxComplexity)
+                ? clamp(Math.round(agent.maxComplexity), 1, 10)
+                : 5;
             candidates.push({
                 agent,
                 capabilities,
@@ -680,6 +1020,7 @@ export class GatewayAgentService {
                 reasoning,
                 usageScore,
                 cost,
+                maxComplexity,
             });
         }
         return candidates;
@@ -688,10 +1029,42 @@ export class GatewayAgentService {
         if (candidates.length === 0) {
             throw new Error("No eligible agents available for this job");
         }
-        const sortedQuality = candidates.map((c) => c.quality);
+        const normalizedComplexity = clamp(Math.round(complexity), 1, 10);
+        const eligible = candidates.filter((c) => c.maxComplexity >= normalizedComplexity);
+        let pool = eligible;
+        let gatingNote = "";
+        if (!eligible.length) {
+            const fallback = normalizedComplexity > 1
+                ? candidates.filter((c) => c.maxComplexity >= normalizedComplexity - 1)
+                : candidates;
+            pool = fallback.length ? fallback : candidates;
+            gatingNote = fallback.length
+                ? ` No agents meet max complexity ${normalizedComplexity}; allowing ${normalizedComplexity - 1} fallback.`
+                : ` No agents meet max complexity ${normalizedComplexity}; using best available.`;
+        }
+        if (Math.random() < EXPLORATION_RATE) {
+            const stretchPool = normalizedComplexity > 1
+                ? candidates.filter((c) => c.maxComplexity < normalizedComplexity && c.maxComplexity >= normalizedComplexity - 1)
+                : [];
+            const allowRedemption = normalizedComplexity <= 4;
+            const sortedByQuality = pool.slice().sort((a, b) => a.quality - b.quality);
+            const redemptionPool = allowRedemption ? sortedByQuality.slice(0, Math.max(1, Math.ceil(pool.length * 0.2))) : [];
+            const canUseStretch = stretchPool.length > 0;
+            const canUseRedemption = redemptionPool.length > 0;
+            if (canUseStretch || canUseRedemption) {
+                const useStretch = canUseStretch && (!canUseRedemption || Math.random() < 0.5);
+                const explorePool = useStretch ? stretchPool : redemptionPool;
+                const pick = explorePool[Math.floor(Math.random() * explorePool.length)];
+                const rationale = useStretch
+                    ? `Exploration: stretching an agent (max complexity ${pick.maxComplexity}) for task complexity ${normalizedComplexity}/10.${gatingNote}`
+                    : `Exploration: redemption run for a lower-rated agent to reassess performance.${gatingNote}`;
+                return { pick, rationale };
+            }
+        }
+        const sortedQuality = pool.map((c) => c.quality);
         const maxQuality = Math.max(...sortedQuality);
-        if (complexity >= 9) {
-            const pick = candidates
+        if (normalizedComplexity >= 9) {
+            const pick = pool
                 .slice()
                 .sort((a, b) => {
                 if (b.quality !== a.quality)
@@ -704,12 +1077,12 @@ export class GatewayAgentService {
             })[0];
             return {
                 pick,
-                rationale: `Complexity ${complexity}/10 requires the highest capability; selected top-rated agent with best fit for ${discipline}.`,
+                rationale: `Complexity ${normalizedComplexity}/10 requires the highest capability; selected top-rated agent with best fit for ${discipline}.${gatingNote}`,
             };
         }
-        if (complexity >= 8) {
-            const pool = candidates.filter((c) => c.quality >= maxQuality - 1);
-            const pick = (pool.length ? pool : candidates)
+        if (normalizedComplexity >= 8) {
+            const qualityPool = pool.filter((c) => c.quality >= maxQuality - 1);
+            const pick = (qualityPool.length ? qualityPool : pool)
                 .slice()
                 .sort((a, b) => {
                 if (b.usageScore !== a.usageScore)
@@ -720,12 +1093,12 @@ export class GatewayAgentService {
             })[0];
             return {
                 pick,
-                rationale: `Complexity ${complexity}/10 favors strong agents with good cost/fit balance; selected best-fit candidate.`,
+                rationale: `Complexity ${normalizedComplexity}/10 favors strong agents with good cost/fit balance; selected best-fit candidate.${gatingNote}`,
             };
         }
-        const target = complexity;
-        const pool = candidates.filter((c) => c.quality >= target);
-        const base = pool.length ? pool : candidates;
+        const target = normalizedComplexity;
+        const qualityPool = pool.filter((c) => c.quality >= target);
+        const base = qualityPool.length ? qualityPool : pool;
         const pick = base
             .slice()
             .sort((a, b) => {
@@ -741,14 +1114,49 @@ export class GatewayAgentService {
         })[0];
         return {
             pick,
-            rationale: `Complexity ${complexity}/10 targets a comparable tier agent; selected closest match with discipline fit and cost awareness.`,
+            rationale: `Complexity ${normalizedComplexity}/10 targets a comparable tier agent; selected closest match with discipline fit and cost awareness.${gatingNote}`,
         };
     }
-    async selectAgentForJob(job, analysis) {
+    async selectAgentForJob(job, analysis, avoidAgents = [], forceStronger = false, forceTier, warnings = []) {
         const normalizedJob = canonicalizeCommandName(job);
         const requiredCaps = getCommandRequiredCapabilities(normalizedJob);
-        const candidates = await this.listCandidates(requiredCaps, analysis.discipline);
-        const { pick, rationale } = this.chooseCandidate(candidates, analysis.complexity, analysis.discipline);
+        let candidates = await this.listCandidates(requiredCaps, analysis.discipline, avoidAgents);
+        if (!candidates.length && avoidAgents.length) {
+            const fallback = await this.listCandidates(requiredCaps, analysis.discipline, []);
+            if (fallback.length) {
+                candidates = fallback;
+                warnings.push("Avoid list removed all eligible agents; reusing available agent.");
+            }
+        }
+        const baseComplexity = Number.isFinite(analysis.complexity)
+            ? clamp(Math.round(analysis.complexity), 1, 10)
+            : 5;
+        const forcedMin = forceTier === "specialist"
+            ? SPECIALIST_TIER_MIN_COMPLEXITY
+            : forceTier === "strong"
+                ? STRONG_TIER_MIN_COMPLEXITY
+                : undefined;
+        let boostedComplexity = baseComplexity;
+        if (typeof forcedMin === "number") {
+            boostedComplexity = Math.max(baseComplexity, forcedMin);
+        }
+        else if (forceStronger) {
+            if (baseComplexity < STRONG_TIER_MIN_COMPLEXITY) {
+                boostedComplexity = STRONG_TIER_MIN_COMPLEXITY;
+            }
+            else if (baseComplexity < SPECIALIST_TIER_MIN_COMPLEXITY) {
+                boostedComplexity = SPECIALIST_TIER_MIN_COMPLEXITY;
+            }
+            else {
+                boostedComplexity = clamp(baseComplexity + 1, 1, 10);
+            }
+        }
+        const { pick, rationale } = this.chooseCandidate(candidates, boostedComplexity, analysis.discipline);
+        const finalRationale = forceTier
+            ? `${rationale} (force_tier:${forceTier})`
+            : forceStronger
+                ? `${rationale} (force_stronger applied)`
+                : rationale;
         return {
             agentId: pick.agent.id,
             agentSlug: pick.agent.slug ?? pick.agent.id,
@@ -756,9 +1164,26 @@ export class GatewayAgentService {
             reasoningRating: pick.agent.reasoningRating ?? undefined,
             bestUsage: pick.agent.bestUsage ?? undefined,
             costPerMillion: Number.isFinite(pick.cost) ? pick.cost : undefined,
-            rationale,
+            rationale: finalRationale,
         };
     }
+    async preflightExecutionAgents(job, overrideAgent) {
+        const normalizedJob = canonicalizeCommandName(job);
+        const requiredCaps = getCommandRequiredCapabilities(normalizedJob);
+        if (overrideAgent) {
+            const resolved = await this.deps.agentService.resolveAgent(overrideAgent);
+            const capabilities = await this.deps.globalRepo.getAgentCapabilities(resolved.id);
+            const missing = requiredCaps.filter((cap) => !capabilities.includes(cap));
+            if (missing.length) {
+                throw new Error(`Agent ${overrideAgent} is missing required capabilities for ${normalizedJob}: ${missing.join(", ")}`);
+            }
+            return;
+        }
+        const candidates = await this.listCandidates(requiredCaps, "other");
+        if (!candidates.length) {
+            throw new Error(`No eligible execution agents available for ${normalizedJob}.`);
+        }
+    }
     async run(request) {
         const warnings = [];
         const normalizedJob = canonicalizeCommandName(request.job);
@@ -776,7 +1201,7 @@ export class GatewayAgentService {
             ]
                 .filter(Boolean)
                 .join("\n\n");
-            const recordUsage = async (promptText, outputText, durationSeconds, action) => {
+            const recordUsage = async (promptText, outputText, durationSeconds, action, attempt) => {
                 const promptTokens = estimateTokens(promptText);
                 const completionTokens = estimateTokens(outputText ?? "");
                 await this.deps.jobService.recordTokenUsage({
@@ -792,20 +1217,24 @@ export class GatewayAgentService {
                     tokensCompletion: completionTokens,
                     tokensTotal: promptTokens + completionTokens,
                     durationSeconds,
-                    metadata: { action, job: normalizedJob },
+                    metadata: { action, job: normalizedJob, phase: action, attempt },
                 });
             };
             const response = await this.invokeGatewayAgent(gatewayAgent, prompt, normalizedJob, {
                 stream: request.agentStream !== false,
                 onChunk: request.onStreamChunk,
             });
-            await recordUsage(prompt, response.output ?? "", response.durationSeconds, "gateway_summary");
-            let parsed = extractJson(response.output);
+            await recordUsage(prompt, response.output ?? "", response.durationSeconds, "gateway_summary", 1);
+            const jsonResult = extractJsonOnly(response.output);
+            let parsed = jsonResult.payload;
+            let fileListCoverage = parsed ? assessFileListCoverage(parsed) : { empty: false, justified: false };
             let missingFields = parsed
-                ? listMissingFields(parsed)
-                : ["summary", "reasoningSummary", "currentState", "todo", "understanding", "plan", "files"];
+                ? listMissingFields(parsed, { allowEmptyFiles: true })
+                : ["summary", "reasoningSummary", "currentState", "todo", "understanding", "plan", "filesLikelyTouched", "filesToCreate", "dirsToCreate"];
             if (!parsed) {
-                warnings.push("Gateway analysis response was not valid JSON; falling back to defaults.");
+                warnings.push(jsonResult.jsonOnly
+                    ? "Gateway analysis response was invalid JSON."
+                    : "Gateway analysis response was not JSON-only.");
             }
             if (missingFields.length) {
                 const repairPrompt = [
@@ -813,7 +1242,8 @@ export class GatewayAgentService {
                     "",
                     "Your previous response was incomplete or invalid. Return JSON only with the exact schema.",
                     `Missing fields: ${missingFields.join(", ")}.`,
-                    "Ensure reasoningSummary, currentState, todo, understanding, plan, and filesLikelyTouched/filesToCreate are populated.",
+                    "Ensure reasoningSummary, currentState, todo, understanding, and plan are populated.",
+                    "If file paths are unknown, leave filesLikelyTouched/filesToCreate/dirsToCreate empty and explain the gap in assumptions or docdexNotes.",
                     "Use real file paths only (no placeholders like (unknown), TBD, or glob patterns).",
                     "If docdex returned no results, say so in docdexNotes.",
                 ].join("\n");
@@ -824,18 +1254,29 @@ export class GatewayAgentService {
                     stream: request.agentStream !== false,
                     onChunk: request.onStreamChunk,
                 });
-                await recordUsage(repairPrompt, repairResponse.output ?? "", repairResponse.durationSeconds, "gateway_summary_repair");
-                const repaired = extractJson(repairResponse.output);
-                if (repaired) {
-                    parsed = repaired;
-                    missingFields = listMissingFields(parsed);
+                await recordUsage(repairPrompt, repairResponse.output ?? "", repairResponse.durationSeconds, "gateway_summary_repair", 2);
+                const repaired = extractJsonOnly(repairResponse.output);
+                if (repaired.payload) {
+                    parsed = repaired.payload;
+                    fileListCoverage = assessFileListCoverage(parsed);
+                    missingFields = listMissingFields(parsed, { allowEmptyFiles: true });
                 }
                 else {
-                    warnings.push("Gateway repair response was not valid JSON; using fallback analysis.");
+                    warnings.push(repaired.jsonOnly
+                        ? "Gateway repair response was invalid JSON."
+                        : "Gateway repair response was not JSON-only.");
                 }
             }
             if (missingFields.length) {
-                warnings.push(`Gateway analysis missing fields: ${missingFields.join(", ")}.`);
+                throw new Error(`Gateway analysis missing required fields: ${missingFields.join(", ")}.`);
+            }
+            if (!parsed) {
+                throw new Error(jsonResult.jsonOnly
+                    ? "Gateway analysis response was invalid JSON."
+                    : "Gateway analysis response was not JSON-only.");
+            }
+            if (fileListCoverage.empty && !fileListCoverage.justified) {
+                warnings.push("Gateway analysis returned no file paths; proceeding without file context.");
             }
             let analysis = this.normalizeAnalysis(parsed ?? {}, normalizedJob, tasks, request.inputText);
             if (analysis.docdexNotes.length === 0) {
@@ -849,7 +1290,7 @@ export class GatewayAgentService {
                 analysis.docdexNotes.push(...docdexWarnings);
             }
             analysis = await this.validateFilePlan(analysis, warnings);
-            const chosenAgent = await this.selectAgentForJob(normalizedJob, analysis);
+            const chosenAgent = await this.selectAgentForJob(normalizedJob, analysis, request.avoidAgents ?? [], request.forceStronger ?? false, request.forceTier, warnings);
             await this.deps.jobService.finishCommandRun(commandRun.id, "succeeded");
             return {
                 commandRunId: commandRun.id,