create-quorum-router 0.1.5

package/templates/basic/src/best_route.ts

@@ -0,0 +1,322 @@
+import {
+  discoverInventoryWithModelListing,
+  invokableEntries,
+} from "./auth_session.ts";
+import { callEnvFallback } from "./auth_env_fallback.ts";
+import { readRouterEnv } from "./env.ts";
+import {
+  modelSelectionMatches,
+  normalizeSelectionValue,
+  providerSelectionMatches,
+  type ProviderSelectionRequest,
+  readProviderSelectionRequest,
+} from "./provider_registry.ts";
+import {
+  assertOptIn,
+  type DogfoodTrace,
+  type ModelInventoryEntry,
+  parseAuthMode,
+  type ProviderResult,
+} from "./schema.ts";
+import { buildTrace, score, writeTrace } from "./trace.ts";
+import { callWrapper } from "./wrapper_client.ts";
+import { preparePromptWithContext } from "./context.ts";
+
+function hasExplicitSelection(request: ProviderSelectionRequest): boolean {
+  return Boolean(
+    normalizeSelectionValue(request.providerLabel) ||
+      normalizeSelectionValue(request.model),
+  );
+}
+
+function availableCandidate(entry: ModelInventoryEntry): string {
+  const listed = entry.listed_models?.length
+    ? ` listed=[${entry.listed_models.join(",")}]`
+    : "";
+  const blocker = entry.available && entry.can_invoke
+    ? "ready"
+    : `blocked=${
+      entry.blocked_reason ?? entry.list_blocked_reason ?? "not invokable"
+    }`;
+  return `${entry.provider}/${entry.model} (${entry.model_id}) source=${entry.source} command=${
+    entry.command ?? "n/a"
+  } ${blocker}${listed}`;
+}
+
+function requestedText(request: ProviderSelectionRequest): string {
+  return [
+    request.providerLabel ? `provider=${request.providerLabel}` : undefined,
+    request.model ? `model=${request.model}` : undefined,
+  ].filter(Boolean).join(" ") || "none";
+}
+
+function autoCandidateScore(entry: ModelInventoryEntry): number {
+  let score = 0;
+  if (entry.can_list_models && entry.listed_models?.length) score += 100;
+  if (entry.source === "env_fallback") score -= 100;
+  return score;
+}
+
+function rankAutoCandidates(
+  candidates: ModelInventoryEntry[],
+): ModelInventoryEntry[] {
+  return candidates.map((entry, index) => ({ entry, index })).sort((a, b) =>
+    autoCandidateScore(b.entry) - autoCandidateScore(a.entry) ||
+    a.index - b.index
+  ).map(({ entry }) => entry);
+}
+
+function withPreferredAutoListedModel(
+  entry: ModelInventoryEntry,
+): ModelInventoryEntry {
+  const listed = entry.listed_models ?? [];
+  const preferred =
+    listed.find((model) => model === "grok-composer-2.5-fast") ??
+      listed.find((model) => model.includes("composer"));
+  if (!preferred) return entry;
+  return {
+    ...entry,
+    model: preferred,
+    model_id: `${entry.provider.toLowerCase()}/${preferred}`,
+    invocation_model: preferred,
+    notes: [
+      ...entry.notes,
+      `Auto-selected listed model ${preferred} for this invocation.`,
+    ],
+  };
+}
+
+function withRequestedListedModel(
+  entry: ModelInventoryEntry,
+  requestedModel: string | undefined,
+): ModelInventoryEntry {
+  const normalizedRequested = normalizeSelectionValue(requestedModel);
+  if (!normalizedRequested) return entry;
+  const providerPrefix = normalizeSelectionValue(entry.provider)?.replace(
+    /[^a-z0-9]+/g,
+    "",
+  );
+  const listed = entry.listed_models ?? [];
+  const selected = listed.find((model) => {
+    const normalizedModel = normalizeSelectionValue(model);
+    return normalizedModel === normalizedRequested ||
+      (providerPrefix
+        ? `${providerPrefix}/${normalizedModel}` === normalizedRequested
+        : false);
+  });
+  if (!selected) return entry;
+  return {
+    ...entry,
+    model: selected,
+    model_id: `${entry.provider.toLowerCase()}/${selected}`,
+    invocation_model: selected,
+    notes: [
+      ...entry.notes,
+      `Selected listed model ${selected} for this invocation.`,
+    ],
+  };
+}
+
+export function selectionHonored(
+  request: ProviderSelectionRequest,
+  selected: ModelInventoryEntry | ProviderResult,
+): boolean {
+  if (!hasExplicitSelection(request)) return true;
+  const entryLike = {
+    provider: selected.provider,
+    model: selected.model,
+    model_id: selected.model_id ?? selected.model,
+    source: selected.source ?? "selected",
+    command: selected.command,
+    listed_models: selected.listed_models,
+  };
+  return providerSelectionMatches(entryLike, request.providerLabel) &&
+    modelSelectionMatches(entryLike, request.model);
+}
+
+export function selectInvokableCandidates(
+  candidates: ModelInventoryEntry[],
+  _authMode = parseAuthMode(readRouterEnv("QUORUM_ROUTER_AUTH_MODE")),
+  request = readProviderSelectionRequest(),
+  allEntries: ModelInventoryEntry[] = candidates,
+): ModelInventoryEntry[] {
+  if (!hasExplicitSelection(request)) {
+    return rankAutoCandidates(candidates).map(withPreferredAutoListedModel);
+  }
+
+  const filtered = candidates.filter((entry) => {
+    const providerMatches = providerSelectionMatches(
+      entry,
+      request.providerLabel,
+    );
+    const modelMatches = modelSelectionMatches(entry, request.model);
+    return providerMatches && modelMatches;
+  }).map((entry) => withRequestedListedModel(entry, request.model));
+
+  if (filtered.length > 0) return filtered;
+
+  throw new Error(
+    `QuorumRouter blocked: requested provider/model unavailable (${
+      requestedText(request)
+    }). ` +
+      `Available candidates and blockers: ${
+        allEntries.map(availableCandidate).join("; ")
+      }`,
+  );
+}
+
+async function discoverCandidates(authMode: ReturnType<typeof parseAuthMode>) {
+  const request = readProviderSelectionRequest();
+  const inventory = await discoverInventoryWithModelListing(authMode, request);
+  const candidates = selectInvokableCandidates(
+    invokableEntries(inventory),
+    authMode,
+    request,
+    inventory.entries,
+  );
+  return { request, inventory, candidates };
+}
+
+export async function invokeSelected(
+  prompt: string,
+): Promise<
+  { results: ProviderResult[]; tracePath: string; trace: DogfoodTrace }
+> {
+  assertOptIn();
+  const authMode = parseAuthMode(readRouterEnv("QUORUM_ROUTER_AUTH_MODE"));
+  const { request, candidates } = await discoverCandidates(authMode);
+  if (candidates.length === 0) {
+    throw new Error(
+      "OAuth/session-first provider unavailable. No usable OAuth/session/wrapper provider is available yet. Next: deno task auth:login",
+    );
+  }
+  const prepared = await preparePromptWithContext(prompt);
+  const safePrompt = prepared.prompt;
+  const errors: string[] = [];
+  let selected: ModelInventoryEntry | undefined;
+  let result: ProviderResult | undefined;
+  let selectedIndex = -1;
+
+  for (const [index, candidate] of candidates.entries()) {
+    if (!selectionHonored(request, candidate)) {
+      throw new Error(
+        `QuorumRouter blocked: provider selection was not honored (${
+          requestedText(request)
+        } selected=${candidate.provider}/${candidate.model})`,
+      );
+    }
+    try {
+      const candidateResult = candidate.source === "env_fallback"
+        ? await callEnvFallback(safePrompt)
+        : await callWrapper(candidate, safePrompt);
+      if (!selectionHonored(request, candidateResult)) {
+        throw new Error(
+          `provider selection was ignored after invocation (${
+            requestedText(request)
+          } selected=${candidateResult.provider}/${candidateResult.model})`,
+        );
+      }
+      selected = candidate;
+      result = candidateResult;
+      selectedIndex = index;
+      break;
+    } catch (error) {
+      errors.push(error instanceof Error ? error.message : String(error));
+      if (hasExplicitSelection(request)) break;
+    }
+  }
+
+  if (!selected || !result) {
+    throw new Error(
+      `OAuth/session-first provider unavailable. all candidates failed safely: ${
+        errors.join("; ")
+      }`,
+    );
+  }
+
+  const row = score(result);
+  const trace = await buildTrace({
+    command: "route:once",
+    mode: "route_once",
+    authMode,
+    prompt: safePrompt,
+    promptContext: prepared.context,
+    results: [result],
+    selected: row,
+    scores: [row],
+    errors,
+    requestedProviderLabel: request.providerLabel,
+    requestedModel: request.model,
+    providerSelectionHonored: selectionHonored(request, result),
+    fallbackUsed: selectedIndex > 0 || selected.source === "env_fallback",
+  });
+  const tracePath = await writeTrace("route-once-trace", trace);
+  return { results: [result], tracePath, trace };
+}
+
+export async function runBestRoute(
+  prompt: string,
+): Promise<
+  { results: ProviderResult[]; tracePath: string; trace: DogfoodTrace }
+> {
+  assertOptIn();
+  const authMode = parseAuthMode(readRouterEnv("QUORUM_ROUTER_AUTH_MODE"));
+  const { request, candidates } = await discoverCandidates(authMode);
+  if (candidates.length === 0) {
+    throw new Error(
+      "OAuth/session-first provider unavailable. best-route has no usable OAuth/session/wrapper provider yet. Next: deno task auth:login",
+    );
+  }
+  const prepared = await preparePromptWithContext(prompt);
+  const safePrompt = prepared.prompt;
+  const results: ProviderResult[] = [];
+  const errors: string[] = [];
+  let usedEnvFallback = false;
+  for (const candidate of candidates) {
+    try {
+      const result = candidate.source === "env_fallback"
+        ? await callEnvFallback(safePrompt)
+        : await callWrapper(candidate, safePrompt);
+      usedEnvFallback = usedEnvFallback || candidate.source === "env_fallback";
+      if (!selectionHonored(request, result)) {
+        throw new Error(
+          `provider selection ignored (${
+            requestedText(request)
+          } selected=${result.provider}/${result.model})`,
+        );
+      }
+      results.push(result);
+    } catch (error) {
+      errors.push(error instanceof Error ? error.message : String(error));
+    }
+  }
+  if (results.length === 0) {
+    throw new Error(
+      `OAuth/session-first provider unavailable. all candidates failed safely: ${
+        errors.join("; ")
+      }`,
+    );
+  }
+  const scores = results.map(score).sort((a, b) =>
+    b.final_score - a.final_score
+  );
+  const trace = await buildTrace({
+    command: "best-route",
+    mode: "best_route",
+    authMode,
+    prompt: safePrompt,
+    promptContext: prepared.context,
+    results,
+    selected: scores[0],
+    scores,
+    errors,
+    requestedProviderLabel: request.providerLabel,
+    requestedModel: request.model,
+    providerSelectionHonored: results.every((result) =>
+      selectionHonored(request, result)
+    ),
+    fallbackUsed: usedEnvFallback,
+  });
+  const tracePath = await writeTrace("best-route-trace", trace);
+  return { results, tracePath, trace };
+}

package/templates/basic/src/cli.ts

@@ -0,0 +1,158 @@
+import { runAgentChat } from "./agent_chat.ts";
+import { readRouterEnv } from "./env.ts";
+import { runAuthLogin, runAuthLogout, runAuthStatus } from "./auth.ts";
+import { discoverInventory, invokableEntries } from "./auth_session.ts";
+import { invokeSelected, runBestRoute } from "./best_route.ts";
+import { runIntake } from "./intake.ts";
+import {
+  inventoryCommand,
+  printInventoryTable,
+  writeInventory,
+} from "./model_inventory.ts";
+import { redact, summarize } from "./redact.ts";
+import { parseAuthMode } from "./schema.ts";
+import { buildTrace, writeTrace } from "./trace.ts";
+
+const DEFAULT_PROMPT = "Review this README for risky claims.";
+
+type CommandName =
+  | "intake"
+  | "auth:status"
+  | "auth:login"
+  | "auth:logout"
+  | "models:list"
+  | "health"
+  | "route:once"
+  | "best-route"
+  | "agent-chat";
+
+function commandName(): CommandName {
+  const command = Deno.args[0];
+  if (
+    command === "intake" || command === "auth:status" ||
+    command === "auth:login" || command === "auth:logout" ||
+    command === "models:list" || command === "health" ||
+    command === "route:once" || command === "best-route" ||
+    command === "agent-chat"
+  ) return command;
+  throw new Error(
+    "usage: deno task intake|auth:status|auth:login|auth:logout|models:list|health|route:once|best-route|agent-chat",
+  );
+}
+
+function promptFromArgs(): string {
+  const promptIndex = Deno.args.indexOf("--prompt");
+  if (promptIndex >= 0) {
+    const value = Deno.args[promptIndex + 1]?.trim();
+    if (value) return value;
+  }
+  const passthroughIndex = Deno.args.indexOf("--");
+  if (passthroughIndex >= 0) {
+    const value = Deno.args.slice(passthroughIndex + 1).join(" ").trim();
+    if (value) return value;
+  }
+  return DEFAULT_PROMPT;
+}
+
+async function runModelsList(): Promise<void> {
+  const inventory = await inventoryCommand();
+  console.log("QuorumRouter model inventory");
+  console.log("Provider request sent: false");
+  console.log("Generation endpoint called: false");
+  console.log("Credential values printed: false");
+  printInventoryTable(inventory);
+}
+
+async function runHealth(): Promise<void> {
+  const authMode = parseAuthMode(readRouterEnv("QUORUM_ROUTER_AUTH_MODE"));
+  const inventory = discoverInventory(authMode);
+  await writeInventory(inventory);
+  const invokable = invokableEntries(inventory);
+  const trace = await buildTrace({
+    command: "health",
+    mode: "health",
+    authMode,
+    errors: invokable.length === 0
+      ? ["no invokable OAuth/session/wrapper provider discovered"]
+      : [],
+  });
+  const tracePath = await writeTrace("health-trace", trace);
+  console.log("QuorumRouter health");
+  console.log(
+    `Config example present: ${await Deno.stat("router.config.example.json")
+      .then((s) => s.isFile).catch(
+        () => false,
+      )}`,
+  );
+  console.log(`Model inventory entries: ${inventory.entries.length}`);
+  console.log(`Usable providers: ${invokable.length}`);
+  console.log("Provider request sent: false");
+  console.log(`Trace: ${tracePath}`);
+  console.log(
+    'Recommended dogfood: RUN_EXTERNAL_MODEL_DOGFOOD=1 deno task route:once --prompt "Review this README for risky claims."',
+  );
+}
+
+try {
+  const command = commandName();
+  if (command === "intake") await runIntake();
+  else if (command === "auth:status") await runAuthStatus();
+  else if (command === "auth:login") runAuthLogin();
+  else if (command === "auth:logout") await runAuthLogout();
+  else if (command === "models:list") await runModelsList();
+  else if (command === "health") await runHealth();
+  else if (command === "route:once") {
+    const { results, tracePath, trace } = await invokeSelected(
+      promptFromArgs(),
+    );
+    console.log("QuorumRouter route:once");
+    console.log(`provider: ${results[0].provider}`);
+    console.log(`model: ${results[0].model}`);
+    console.log(`response_received: ${results[0].response_received}`);
+    console.log(`schema_valid: ${results[0].schema_valid}`);
+    console.log(`redaction_ok: ${trace.redaction_ok}`);
+    console.log(`credential_value_present: ${trace.credential_value_present}`);
+    console.log(`sensitive_value_present: ${trace.sensitive_value_present}`);
+    console.log(`final: ${summarize(results[0].response_summary, 500)}`);
+    console.log(`trace: ${tracePath}`);
+  } else if (command === "best-route") {
+    const { results, tracePath } = await runBestRoute(promptFromArgs());
+    console.log("QuorumRouter best-route");
+    console.log(`models_called: ${results.length}`);
+    console.log(`trace: ${tracePath}`);
+  } else if (command === "agent-chat") {
+    console.log("QuorumRouter live multi-model Agent Chat");
+    const { tracePath, turns } = await runAgentChat(
+      promptFromArgs(),
+      (progress) => {
+        if (progress.type === "start") {
+          console.log(
+            `agents: ${
+              progress.agents.map((agent) => `${agent.provider}/${agent.model}`)
+                .join(" ↔ ")
+            }`,
+          );
+          console.log("");
+        } else if (progress.type === "turn") {
+          const turn = progress.turn;
+          console.log(`Round ${turn.round}`);
+          console.log(
+            `${turn.provider}/${turn.model}${
+              turn.reply_to
+                ? ` → replying to ${turn.reply_to.provider}/${turn.reply_to.model} (round ${turn.reply_to.round})`
+                : " → opening proposal"
+            }`,
+          );
+          console.log(turn.content);
+          console.log("");
+        }
+      },
+    );
+    console.log(`dialogue_complete: true`);
+    console.log(`turns: ${turns.length}`);
+    console.log(`trace: ${tracePath}`);
+  }
+} catch (error) {
+  console.error(redact(error instanceof Error ? error.message : String(error)));
+  Deno.exit(1);
+}