kimi-agent-swarm-cli 0.7.0

package/src/leaderboard.ts

@@ -0,0 +1,201 @@
+import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+import type { BenchmarkResult, LeaderboardEntry } from "./types";
+
+export interface ComparisonResult {
+  runIds: string[];
+  entries: LeaderboardEntry[];
+}
+
+export function getLeaderboardDir(): string {
+  return join(homedir(), ".kasw");
+}
+
+export function getLeaderboardPath(): string {
+  return join(getLeaderboardDir(), "leaderboard.jsonl");
+}
+
+export async function recordEntry(entry: LeaderboardEntry): Promise<void> {
+  const dir = getLeaderboardDir();
+  await mkdir(dir, { recursive: true });
+  await appendFile(getLeaderboardPath(), `${JSON.stringify(entry)}\n`);
+}
+
+export async function getLeaderboard(profile?: string): Promise<LeaderboardEntry[]> {
+  try {
+    const text = await readFile(getLeaderboardPath(), "utf8");
+    const entries = text
+      .split(/\r?\n/)
+      .filter(Boolean)
+      .map((line) => JSON.parse(line) as LeaderboardEntry);
+
+    if (profile) {
+      return entries.filter((e) => e.profile === profile);
+    }
+    return entries;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return [];
+    }
+    throw error;
+  }
+}
+
+export async function compareRuns(runIds: string[]): Promise<ComparisonResult> {
+  const all = await getLeaderboard();
+  const entries = all.filter((e) => runIds.includes(e.runId));
+  return { runIds, entries };
+}
+
+export async function clearLeaderboard(): Promise<void> {
+  await writeFile(getLeaderboardPath(), "");
+}
+
+function escapeHtml(text: string): string {
+  return text
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&#039;");
+}
+
+function trendLine(values: number[], width: number, height: number): string {
+  if (values.length < 2) return "";
+  const min = Math.min(...values);
+  const max = Math.max(...values);
+  const range = max - min || 1;
+  const stepX = width / (values.length - 1);
+  const points = values
+    .map((v, i) => {
+      const x = i * stepX;
+      const y = height - ((v - min) / range) * height;
+      return `${x},${y}`;
+    })
+    .join(" ");
+  return `<polyline points="${points}" fill="none" stroke="#2563eb" stroke-width="2" />`;
+}
+
+function renderProfileSection(profile: string, entries: LeaderboardEntry[]): string {
+  const sorted = [...entries].sort(
+    (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(),
+  );
+  const precisionValues = sorted.map((e) => e.scores.precision);
+  const recallValues = sorted.map((e) => e.scores.recall);
+  const f1Values = sorted.map((e) => e.scores.f1);
+
+  const bestF1 = sorted.reduce((best, e) => (e.scores.f1 > best.scores.f1 ? e : best), sorted[0]);
+
+  const rows = sorted
+    .map(
+      (e) => `
+    <tr>
+      <td>${escapeHtml(e.timestamp)}</td>
+      <td>${escapeHtml(e.runId)}</td>
+      <td>${e.scores.precision.toFixed(4)}</td>
+      <td>${e.scores.recall.toFixed(4)}</td>
+      <td>${e.scores.citationAccuracy.toFixed(4)}</td>
+      <td>${e.scores.f1.toFixed(4)}</td>
+      <td>${e.scores.passed ? "✅" : "❌"}</td>
+    </tr>
+  `,
+    )
+    .join("");
+
+  return `
+    <section>
+      <h2>${escapeHtml(profile)}</h2>
+      <p>Best F1: ${bestF1.scores.f1.toFixed(4)} (run ${escapeHtml(bestF1.runId)})</p>
+      <div class="charts">
+        <div class="chart">
+          <h3>Precision</h3>
+          <svg viewBox="0 0 300 100" preserveAspectRatio="none">${trendLine(precisionValues, 300, 100)}</svg>
+        </div>
+        <div class="chart">
+          <h3>Recall</h3>
+          <svg viewBox="0 0 300 100" preserveAspectRatio="none">${trendLine(recallValues, 300, 100)}</svg>
+        </div>
+        <div class="chart">
+          <h3>F1</h3>
+          <svg viewBox="0 0 300 100" preserveAspectRatio="none">${trendLine(f1Values, 300, 100)}</svg>
+        </div>
+      </div>
+      <table>
+        <thead>
+          <tr>
+            <th>Timestamp</th>
+            <th>Run ID</th>
+            <th>Precision</th>
+            <th>Recall</th>
+            <th>Citation</th>
+            <th>F1</th>
+            <th>Passed</th>
+          </tr>
+        </thead>
+        <tbody>${rows}</tbody>
+      </table>
+    </section>
+  `;
+}
+
+export async function generateHtmlReport(
+  entries: LeaderboardEntry[],
+  outPath: string,
+): Promise<string> {
+  const byProfile = entries.reduce<Record<string, LeaderboardEntry[]>>((acc, entry) => {
+    acc[entry.profile] = acc[entry.profile] ?? [];
+    acc[entry.profile].push(entry);
+    return acc;
+  }, {});
+
+  const sections = Object.entries(byProfile)
+    .map(([profile, profileEntries]) => renderProfileSection(profile, profileEntries))
+    .join("");
+
+  const html = `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>KASW Benchmark Leaderboard</title>
+  <style>
+    body { font-family: system-ui, -apple-system, sans-serif; max-width: 960px; margin: 0 auto; padding: 2rem; color: #111; }
+    h1 { font-size: 1.75rem; margin-bottom: 1rem; }
+    section { margin-bottom: 3rem; }
+    table { width: 100%; border-collapse: collapse; margin-top: 1rem; }
+    th, td { text-align: left; padding: 0.5rem; border-bottom: 1px solid #ddd; }
+    th { font-weight: 600; }
+    .charts { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 1rem; margin: 1rem 0; }
+    .chart { background: #f8fafc; border-radius: 0.5rem; padding: 0.75rem; }
+    .chart h3 { margin: 0 0 0.5rem; font-size: 0.875rem; color: #475569; }
+    svg { width: 100%; height: 100px; }
+  </style>
+</head>
+<body>
+  <h1>KASW Benchmark Leaderboard</h1>
+  <p>Generated at ${new Date().toISOString()}</p>
+  ${sections}
+</body>
+</html>`;
+
+  await writeFile(outPath, html);
+  return outPath;
+}
+
+export async function getGitCommit(): Promise<string | undefined> {
+  try {
+    const result = await new Promise<string>((resolve, reject) => {
+      import("node:child_process").then(({ exec }) => {
+        exec("git rev-parse HEAD", (error, stdout) => {
+          if (error) reject(error);
+          else resolve(stdout.trim());
+        });
+      });
+    });
+    return result;
+  } catch {
+    return process.env.KASW_GIT_COMMIT;
+  }
+}

package/src/providers/brave-provider.ts

@@ -0,0 +1,161 @@
+import type { SearchOptions, SearchProvider } from "./search-provider";
+import type { Source, SourceScores, UsageMetrics } from "../types";
+
+interface BraveResult {
+  title: string;
+  url: string;
+  description?: string;
+  age?: string;
+  extra_snippets?: string[];
+}
+
+interface BraveResponse {
+  web?: {
+    results?: BraveResult[];
+  };
+}
+
+function inferSourceClass(url: string): "primary" | "secondary" {
+  try {
+    const hostname = new URL(url).hostname.toLowerCase();
+    if (
+      hostname === "github.com" ||
+      hostname.endsWith(".github.io") ||
+      hostname === "arxiv.org" ||
+      hostname.endsWith(".gov") ||
+      hostname.endsWith(".edu") ||
+      hostname.endsWith(".ac.uk")
+    ) {
+      return "primary";
+    }
+  } catch {
+    // invalid URL, fall through to secondary
+  }
+  return "secondary";
+}
+
+function parsePublishedAt(dateText?: string): string {
+  if (!dateText) return new Date().toISOString().split("T")[0];
+
+  if (/^\d{4}-\d{2}-\d{2}$/.test(dateText)) {
+    return dateText;
+  }
+
+  const parsed = new Date(dateText);
+  if (!Number.isNaN(parsed.getTime())) {
+    return parsed.toISOString().split("T")[0];
+  }
+
+  return new Date().toISOString().split("T")[0];
+}
+
+function buildScores(sourceClass: "primary" | "secondary", rank: number): SourceScores {
+  // Slight relevance decay by rank.
+  const relevance = Math.max(2, 5 - Math.floor(rank / 5));
+  return {
+    relevance,
+    authority: sourceClass === "primary" ? 4 : 3,
+    freshness: 4,
+    diversity: 3,
+    extractionValue: sourceClass === "primary" ? 4 : 3,
+  };
+}
+
+function mockResults(objective: string): Source[] {
+  const now = new Date().toISOString().split("T")[0];
+  return [
+    {
+      id: "BRAVE-001",
+      url: "https://example.com/mock/brave-result-1",
+      title: `Brave mock result for: ${objective}`,
+      sourceClass: "primary",
+      publishedAt: now,
+      discoveredBy: "brave-search-provider-mock",
+      scores: { relevance: 5, authority: 4, freshness: 5, diversity: 3, extractionValue: 4 },
+      claims: [
+        "Brave mock search returned a high-relevance primary source.",
+        "This is a deterministic fixture for CI and development.",
+      ],
+    },
+    {
+      id: "BRAVE-002",
+      url: "https://example.com/mock/brave-result-2",
+      title: "Brave mock secondary perspective",
+      sourceClass: "secondary",
+      publishedAt: now,
+      discoveredBy: "brave-search-provider-mock",
+      scores: { relevance: 3, authority: 3, freshness: 4, diversity: 4, extractionValue: 3 },
+      claims: ["Secondary sources broaden coverage in a wide search."],
+    },
+  ];
+}
+
+export class BraveSearchProvider implements SearchProvider {
+  readonly name = "brave";
+  private readonly apiKey: string;
+  private readonly metrics?: UsageMetrics;
+
+  constructor(apiKey: string, metrics?: UsageMetrics) {
+    this.apiKey = apiKey;
+    this.metrics = metrics;
+  }
+
+  async search({ objective, maxResults }: SearchOptions): Promise<Source[]> {
+    if (this.metrics) {
+      this.metrics.providerCalls += 1;
+      this.metrics.apiCalls += 1;
+    }
+
+    if (!this.apiKey) {
+      if (process.env.BRAVE_MOCK === "1") {
+        return mockResults(objective).slice(0, maxResults);
+      }
+      throw new Error(
+        "BRAVE_API_KEY environment variable is required for the brave provider (or set BRAVE_MOCK=1 for CI)",
+      );
+    }
+
+    const url = new URL("https://api.search.brave.com/res/v1/web/search");
+    url.searchParams.set("q", objective);
+    url.searchParams.set("count", String(Math.min(Math.max(maxResults, 1), 20)));
+
+    const response = await fetch(url.toString(), {
+      method: "GET",
+      headers: {
+        Accept: "application/json",
+        "X-Subscription-Token": this.apiKey,
+      },
+    });
+
+    if (!response.ok) {
+      const body = await response.text().catch(() => "unknown");
+      if (response.status === 429) {
+        throw new Error(`Brave API rate limit exceeded (429): ${body}`);
+      }
+      if (response.status === 401) {
+        throw new Error(`Brave API unauthorized (401): check BRAVE_API_KEY`);
+      }
+      throw new Error(`Brave API error: ${response.status} ${body}`);
+    }
+
+    const data = (await response.json()) as BraveResponse;
+    const results = data.web?.results ?? [];
+
+    return results.slice(0, maxResults).map((result, index) => {
+      const sourceClass = inferSourceClass(result.url);
+      const claim = [result.description, ...(result.extra_snippets ?? [])]
+        .filter(Boolean)
+        .join(" ");
+      return {
+        id: `BRAVE-${String(index + 1).padStart(3, "0")}`,
+        url: result.url,
+        title: result.title,
+        sourceClass,
+        publishedAt: parsePublishedAt(result.age),
+        discoveredBy: "brave-search-provider",
+        scores: buildScores(sourceClass, index),
+        claims: [claim || result.title],
+      };
+    });
+  }
+}

package/src/providers/github-provider.ts

@@ -0,0 +1,151 @@
+import type { SearchOptions, SearchProvider } from "./search-provider";
+import type { Source, SourceScores, UsageMetrics } from "../types";
+
+interface GitHubRepoItem {
+  full_name: string;
+  html_url: string;
+  description?: string | null;
+  stargazers_count: number;
+  updated_at: string;
+  language?: string | null;
+}
+
+interface GitHubResponse {
+  items?: GitHubRepoItem[];
+}
+
+function parseUpdatedAt(dateText?: string): string {
+  if (!dateText) return new Date().toISOString().split("T")[0];
+  const parsed = new Date(dateText);
+  if (Number.isNaN(parsed.getTime())) {
+    return new Date().toISOString().split("T")[0];
+  }
+  return parsed.toISOString().split("T")[0];
+}
+
+function authorityFromStars(stars: number): number {
+  if (stars >= 50000) return 5;
+  if (stars >= 10000) return 4;
+  if (stars >= 1000) return 3;
+  if (stars >= 100) return 2;
+  return 1;
+}
+
+function buildScores(stars: number): SourceScores {
+  return {
+    relevance: 5,
+    authority: authorityFromStars(stars),
+    freshness: 4,
+    diversity: 3,
+    extractionValue: 5,
+  };
+}
+
+function mockResults(objective: string): Source[] {
+  const now = new Date().toISOString().split("T")[0];
+  return [
+    {
+      id: "GITHUB-001",
+      url: "https://github.com/mock/example-agent",
+      title: `Mock repo for: ${objective}`,
+      sourceClass: "primary",
+      publishedAt: now,
+      discoveredBy: "github-search-provider-mock",
+      scores: { relevance: 5, authority: 4, freshness: 5, diversity: 3, extractionValue: 5 },
+      claims: [
+        "Mock GitHub repository for CI and development.",
+        "Language: TypeScript, Stars: 15000.",
+      ],
+    },
+    {
+      id: "GITHUB-002",
+      url: "https://github.com/mock/secondary-agent",
+      title: "Mock secondary agent repo",
+      sourceClass: "primary",
+      publishedAt: now,
+      discoveredBy: "github-search-provider-mock",
+      scores: { relevance: 4, authority: 3, freshness: 4, diversity: 4, extractionValue: 4 },
+      claims: ["Secondary mock repository with lower star count."],
+    },
+  ];
+}
+
+export class GitHubSearchProvider implements SearchProvider {
+  readonly name = "github";
+  private readonly token: string;
+  private readonly metrics?: UsageMetrics;
+
+  constructor(token: string, metrics?: UsageMetrics) {
+    this.token = token;
+    this.metrics = metrics;
+  }
+
+  async search({ objective, maxResults }: SearchOptions): Promise<Source[]> {
+    if (this.metrics) {
+      this.metrics.providerCalls += 1;
+      this.metrics.apiCalls += 1;
+    }
+
+    if (!this.token) {
+      if (process.env.GITHUB_MOCK === "1") {
+        return mockResults(objective).slice(0, maxResults);
+      }
+      throw new Error(
+        "GITHUB_TOKEN environment variable is required for the github provider (or set GITHUB_MOCK=1 for CI)",
+      );
+    }
+
+    const url = new URL("https://api.github.com/search/repositories");
+    url.searchParams.set("q", objective);
+    url.searchParams.set("per_page", String(Math.min(Math.max(maxResults, 1), 100)));
+    url.searchParams.set("sort", "stars");
+    url.searchParams.set("order", "desc");
+
+    const headers: Record<string, string> = {
+      Accept: "application/vnd.github+json",
+      "X-GitHub-Api-Version": "2022-11-28",
+    };
+    if (this.token) {
+      headers.Authorization = `Bearer ${this.token}`;
+    }
+
+    const response = await fetch(url.toString(), {
+      method: "GET",
+      headers,
+    });
+
+    if (!response.ok) {
+      const body = await response.text().catch(() => "unknown");
+      if (response.status === 403 || response.status === 429) {
+        throw new Error(`GitHub API rate limit exceeded (${response.status}): ${body}`);
+      }
+      if (response.status === 401) {
+        throw new Error(`GitHub API unauthorized (401): check GITHUB_TOKEN`);
+      }
+      throw new Error(`GitHub API error: ${response.status} ${body}`);
+    }
+
+    const data = (await response.json()) as GitHubResponse;
+    const results = data.items ?? [];
+
+    return results.slice(0, maxResults).map((result, index) => {
+      const description = result.description ?? "";
+      const language = result.language ?? "unknown";
+      const stars = result.stargazers_count ?? 0;
+      return {
+        id: `GITHUB-${String(index + 1).padStart(3, "0")}`,
+        url: result.html_url,
+        title: result.full_name,
+        sourceClass: "primary",
+        publishedAt: parseUpdatedAt(result.updated_at),
+        discoveredBy: "github-search-provider",
+        scores: buildScores(stars),
+        claims: [
+          [description, `Language: ${language}`, `Stars: ${stars.toLocaleString()}`]
+            .filter(Boolean)
+            .join(" "),
+        ],
+      };
+    });
+  }
+}

package/src/providers/index.ts

@@ -0,0 +1,49 @@
+import { BraveSearchProvider } from "./brave-provider";
+import { GitHubSearchProvider } from "./github-provider";
+import { MockSearchProvider } from "./mock-search-provider";
+import { SerperSearchProvider } from "./serper-provider";
+import { TavilySearchProvider } from "./tavily-provider";
+import type { SearchProvider } from "./search-provider";
+import type { UsageMetrics } from "../types";
+
+export * from "./search-provider";
+export { BraveSearchProvider } from "./brave-provider";
+export { GitHubSearchProvider } from "./github-provider";
+export { MockSearchProvider } from "./mock-search-provider";
+export { SerperSearchProvider } from "./serper-provider";
+export { TavilySearchProvider } from "./tavily-provider";
+
+export interface CreateSearchProviderOptions {
+  credential?: string;
+  metrics?: UsageMetrics;
+}
+
+export function createSearchProvider(
+  name: string,
+  options: CreateSearchProviderOptions = {},
+): SearchProvider {
+  const { credential, metrics } = options;
+
+  switch (name) {
+    case "mock":
+      return new MockSearchProvider(metrics);
+    case "serper": {
+      const apiKey = credential ?? process.env.SERPER_API_KEY ?? "";
+      return new SerperSearchProvider(apiKey, metrics);
+    }
+    case "tavily": {
+      const apiKey = credential ?? process.env.TAVILY_API_KEY ?? "";
+      return new TavilySearchProvider(apiKey, metrics);
+    }
+    case "brave": {
+      const apiKey = credential ?? process.env.BRAVE_API_KEY ?? "";
+      return new BraveSearchProvider(apiKey, metrics);
+    }
+    case "github": {
+      const token = credential ?? process.env.GITHUB_TOKEN ?? "";
+      return new GitHubSearchProvider(token, metrics);
+    }
+    default:
+      throw new Error(`Unknown search provider: ${name}`);
+  }
+}

package/src/providers/mock-search-provider.ts

@@ -0,0 +1,45 @@
+import type { SearchOptions, SearchProvider } from "./search-provider";
+import type { Source, UsageMetrics } from "../types";
+
+export class MockSearchProvider implements SearchProvider {
+  readonly name = "mock";
+  private readonly metrics?: UsageMetrics;
+
+  constructor(metrics?: UsageMetrics) {
+    this.metrics = metrics;
+  }
+
+  async search({ objective, depth }: SearchOptions): Promise<Source[]> {
+    if (this.metrics) {
+      this.metrics.providerCalls += 1;
+      this.metrics.apiCalls += 1;
+    }
+    const now = new Date().toISOString().split("T")[0];
+    return [
+      {
+        id: "MOCK-001",
+        url: "https://example.com/mock/result-1",
+        title: `Mock result for: ${objective}`,
+        sourceClass: "primary-analysis",
+        publishedAt: now,
+        discoveredBy: "mock-search-provider",
+        scores: { relevance: 5, authority: 4, freshness: 5, diversity: 3, extractionValue: 4 },
+        claims: [
+          `Mock search was executed with depth '${depth}'.`,
+          "This is a placeholder result for development and CI.",
+          "Replace with a real provider (serper, tavily, etc.) for live searches.",
+        ],
+      },
+      {
+        id: "MOCK-002",
+        url: "https://example.com/mock/result-2",
+        title: "Mock secondary perspective",
+        sourceClass: "secondary",
+        publishedAt: now,
+        discoveredBy: "mock-search-provider",
+        scores: { relevance: 3, authority: 2, freshness: 4, diversity: 4, extractionValue: 2 },
+        claims: ["Secondary sources can provide alternative viewpoints in a wide search."],
+      },
+    ];
+  }
+}

package/src/providers/search-provider.ts

@@ -0,0 +1,12 @@
+import type { SearchDepth, Source } from "../types";
+
+export interface SearchOptions {
+  objective: string;
+  depth: SearchDepth;
+  maxResults: number;
+}
+
+export interface SearchProvider {
+  readonly name: string;
+  search(options: SearchOptions): Promise<Source[]>;
+}