kimi-agent-swarm-cli 0.7.0

package/src/scorer.ts

@@ -0,0 +1,103 @@
+import type { EnrichedSource, Source } from "./types";
+
+const SCORE_WEIGHTS = {
+  relevance: 0.35,
+  authority: 0.25,
+  freshness: 0.2,
+  diversity: 0.1,
+  extractionValue: 0.1,
+};
+
+const HIGH_AUTHORITY_DOMAINS = new Set([
+  "github.com",
+  "arxiv.org",
+  "ssrn.com",
+  "crunchbase.com",
+  "sec.gov",
+  "who.int",
+  "imf.org",
+  "worldbank.org",
+  "bloomberg.com",
+  "reuters.com",
+  "ft.com",
+  "wsj.com",
+  "economist.com",
+  "nature.com",
+  "science.org",
+]);
+
+function extractDomain(url: string): string | undefined {
+  try {
+    return new URL(url).hostname.toLowerCase();
+  } catch {
+    return undefined;
+  }
+}
+
+function domainAuthorityBonus(url: string): number {
+  const domain = extractDomain(url);
+  if (!domain) return 0;
+
+  if (HIGH_AUTHORITY_DOMAINS.has(domain)) return 0.5;
+  if (domain.endsWith(".edu") || domain.endsWith(".gov") || domain.endsWith(".ac.uk")) {
+    return 0.4;
+  }
+  if (domain.endsWith(".org")) return 0.15;
+  return 0;
+}
+
+function freshnessPenalty(publishedAt?: string): number {
+  if (!publishedAt || publishedAt === "unknown") return -0.3;
+  if (publishedAt >= "2026-01-01") return 0;
+  if (publishedAt >= "2025-01-01") return -0.2;
+  if (publishedAt >= "2024-01-01") return -0.5;
+  return -0.8;
+}
+
+export function calculateWeightedScore(source: Source): number {
+  const scores = source.scores ?? {};
+  const baseScore =
+    (scores.relevance ?? 0) * SCORE_WEIGHTS.relevance +
+    (scores.authority ?? 0) * SCORE_WEIGHTS.authority +
+    (scores.freshness ?? 0) * SCORE_WEIGHTS.freshness +
+    (scores.diversity ?? 0) * SCORE_WEIGHTS.diversity +
+    (scores.extractionValue ?? 0) * SCORE_WEIGHTS.extractionValue;
+
+  const domainBonus = domainAuthorityBonus(source.url);
+  const freshness = freshnessPenalty(source.publishedAt);
+
+  return Math.max(0, Math.min(5, baseScore + domainBonus + freshness));
+}
+
+export function scoreSource(source: Source): EnrichedSource {
+  const weightedScore = calculateWeightedScore(source);
+  const scores = source.scores ?? {};
+  const relevance = scores.relevance ?? 0;
+  const authority = scores.authority ?? 0;
+
+  let reason: string;
+  let accepted: boolean;
+
+  if (weightedScore >= 3.0 && relevance >= 2 && authority >= 2) {
+    accepted = true;
+    reason = `weighted score ${weightedScore.toFixed(2)} meets acceptance threshold`;
+  } else if (relevance < 2) {
+    accepted = false;
+    reason = "low relevance";
+  } else if (authority < 2) {
+    accepted = false;
+    reason = "low authority";
+  } else if (weightedScore < 3.0) {
+    accepted = false;
+    reason = `weighted score ${weightedScore.toFixed(2)} below acceptance threshold`;
+  } else {
+    accepted = false;
+    reason = "duplicate or low-value source";
+  }
+
+  return {
+    ...source,
+    decision: accepted ? "accepted" : "rejected",
+    reason,
+  };
+}

package/src/types.ts

@@ -0,0 +1,246 @@
+export type SearchDepth = "light" | "standard" | "deep" | "maximum";
+
+export type ExecutionProfile =
+  | "fixture"
+  | "fixture-asset-mgmt"
+  | "fixture-sellside-research"
+  | "fixture-youtube-niche"
+  | "fixture-paul-graham-corpus"
+  | "fixture-github-repo-landscape"
+  | "fixture-market-scan"
+  | "local-command"
+  | "web-search";
+
+export interface SourceScores {
+  relevance: number;
+  authority: number;
+  freshness?: number;
+  diversity?: number;
+  extractionValue?: number;
+}
+
+export interface Source {
+  id: string;
+  url: string;
+  title: string;
+  sourceClass: string;
+  publishedAt?: string;
+  discoveredBy: string;
+  scores: SourceScores;
+  claims?: string[];
+}
+
+export interface EnrichedSource extends Source {
+  decision: "accepted" | "rejected";
+  reason: string;
+}
+
+export type ClaimConfidence = "high" | "medium" | "low";
+
+export type ClaimFreshness = "current" | "stale" | "unknown";
+
+export interface Claim {
+  id: string;
+  claim: string;
+  sourceIds: string[];
+  confidence: ClaimConfidence;
+  freshness: ClaimFreshness;
+}
+
+export interface UsageMetrics {
+  providerCalls: number;
+  apiCalls: number;
+  estimatedTokens?: number;
+  estimatedCostUsd?: number;
+  actualCostUsd?: number;
+  notes?: string;
+}
+
+export interface Run {
+  runId: string;
+  objective: string;
+  executionProfile: ExecutionProfile;
+  status: "completed" | "failed";
+  createdAt: string;
+  usageMetrics: UsageMetrics;
+  replayedFrom?: string;
+  cached?: boolean;
+}
+
+export interface ResearchPlan {
+  objective: string;
+  searchDepth: SearchDepth;
+  executionProfile: ExecutionProfile;
+  queryFamilies: string[];
+  sourceTargets: string[];
+  stopConditions: string[];
+}
+
+export interface DuplicateClaimGroup {
+  representativeClaimId: string;
+  claimIds: string[];
+  similarityReason: string;
+}
+
+export interface ConflictingClaimPair {
+  claimIdA: string;
+  claimIdB: string;
+  entity: string;
+  reason: string;
+}
+
+export interface VerificationReport {
+  status: "passed" | "failed";
+  acceptedSources: number;
+  rejectedSources: number;
+  unsupportedClaims: number;
+  staleClaims: number;
+  unknownFreshnessClaims: number;
+  lowConfidenceClaims: number;
+  duplicateClaimGroups: DuplicateClaimGroup[];
+  conflictingClaimPairs: ConflictingClaimPair[];
+  coverageGaps: string[];
+  failures: string[];
+  warnings: string[];
+}
+
+export interface RunWideSearchOptions {
+  objective?: string;
+  profile?: ExecutionProfile;
+  providerCommand?: string;
+  providerArgs?: string[];
+  providerName?: string;
+  searchDepth?: SearchDepth;
+  workDir?: string;
+  budget?: BudgetOptions;
+  useCache?: boolean;
+  replayRunId?: string;
+  distributed?: DistributedRunOptions;
+}
+
+export interface DistributedRunOptions {
+  enabled: boolean;
+  workers?: number;
+  maxRetries?: number;
+  resumeJobId?: string;
+  queueType?: "memory" | "redis";
+}
+
+export interface DistributedJob {
+  jobId: string;
+  objective: string;
+  executionProfile: ExecutionProfile;
+  providerName: string;
+  searchDepth: SearchDepth;
+  queueType: "memory" | "redis";
+  status: DistributedJobStatus;
+  tasks: DistributedTask[];
+  createdAt: string;
+  updatedAt: string;
+}
+
+export type DistributedJobStatus = "pending" | "running" | "completed" | "failed";
+
+export interface DistributedTask {
+  taskId: string;
+  jobId: string;
+  queryFamily: string;
+  query: string;
+  status: DistributedTaskStatus;
+  attempts: number;
+  maxRetries: number;
+  workerId?: string;
+  result?: WorkerResult;
+  error?: string;
+  startedAt?: string;
+  completedAt?: string;
+}
+
+export type DistributedTaskStatus = "pending" | "running" | "completed" | "failed";
+
+export interface WorkerResult {
+  sources: Source[];
+  usageMetrics: UsageMetrics;
+}
+
+export interface LoadSourcesOptions {
+  profile: ExecutionProfile;
+  objective: string;
+  providerCommand?: string;
+  providerArgs?: string[];
+  providerName?: string;
+  searchDepth?: SearchDepth;
+  metrics?: UsageMetrics;
+  useCache?: boolean;
+}
+
+export interface VerifyRunOptions {
+  runDir?: string;
+  minAcceptedSources?: number;
+}
+
+export interface RunWideSearchResult {
+  runId: string;
+  runDir: string;
+  verification: VerificationReport;
+}
+
+export interface ProviderPricing {
+  perCallUsd: number;
+  per1kTokensUsd?: number;
+}
+
+export interface BudgetOptions {
+  maxCostUsd?: number;
+  maxProviderCalls?: number;
+  maxApiCalls?: number;
+  dryRun?: boolean;
+}
+
+export type ExportFormat = "json" | "csv";
+
+export interface ExportOptions {
+  runDir: string;
+  format: ExportFormat;
+  outPath?: string;
+}
+
+export interface GoldenAnswer {
+  expectedClaims: string[];
+  expectedSourceUrls?: string[];
+}
+
+export interface BenchmarkResult {
+  profile: string;
+  runId: string;
+  runDir: string;
+  precision: number;
+  recall: number;
+  citationAccuracy: number;
+  f1: number;
+  passed: boolean;
+}
+
+export interface LeaderboardEntry {
+  runId: string;
+  profile: string;
+  runDir: string;
+  timestamp: string;
+  gitCommit?: string;
+  scores: BenchmarkResult;
+}
+
+export interface CostEstimate {
+  providerName: string;
+  depth: SearchDepth;
+  estimatedProviderCalls: number;
+  estimatedApiCalls: number;
+  estimatedCostUsd: number;
+}
+
+export interface CacheKey {
+  provider: string;
+  objective: string;
+  depth: SearchDepth;
+  maxResults: number;
+}

package/src/verifier.ts

@@ -0,0 +1,369 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type {
+  Claim,
+  ClaimConfidence,
+  ClaimFreshness,
+  ConflictingClaimPair,
+  DuplicateClaimGroup,
+  EnrichedSource,
+  VerificationReport,
+} from "./types";
+
+async function readJsonl<T>(path: string): Promise<T[] | null> {
+  try {
+    const text = await readFile(path, "utf8");
+    return text
+      .split(/\r?\n/)
+      .filter(Boolean)
+      .map((line) => JSON.parse(line) as T);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return null;
+    }
+    throw error;
+  }
+}
+
+function normalizeClaimText(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]/gu, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function tokenSet(text: string): Set<string> {
+  return new Set(normalizeClaimText(text).split(" ").filter(Boolean));
+}
+
+function jaccardSimilarity(a: string, b: string): number {
+  const setA = tokenSet(a);
+  const setB = tokenSet(b);
+  if (setA.size === 0 && setB.size === 0) return 1;
+  const intersection = new Set([...setA].filter((x) => setB.has(x)));
+  const union = new Set([...setA, ...setB]);
+  return intersection.size / union.size;
+}
+
+function findDuplicateClaims(claims: Claim[]): DuplicateClaimGroup[] {
+  const groups: DuplicateClaimGroup[] = [];
+  const assigned = new Set<string>();
+  const sorted = [...claims].sort((a, b) => a.claim.length - b.claim.length);
+
+  for (let i = 0; i < sorted.length; i++) {
+    const base = sorted[i];
+    if (assigned.has(base.id)) continue;
+
+    const duplicates: Claim[] = [base];
+    for (let j = i + 1; j < sorted.length; j++) {
+      const other = sorted[j];
+      if (assigned.has(other.id)) continue;
+      const similarity = jaccardSimilarity(base.claim, other.claim);
+      const normalizedBase = normalizeClaimText(base.claim);
+      const normalizedOther = normalizeClaimText(other.claim);
+      const isSubstring =
+        normalizedBase.length > 10 &&
+        (normalizedOther.includes(normalizedBase) || normalizedBase.includes(normalizedOther));
+      if (similarity >= 0.7 || isSubstring) {
+        duplicates.push(other);
+      }
+    }
+
+    if (duplicates.length > 1) {
+      const claimIds = duplicates.map((c) => c.id);
+      claimIds.forEach((id) => assigned.add(id));
+      groups.push({
+        representativeClaimId: base.id,
+        claimIds,
+        similarityReason: "jaccard token similarity >= 0.7 or substring containment",
+      });
+    }
+  }
+
+  return groups;
+}
+
+const POSITIVE_POLARITY = [
+  "increase",
+  "increases",
+  "increased",
+  "rising",
+  "rises",
+  "rose",
+  "grows",
+  "grew",
+  "growth",
+  "up",
+  "higher",
+  "more",
+  "positive",
+  "good",
+  "bullish",
+  "strong",
+  "buy",
+  "outperform",
+  "above",
+  "exceeds",
+];
+
+const NEGATIVE_POLARITY = [
+  "decrease",
+  "decreases",
+  "decreased",
+  "falling",
+  "falls",
+  "fell",
+  "shrinks",
+  "shrank",
+  "shrunk",
+  "down",
+  "lower",
+  "less",
+  "negative",
+  "bad",
+  "bearish",
+  "weak",
+  "sell",
+  "underperform",
+  "below",
+  "misses",
+];
+
+function extractEntities(claim: string): string[] {
+  const entities: string[] = [];
+
+  // Quoted phrases
+  const quoted = claim.match(/"([^"]{3,80})"/g) ?? [];
+  entities.push(...quoted.map((m) => m.slice(1, -1)));
+
+  // Capitalized phrases (1-4 words) - likely proper nouns or product names
+  const capitalized = claim.match(/\b[A-Z][a-zA-Z0-9]*(?:\s+[A-Z][a-zA-Z0-9]*){0,3}\b/g) ?? [];
+  entities.push(...capitalized);
+
+  // Numbers with units
+  const numbers =
+    claim.match(
+      /\d+(?:\.\d+)?\s*(?:%|percent|bp|basis points|million|billion|trillion|KRW|USD|EUR|GBP)/gi,
+    ) ?? [];
+  entities.push(...numbers);
+
+  return [...new Set(entities.map((e) => e.toLowerCase().trim()))].filter((e) => e.length > 2);
+}
+
+function detectPolarity(claim: string): "positive" | "negative" | "neutral" {
+  const normalized = normalizeClaimText(claim);
+  const positiveHits = POSITIVE_POLARITY.filter((word) => normalized.includes(word)).length;
+  const negativeHits = NEGATIVE_POLARITY.filter((word) => normalized.includes(word)).length;
+
+  if (positiveHits > 0 && negativeHits === 0) return "positive";
+  if (negativeHits > 0 && positiveHits === 0) return "negative";
+  return "neutral";
+}
+
+function findConflictingClaims(claims: Claim[]): ConflictingClaimPair[] {
+  const pairs: ConflictingClaimPair[] = [];
+  const seen = new Set<string>();
+
+  for (let i = 0; i < claims.length; i++) {
+    const claimA = claims[i];
+    const polarityA = detectPolarity(claimA.claim);
+    if (polarityA === "neutral") continue;
+
+    const entitiesA = extractEntities(claimA.claim);
+    if (entitiesA.length === 0) continue;
+
+    for (let j = i + 1; j < claims.length; j++) {
+      const claimB = claims[j];
+      const polarityB = detectPolarity(claimB.claim);
+      if (polarityB === "neutral" || polarityA === polarityB) continue;
+
+      const entitiesB = extractEntities(claimB.claim);
+      const sharedEntities = entitiesA.filter((entity) => entitiesB.includes(entity));
+      if (sharedEntities.length === 0) continue;
+
+      const pairKey = [claimA.id, claimB.id].sort().join("::");
+      if (seen.has(pairKey)) continue;
+      seen.add(pairKey);
+
+      pairs.push({
+        claimIdA: claimA.id,
+        claimIdB: claimB.id,
+        entity: sharedEntities[0],
+        reason: `opposing polarity (${polarityA} vs ${polarityB}) on shared entity "${sharedEntities[0]}"`,
+      });
+    }
+  }
+
+  return pairs;
+}
+
+function countByFreshness(claims: Claim[], freshness: ClaimFreshness): number {
+  return claims.filter((claim) => claim.freshness === freshness).length;
+}
+
+function countByConfidence(claims: Claim[], confidence: ClaimConfidence): number {
+  return claims.filter((claim) => claim.confidence === confidence).length;
+}
+
+function findCoverageGaps(
+  sources: EnrichedSource[],
+  acceptedSources: EnrichedSource[],
+): string[] {
+  const gaps: string[] = [];
+  const sourceClasses = new Set(sources.map((s) => s.sourceClass));
+  const acceptedClasses = new Set(acceptedSources.map((s) => s.sourceClass));
+
+  if (!acceptedClasses.has("primary-analysis") && sourceClasses.has("primary-analysis")) {
+    gaps.push("no accepted primary-analysis sources");
+  }
+  if (!acceptedClasses.has("secondary") && sourceClasses.has("secondary")) {
+    gaps.push("no accepted secondary sources");
+  }
+
+  const acceptedRatio = acceptedSources.length / Math.max(sources.length, 1);
+  if (acceptedRatio < 0.25) {
+    gaps.push("accepted source ratio below 25%");
+  }
+
+  return gaps;
+}
+
+function findBrokenSourceReferences(claims: Claim[], sourceIds: Set<string>): string[] {
+  const broken: string[] = [];
+  for (const claim of claims) {
+    for (const sourceId of claim.sourceIds ?? []) {
+      if (!sourceIds.has(sourceId)) {
+        broken.push(`${claim.id} -> ${sourceId}`);
+      }
+    }
+  }
+  return broken;
+}
+
+export interface VerifyRunOptions {
+  runDir?: string;
+  minAcceptedSources?: number;
+  maxLowConfidenceRatio?: number;
+  maxStaleRatio?: number;
+  maxDuplicateClaimGroups?: number;
+}
+
+export async function verifyRun({
+  runDir,
+  minAcceptedSources = 1,
+  maxLowConfidenceRatio = 0.5,
+  maxStaleRatio = 0.5,
+  maxDuplicateClaimGroups = Number.MAX_SAFE_INTEGER,
+}: VerifyRunOptions = {}): Promise<VerificationReport> {
+  if (!runDir) {
+    throw new Error("verifyRun requires runDir");
+  }
+
+  const failures: string[] = [];
+  const warnings: string[] = [];
+  const sources = await readJsonl<EnrichedSource>(join(runDir, "source-ledger.jsonl"));
+  const claims = await readJsonl<Claim>(join(runDir, "claim-ledger.jsonl"));
+
+  if (!sources) {
+    failures.push("missing source ledger");
+  }
+
+  if (!claims) {
+    failures.push("missing claim ledger");
+  }
+
+  const acceptedSources = sources?.filter((source) => source.decision === "accepted") ?? [];
+  const rejectedSources = sources?.filter((source) => source.decision === "rejected") ?? [];
+
+  if (sources && acceptedSources.length < minAcceptedSources) {
+    failures.push(
+      `accepted source count ${acceptedSources.length} below minimum ${minAcceptedSources}`,
+    );
+  }
+
+  const unsupportedClaims: string[] = [];
+  for (const claim of claims ?? []) {
+    if (!Array.isArray(claim.sourceIds) || claim.sourceIds.length === 0) {
+      unsupportedClaims.push(claim.id ?? claim.claim ?? "unknown claim");
+    }
+  }
+
+  if (unsupportedClaims.length > 0) {
+    failures.push(`unsupported claims: ${unsupportedClaims.join(", ")}`);
+  }
+
+  const duplicateSources =
+    sources?.filter((source) => source.reason === "duplicate or low-value source") ?? [];
+  if (sources && duplicateSources.length / Math.max(sources.length, 1) > 0.5) {
+    warnings.push("duplicate or low-value source ratio is high");
+  }
+
+  const staleClaims = countByFreshness(claims ?? [], "stale");
+  const unknownFreshnessClaims = countByFreshness(claims ?? [], "unknown");
+  const lowConfidenceClaims = countByConfidence(claims ?? [], "low");
+
+  const totalClaims = claims?.length ?? 0;
+  if (totalClaims > 0) {
+    if (staleClaims / totalClaims > maxStaleRatio) {
+      failures.push(
+        `stale claim ratio ${(staleClaims / totalClaims).toFixed(2)} exceeds ${maxStaleRatio}`,
+      );
+    } else if (staleClaims / totalClaims > maxStaleRatio / 2) {
+      warnings.push(`stale claim ratio is ${(staleClaims / totalClaims).toFixed(2)}`);
+    }
+
+    if (lowConfidenceClaims / totalClaims > maxLowConfidenceRatio) {
+      failures.push(
+        `low-confidence claim ratio ${(lowConfidenceClaims / totalClaims).toFixed(2)} exceeds ${maxLowConfidenceRatio}`,
+      );
+    } else if (lowConfidenceClaims / totalClaims > maxLowConfidenceRatio / 2) {
+      warnings.push(`low-confidence claim ratio is ${(lowConfidenceClaims / totalClaims).toFixed(2)}`);
+    }
+  }
+
+  const duplicateClaimGroups = findDuplicateClaims(claims ?? []);
+  if (duplicateClaimGroups.length > maxDuplicateClaimGroups) {
+    failures.push(
+      `duplicate claim groups ${duplicateClaimGroups.length} exceeds ${maxDuplicateClaimGroups}`,
+    );
+  } else if (duplicateClaimGroups.length > 0) {
+    warnings.push(`${duplicateClaimGroups.length} duplicate claim groups detected`);
+  }
+
+  const conflictingClaimPairs = findConflictingClaims(claims ?? []);
+  if (conflictingClaimPairs.length > 0) {
+    warnings.push(
+      `${conflictingClaimPairs.length} conflicting claim pairs detected; review needed`,
+    );
+  }
+
+  const coverageGaps = findCoverageGaps(sources ?? [], acceptedSources);
+  if (coverageGaps.length > 0) {
+    warnings.push(`coverage gaps: ${coverageGaps.join("; ")}`);
+  }
+
+  const sourceIds = new Set(sources?.map((s) => s.id) ?? []);
+  const brokenReferences = findBrokenSourceReferences(claims ?? [], sourceIds);
+  if (brokenReferences.length > 0) {
+    failures.push(`broken source references: ${brokenReferences.join(", ")}`);
+  }
+
+  const report: VerificationReport = {
+    status: failures.length === 0 ? "passed" : "failed",
+    acceptedSources: acceptedSources.length,
+    rejectedSources: rejectedSources.length,
+    unsupportedClaims: unsupportedClaims.length,
+    staleClaims,
+    unknownFreshnessClaims,
+    lowConfidenceClaims,
+    duplicateClaimGroups,
+    conflictingClaimPairs,
+    coverageGaps,
+    failures,
+    warnings,
+  };
+
+  await writeFile(join(runDir, "verification-report.json"), `${JSON.stringify(report, null, 2)}\n`);
+  return report;
+}