github-search-cli 0.7.0 → 0.8.0

package/src/provider.ts

@@ -1,165 +0,0 @@
-/**
- * Search provider: talks to the GitHub REST search API.
- *
- * This is the only layer that performs network I/O. It implements the
- * {@link SearchProvider} interface so alternative backends (a local index, a
- * different forge, or an AI-powered source) can be swapped in without touching
- * the CLI, query, ranking, or output layers.
- */
-import type { ParsedQuery, SearchOptions, SearchProvider, SearchResponse } from "./types.ts";
-import { buildSearchUrl } from "./query.ts";
-import { normalizeEnvelope, type GitHubSearchEnvelope } from "./normalizer.ts";
-import { NetworkError, RateLimitError, ParseError } from "./errors.ts";
-import { MemoryCache } from "./cache.ts";
-
-/** A minimal logger so the CLI can surface outgoing queries, errors, and
- *  rate-limit events without coupling to a specific logging framework. */
-export interface Logger {
-  debug(msg: string): void;
-  info(msg: string): void;
-  warn(msg: string): void;
-  error(msg: string): void;
-}
-
-const noopLogger: Logger = {
-  debug: () => {},
-  info: () => {},
-  warn: () => {},
-  error: () => {},
-};
-
-export class GitHubSearchProvider implements SearchProvider {
-  readonly name = "github";
-  private readonly logger: Logger;
-  private tokens: string[];
-  private tokenIndex = 0;
-  /** Shared cache across instances. */
-  static cache = new MemoryCache<SearchResponse>(300_000);
-
-  constructor(logger: Logger = noopLogger, tokens: string[] = []) {
-    this.logger = logger;
-    this.tokens = tokens.filter(Boolean);
-  }
-
-  private nextToken(): string | undefined {
-    if (this.tokens.length === 0) return undefined;
-    const token = this.tokens[this.tokenIndex % this.tokens.length];
-    this.tokenIndex++;
-    return token;
-  }
-
-  async search(query: ParsedQuery, options: SearchOptions): Promise<SearchResponse> {
-    // Check cache first
-    const cacheKey = MemoryCache.key(query.raw, options.sort, String(options.limit), String(options.page ?? 1));
-    const cached = GitHubSearchProvider.cache.get(cacheKey);
-    if (cached) {
-      this.logger.debug("[github] cache hit");
-      return cached;
-    }
-    this.logger.debug("[github] cache miss");
-    const url = buildSearchUrl(query, options);
-    this.logger.debug(`[github] outgoing query: ${query.raw}`);
-    this.logger.debug(`[github] request url: ${url}`);
-
-    const all: SearchResponse["repos"] = [];
-    let totalCount = 0;
-    let rateLimited = false;
-    let rateLimitRemaining: number | undefined;
-
-    // Fetch one page (or multiple pages to reach limit if page=1)
-    const perPage = Math.min(Math.max(options.limit, 1), 100);
-    const startPage = options.page ?? 1;
-    // ponytail: when page > 1, fetch only that specific page
-    const maxPages = startPage > 1 ? startPage : Math.ceil(options.limit / perPage);
-
-    for (let page = startPage > 1 ? startPage : 1; page <= maxPages; page++) {
-      const pagedUrl = appendPage(url, page, perPage);
-      let lastErr: Error | undefined;
-
-      // Try with current token; if 403, retry with next token
-      for (let attempt = 0; attempt <= this.tokens.length; attempt++) {
-        const token = this.tokens.length > 0 ? this.nextToken() : options.token;
-        try {
-          const res = await fetch(pagedUrl, {
-            headers: {
-              Accept: "application/vnd.github+json",
-              "X-GitHub-Api-Version": "2022-11-28",
-              "User-Agent": "search-cli",
-              ...(token ? { Authorization: `Bearer ${token}` } : {}),
-            },
-          });
-
-          const remainingStr = res.headers.get("x-ratelimit-remaining");
-          rateLimitRemaining = remainingStr !== null ? Number(remainingStr) : undefined;
-          if (res.status === 403) {
-            if (remainingStr === "0" || remainingStr === null) {
-              // Rate limited
-              if (this.tokens.length > 0 && attempt < this.tokens.length) {
-                continue;
-              }
-              this.logger.warn("[github] rate limit exceeded");
-              rateLimited = true;
-              throw new RateLimitError(!!token);
-            }
-            // Non-rate-limit 403 (e.g. private repo, permissions)
-            const body = await res.text().catch(() => "");
-            this.logger.error(`[github] non-rate-limit 403: ${body.slice(0, 200)}`);
-            throw new NetworkError();
-          }
-          if (!res.ok) {
-            const body = await res.text().catch(() => "");
-            this.logger.error(`[github] API error ${res.status}: ${body.slice(0, 200)}`);
-            throw new NetworkError();
-          }
-
-          // Validate response is valid JSON before trusting the cast
-          const raw = await res.text();
-          let parsed: any;
-          try {
-            parsed = JSON.parse(raw);
-          } catch {
-            const snippet = raw.slice(0, 200);
-            this.logger.error(`[github] invalid JSON response: ${snippet}`);
-            throw new ParseError("GitHub API", snippet.includes("<!DOCTYPE")
-              ? "GitHub returned an HTML page (maintenance or CAPTCHA?)"
-              : "Invalid JSON response");
-          }
-          if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.items)) {
-            this.logger.error(`[github] unexpected response shape: ${JSON.stringify(parsed).slice(0, 200)}`);
-            throw new ParseError("GitHub API", "Unexpected response shape — missing items array");
-          }
-          const env = parsed as GitHubSearchEnvelope;
-          totalCount = env.total_count;
-          const repos = normalizeEnvelope(env);
-          all.push(...repos);
-          lastErr = undefined;
-          break; // success, exit token retry loop
-        } catch (err) {
-          lastErr = err as Error;
-          if (rateLimited) break;
-        }
-      }
-
-      if (lastErr && !rateLimited) throw lastErr;
-      if (rateLimited) break;
-      if (all.length >= options.limit) break;
-    }
-
-    const repos = all.slice(0, options.limit);
-    if (repos.length === 0) {
-      this.logger.debug("[github] empty results");
-    }
-    const response: SearchResponse = { totalCount, repos, rateLimited, rateLimitRemaining };
-    if (!rateLimited) {
-      GitHubSearchProvider.cache.set(cacheKey, response);
-    }
-    return response;
-  }
-}
-
-function appendPage(url: string, page: number, perPage: number): string {
-  const u = new URL(url);
-  u.searchParams.set("page", String(page));
-  u.searchParams.set("per_page", String(perPage));
-  return u.toString();
-}

package/src/query.ts

@@ -1,249 +0,0 @@
-/**
- * Query layer: parse a raw user query into structured qualifiers/keywords and
- * build the GitHub search syntax string.
- *
- * This layer is deliberately free of any I/O. It only transforms strings into
- * the {@link ParsedQuery} shape and then into a GitHub `q` parameter. Keeping it
- * pure makes it trivially unit-testable and lets a future AI query-rewriter sit
- * in front of it without side effects.
- */
-import type { ParsedQuery, Qualifier, SearchOptions, SortStrategy } from "./types.ts";
-
-/** Qualifier keys we explicitly understand and can also accept as CLI flags. */
-export const KNOWN_QUALIFIERS = [
-  "language",
-  "stars",
-  "fork",
-  "archived",
-  "topic",
-  "user",
-  "org",
-  "repo",
-  "updated",
-  "pushed",
-  "visibility",
-  "in",
-  "size",
-  "license",
-  "created",
-] as const;
-
-export type KnownQualifier = (typeof KNOWN_QUALIFIERS)[number];
-
-/** Qualifier suggestions for Tab auto-complete. Maps a prefix to matching qualifiers. */
-export const QUALIFIER_SUGGESTIONS: Record<string, string[]> = {
-  l: ["language:"],
-  s: ["stars:", "size:"],
-  t: ["topic:"],
-  u: ["user:"],
-  o: ["org:"],
-  r: ["repo:"],
-  c: ["created:"],
-  p: ["pushed:"],
-  v: ["visibility:"],
-  i: ["in:"],
-  f: ["fork:"],
-  a: ["archived:"],
-};
-
-/** Find matching qualifier suggestions for a partial word. */
-export function suggestFor(input: string): string[] {
-  const trimmed = input.trim().toLowerCase();
-  if (trimmed.length === 0) return [];
-  // Single letter prefix
-  if (trimmed.length === 1 && QUALIFIER_SUGGESTIONS[trimmed]) {
-    return QUALIFIER_SUGGESTIONS[trimmed];
-  }
-  // Partial qualifier key (e.g. "lang" → "language:")
-  const matches = KNOWN_QUALIFIERS.filter(k => k.startsWith(trimmed));
-  return matches.map(k => k + ":");
-}
-
-/**
- * Tokenize a raw query string while respecting double quotes so that phrases
- * like `"machine learning"` or `repo:"my cool repo"` stay together as one
- * token even when the quote is not at a token boundary.
- */
-export function tokenize(input: string): string[] {
-  const tokens: string[] = [];
-  let current = "";
-  let inQuotes = false;
-  for (let i = 0; i < input.length; i++) {
-    const ch = input[i];
-    if (ch === '"') {
-      inQuotes = !inQuotes;
-      current += ch; // keep the quote so callers can detect quoted values
-    } else if (/\s/.test(ch) && !inQuotes) {
-      if (current) {
-        tokens.push(current);
-        current = "";
-      }
-    } else {
-      current += ch;
-    }
-  }
-  if (current) {
-    // Strip quotes only when the whole token is wrapped in them
-    // (e.g. `"machine learning"`); leave `repo:"my repo"` intact for the
-    // qualifier parser to handle.
-    if (current.startsWith('"') && current.endsWith('"') && current.length >= 2) {
-      current = current.slice(1, -1);
-    }
-    tokens.push(current);
-  }
-  return tokens;
-}
-
-/**
- * Parse a raw query string into a {@link ParsedQuery}.
- *
- * Rules:
- *  - `key:value` or `key:"quoted value"` becomes a qualifier.
- *  - `-key:value` becomes a negated qualifier.
- *  - Anything else is a free-text keyword (quotes are stripped).
- *  - The original string is preserved verbatim in `raw`.
- */
-export function parseQuery(raw: string): ParsedQuery {
-  const trimmed = raw.trim();
-  if (trimmed === "") {
-    return { keywords: [], qualifiers: [], raw };
-  }
-
-  const keywords: string[] = [];
-  const qualifiers: Qualifier[] = [];
-
-  for (const token of tokenize(trimmed)) {
-    const qualifier = matchQualifier(token);
-    if (qualifier) {
-      qualifiers.push(qualifier);
-    } else {
-      keywords.push(token);
-    }
-  }
-
-  return { keywords, qualifiers, raw: trimmed };
-}
-
-/** Match a single token against `[-]key:value` (value may be quoted). */
-function matchQualifier(token: string): Qualifier | null {
-  const negated = token.startsWith("-");
-  const body = negated ? token.slice(1) : token;
-  const idx = body.indexOf(":");
-  if (idx <= 0) return null; // no key, or starts with ':'
-  const key = body.slice(0, idx).toLowerCase();
-  let value = body.slice(idx + 1);
-  if (value === "") return null;
-  // A qualifier key must be a plain word (letters only) so that URLs like
-  // `http://example.com` are treated as free-text keywords, not qualifiers.
-  if (!/^[a-z]+$/i.test(key)) return null;
-  // A value starting with `//` is a URL scheme, not a qualifier value.
-  if (value.startsWith("//")) return null;
-  // Strip surrounding quotes from a quoted value (e.g. `repo:"my repo"`).
-  if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) {
-    value = value.slice(1, -1);
-  }
-  return { key, value, negated };
-}
-
-/** CLI flag filters that map onto qualifiers. */
-export interface FlagFilters {
-  language?: string;
-  stars?: string;
-  org?: string;
-  user?: string;
-  topic?: string;
-  archived?: boolean;
-  fork?: boolean;
-}
-
-/**
- * Merge flag-based filters into a parsed query. Flags are appended as
- * qualifiers. Returns a new {@link ParsedQuery} (does not mutate the input).
- */
-export function applyFlagFilters(
-  query: ParsedQuery,
-  flags: FlagFilters,
-): ParsedQuery {
-  const qualifiers = [...query.qualifiers];
-
-  if (flags.language) qualifiers.push({ key: "language", value: flags.language, negated: false });
-  if (flags.stars) qualifiers.push({ key: "stars", value: flags.stars, negated: false });
-  if (flags.org) qualifiers.push({ key: "org", value: flags.org, negated: false });
-  if (flags.user) qualifiers.push({ key: "user", value: flags.user, negated: false });
-  if (flags.topic) qualifiers.push({ key: "topic", value: flags.topic, negated: false });
-  if (flags.archived !== undefined)
-    qualifiers.push({ key: "archived", value: flags.archived ? "true" : "false", negated: false });
-  if (flags.fork !== undefined)
-    qualifiers.push({ key: "fork", value: flags.fork ? "true" : "false", negated: false });
-
-  return { ...query, qualifiers };
-}
-
-/**
- * Validate a parsed query for conflicting or nonsensical filters.
- * Throws an Error describing the first conflict found.
- */
-export function validateQuery(query: ParsedQuery): void {
-  const has = (key: string) => query.qualifiers.some((q) => q.key === key && !q.negated);
-
-  // ponytail: removed fork+archived and user+org validation — both are valid
-  // GitHub search queries (a fork can be archived; user: acts as author/committer
-  // not owner, so user+org is a valid scoping query).
-  if (has("visibility") && has("private")) {
-    throw new Error("Cannot combine `visibility:` and `private:` filters.");
-  }
-}
-
-/**
- * Build the GitHub search `q` string from a parsed query.
- * Keywords are joined with spaces; qualifiers are emitted as `key:value`
- * (or `-key:value` when negated). Quoted values are re-quoted when they
- * contain spaces.
- */
-export function buildGitHubQuery(query: ParsedQuery): string {
-  const parts: string[] = [];
-
-  for (const kw of query.keywords) {
-    const alreadyQuoted = kw.startsWith('"') && kw.endsWith('"');
-    parts.push(kw.includes(" ") && !alreadyQuoted ? `"${kw}"` : kw);
-  }
-
-  for (const q of query.qualifiers) {
-    const needsQuote = q.value.includes(" ") && !q.value.startsWith('"');
-    const value = needsQuote ? `"${q.value}"` : q.value;
-    parts.push(`${q.negated ? "-" : ""}${q.key}:${value}`);
-  }
-
-  return parts.join(" ");
-}
-
-/**
- * Map our sort strategy to the GitHub `sort` param and the in-query `sort:`
- * qualifier. GitHub's repository search supports `stars`, `forks`, `updated`
- * and (implicitly) best-match. We encode best-match by omitting the param.
- */
-export function githubSortParam(sort: SortStrategy): { sort?: string; order?: string } {
-  switch (sort) {
-    case "stars":
-      return { sort: "stars", order: "desc" };
-    case "forks":
-      return { sort: "forks", order: "desc" };
-    case "updated":
-      return { sort: "updated", order: "desc" };
-    case "best-match":
-    default:
-      return {}; // GitHub default relevance ranking
-  }
-}
-
-/** Build the full GitHub search URL for the given query + options. */
-export function buildSearchUrl(query: ParsedQuery, options: SearchOptions): string {
-  const q = buildGitHubQuery(query);
-  const { sort, order } = githubSortParam(options.sort);
-  const perPage = Math.min(Math.max(options.limit, 1), 100); // GitHub max page size
-  // ponytail: per_page is set by appendPage in provider, not here
-  const params = new URLSearchParams({ q: q || " " });
-  if (sort) params.set("sort", sort);
-  if (order) params.set("order", order);
-  return `https://api.github.com/search/repositories?${params.toString()}`;
-}

package/src/ranking.ts

@@ -1,57 +0,0 @@
-/**
- * Ranking layer: order normalized {@link Repo} results deterministically.
- *
- * All strategies are pure functions of the repo data plus a stable tie-breaker
- * (repo id) so that identical inputs always produce identical output. A future
- * AI reranker can be added as an additional strategy or as a post-step without
- * changing the rest of the pipeline.
- */
-import type { Repo, SortStrategy } from "./types.ts";
-
-/**
- * Rank repos according to the chosen strategy.
- * The sort is stable and deterministic: when two repos compare equal on the
- * primary key, we fall back to descending stars, then descending id.
- */
-export function rankRepos(repos: Repo[], strategy: SortStrategy): Repo[] {
-  const copy = [...repos];
-  switch (strategy) {
-    case "stars":
-      copy.sort(byKey((r) => r.stars));
-      break;
-    case "forks":
-      copy.sort(byKey((r) => r.forks));
-      break;
-    case "updated":
-      copy.sort(byKey((r) => Date.parse(r.updatedAt)));
-      break;
-    case "best-match":
-    default:
-      // GitHub already returns best-match order; preserve it unchanged.
-      // Re-sorting by score would be redundant for first-page results and
-      // misleading for paginated (page 2 repos would intermix with page 1).
-      // ponytail: if deterministic output is needed, sort by id:
-      // copy.sort((a, b) => a.id - b.id);
-      break;
-  }
-  return copy;
-}
-
-/** Comparator factory: descending by `key`, then descending stars, then
- *  ascending id (stable, deterministic tie-break). */
-function byKey(key: (r: Repo) => number): (a: Repo, b: Repo) => number {
-  return (a, b) => key(b) - key(a) || b.stars - a.stars || a.id - b.id;
-}
-
-/**
- * A simple, explainable composite score used for the (optional) "best" blend.
- * Not used by default but available for an AI/rerank layer later.
- * Range is roughly 0..1 after normalization by the caller.
- */
-export function compositeScore(r: Repo): number {
-  // Log-scaled stars + recency bonus. Deterministic and transparent.
-  const starScore = Math.log10(r.stars + 1);
-  const recencyMs = Date.now() - Date.parse(r.pushedAt);
-  const recencyScore = Math.max(0, 1 - recencyMs / (1000 * 60 * 60 * 24 * 365));
-  return starScore * 0.7 + recencyScore * 0.3;
-}