@oss-scout/core 0.1.0

package/dist/core/http-cache.js

@@ -0,0 +1,314 @@
+/**
+ * HTTP caching with ETags for GitHub API responses.
+ *
+ * Stores ETags and response bodies for cacheable GET endpoints in
+ * `~/.oss-scout/cache/`. On subsequent requests, sends `If-None-Match`
+ * headers — 304 responses don't count against GitHub rate limits.
+ *
+ * Also provides in-flight request deduplication so that concurrent calls
+ * for the same endpoint (e.g., star counts for two PRs in the same repo)
+ * share a single HTTP round-trip.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+import { getCacheDir } from './utils.js';
+import { debug, warn } from './logger.js';
+import { errorMessage, getHttpStatusCode } from './errors.js';
+const MODULE = 'http-cache';
+/**
+ * Maximum age (in ms) before a cache entry is considered stale and eligible for
+ * eviction during cleanup. Defaults to 24 hours. Entries older than this are
+ * still *usable* for conditional requests (the ETag may still be valid), but
+ * `evictStale()` will remove them.
+ */
+const DEFAULT_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+/**
+ * File-based HTTP cache backed by `~/.oss-scout/cache/`.
+ *
+ * Each cache entry is stored as a separate JSON file keyed by the SHA-256
+ * hash of the request URL. This avoids filesystem issues with URL-based
+ * filenames and keeps lookup O(1).
+ */
+export class HttpCache {
+    cacheDir;
+    /** In-flight request deduplication map: URL -> Promise<response>. */
+    inflightRequests = new Map();
+    constructor(cacheDir) {
+        this.cacheDir = cacheDir ?? getCacheDir();
+    }
+    /** Derive a filesystem-safe cache key from a URL. */
+    keyFor(url) {
+        return crypto.createHash('sha256').update(url).digest('hex');
+    }
+    /** Full path to the cache file for a given URL. */
+    pathFor(url) {
+        return path.join(this.cacheDir, `${this.keyFor(url)}.json`);
+    }
+    /**
+     * Return the cached body if the entry exists and is younger than `maxAgeMs`.
+     * Useful for time-based caching where ETag validation isn't applicable
+     * (e.g., caching aggregated results from paginated API calls).
+     */
+    getIfFresh(key, maxAgeMs) {
+        const entry = this.get(key);
+        if (!entry)
+            return null;
+        const age = Date.now() - new Date(entry.cachedAt).getTime();
+        if (!Number.isFinite(age) || age < 0 || age > maxAgeMs)
+            return null;
+        return entry.body;
+    }
+    /**
+     * Look up a cached response. Returns `null` if no cache entry exists.
+     */
+    get(url) {
+        const filePath = this.pathFor(url);
+        try {
+            const raw = fs.readFileSync(filePath, 'utf-8');
+            const entry = JSON.parse(raw);
+            // Sanity-check: the file should contain the URL we asked for
+            if (entry.url !== url) {
+                debug(MODULE, `Cache collision detected for ${url}, ignoring`);
+                return null;
+            }
+            return entry;
+        }
+        catch (err) {
+            const code = err?.code;
+            if (code === 'ENOENT')
+                return null;
+            if (err instanceof SyntaxError) {
+                debug(MODULE, `Corrupt cache entry, deleting: ${url}`);
+                try {
+                    fs.unlinkSync(filePath);
+                }
+                catch (unlinkErr) {
+                    debug(MODULE, `Failed to delete corrupt cache entry: ${errorMessage(unlinkErr)}`);
+                }
+                return null;
+            }
+            warn(MODULE, `Cache read failed for ${url}: ${errorMessage(err)}`);
+            return null;
+        }
+    }
+    /**
+     * Store a response with its ETag.
+     */
+    set(url, etag, body) {
+        const entry = {
+            etag,
+            url,
+            body,
+            cachedAt: new Date().toISOString(),
+        };
+        try {
+            fs.writeFileSync(this.pathFor(url), JSON.stringify(entry), { encoding: 'utf-8', mode: 0o600 });
+            debug(MODULE, `Cached response for ${url}`);
+        }
+        catch (err) {
+            // Non-fatal: cache write failure should not break the request
+            warn(MODULE, `Failed to write cache for ${url}: ${errorMessage(err)}`);
+        }
+    }
+    /**
+     * Get the in-flight promise for a URL (for deduplication).
+     */
+    getInflight(url) {
+        return this.inflightRequests.get(url);
+    }
+    /**
+     * Register an in-flight request for deduplication.
+     * Returns a cleanup function to call when the request completes.
+     */
+    setInflight(url, promise) {
+        this.inflightRequests.set(url, promise);
+        return () => {
+            this.inflightRequests.delete(url);
+        };
+    }
+    /**
+     * Remove stale entries older than `maxAgeMs` from the cache directory.
+     * Intended to be called periodically (e.g., once per search invocation).
+     */
+    evictStale(maxAgeMs = DEFAULT_MAX_AGE_MS) {
+        let evicted = 0;
+        try {
+            const files = fs.readdirSync(this.cacheDir);
+            const now = Date.now();
+            for (const file of files) {
+                if (!file.endsWith('.json'))
+                    continue;
+                const filePath = path.join(this.cacheDir, file);
+                try {
+                    const raw = fs.readFileSync(filePath, 'utf-8');
+                    const entry = JSON.parse(raw);
+                    const age = now - new Date(entry.cachedAt).getTime();
+                    if (age > maxAgeMs) {
+                        fs.unlinkSync(filePath);
+                        evicted++;
+                    }
+                }
+                catch {
+                    debug(MODULE, `Removing unreadable cache entry ${file}`);
+                    try {
+                        fs.unlinkSync(filePath);
+                        evicted++;
+                    }
+                    catch (unlinkErr) {
+                        debug(MODULE, `Failed to remove stale cache entry ${file}: ${errorMessage(unlinkErr)}`);
+                    }
+                }
+            }
+        }
+        catch (err) {
+            const code = err?.code;
+            if (code !== 'ENOENT') {
+                warn(MODULE, `Failed to evict stale cache entries: ${errorMessage(err)}`);
+            }
+        }
+        if (evicted > 0) {
+            debug(MODULE, `Evicted ${evicted} stale cache entries`);
+        }
+        return evicted;
+    }
+    /**
+     * Remove all entries from the cache.
+     */
+    clear() {
+        try {
+            const files = fs.readdirSync(this.cacheDir);
+            for (const file of files) {
+                if (!file.endsWith('.json'))
+                    continue;
+                fs.unlinkSync(path.join(this.cacheDir, file));
+            }
+            debug(MODULE, 'Cache cleared');
+        }
+        catch (err) {
+            const code = err?.code;
+            if (code !== 'ENOENT') {
+                warn(MODULE, `Failed to clear cache: ${errorMessage(err)}`);
+            }
+        }
+    }
+    /**
+     * Return the number of entries currently in the cache.
+     */
+    size() {
+        try {
+            return fs.readdirSync(this.cacheDir).filter((f) => f.endsWith('.json')).length;
+        }
+        catch (err) {
+            const code = err?.code;
+            if (code !== 'ENOENT') {
+                debug(MODULE, `Failed to read cache size: ${errorMessage(err)}`);
+            }
+            return 0;
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Singleton
+// ---------------------------------------------------------------------------
+let _httpCache = null;
+/**
+ * Get (or create) the shared HttpCache singleton.
+ * The singleton is lazily initialized on first access.
+ */
+export function getHttpCache() {
+    if (!_httpCache) {
+        _httpCache = new HttpCache();
+    }
+    return _httpCache;
+}
+// ---------------------------------------------------------------------------
+// Octokit integration helpers
+// ---------------------------------------------------------------------------
+/**
+ * Wraps an Octokit `repos.get`-style call with ETag caching and request
+ * deduplication.
+ *
+ * Usage:
+ * ```ts
+ * const data = await cachedRequest(cache, octokit, '/repos/owner/repo', () =>
+ *   octokit.repos.get({ owner, repo: name }),
+ * );
+ * ```
+ *
+ * 1. If an identical request is already in-flight, returns the existing promise
+ *    (request deduplication).
+ * 2. If a cached ETag exists, sends `If-None-Match`. On 304, returns the
+ *    cached body without consuming a rate-limit point.
+ * 3. On a fresh 200, caches the ETag + body for next time.
+ */
+export async function cachedRequest(cache, url, fetcher) {
+    // --- Deduplication ---
+    const existing = cache.getInflight(url);
+    if (existing) {
+        debug(MODULE, `Dedup hit for ${url}`);
+        return (await existing);
+    }
+    const doFetch = async () => {
+        const extraHeaders = {};
+        const cached = cache.get(url);
+        if (cached) {
+            extraHeaders['if-none-match'] = cached.etag;
+        }
+        try {
+            const response = await fetcher(extraHeaders);
+            // Store ETag if present (headers may be absent in test mocks)
+            const etag = response.headers?.['etag'];
+            if (etag) {
+                cache.set(url, etag, response.data);
+            }
+            return response.data;
+        }
+        catch (err) {
+            // Check for 304 Not Modified — re-read cache to avoid stale closure snapshot
+            if (isNotModifiedError(err)) {
+                const freshCached = cache.get(url);
+                if (freshCached) {
+                    debug(MODULE, `304 cache hit for ${url}`);
+                    return freshCached.body;
+                }
+            }
+            throw err;
+        }
+    };
+    const promise = doFetch();
+    const cleanup = cache.setInflight(url, promise);
+    try {
+        const result = await promise;
+        return result;
+    }
+    finally {
+        cleanup();
+    }
+}
+/**
+ * Time-based cache wrapper (no ETag / conditional requests).
+ *
+ * If a cached result exists and is younger than `maxAgeMs`, returns it.
+ * Otherwise calls `fetcher`, caches the result, and returns it.
+ *
+ * Use this for expensive operations whose results change slowly
+ * (e.g. search queries, project health checks).
+ */
+export async function cachedTimeBased(cache, key, maxAgeMs, fetcher) {
+    const cached = cache.getIfFresh(key, maxAgeMs);
+    if (cached) {
+        debug(MODULE, `Time-based cache hit for ${key}`);
+        return cached;
+    }
+    const result = await fetcher();
+    cache.set(key, '', result);
+    return result;
+}
+/**
+ * Detect whether an error is a 304 Not Modified response.
+ * Octokit throws a RequestError with status 304 for conditional requests.
+ */
+function isNotModifiedError(err) {
+    return getHttpStatusCode(err) === 304;
+}

package/dist/core/issue-discovery.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Issue Discovery — orchestrates multi-phase issue search across GitHub.
+ *
+ * Delegates filtering, scoring, vetting, and search infrastructure to focused modules:
+ * - issue-filtering.ts  — spam detection, doc-only filtering, per-repo caps
+ * - issue-scoring.ts   — viability scores, repo quality bonuses
+ * - issue-vetting.ts   — vetting orchestration, recommendation + viability scoring
+ * - issue-eligibility.ts — PR existence, claim detection, requirements analysis
+ * - repo-health.ts     — project health checks, contribution guidelines
+ * - search-phases.ts   — search helpers, caching, batched repo search
+ *
+ * All state is injected via constructor parameters (ScoutStateReader + ScoutPreferences).
+ */
+import { type IssueCandidate } from './types.js';
+import type { ScoutPreferences, SearchStrategy } from './schemas.js';
+import { type ScoutStateReader } from './issue-vetting.js';
+/**
+ * Multi-phase issue discovery engine that searches GitHub for contributable issues.
+ *
+ * Search phases (in priority order):
+ * 0. Repos where user has merged PRs (highest merge probability)
+ * 0.5. Preferred organizations
+ * 1. Starred repos
+ * 2. General label-filtered search
+ * 3. Actively maintained repos
+ *
+ * Each candidate is vetted for claimability and scored 0-100 for viability.
+ */
+export declare class IssueDiscovery {
+    private preferences;
+    private stateReader;
+    private octokit;
+    private githubToken;
+    private vetter;
+    /** Set after searchIssues() runs if rate limits affected the search (low pre-flight quota or mid-search rate limit hits). */
+    rateLimitWarning: string | null;
+    /**
+     * @param githubToken  - GitHub personal access token or token from `gh auth token`
+     * @param preferences  - User's search preferences (languages, labels, scopes, etc.)
+     * @param stateReader  - Read-only interface for accessing scout state (merged PRs, starred repos, etc.)
+     */
+    constructor(githubToken: string, preferences: ScoutPreferences, stateReader: ScoutStateReader);
+    /**
+     * Get starred repos from the state reader.
+     * @returns Array of starred repo names in "owner/repo" format
+     */
+    getStarredRepos(): string[];
+    /**
+     * Search for issues matching our criteria.
+     * Searches in priority order: merged-PR repos first (no label filter), then preferred
+     * organizations, then starred repos, then general search, then actively maintained repos.
+     * Filters out issues from low-scoring and excluded repos.
+     *
+     * @param options - Search configuration
+     * @param options.languages - Programming languages to filter by
+     * @param options.labels - Issue labels to search for
+     * @param options.maxResults - Maximum candidates to return (default: 10)
+     * @returns Scored and sorted issue candidates
+     * @throws {ValidationError} If no candidates found and no rate limits prevented the search
+     *
+     * @example
+     * ```typescript
+     * import { IssueDiscovery } from '@oss-scout/core';
+     *
+     * const discovery = new IssueDiscovery(token, preferences, stateReader);
+     * const candidates = await discovery.searchIssues({ maxResults: 5 });
+     * for (const c of candidates) {
+     *   console.log(`${c.issue.repo}#${c.issue.number}: ${c.viabilityScore}/100`);
+     * }
+     * ```
+     */
+    searchIssues(options?: {
+        languages?: string[];
+        labels?: string[];
+        maxResults?: number;
+        strategies?: SearchStrategy[];
+    }): Promise<{
+        candidates: IssueCandidate[];
+        strategiesUsed: SearchStrategy[];
+    }>;
+    /**
+     * Vet a specific issue for claimability and project health.
+     * @param issueUrl - Full GitHub issue URL
+     * @returns The vetted issue candidate with recommendation and scores
+     * @throws {ValidationError} If the URL is invalid or the issue cannot be fetched
+     */
+    vetIssue(issueUrl: string): Promise<IssueCandidate>;
+    /**
+     * Derive low-scoring repos from the state reader.
+     * A repo is considered "low-scoring" if its score is at or below the threshold.
+     */
+    private deriveLowScoringRepos;
+}