@oss-scout/core 0.1.0

package/dist/core/gist-state-store.js

@@ -0,0 +1,302 @@
+/**
+ * Gist-backed state persistence for oss-scout.
+ *
+ * Stores ScoutState as a private GitHub Gist, with a local file cache
+ * as fallback when the API is unavailable.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { ScoutStateSchema } from './schemas.js';
+import { getDataDir } from './utils.js';
+import { debug, warn } from './logger.js';
+import { errorMessage } from './errors.js';
+const MODULE = 'gist-state';
+const GIST_DESCRIPTION = 'oss-scout-state';
+const GIST_FILENAME = 'state.json';
+const GIST_ID_FILE = 'gist-id';
+const CACHE_FILE = 'state-cache.json';
+const SEARCH_MAX_PAGES = 5;
+function getGistIdPath() {
+    return path.join(getDataDir(), GIST_ID_FILE);
+}
+function getCachePath() {
+    return path.join(getDataDir(), CACHE_FILE);
+}
+export class GistStateStore {
+    octokit;
+    gistId = null;
+    constructor(octokit) {
+        this.octokit = octokit;
+    }
+    /**
+     * Bootstrap: find an existing gist or create a new one.
+     * Falls back to local cache if the API is unavailable.
+     */
+    async bootstrap() {
+        try {
+            return await this.bootstrapFromApi();
+        }
+        catch (err) {
+            warn(MODULE, `API bootstrap failed: ${errorMessage(err)}`);
+            return this.bootstrapFromCache();
+        }
+    }
+    /**
+     * Push state to the gist. Also writes to local cache as fallback.
+     */
+    async push(state) {
+        this.writeCache(state);
+        if (!this.gistId) {
+            warn(MODULE, 'No gist ID — cannot push');
+            return false;
+        }
+        try {
+            await this.octokit.gists.update({
+                gist_id: this.gistId,
+                files: {
+                    [GIST_FILENAME]: { content: JSON.stringify(state, null, 2) },
+                },
+            });
+            debug(MODULE, 'State pushed to gist');
+            return true;
+        }
+        catch (err) {
+            warn(MODULE, `Failed to push: ${errorMessage(err)}`);
+            return false;
+        }
+    }
+    /**
+     * Pull state from the gist and merge with local state.
+     */
+    async pull() {
+        if (!this.gistId)
+            return null;
+        try {
+            const state = await this.fetchGistState(this.gistId);
+            if (state) {
+                this.writeCache(state);
+            }
+            return state;
+        }
+        catch (err) {
+            warn(MODULE, `Failed to pull: ${errorMessage(err)}`);
+            return null;
+        }
+    }
+    /** Get the current gist ID (if known). */
+    getGistId() {
+        return this.gistId;
+    }
+    // ── API bootstrap flow ───────────────────────────────────────────────
+    async bootstrapFromApi() {
+        // 1. Check local gist-id cache
+        const cachedId = this.readCachedGistId();
+        if (cachedId) {
+            debug(MODULE, `Trying cached gist ID: ${cachedId}`);
+            try {
+                const state = await this.fetchGistState(cachedId);
+                if (state) {
+                    this.gistId = cachedId;
+                    this.writeCache(state);
+                    return { gistId: cachedId, state, created: false };
+                }
+            }
+            catch (err) {
+                debug(MODULE, `Cached gist ID invalid: ${errorMessage(err)}`);
+            }
+            debug(MODULE, 'Cached gist ID invalid, searching...');
+        }
+        // 2. Search user's gists
+        const foundId = await this.searchForGist();
+        if (foundId) {
+            debug(MODULE, `Found gist via search: ${foundId}`);
+            this.saveGistId(foundId);
+            this.gistId = foundId;
+            const state = await this.fetchGistState(foundId);
+            if (state) {
+                this.writeCache(state);
+                return { gistId: foundId, state, created: false };
+            }
+        }
+        // 3. Create new gist
+        debug(MODULE, 'No existing gist found, creating new one');
+        const freshState = ScoutStateSchema.parse({ version: 1 });
+        const newId = await this.createGist(freshState);
+        this.saveGistId(newId);
+        this.gistId = newId;
+        this.writeCache(freshState);
+        return { gistId: newId, state: freshState, created: true };
+    }
+    bootstrapFromCache() {
+        const cached = this.readCache();
+        if (cached) {
+            debug(MODULE, 'Bootstrapped from local cache (degraded mode)');
+            const cachedId = this.readCachedGistId();
+            if (cachedId)
+                this.gistId = cachedId;
+            return {
+                gistId: cachedId ?? '',
+                state: cached,
+                created: false,
+                degraded: true,
+            };
+        }
+        debug(MODULE, 'No cache available, using fresh state (degraded mode)');
+        const fresh = ScoutStateSchema.parse({ version: 1 });
+        return { gistId: '', state: fresh, created: false, degraded: true };
+    }
+    // ── Gist API operations ──────────────────────────────────────────────
+    async fetchGistState(gistId) {
+        const { data } = await this.octokit.gists.get({ gist_id: gistId });
+        const file = data.files?.[GIST_FILENAME];
+        if (!file?.content)
+            return null;
+        try {
+            const parsed = JSON.parse(file.content);
+            return ScoutStateSchema.parse(parsed);
+        }
+        catch (err) {
+            warn(MODULE, `Gist content failed validation: ${errorMessage(err)}`);
+            return null;
+        }
+    }
+    async searchForGist() {
+        for (let page = 1; page <= SEARCH_MAX_PAGES; page++) {
+            const { data: gists } = await this.octokit.gists.list({
+                per_page: 100,
+                page,
+            });
+            if (gists.length === 0)
+                break;
+            const match = gists.find((g) => g.description === GIST_DESCRIPTION);
+            if (match)
+                return match.id;
+        }
+        return null;
+    }
+    async createGist(state) {
+        const { data } = await this.octokit.gists.create({
+            description: GIST_DESCRIPTION,
+            public: false,
+            files: {
+                [GIST_FILENAME]: { content: JSON.stringify(state, null, 2) },
+            },
+        });
+        return data.id;
+    }
+    // ── Local file helpers ───────────────────────────────────────────────
+    readCachedGistId() {
+        try {
+            const id = fs.readFileSync(getGistIdPath(), 'utf-8').trim();
+            return id || null;
+        }
+        catch (err) {
+            const code = err?.code;
+            if (code !== 'ENOENT') {
+                warn(MODULE, `Failed to read cached gist ID: ${errorMessage(err)}`);
+            }
+            return null;
+        }
+    }
+    saveGistId(id) {
+        fs.writeFileSync(getGistIdPath(), id + '\n', { mode: 0o600 });
+    }
+    readCache() {
+        try {
+            const raw = fs.readFileSync(getCachePath(), 'utf-8');
+            return ScoutStateSchema.parse(JSON.parse(raw));
+        }
+        catch (err) {
+            const code = err?.code;
+            if (code !== 'ENOENT') {
+                warn(MODULE, `Failed to read state cache: ${errorMessage(err)}`);
+            }
+            return null;
+        }
+    }
+    writeCache(state) {
+        try {
+            fs.writeFileSync(getCachePath(), JSON.stringify(state, null, 2) + '\n', { mode: 0o600 });
+        }
+        catch (err) {
+            warn(MODULE, `Failed to write cache: ${errorMessage(err)}`);
+        }
+    }
+}
+// ── State merging ────────────────────────────────────────────────────
+/**
+ * Merge two ScoutState objects with conflict resolution:
+ * - repoScores: per-repo, keep the one with more total PR activity
+ * - mergedPRs/closedPRs: union by URL
+ * - preferences: remote wins
+ * - starredRepos: keep the list with the fresher timestamp
+ * - savedResults: union by issueUrl, keep newer lastSeenAt
+ */
+export function mergeStates(local, remote) {
+    return {
+        version: 1,
+        preferences: remote.preferences,
+        repoScores: mergeRepoScores(local.repoScores, remote.repoScores),
+        starredRepos: mergeStarredRepos(local, remote),
+        starredReposLastFetched: pickFresherTimestamp(local.starredReposLastFetched, remote.starredReposLastFetched),
+        mergedPRs: unionByUrl(local.mergedPRs, remote.mergedPRs),
+        closedPRs: unionByUrl(local.closedPRs, remote.closedPRs),
+        savedResults: mergeSavedResults(local.savedResults ?? [], remote.savedResults ?? []),
+        lastSearchAt: pickFresherTimestamp(local.lastSearchAt, remote.lastSearchAt),
+        lastRunAt: pickFresherTimestamp(local.lastRunAt, remote.lastRunAt) ?? new Date().toISOString(),
+        gistId: remote.gistId ?? local.gistId,
+    };
+}
+function mergeRepoScores(local, remote) {
+    const merged = { ...local };
+    for (const [repo, remoteScore] of Object.entries(remote)) {
+        const localScore = merged[repo];
+        if (!localScore) {
+            merged[repo] = remoteScore;
+        }
+        else {
+            const localActivity = localScore.mergedPRCount + localScore.closedWithoutMergeCount;
+            const remoteActivity = remoteScore.mergedPRCount + remoteScore.closedWithoutMergeCount;
+            merged[repo] = remoteActivity >= localActivity ? remoteScore : localScore;
+        }
+    }
+    return merged;
+}
+function mergeStarredRepos(local, remote) {
+    const localTs = local.starredReposLastFetched;
+    const remoteTs = remote.starredReposLastFetched;
+    if (!localTs && !remoteTs)
+        return remote.starredRepos.length >= local.starredRepos.length ? remote.starredRepos : local.starredRepos;
+    if (!localTs)
+        return remote.starredRepos;
+    if (!remoteTs)
+        return local.starredRepos;
+    return remoteTs >= localTs ? remote.starredRepos : local.starredRepos;
+}
+function unionByUrl(local, remote) {
+    const seen = new Map();
+    for (const item of local)
+        seen.set(item.url, item);
+    for (const item of remote)
+        seen.set(item.url, item);
+    return [...seen.values()];
+}
+function mergeSavedResults(local, remote) {
+    const merged = new Map();
+    for (const item of local)
+        merged.set(item.issueUrl, item);
+    for (const item of remote) {
+        const existing = merged.get(item.issueUrl);
+        if (!existing || item.lastSeenAt > existing.lastSeenAt) {
+            merged.set(item.issueUrl, item);
+        }
+    }
+    return [...merged.values()];
+}
+function pickFresherTimestamp(a, b) {
+    if (!a)
+        return b;
+    if (!b)
+        return a;
+    return a >= b ? a : b;
+}

package/dist/core/github.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Shared GitHub API client with rate limiting and throttling.
+ */
+import { Octokit } from '@octokit/rest';
+interface RateLimitInfo {
+    remaining: number;
+    limit: number;
+    resetAt: string;
+}
+export declare function getRateLimitCallbacks(): {
+    onRateLimit: (retryAfter: number, options: unknown, _octokit: unknown, retryCount: number) => boolean;
+    onSecondaryRateLimit: (retryAfter: number, options: unknown, _octokit: unknown, retryCount: number) => boolean;
+};
+export declare function getOctokit(token: string): Octokit;
+export declare function checkRateLimit(token: string): Promise<RateLimitInfo>;
+export {};

package/dist/core/github.js

@@ -0,0 +1,58 @@
+/**
+ * Shared GitHub API client with rate limiting and throttling.
+ */
+import { Octokit } from '@octokit/rest';
+import { throttling } from '@octokit/plugin-throttling';
+import { warn } from './logger.js';
+const MODULE = 'github';
+const ThrottledOctokit = Octokit.plugin(throttling);
+let _octokit = null;
+let _currentToken = null;
+function formatResetTime(date) {
+    return date.toLocaleTimeString('en-US', { hour12: false });
+}
+export function getRateLimitCallbacks() {
+    return {
+        onRateLimit: (retryAfter, options, _octokit, retryCount) => {
+            const opts = options;
+            const resetAt = new Date(Date.now() + retryAfter * 1000);
+            if (retryCount < 2) {
+                warn(MODULE, `Rate limit hit (retry ${retryCount + 1}/2, waiting ${retryAfter}s, resets at ${formatResetTime(resetAt)}) — ${opts.method} ${opts.url}`);
+                return true;
+            }
+            warn(MODULE, `Rate limit exceeded, not retrying — ${opts.method} ${opts.url} (resets at ${formatResetTime(resetAt)})`);
+            return false;
+        },
+        onSecondaryRateLimit: (retryAfter, options, _octokit, retryCount) => {
+            const opts = options;
+            const resetAt = new Date(Date.now() + retryAfter * 1000);
+            if (retryCount < 3) {
+                warn(MODULE, `Secondary rate limit hit (retry ${retryCount + 1}/3, waiting ${retryAfter}s, resets at ${formatResetTime(resetAt)}) — ${opts.method} ${opts.url}`);
+                return true;
+            }
+            warn(MODULE, `Secondary rate limit exceeded, not retrying — ${opts.method} ${opts.url} (resets at ${formatResetTime(resetAt)})`);
+            return false;
+        },
+    };
+}
+export function getOctokit(token) {
+    if (_octokit && _currentToken === token)
+        return _octokit;
+    const callbacks = getRateLimitCallbacks();
+    _octokit = new ThrottledOctokit({
+        auth: token,
+        throttle: callbacks,
+    });
+    _currentToken = token;
+    return _octokit;
+}
+export async function checkRateLimit(token) {
+    const octokit = getOctokit(token);
+    const { data } = await octokit.rateLimit.get();
+    const search = data.resources.search;
+    return {
+        remaining: search.remaining,
+        limit: search.limit,
+        resetAt: new Date(search.reset * 1000).toISOString(),
+    };
+}

package/dist/core/http-cache.d.ts

@@ -0,0 +1,108 @@
+/**
+ * HTTP caching with ETags for GitHub API responses.
+ *
+ * Stores ETags and response bodies for cacheable GET endpoints in
+ * `~/.oss-scout/cache/`. On subsequent requests, sends `If-None-Match`
+ * headers — 304 responses don't count against GitHub rate limits.
+ *
+ * Also provides in-flight request deduplication so that concurrent calls
+ * for the same endpoint (e.g., star counts for two PRs in the same repo)
+ * share a single HTTP round-trip.
+ */
+/** Shape of a single cache entry on disk. */
+interface CacheEntry {
+    etag: string;
+    url: string;
+    body: unknown;
+    cachedAt: string;
+}
+/**
+ * File-based HTTP cache backed by `~/.oss-scout/cache/`.
+ *
+ * Each cache entry is stored as a separate JSON file keyed by the SHA-256
+ * hash of the request URL. This avoids filesystem issues with URL-based
+ * filenames and keeps lookup O(1).
+ */
+export declare class HttpCache {
+    private readonly cacheDir;
+    /** In-flight request deduplication map: URL -> Promise<response>. */
+    private readonly inflightRequests;
+    constructor(cacheDir?: string);
+    /** Derive a filesystem-safe cache key from a URL. */
+    private keyFor;
+    /** Full path to the cache file for a given URL. */
+    private pathFor;
+    /**
+     * Return the cached body if the entry exists and is younger than `maxAgeMs`.
+     * Useful for time-based caching where ETag validation isn't applicable
+     * (e.g., caching aggregated results from paginated API calls).
+     */
+    getIfFresh(key: string, maxAgeMs: number): unknown | null;
+    /**
+     * Look up a cached response. Returns `null` if no cache entry exists.
+     */
+    get(url: string): CacheEntry | null;
+    /**
+     * Store a response with its ETag.
+     */
+    set(url: string, etag: string, body: unknown): void;
+    /**
+     * Get the in-flight promise for a URL (for deduplication).
+     */
+    getInflight(url: string): Promise<unknown> | undefined;
+    /**
+     * Register an in-flight request for deduplication.
+     * Returns a cleanup function to call when the request completes.
+     */
+    setInflight(url: string, promise: Promise<unknown>): () => void;
+    /**
+     * Remove stale entries older than `maxAgeMs` from the cache directory.
+     * Intended to be called periodically (e.g., once per search invocation).
+     */
+    evictStale(maxAgeMs?: number): number;
+    /**
+     * Remove all entries from the cache.
+     */
+    clear(): void;
+    /**
+     * Return the number of entries currently in the cache.
+     */
+    size(): number;
+}
+/**
+ * Get (or create) the shared HttpCache singleton.
+ * The singleton is lazily initialized on first access.
+ */
+export declare function getHttpCache(): HttpCache;
+/**
+ * Wraps an Octokit `repos.get`-style call with ETag caching and request
+ * deduplication.
+ *
+ * Usage:
+ * ```ts
+ * const data = await cachedRequest(cache, octokit, '/repos/owner/repo', () =>
+ *   octokit.repos.get({ owner, repo: name }),
+ * );
+ * ```
+ *
+ * 1. If an identical request is already in-flight, returns the existing promise
+ *    (request deduplication).
+ * 2. If a cached ETag exists, sends `If-None-Match`. On 304, returns the
+ *    cached body without consuming a rate-limit point.
+ * 3. On a fresh 200, caches the ETag + body for next time.
+ */
+export declare function cachedRequest<T>(cache: HttpCache, url: string, fetcher: (headers: Record<string, string>) => Promise<{
+    data: T;
+    headers?: Record<string, string>;
+}>): Promise<T>;
+/**
+ * Time-based cache wrapper (no ETag / conditional requests).
+ *
+ * If a cached result exists and is younger than `maxAgeMs`, returns it.
+ * Otherwise calls `fetcher`, caches the result, and returns it.
+ *
+ * Use this for expensive operations whose results change slowly
+ * (e.g. search queries, project health checks).
+ */
+export declare function cachedTimeBased<T>(cache: HttpCache, key: string, maxAgeMs: number, fetcher: () => Promise<T>): Promise<T>;
+export {};