@rulemetric/skills-registry 0.15.1 → 0.17.0

package/dist/github-fetcher.d.ts

@@ -48,12 +48,27 @@ export interface DraftEntry extends SkillEntry {
  * expected to keep its existing entries for that source. Pass `undefined` to
  * always fetch.
  */
-export declare function fetchSource(source: RegistrySource, previousTreeSha?: string): Promise<FetchSourceResult>;
+export declare function fetchSource(source: RegistrySource, previousTreeSha?: string, signal?: AbortSignal): Promise<FetchSourceResult>;
 export interface FetchAllOptions {
     /** Map of repo → previous tree SHA. Sources with matching SHA are skipped. */
     previousShas?: Map<string, string>;
     /** Called after each source completes (successful or not). */
     onProgress?: (source: string, count: number, skipped: boolean) => void;
+    /**
+     * Stops the crawl. Sources not yet started are not started; a source in
+     * flight stops at its next network hop. Entries already drafted are kept,
+     * so the caller decides what a partial registry is worth (`aborted` says
+     * it is one). Without this a caller had no way to end a crawl short of
+     * killing the process — which is how one kept running for ~16 hours after
+     * its job lease was gone (2026-09-10).
+     */
+    signal?: AbortSignal;
+    /**
+     * Test seam: replaces the per-source fetch so the scheduling contract
+     * (concurrency, abort, accounting) can be exercised without GitHub.
+     * Production callers never set it.
+     */
+    fetchSourceImpl?: typeof fetchSource;
 }
 export interface FetchAllResult {
     entries: SkillEntry[];
@@ -66,6 +81,14 @@ export interface FetchAllResult {
         path: string;
         reason: string;
     }>;
+    /** True when `options.signal` fired before every source had been visited.
+     *  `entries` is then a PARTIAL registry and must not replace a full one. */
+    aborted: boolean;
+    /** Sources whose fetch ran to a result (including skipped-unchanged and
+     *  failed-and-logged); `sourcesUnfetched` is the remainder the abort
+     *  prevented. They sum to `sources.length`. */
+    sourcesFetched: number;
+    sourcesUnfetched: number;
 }
 /**
  * Assign a final, unique id to every drafted entry.

package/dist/github-fetcher.js

@@ -92,7 +92,7 @@ const rawEtagCache = new Map();
  * GETs and basic retry. Raw URLs don't count against the GitHub API quota, so
  * we don't go through octokit here — but we still want timeouts and retries.
  */
-async function fetchRaw(url, timeoutMs = 10_000, retries = 2) {
+async function fetchRaw(url, timeoutMs = 10_000, retries = 2, signal) {
     const cached = rawEtagCache.get(url);
     const headers = { 'User-Agent': USER_AGENT };
     if (process.env.GITHUB_TOKEN)
@@ -101,8 +101,12 @@ async function fetchRaw(url, timeoutMs = 10_000, retries = 2) {
         headers['If-None-Match'] = cached.etag;
     let lastErr;
     for (let attempt = 0; attempt <= retries; attempt++) {
+        // A caller-side abort (lease lost, budget spent) ends the retry loop; the
+        // per-attempt timeout alone would let a dead crawl keep re-trying.
+        if (signal?.aborted)
+            return { status: 0, body: null, fromCache: false };
         try {
-            const resp = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) });
+            const resp = await fetch(url, { headers, signal: withTimeout(timeoutMs, signal) });
             if (resp.status === 304 && cached) {
                 return { status: 304, body: cached.body, fromCache: true };
             }
@@ -126,6 +130,8 @@ async function fetchRaw(url, timeoutMs = 10_000, retries = 2) {
         }
         catch (err) {
             lastErr = err;
+            if (signal?.aborted)
+                return { status: 0, body: null, fromCache: false };
             await sleep(250 * 2 ** attempt);
         }
     }
@@ -133,18 +139,35 @@ async function fetchRaw(url, timeoutMs = 10_000, retries = 2) {
     return { status: 0, body: null, fromCache: false };
 }
 const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+/**
+ * Per-request timeout combined with the caller's abort signal, so a crawl the
+ * caller has stopped does not wait out its own timeout first. `AbortSignal.any`
+ * is Node 20.3+; the package still declares >=18.18, so fall back to the
+ * timeout alone (the caller-side `aborted` checks between hops still bound
+ * the work — just one request later).
+ */
+function withTimeout(timeoutMs, signal) {
+    const timeout = AbortSignal.timeout(timeoutMs);
+    if (!signal)
+        return timeout;
+    const any = AbortSignal.any;
+    return typeof any === 'function' ? any([timeout, signal]) : timeout;
+}
 /**
  * Fetch repo metadata + tree SHA in one round trip via the API. Lets callers
  * compare treeSha against a previous run to skip unchanged repos entirely.
  */
-async function fetchRepoMeta(repo) {
+async function fetchRepoMeta(repo, signal) {
     const [owner, name] = repo.split('/');
     if (!owner || !name)
         return null;
     try {
-        const { data } = await getOctokit().rest.repos.get({ owner, repo: name });
+        const request = signal ? { signal } : undefined;
+        const { data } = await getOctokit().rest.repos.get({ owner, repo: name, request });
+        if (signal?.aborted)
+            return null;
         const branchData = await getOctokit().rest.repos.getBranch({
-            owner, repo: name, branch: data.default_branch,
+            owner, repo: name, branch: data.default_branch, request,
         });
         return {
             defaultBranch: data.default_branch,
@@ -155,7 +178,7 @@ async function fetchRepoMeta(repo) {
     }
     catch (err) {
         const status = err.status;
-        if (status !== 404)
+        if (status !== 404 && !signal?.aborted)
             console.warn(`fetchRepoMeta failed for ${repo}:`, err);
         return null;
     }
@@ -171,30 +194,30 @@ async function fetchRepoMeta(repo) {
  * Returns null on any failure (404, rate-limit give-up, network) so the caller
  * can fall back to the repo pushed_at without aborting the whole build.
  */
-async function fetchFileCommitDate(repo, filePath) {
+async function fetchFileCommitDate(repo, filePath, signal) {
     const [owner, name] = repo.split('/');
-    if (!owner || !name)
+    if (!owner || !name || signal?.aborted)
         return null;
     try {
         const { data } = await getOctokit().rest.repos.listCommits({
-            owner, repo: name, path: filePath, per_page: 1,
+            owner, repo: name, path: filePath, per_page: 1, request: signal ? { signal } : undefined,
         });
         return data[0]?.commit?.committer?.date ?? data[0]?.commit?.author?.date ?? null;
     }
     catch (err) {
         const status = err.status;
-        if (status !== 404)
+        if (status !== 404 && !signal?.aborted)
             console.warn(`fetchFileCommitDate failed for ${repo}/${filePath}:`, err);
         return null;
     }
 }
-async function fetchTree(repo, treeSha) {
+async function fetchTree(repo, treeSha, signal) {
     const [owner, name] = repo.split('/');
-    if (!owner || !name)
+    if (!owner || !name || signal?.aborted)
         return [];
     try {
         const { data } = await getOctokit().rest.git.getTree({
-            owner, repo: name, tree_sha: treeSha, recursive: 'true',
+            owner, repo: name, tree_sha: treeSha, recursive: 'true', request: signal ? { signal } : undefined,
         });
         if (data.truncated)
             console.warn(`Tree truncated for ${repo} — large repo, some files skipped`);
@@ -207,7 +230,8 @@ async function fetchTree(repo, treeSha) {
         return out;
     }
     catch (err) {
-        console.warn(`fetchTree failed for ${repo}@${treeSha}:`, err);
+        if (!signal?.aborted)
+            console.warn(`fetchTree failed for ${repo}@${treeSha}:`, err);
         return [];
     }
 }
@@ -286,15 +310,15 @@ const FILE_CONCURRENCY = pLimit(8);
  * expected to keep its existing entries for that source. Pass `undefined` to
  * always fetch.
  */
-export async function fetchSource(source, previousTreeSha) {
-    const meta = await fetchRepoMeta(source.repo);
+export async function fetchSource(source, previousTreeSha, signal) {
+    const meta = await fetchRepoMeta(source.repo, signal);
     if (!meta)
         return { entries: [], treeSha: null, skipped: false, dropped: [] };
     if (previousTreeSha && previousTreeSha === meta.treeSha) {
         return { entries: [], treeSha: meta.treeSha, skipped: true, dropped: [] };
     }
     const branch = source.branch ?? meta.defaultBranch;
-    const tree = await fetchTree(source.repo, meta.treeSha);
+    const tree = await fetchTree(source.repo, meta.treeSha, signal);
     const matchedFiles = tree.filter(item => {
         const ext = path.extname(item.path).toLowerCase();
         if (SKIP_EXTENSIONS.has(ext))
@@ -305,8 +329,12 @@ export async function fetchSource(source, previousTreeSha) {
     });
     const dropped = [];
     const entries = await Promise.all(matchedFiles.map(file => FILE_CONCURRENCY(async () => {
+        // Checked per file, not just per source: a large repo is hundreds of
+        // raw fetches, and the caller's stop must land inside it.
+        if (signal?.aborted)
+            return null;
         const url = `${GITHUB_RAW}/${source.repo}/${branch}/${file.path}`;
-        const result = await fetchRaw(url);
+        const result = await fetchRaw(url, undefined, undefined, signal);
         if (!result.body || result.body.trim().length < 20)
             return null;
         const content = result.body;
@@ -327,7 +355,7 @@ export async function fetchSource(source, previousTreeSha) {
         // Real "last updated" date: the file's most recent commit. Falls back
         // to the repo's pushed_at if the per-file lookup fails (rate limit /
         // error), so the field is populated even on a degraded run.
-        const sourceUpdatedAt = (await fetchFileCommitDate(source.repo, file.path)) ?? meta.pushedAt ?? undefined;
+        const sourceUpdatedAt = (await fetchFileCommitDate(source.repo, file.path, signal)) ?? meta.pushedAt ?? undefined;
         return {
             // Provisional. Final ids are assigned across all sources at once so
             // two repos racing for the same id cannot decide it between them.
@@ -420,10 +448,20 @@ export async function fetchAllSources(sources, options = {}) {
     const drafts = [];
     const treeShas = new Map();
     const dropped = [];
+    const fetchOne = options.fetchSourceImpl ?? fetchSource;
+    const signal = options.signal;
+    let sourcesFetched = 0;
+    let sourcesUnfetched = 0;
     await Promise.all(sources.map(source => SOURCE_CONCURRENCY(async () => {
+        // The limiter releases queued sources one at a time after the abort;
+        // each must decline rather than start a fresh crawl.
+        if (signal?.aborted) {
+            sourcesUnfetched++;
+            return;
+        }
         try {
             const previous = options.previousShas?.get(source.repo);
-            const result = await fetchSource(source, previous);
+            const result = await fetchOne(source, previous, signal);
             if (result.treeSha)
                 treeShas.set(source.repo, result.treeSha);
             drafts.push(...result.entries);
@@ -431,11 +469,24 @@ export async function fetchAllSources(sources, options = {}) {
             options.onProgress?.(source.repo, result.entries.length, result.skipped);
         }
         catch (err) {
-            console.error(`Failed to fetch ${source.repo}:`, err);
+            if (!signal?.aborted)
+                console.error(`Failed to fetch ${source.repo}:`, err);
             options.onProgress?.(source.repo, 0, false);
         }
+        // A source cut off mid-flight is counted as fetched: it ran, and its
+        // partial entries are in `drafts`. `aborted` is what tells the caller
+        // the whole result is partial.
+        sourcesFetched++;
     })));
     const { entries, collisions } = assignIds(drafts);
-    return { entries, treeShas, collisions, dropped };
+    return {
+        entries,
+        treeShas,
+        collisions,
+        dropped,
+        aborted: Boolean(signal?.aborted),
+        sourcesFetched,
+        sourcesUnfetched,
+    };
 }
 //# sourceMappingURL=github-fetcher.js.map

package/dist/github-fetcher.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=github-fetcher.test.d.ts.map

package/dist/github-fetcher.test.js

@@ -0,0 +1,119 @@
+import { describe, it, expect, vi } from 'vitest';
+import { fetchAllSources } from './github-fetcher.js';
+/**
+ * The scheduling contract of fetchAllSources, exercised through the
+ * `fetchSourceImpl` seam so no GitHub call is made.
+ *
+ * Origin 2026-09-10: a caller had no way to end a crawl short of killing the
+ * process, and one ran ~16 hours past the point its job lease was gone. The
+ * contract: `signal` stops sources from STARTING, entries already drafted are
+ * kept, and the result says it is partial.
+ */
+function source(repo) {
+    return { repo, patterns: ['**/*.md'], defaultTool: 'claude_code', defaultCategory: 'other' };
+}
+function draft(repo, n) {
+    const id = `${repo.replace('/', '--')}--${n}`;
+    return {
+        id,
+        _idCandidates: [id],
+        _rank: 0,
+        kind: 'skill',
+        name: `Skill ${n}`,
+        description: 'd',
+        category: 'other',
+        tags: [],
+        tools: ['claude_code'],
+        source: { repo, path: `${n}.md`, url: 'u' },
+        content: `# ${n}`,
+        popularity: { stars: 0 },
+        updatedAt: '2026-09-10T00:00:00.000Z',
+    };
+}
+function fetched(repo, count) {
+    return {
+        entries: Array.from({ length: count }, (_, i) => draft(repo, i)),
+        treeSha: `sha-${repo}`,
+        skipped: false,
+        dropped: [],
+    };
+}
+function deferred() {
+    let resolve;
+    const promise = new Promise((r) => { resolve = r; });
+    return { promise, resolve };
+}
+describe('fetchAllSources abort contract', () => {
+    it('without a signal visits every source and reports a complete result', async () => {
+        const sources = ['a/1', 'b/2', 'c/3'].map(source);
+        const impl = vi.fn(async (s) => fetched(s.repo, 2));
+        const result = await fetchAllSources(sources, { fetchSourceImpl: impl });
+        expect(impl).toHaveBeenCalledTimes(3);
+        expect(result.entries).toHaveLength(6);
+        expect(result).toMatchObject({ aborted: false, sourcesFetched: 3, sourcesUnfetched: 0 });
+    });
+    it('with an already-aborted signal starts no source at all', async () => {
+        const sources = ['a/1', 'b/2', 'c/3'].map(source);
+        const impl = vi.fn(async (s) => fetched(s.repo, 2));
+        const ctrl = new AbortController();
+        ctrl.abort();
+        const result = await fetchAllSources(sources, { fetchSourceImpl: impl, signal: ctrl.signal });
+        expect(impl).not.toHaveBeenCalled();
+        expect(result.entries).toEqual([]);
+        expect(result).toMatchObject({ aborted: true, sourcesFetched: 0, sourcesUnfetched: 3 });
+    });
+    it('aborting mid-crawl keeps what was drafted, starts nothing further, and marks the result partial', async () => {
+        // Six sources against the fetcher's concurrency of four: four start, two
+        // wait in the limiter's queue. The abort lands while all four are in
+        // flight; the two queued ones must decline when their turn comes.
+        const sources = ['a/1', 'b/2', 'c/3', 'd/4', 'e/5', 'f/6'].map(source);
+        const gates = new Map();
+        const impl = vi.fn((s, _prev, signal) => {
+            expect(signal).toBeInstanceOf(AbortSignal);
+            const gate = deferred();
+            gates.set(s.repo, gate);
+            return gate.promise;
+        });
+        const ctrl = new AbortController();
+        const progress = [];
+        const run = fetchAllSources(sources, {
+            fetchSourceImpl: impl,
+            signal: ctrl.signal,
+            onProgress: (repo) => progress.push(repo),
+        });
+        await vi.waitFor(() => expect(impl).toHaveBeenCalledTimes(4));
+        // One source completes normally, then the caller stops the crawl while
+        // three are still in flight.
+        gates.get('a/1').resolve(fetched('a/1', 3));
+        ctrl.abort();
+        // The in-flight three return whatever they had when they noticed.
+        gates.get('b/2').resolve(fetched('b/2', 1));
+        gates.get('c/3').resolve({ entries: [], treeSha: null, skipped: false, dropped: [] });
+        gates.get('d/4').resolve({ entries: [], treeSha: null, skipped: false, dropped: [] });
+        const result = await run;
+        expect(impl).toHaveBeenCalledTimes(4);
+        expect(result.entries.map((e) => e.id).sort()).toEqual(['a--1--0', 'a--1--1', 'a--1--2', 'b--2--0'].sort());
+        expect(result).toMatchObject({ aborted: true, sourcesFetched: 4, sourcesUnfetched: 2 });
+        expect(progress.sort()).toEqual(['a/1', 'b/2', 'c/3', 'd/4']);
+        // The two that never ran are not reported as progress either.
+        expect(progress).not.toContain('e/5');
+    });
+    it('a source that throws after the abort is counted as fetched and does not surface as an error', async () => {
+        const sources = ['a/1'].map(source);
+        const ctrl = new AbortController();
+        const impl = vi.fn(async () => {
+            ctrl.abort();
+            throw new Error('The operation was aborted');
+        });
+        const errors = vi.spyOn(console, 'error').mockImplementation(() => undefined);
+        try {
+            const result = await fetchAllSources(sources, { fetchSourceImpl: impl, signal: ctrl.signal });
+            expect(result).toMatchObject({ aborted: true, sourcesFetched: 1, sourcesUnfetched: 0, entries: [] });
+            expect(errors).not.toHaveBeenCalled();
+        }
+        finally {
+            errors.mockRestore();
+        }
+    });
+});
+//# sourceMappingURL=github-fetcher.test.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rulemetric/skills-registry",
-  "version": "0.15.1",
+  "version": "0.17.0",
   "publishConfig": {
     "access": "public"
   },