@seojin08/skill-finder 0.3.3 → 0.3.5

package/server/src/routes/translate.js

@@ -0,0 +1,35 @@
+import express, { Router } from 'express';
+import { MODEL_LANGS, MODEL_URL_PATH } from "../translate/models.js";
+const langOf = (v) => (MODEL_LANGS.includes(v) ? v : null);
+/** The translation model for browsers without a built-in translator: its state, and the one-time download. */
+export function translateRouter(ctx) {
+    const r = Router();
+    r.get('/translate/models/:lang', (req, res) => {
+        const lang = langOf(req.params.lang);
+        if (!lang)
+            return void res.status(404).json({ error: '번역 모델이 없는 언어입니다' });
+        res.json(ctx.translateModels.status(lang));
+    });
+    r.post('/translate/models/:lang', (req, res) => {
+        const lang = langOf(req.params.lang);
+        if (!lang)
+            return void res.status(404).json({ error: '번역 모델이 없는 언어입니다' });
+        res.json(ctx.translateModels.start(lang));
+    });
+    return r;
+}
+/**
+ * The downloaded files and the engine's registry, outside /api: the engine fetches them without the API header.
+ * Read only; the registry's addresses carry each file's hash (?v=), so the files are cached for good.
+ */
+export function translateFiles(ctx) {
+    const r = Router();
+    r.get(`${MODEL_URL_PATH}/index.json`, (_req, res) => {
+        res.setHeader('Cache-Control', 'no-cache');
+        res.json(ctx.translateModels.registry());
+    });
+    r.use(MODEL_URL_PATH, express.static(ctx.translateModels.root, { index: false, dotfiles: 'deny', maxAge: '30d', immutable: true }));
+    // a file that is not there is a 404, not the page's index.html (the engine saw that only as an integrity error)
+    r.use(MODEL_URL_PATH, (_req, res) => void res.status(404).end());
+    return r;
+}

package/server/src/search/result.js

@@ -2,7 +2,7 @@ import { RISK_ORDER } from "../analyze/risk.js";
 import { planInstall } from "../install/plan.js";
 import { truncate } from "../util/sanitize.js";
 import { categorize } from "./category.js";
-import { trustTier } from "./rank.js";
+import { freshInstalls, freshness, TRUST_SCORE, trustTier } from "./rank.js";
 export const TRUST_LABEL = { official: '공식', partner: '파트너', community: '커뮤니티' };
 export function treeUrl(repo, ref, dir) {
     const base = `https://github.com/${repo}`;
@@ -81,6 +81,14 @@ export function storeLookup(store) {
         },
     };
 }
+/** An indexed skill as a card outside a search (favorites, installed): no relevance or popularity score. */
+export function indexItem(store, lookup, skill) {
+    const plugin = lookup.plugin(skill.plugin_id);
+    const source = plugin ? marketplaceSource(skill, plugin) : githubSource(skill);
+    const breakdown = { R: 0, P: 0, F: freshness(skill.last_commit_at), T: TRUST_SCORE[trustTier(skill, plugin)] };
+    const stars = skill.repo ? (store.getRepo(skill.repo)?.stars ?? null) : null;
+    return buildResultItem({ skill, lookup, sources: source ? [source] : [], installs: freshInstalls(skill), stars, breakdown, score: 0 });
+}
 export function buildResultItem(a) {
     const { skill } = a;
     const plugin = a.lookup.plugin(skill.plugin_id);

package/server/src/security.js

@@ -49,5 +49,8 @@ export function securityMiddleware(opts) {
         next();
     };
 }
-export const CSP = "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; " +
+// 'wasm-unsafe-eval': the translation engine is WebAssembly (served from here); it allows compiling WASM, not eval.
+// Its worker takes the policy of its own script's response, but the page has it too for browsers that give workers
+// the page's policy.
+export const CSP = "default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; " +
     "connect-src 'self'; font-src 'self' data:; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'";

package/server/src/sources/gitTree.js

@@ -2,6 +2,13 @@ import { createHash, randomBytes } from 'node:crypto';
 import { mkdir, rm } from 'node:fs/promises';
 import { join } from 'node:path';
 import { run } from "../util/proc.js";
+/**
+ * git asked for a login: GitHub answers so for a repo that is deleted or private (credentials are turned off
+ * here), so it is gone, not a network hiccup. It used to count as "could not read" in searches and was asked
+ * again on every sync (2026-11-03).
+ */
+export class RepoUnreachableError extends Error {
+}
 const REPO_RE = /^[A-Za-z0-9-]{1,39}\/[A-Za-z0-9._-]{1,100}$/;
 const SHA_RE = /^[0-9a-f]{40}$/i;
 const REF_RE = /^[A-Za-z0-9._\/-]{1,200}$/;
@@ -75,8 +82,11 @@ export async function gitTree(repo, sha, workRoot) {
         if (init.exitCode !== 0)
             throw new Error(`git init failed: ${init.stderr.trim()}`);
         const fetch = await git(['fetch', '-q', '--depth', '1', '--filter=blob:none', '--no-tags', repoUrl(repo), sha], dir);
-        if (fetch.exitCode !== 0)
+        if (fetch.exitCode !== 0) {
+            if (/could not read Username|Repository not found/i.test(fetch.stderr))
+                throw new RepoUnreachableError(`저장소를 찾을 수 없습니다: ${repo}`);
             throw new Error(`git fetch failed: ${fetch.stderr.trim().split('\n')[0] || fetch.exitCode}`);
+        }
         if (sha === 'HEAD') {
             const rev = await git(['rev-parse', 'FETCH_HEAD'], dir);
             sha = rev.stdout.trim();

package/server/src/sources/github.js

@@ -27,6 +27,8 @@ function isSafeRepoPath(path) {
 }
 /** Raw files (SKILL.md included) are read up to this many bytes; the rest is cut off. */
 export const RAW_MAX_BYTES = 512 * 1024;
+/** First wait after a raw download got 429, doubled on each retry. */
+const RAW_THROTTLE_WAIT_MS = 1000;
 export class GitHub {
     store;
     config;
@@ -44,6 +46,8 @@ export class GitHub {
     refCache = new Map();
     /** raw.githubusercontent.com answers bursts of hundreds of parallel downloads with 429s; keep it civil. */
     rawSlots = new Semaphore(24);
+    /** raw.githubusercontent.com said 429: no download starts before this time. */
+    rawPausedUntil = 0;
     metaInFlight = new Map();
     constructor(store, config) {
         this.store = store;
@@ -244,8 +248,19 @@ export class GitHub {
         const url = `https://raw.githubusercontent.com/${repo}/${ref}/${path.split('/').map(encodeURIComponent).join('/')}`;
         for (let attempt = 0;; attempt++) {
             try {
-                const res = await this.rawSlots.run(() => fetchText(url, { maxBytes, timeoutMs: 15_000 }));
-                if ((res.status === 429 || res.status >= 500) && attempt < 2) {
+                const res = await this.rawSlots.run(async () => {
+                    const pause = this.rawPausedUntil - Date.now();
+                    if (pause > 0)
+                        await sleep(pause);
+                    return fetchText(url, { maxBytes, timeoutMs: 15_000 });
+                });
+                if (res.status === 429 && attempt < 3) {
+                    // Throttled: every download waits (1, 2, 4 s), not only this one. Retrying after 0.4 s still failed
+                    // while a repo with hundreds of skills was read (2026-11-03).
+                    this.rawPausedUntil = Math.max(this.rawPausedUntil, Date.now() + RAW_THROTTLE_WAIT_MS * 2 ** attempt);
+                    continue;
+                }
+                if (res.status >= 500 && attempt < 2) {
                     await sleep(400 * (attempt + 1));
                     continue;
                 }

package/server/src/tasks.js

@@ -1,7 +1,7 @@
 import { RISK_RULES_VERSION, riskLevel } from "./analyze/risk.js";
 import { skillMdPath } from "./discovery/tree.js";
 import { PluginUnavailableError, recheckRisk } from "./indexer.js";
-import { GRAPHQL_BATCH } from "./sources/github.js";
+import { GitHubError, GRAPHQL_BATCH } from "./sources/github.js";
 import { syncMarketplace } from "./sources/marketplace.js";
 import { collectPopularRepos, popularState } from "./sources/popularRepos.js";
 import { searchActivity } from "./search/activity.js";
@@ -41,18 +41,25 @@ export function startSync(ctx, marketplace, force = false) {
             .listMarketplaces()
             .filter((m) => (marketplace ? m.name === marketplace : m.enabled === 1));
         ctl.setTotal(targets.length);
+        const eager = [];
         for (const m of targets) {
             await ctl.checkpoint();
             try {
                 await syncMarketplace(ctx.store, ctx.github, m.name, force);
                 ctl.tick();
+                if (m.index_mode === 'eager')
+                    eager.push(m.name);
             }
             catch (err) {
                 ctl.tick(`${m.name}: ${String(err.message ?? err).slice(0, 200)}`);
-                continue;
             }
-            if (m.index_mode === 'eager')
-                await indexPending(ctx, m.name, ctl);
+        }
+        // Every list first, then the skills of the marketplace with the fewest plugins to read: Anthropic's own skills
+        // (5 plugins) used to wait about 100 s for the 300-plugin official list on a first run (2026-11-03).
+        const pending = new Map(eager.map((name) => [name, ctx.store.listPluginsNeedingIndex(name).length]));
+        for (const name of eager.sort((a, b) => pending.get(a) - pending.get(b))) {
+            await indexPending(ctx, name, ctl);
+            ctx.store.clearSearchCache();
         }
         ctx.store.clearSearchCache();
         if (!marketplace && getSettings(ctx.store).autoCollect) {
@@ -68,6 +75,30 @@ const POPULAR_EVERY_MS = 24 * 60 * 60 * 1000;
 export const REPO_META_EVERY_MS = 6 * 60 * 60 * 1000;
 const REPO_META_STALE_MS = 18 * 60 * 60 * 1000;
 let repoMetaRunning = false;
+/** The smallest group asked again in halves; a group this small that still fails waits for the next run. */
+const REPO_META_MIN_SPLIT = 25;
+/** GitHub gave up on the query (502/503/504, usually when it takes too long) or our own 30 s limit ran out. */
+const gaveUp = (err) => (err instanceof GitHubError && [502, 503, 504].includes(err.status)) || err?.name === 'TimeoutError';
+/**
+ * One group of repos, asked again in halves when GitHub gives up on it: a hundred lookups in one query can take
+ * too long, and one 502 used to end the whole run, leaving the rest stale until the next one 6 hours later
+ * (2026-11-03). Other errors (limit reached, network) still end the run.
+ */
+async function refreshRepoGroup(ctx, repos) {
+    try {
+        return { stored: await ctx.github.refreshRepoMeta(repos), left: 0 };
+    }
+    catch (err) {
+        if (!gaveUp(err))
+            throw err;
+        if (repos.length <= REPO_META_MIN_SPLIT)
+            return { stored: 0, left: repos.length, error: err };
+        const half = Math.ceil(repos.length / 2);
+        const a = await refreshRepoGroup(ctx, repos.slice(0, half));
+        const b = await refreshRepoGroup(ctx, repos.slice(half));
+        return { stored: a.stored + b.stored, left: a.left + b.left, error: b.error ?? a.error };
+    }
+}
 /**
  * Keeps star counts of the repos skills come from current (GraphQL, 100 per call), so a search reads them
  * from the database instead of asking GitHub per repo, and fills repos never fetched (indexed without a
@@ -80,11 +111,20 @@ export async function refreshRepoMeta(ctx) {
     let stored = 0;
     try {
         const repos = ctx.store.listReposToRefresh(new Date(Date.now() - REPO_META_STALE_MS).toISOString(), 10_000);
+        let left = 0;
         for (let i = 0; i < repos.length; i += GRAPHQL_BATCH) {
             while (searchActivity.busy())
                 await sleep(500);
-            stored += await ctx.github.refreshRepoMeta(repos.slice(i, i + GRAPHQL_BATCH));
+            const group = repos.slice(i, i + GRAPHQL_BATCH);
+            const r = await refreshRepoGroup(ctx, group);
+            stored += r.stored;
+            left += r.left;
+            // not one piece answered: GitHub is down rather than slow, so stop asking until the next run
+            if (r.left === group.length)
+                throw r.error;
         }
+        if (left)
+            console.warn('[repo-meta]', `GitHub가 일부 요청에 답하지 않아 저장소 ${left}개는 다음 갱신 때 다시 묻습니다`);
     }
     catch (err) {
         console.warn('[repo-meta]', String(err.message ?? err).slice(0, 200));

package/server/src/translate/models.js

@@ -0,0 +1,140 @@
+import { createHash } from 'node:crypto';
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { mkdir, rename, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+import { zstdDecompress } from 'node:zlib';
+export const MODEL_LANGS = ['ko', 'ja'];
+const CDN = 'https://firefox-settings-attachments.cdn.mozilla.net/';
+/**
+ * Pinned from Firefox's Remote Settings collection `translations-models-v2` (desktop records, version 3.0), the
+ * versions tried with the engine. A newer model may need a newer engine, so they are changed together.
+ */
+export const MODEL_FILES = {
+    ko: [
+        { part: 'lex', name: 'lex.50.50.enko.s2t.bin', location: 'main-workspace/translations-models-v2/493336e3-8e7b-4223-86c5-e88846a85d46.zst', size: 2675843, sha256: '7216367b7f524c0ae1f39353e5c4622b230d1c2e47924182a5853418c1e69f64', unpackedSize: 6676356, unpackedSha256: '7734265822ce315c72334ad538e698ab3900ab8e8f04bd43768d0aecad6b6df7' },
+        { part: 'model', name: 'model.enko.intgemm.alphas.bin', location: 'main-workspace/translations-models-v2/e4e7e781-bd8e-4dae-9f8e-d0c8a8a5b07b.zst', size: 44942689, sha256: 'ebe3a360c2a007e31df8595bc01a856376c195718ed3c0a27b336eb9b349d313', unpackedSize: 59504955, unpackedSha256: '1c310a79b61b8824b2eb26b045db043d92722f3e66ea06998f7c89f48da9f6bc' },
+        { part: 'vocab', name: 'vocab.enko.spm', location: 'main-workspace/translations-models-v2/ac6a46cf-1cf0-4ea0-b1ad-9fa40d6f6513.zst', size: 595440, sha256: 'dc5b8d46f6dff08950bf36db741f52431a41466cf33f5a105d1c5154586559d2', unpackedSize: 1406926, unpackedSha256: '709bf0e425345e0e92ce41ed84348a1296a2107c92bdb6624404450ffecbd1f9' },
+    ],
+    ja: [
+        { part: 'lex', name: 'lex.50.50.enja.s2t.bin', location: 'main-workspace/translations-models-v2/83e22dc9-40c4-4ac1-8c25-5d3877b5f628.zst', size: 1751977, sha256: '8954d5b63432366992b22ad2aaa11895e2643014835049fe677655540a705915', unpackedSize: 4128360, unpackedSha256: 'edfb7eb47b98a2689b804ab3614c31b24f1257aa54b1154da3d85e1ae8152d9f' },
+        { part: 'trgvocab', name: 'trgvocab.enja.spm', location: 'main-workspace/translations-models-v2/e54d177c-1131-4fbf-90ea-6741a98d6675.zst', size: 361180, sha256: '5384abf34f0e1298f795bd14909a577272059041e60f91a5eb10500e35d0ee98', unpackedSize: 827144, unpackedSha256: '3b3d9f8f3a034d98d0a476f1794fa79c01e4e98a967ceb6777a66ba2d03ec1e1' },
+        { part: 'srcvocab', name: 'srcvocab.enja.spm', location: 'main-workspace/translations-models-v2/d10e4482-b9ba-4565-ab44-5ba3cebcf94b.zst', size: 345747, sha256: 'dd1e93d625d9a8d8a42f75255343f302ea37527933eda1a490652b66a7408a64', unpackedSize: 796275, unpackedSha256: '970c98d174fc01e0339fbabbf45af36a4be3f26f819ec1a5ea1189f71e091889' },
+        { part: 'model', name: 'model.enja.intgemm.alphas.bin', location: 'main-workspace/translations-models-v2/a01a5318-41c7-4266-8fe1-66c105d6d711.zst', size: 32431590, sha256: 'f92259b9904812350b71f440c3328f3e2e856f31c1a3ab6002756380dd908e00', unpackedSize: 43849787, unpackedSha256: '59ae659f9bb63e4f81f474fe3c03d3f4499434b5f9e779fab7c12a45f31fd562' },
+    ],
+};
+/** The address the page reads the files from (outside /api: the engine's fetch sends no X-Skill-Finder header). */
+export const MODEL_URL_PATH = '/translate-models';
+const FILE_TIMEOUT_MS = 5 * 60 * 1000;
+const sha256 = (b) => createHash('sha256').update(b).digest('hex');
+const pairDir = (lang) => `en${lang}`;
+/** Written last, so a half-finished download is never taken for a model. Its content names the pinned files. */
+const READY = 'ready.json';
+export class TranslateModels {
+    dir;
+    fetchImpl;
+    files;
+    running = new Map();
+    /** `dir`: where the unpacked files live (`<dataDir>/translate-models`), served as {@link MODEL_URL_PATH}. `files`: tests. */
+    constructor(dir, fetchImpl = (...a) => fetch(...a), files = MODEL_FILES) {
+        this.dir = dir;
+        this.fetchImpl = fetchImpl;
+        this.files = files;
+    }
+    get root() {
+        return this.dir;
+    }
+    readyMark(lang) {
+        return JSON.stringify(this.files[lang].map((f) => f.unpackedSha256));
+    }
+    /** The mark, and every file still there at its size (one deleted by hand or quarantined means download again). */
+    ready(lang) {
+        try {
+            const dir = join(this.dir, pairDir(lang));
+            return readFileSync(join(dir, READY), 'utf8') === this.readyMark(lang) && this.files[lang].every((f) => statSync(join(dir, f.name)).size === f.unpackedSize);
+        }
+        catch {
+            return false;
+        }
+    }
+    status(lang) {
+        const run = this.running.get(lang);
+        if (run && run.state !== 'ready')
+            return { ...run };
+        const total = this.files[lang].reduce((n, f) => n + f.size, 0);
+        return this.ready(lang) ? { lang, state: 'ready', received: total, total, error: null } : { lang, state: 'absent', received: 0, total, error: null };
+    }
+    /** Starts the download unless it is there or under way; the returned status is polled with {@link status}. */
+    start(lang) {
+        const now = this.status(lang);
+        if (now.state === 'ready' || now.state === 'downloading')
+            return now;
+        const run = { lang, state: 'downloading', received: 0, total: now.total, error: null };
+        this.running.set(lang, run);
+        this.download(lang, run).then(() => (run.state = 'ready'), (err) => {
+            run.state = 'error';
+            run.error = String(err?.message ?? err).slice(0, 300);
+            console.warn('[translate]', run.error);
+        });
+        return { ...run };
+    }
+    async download(lang, run) {
+        const dir = join(this.dir, pairDir(lang));
+        await mkdir(dir, { recursive: true });
+        await rm(join(dir, READY), { force: true });
+        for (const f of this.files[lang]) {
+            const target = join(dir, f.name);
+            if (existsSync(target) && sha256(readFileSync(target)) === f.unpackedSha256) {
+                run.received += f.size;
+                continue;
+            }
+            // network failures and the time limit in the app's words (they used to show as English runtime messages)
+            const failed = (err) => new Error(`${f.name}: 내려받지 못했습니다 (${err?.name === 'TimeoutError' ? '시간 초과' : '연결 실패'})`);
+            const res = await this.fetchImpl(CDN + f.location, { signal: AbortSignal.timeout(FILE_TIMEOUT_MS) }).catch((err) => {
+                throw failed(err);
+            });
+            if (!res.ok || !res.body)
+                throw new Error(`${f.name}: 내려받지 못했습니다 (${res.status})`);
+            const chunks = [];
+            let size = 0;
+            try {
+                for await (const chunk of res.body) {
+                    size += chunk.byteLength;
+                    if (size > f.size)
+                        throw new Error(`${f.name}: 크기가 맞지 않습니다`);
+                    chunks.push(chunk);
+                    run.received += chunk.byteLength;
+                }
+            }
+            catch (err) {
+                throw size > f.size ? err : failed(err);
+            }
+            const packed = Buffer.concat(chunks);
+            if (sha256(packed) !== f.sha256)
+                throw new Error(`${f.name}: SHA-256이 맞지 않습니다`);
+            const unpacked = await promisify(zstdDecompress)(packed);
+            if (unpacked.length !== f.unpackedSize || sha256(unpacked) !== f.unpackedSha256)
+                throw new Error(`${f.name}: 압축을 푼 파일이 맞지 않습니다`);
+            await writeFile(`${target}.part`, unpacked);
+            await rename(`${target}.part`, target);
+        }
+        await writeFile(join(dir, READY), this.readyMark(lang));
+    }
+    /**
+     * The engine's model registry for the languages that are ready: per pair, its files with their addresses and
+     * SHA-256 (the engine checks them again with subresource integrity). The address carries the file's hash, so a
+     * browser that kept an older model (cached for good) asks for the new one when the pins change.
+     */
+    registry() {
+        const out = {};
+        for (const lang of MODEL_LANGS) {
+            if (!this.ready(lang))
+                continue;
+            out[pairDir(lang)] = Object.fromEntries(this.files[lang].map((f) => [
+                f.part,
+                { name: `${MODEL_URL_PATH}/${pairDir(lang)}/${f.name}?v=${f.unpackedSha256.slice(0, 16)}`, size: f.unpackedSize, expectedSha256Hash: f.unpackedSha256 },
+            ]));
+        }
+        return out;
+    }
+}

package/server/src/update.js

@@ -1,5 +1,5 @@
 import { releaseMessage, signedNotes, verifyText } from "./release.js";
-const DEFAULT_BASE = 'https://skill-finder.a01075011775.workers.dev/download/';
+const DEFAULT_BASE = 'https://seojin-skill-finder.pages.dev/download/';
 const CHECK_EVERY_MS = 6 * 60 * 60 * 1000;
 let latest = null;
 const parse = (v) => v.split('.').map((n) => Number(n) || 0);

package/web/dist/THIRD-PARTY-LICENSES.txt

@@ -73,6 +73,8 @@ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+## @mkljczk/bergamot-translator - 0.4.16 (MPL-2.0)
+
 ## @phosphor-icons/react - 2.1.10 (MIT)
 
 MIT License
@@ -2265,3 +2267,7 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+## @mkljczk/bergamot-translator - 0.4.16 (MPL-2.0)
+
+The translation engine in bergamot/ is shipped unmodified. Its source (version 0.4.16): https://codeberg.org/mkljczk/bergamot-translator (MPL-2.0, https://mozilla.org/MPL/2.0/). Its WebAssembly is built from bergamot-translator with the libraries it includes, such as Marian NMT (MIT), SentencePiece (Apache-2.0) and intgemm (MIT); their notices are in that source. The translation models are Mozilla's (MPL-2.0), downloaded from Mozilla when turned on.