@seojin08/skill-finder 0.3.2 → 0.3.4

package/server/src/routes/favorites.js

@@ -1,14 +1,34 @@
 import { Router } from 'express';
 import { z } from 'zod';
-import { freshInstalls, freshness, TRUST_SCORE, trustTier } from "../search/rank.js";
-import { buildResultItem, githubSource, marketplaceSource, storeLookup } from "../search/result.js";
+import { installSummary } from "../install/summary.js";
+import { isRepo } from "../install/validate.js";
+import { indexItem, storeLookup } from "../search/result.js";
+import { sanitizeLine } from "../util/sanitize.js";
 const FavoriteBody = z.object({ id: z.string().min(1).max(400) });
 /**
  * `name`: a search result not in the index yet (still being checked) is kept by it until it is. Unusable (empty,
  * not text) it is left out rather than refused, so an indexed skill still gets its star.
  */
-const FavoriteAdd = FavoriteBody.extend({ name: z.unknown().optional() });
+const FavoriteAdd = FavoriteBody.extend({ name: z.unknown().optional(), repo: z.unknown().optional() });
 const usableName = (v) => (typeof v === 'string' && v.trim() ? v.trim().slice(0, 200) : undefined);
+/** A favorite GitHub repository (user request 2026-11-03): `repo:` and the name in lower case, so one per repo. */
+const REPO_PREFIX = 'repo:';
+/** What the search showed of the repo; anything unusable is left out rather than refused. */
+function repoMeta(v) {
+    const o = (v && typeof v === 'object' ? v : {});
+    const description = typeof o.description === 'string' && o.description.trim() ? sanitizeLine(o.description).slice(0, 500) : null;
+    const topics = Array.isArray(o.topics) ? o.topics.filter((x) => typeof x === 'string' && /^[a-z0-9][a-z0-9-]{0,49}$/.test(x)).slice(0, 30) : [];
+    return { description, topics };
+}
+/** One damaged row must not take the whole favorites list down with it. */
+function parseJson(text) {
+    try {
+        return text ? JSON.parse(text) : null;
+    }
+    catch {
+        return null;
+    }
+}
 /**
  * This PC's favorite skills (user decision 2026-11-02). No accounts: the app runs for one person on 127.0.0.1, so a
  * login kept only here protected nothing. The API's header and same-origin checks apply as everywhere.
@@ -19,28 +39,52 @@ export function favoritesRouter(ctx) {
     r.get('/favorites/ids', (_req, res) => {
         res.json({ ids: store.listFavorites().map((f) => f.skill_id) });
     });
-    /** The favorites as result cards; ones whose skill left the index are listed by name. */
+    /** The favorites as result cards and repository cards; skills that left the index are listed by name. */
     r.get('/favorites', (_req, res) => {
         const lookup = storeLookup(store);
         const items = [];
+        const repos = [];
         const gone = [];
         for (const f of store.listFavorites()) {
+            if (f.skill_id.startsWith(REPO_PREFIX)) {
+                const repo = f.skill_name;
+                const meta = repoMeta(parseJson(f.meta_json));
+                repos.push({
+                    repo,
+                    url: `https://github.com/${repo}`,
+                    ...meta,
+                    stars: store.getRepo(repo)?.stars ?? null,
+                    // as the search's repository card lists them
+                    skills: store
+                        .findSkillsByRepo(repo)
+                        .filter((s) => s.description && s.origin !== 'wellknown')
+                        .slice(0, 30)
+                        .map((s) => installSummary(s, lookup)),
+                });
+                continue;
+            }
             const s = store.getSkill(f.skill_id);
             if (!s) {
                 gone.push({ id: f.skill_id, name: f.skill_name });
                 continue;
             }
-            const plugin = lookup.plugin(s.plugin_id);
-            const source = plugin ? marketplaceSource(s, plugin) : githubSource(s);
-            const breakdown = { R: 0, P: 0, F: freshness(s.last_commit_at), T: TRUST_SCORE[trustTier(s, plugin)] };
-            items.push(buildResultItem({ skill: s, lookup, sources: source ? [source] : [], installs: freshInstalls(s), stars: s.repo ? (store.getRepo(s.repo)?.stars ?? null) : null, breakdown, score: 0 }));
+            items.push(indexItem(store, lookup, s));
         }
-        res.json({ items, gone });
+        res.json({ items, repos, gone });
     });
     r.post('/favorites', (req, res) => {
         const parsed = FavoriteAdd.safeParse(req.body);
         if (!parsed.success)
             return void res.status(400).json({ error: '스킬 id가 없습니다' });
+        if (parsed.data.id.startsWith(REPO_PREFIX)) {
+            const repo = parsed.data.id.slice(REPO_PREFIX.length);
+            if (!isRepo(repo))
+                return void res.status(400).json({ error: '저장소 이름이 올바르지 않습니다' });
+            // the name as GitHub writes it (morluto/rea), when it is a repo name and the same repo
+            const name = usableName(parsed.data.name);
+            store.addFavorite(REPO_PREFIX + repo.toLowerCase(), isRepo(name) && name.toLowerCase() === repo.toLowerCase() ? name : repo, repoMeta(parsed.data.repo));
+            return void res.json({ ok: true });
+        }
         const skill = store.getSkill(parsed.data.id);
         // not indexed: listed by name on the favorites page, as a card once the index has it
         const name = skill?.name ?? usableName(parsed.data.name);
@@ -54,7 +98,8 @@ export function favoritesRouter(ctx) {
         const parsed = FavoriteBody.safeParse({ id: req.query.id });
         if (!parsed.success)
             return void res.status(400).json({ error: '스킬 id가 없습니다' });
-        store.removeFavorite(parsed.data.id);
+        // repository favorites are kept in lower case (POST)
+        store.removeFavorite(parsed.data.id.startsWith(REPO_PREFIX) ? parsed.data.id.toLowerCase() : parsed.data.id);
         res.json({ ok: true });
     });
     return r;

package/server/src/routes/install.js

@@ -5,7 +5,8 @@ import { Router } from 'express';
 import { z } from 'zod';
 import { buildSteps, planInstall } from "../install/plan.js";
 import { executeInstall } from "../install/run.js";
-import { existingSkillFolder, forgetPlugins, installedPluginCommits, listInstalledPlugins, pluginCommitKey, pluginIndexInfo, pluginSkills, scanInstalledSkills } from "../install/installed.js";
+import { existingSkillFolder, forgetPlugins, installedPluginCommits, listInstalledPlugins, pathKey, pluginCommitKey, pluginIndexInfo, pluginSkills, scanInstalledSkills, scanPluginSkills } from "../install/installed.js";
+import { indexItem, storeLookup } from "../search/result.js";
 import { uninstallPlugin, uninstallSkill } from "../install/uninstall.js";
 import { isProjectDir } from "../install/validate.js";
 import { getSettings } from "../settings.js";
@@ -241,7 +242,8 @@ export function installRouter(ctx) {
         }
         const listed = await listInstalledPlugins(projectPath ?? homedir(), true);
         const commits = installedPluginCommits();
-        const plugins = listed.plugins.map((o) => {
+        // with a folder, another project's project and local plugins are not this one's (user report 2026-11-04)
+        const plugins = listed.plugins.filter((o) => !projectPath || (o.scope !== 'project' && o.scope !== 'local') || pathKey(o.projectPath ?? '') === pathKey(projectPath)).map((o) => {
             const { scope, id: pid } = o;
             const ownProject = o.projectPath;
             // `claude plugin list` reports enabled for the folder it runs in; read the settings file that owns this install instead.
@@ -263,19 +265,43 @@ export function installRouter(ctx) {
             ...(await scanInstalledSkills(store, 'user', null)),
             ...(projectPath ? await scanInstalledSkills(store, 'project', projectPath) : []),
         ];
-        res.json({ plugins, pluginError: listed.error, skills, projects: store.listInstalledProjects() });
+        // The skills of the plugins that apply here: everyone's (user, synced) and this folder's. Without a folder the
+        // project plugins listed above belong to other folders. One row per folder: a plugin installed for the user and
+        // for a project shares its install folder (code review 2026-11-04).
+        const applying = plugins.filter((p) => (p.scope !== 'project' && p.scope !== 'local') || projectPath);
+        const seen = new Set();
+        const pluginSkillRows = (await Promise.all(applying.map((p) => scanPluginSkills(store, { ...p, enabled: typeof p.enabled === 'boolean' ? p.enabled : null }))))
+            .flat()
+            .filter((s) => !seen.has(s.path) && seen.add(s.path));
+        // the index's card for each indexed one: provider, category and install method for the side filters
+        const lookup = storeLookup(store);
+        const items = {};
+        for (const s of [...skills, ...pluginSkillRows]) {
+            const row = s.skillId ? store.getSkill(s.skillId) : undefined;
+            if (row && !items[row.id])
+                items[row.id] = indexItem(store, lookup, row);
+        }
+        res.json({ plugins, pluginError: listed.error, skills, pluginSkills: pluginSkillRows, items, projects: store.listInstalledProjects() });
     });
     // What is installed, by index id, for the "설치됨" badges on results (UI debate 3-1). Project skills are those of
-    // the default project folder in the settings. Unknown ones are simply absent: no badge, never "not installed".
-    r.get('/installed/ids', async (_req, res) => {
-        const project = getSettings(store).defaultProjectPath;
+    // `projectPath` (the folder README 추천 analyzed, user request 2026-11-04), else of the default project folder in
+    // the settings; with `projectPath`, project and local plugins of other folders are left out. Unknown ones are
+    // simply absent: no badge, never "not installed".
+    r.get('/installed/ids', async (req, res) => {
+        const asked = typeof req.query.projectPath === 'string' && req.query.projectPath.trim() ? req.query.projectPath.trim() : null;
+        if (asked && !isProjectDir(asked)) {
+            res.status(400).json({ error: '프로젝트 경로는 존재하는 디렉터리의 절대경로여야 합니다' });
+            return;
+        }
+        const project = asked ?? getSettings(store).defaultProjectPath;
         const listed = await listInstalledPlugins();
         const skills = [
             ...(await scanInstalledSkills(store, 'user', null)),
             ...(isProjectDir(project) ? await scanInstalledSkills(store, 'project', project) : []),
         ];
+        const inFolder = (p) => !asked || (p.scope !== 'project' && p.scope !== 'local') || pathKey(p.projectPath ?? '') === pathKey(asked);
         res.json({
-            plugins: listed.plugins.map((p) => ({ id: p.id, scope: p.scope })),
+            plugins: listed.plugins.filter(inFolder).map((p) => ({ id: p.id, scope: p.scope })),
             skills: skills.filter((s) => s.skillId).map((s) => ({ id: s.skillId, scope: s.scope })),
         });
     });

package/server/src/routes/translate.js

@@ -0,0 +1,35 @@
+import express, { Router } from 'express';
+import { MODEL_LANGS, MODEL_URL_PATH } from "../translate/models.js";
+const langOf = (v) => (MODEL_LANGS.includes(v) ? v : null);
+/** The translation model for browsers without a built-in translator: its state, and the one-time download. */
+export function translateRouter(ctx) {
+    const r = Router();
+    r.get('/translate/models/:lang', (req, res) => {
+        const lang = langOf(req.params.lang);
+        if (!lang)
+            return void res.status(404).json({ error: '번역 모델이 없는 언어입니다' });
+        res.json(ctx.translateModels.status(lang));
+    });
+    r.post('/translate/models/:lang', (req, res) => {
+        const lang = langOf(req.params.lang);
+        if (!lang)
+            return void res.status(404).json({ error: '번역 모델이 없는 언어입니다' });
+        res.json(ctx.translateModels.start(lang));
+    });
+    return r;
+}
+/**
+ * The downloaded files and the engine's registry, outside /api: the engine fetches them without the API header.
+ * Read only; the registry's addresses carry each file's hash (?v=), so the files are cached for good.
+ */
+export function translateFiles(ctx) {
+    const r = Router();
+    r.get(`${MODEL_URL_PATH}/index.json`, (_req, res) => {
+        res.setHeader('Cache-Control', 'no-cache');
+        res.json(ctx.translateModels.registry());
+    });
+    r.use(MODEL_URL_PATH, express.static(ctx.translateModels.root, { index: false, dotfiles: 'deny', maxAge: '30d', immutable: true }));
+    // a file that is not there is a 404, not the page's index.html (the engine saw that only as an integrity error)
+    r.use(MODEL_URL_PATH, (_req, res) => void res.status(404).end());
+    return r;
+}

package/server/src/search/pipeline.js

@@ -1,5 +1,6 @@
 import { createHash } from 'node:crypto';
 import { RISK_ORDER, RISK_RULES_VERSION } from "../analyze/risk.js";
+import { SKILLS_CLI_VERSION } from "../config.js";
 import { basename } from "../discovery/tree.js";
 import { normalizeSkillName, PluginUnavailableError, skillId } from "../indexer.js";
 import { installSummary } from "../install/summary.js";
@@ -132,7 +133,8 @@ export async function runSearch(deps, query, filters, emit) {
     const sources = new Set(filters.sources?.length ? filters.sources : ['marketplace', 'skillssh', 'github']);
     const maxRisk = filters.maxRisk ?? 'medium';
     // `r` is the result-item shape; bump it when items gain fields so cached searches are rebuilt.
-    const key = cacheKey(query, filters, { w: settings.weights, t: !!ctx.config.githubToken, s: minStars, r: RESULTS_VERSION, k: RISK_RULES_VERSION });
+    // c: cached results carry install commands naming the skills CLI version (`npx skills@x.y.z …`)
+    const key = cacheKey(query, filters, { w: settings.weights, t: !!ctx.config.githubToken, s: minStars, r: RESULTS_VERSION, k: RISK_RULES_VERSION, c: SKILLS_CLI_VERSION });
     // The English query for a Korean one goes out first, cached search or not: it comes from the query alone,
     // and the button helps most exactly when there is little else to show.
     const suggestion = koreanSuggestion(query);

package/server/src/search/result.js

@@ -2,7 +2,7 @@ import { RISK_ORDER } from "../analyze/risk.js";
 import { planInstall } from "../install/plan.js";
 import { truncate } from "../util/sanitize.js";
 import { categorize } from "./category.js";
-import { trustTier } from "./rank.js";
+import { freshInstalls, freshness, TRUST_SCORE, trustTier } from "./rank.js";
 export const TRUST_LABEL = { official: '공식', partner: '파트너', community: '커뮤니티' };
 export function treeUrl(repo, ref, dir) {
     const base = `https://github.com/${repo}`;
@@ -81,6 +81,14 @@ export function storeLookup(store) {
         },
     };
 }
+/** An indexed skill as a card outside a search (favorites, installed): no relevance or popularity score. */
+export function indexItem(store, lookup, skill) {
+    const plugin = lookup.plugin(skill.plugin_id);
+    const source = plugin ? marketplaceSource(skill, plugin) : githubSource(skill);
+    const breakdown = { R: 0, P: 0, F: freshness(skill.last_commit_at), T: TRUST_SCORE[trustTier(skill, plugin)] };
+    const stars = skill.repo ? (store.getRepo(skill.repo)?.stars ?? null) : null;
+    return buildResultItem({ skill, lookup, sources: source ? [source] : [], installs: freshInstalls(skill), stars, breakdown, score: 0 });
+}
 export function buildResultItem(a) {
     const { skill } = a;
     const plugin = a.lookup.plugin(skill.plugin_id);

package/server/src/security.js

@@ -49,5 +49,8 @@ export function securityMiddleware(opts) {
         next();
     };
 }
-export const CSP = "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; " +
+// 'wasm-unsafe-eval': the translation engine is WebAssembly (served from here); it allows compiling WASM, not eval.
+// Its worker takes the policy of its own script's response, but the page has it too for browsers that give workers
+// the page's policy.
+export const CSP = "default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; " +
     "connect-src 'self'; font-src 'self' data:; object-src 'none'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'";

package/server/src/sources/gitTree.js

@@ -2,6 +2,13 @@ import { createHash, randomBytes } from 'node:crypto';
 import { mkdir, rm } from 'node:fs/promises';
 import { join } from 'node:path';
 import { run } from "../util/proc.js";
+/**
+ * git asked for a login: GitHub answers so for a repo that is deleted or private (credentials are turned off
+ * here), so it is gone, not a network hiccup. It used to count as "could not read" in searches and was asked
+ * again on every sync (2026-11-03).
+ */
+export class RepoUnreachableError extends Error {
+}
 const REPO_RE = /^[A-Za-z0-9-]{1,39}\/[A-Za-z0-9._-]{1,100}$/;
 const SHA_RE = /^[0-9a-f]{40}$/i;
 const REF_RE = /^[A-Za-z0-9._\/-]{1,200}$/;
@@ -75,8 +82,11 @@ export async function gitTree(repo, sha, workRoot) {
         if (init.exitCode !== 0)
             throw new Error(`git init failed: ${init.stderr.trim()}`);
         const fetch = await git(['fetch', '-q', '--depth', '1', '--filter=blob:none', '--no-tags', repoUrl(repo), sha], dir);
-        if (fetch.exitCode !== 0)
+        if (fetch.exitCode !== 0) {
+            if (/could not read Username|Repository not found/i.test(fetch.stderr))
+                throw new RepoUnreachableError(`저장소를 찾을 수 없습니다: ${repo}`);
             throw new Error(`git fetch failed: ${fetch.stderr.trim().split('\n')[0] || fetch.exitCode}`);
+        }
         if (sha === 'HEAD') {
             const rev = await git(['rev-parse', 'FETCH_HEAD'], dir);
             sha = rev.stdout.trim();

package/server/src/sources/github.js

@@ -27,6 +27,8 @@ function isSafeRepoPath(path) {
 }
 /** Raw files (SKILL.md included) are read up to this many bytes; the rest is cut off. */
 export const RAW_MAX_BYTES = 512 * 1024;
+/** First wait after a raw download got 429, doubled on each retry. */
+const RAW_THROTTLE_WAIT_MS = 1000;
 export class GitHub {
     store;
     config;
@@ -44,6 +46,8 @@ export class GitHub {
     refCache = new Map();
     /** raw.githubusercontent.com answers bursts of hundreds of parallel downloads with 429s; keep it civil. */
     rawSlots = new Semaphore(24);
+    /** raw.githubusercontent.com said 429: no download starts before this time. */
+    rawPausedUntil = 0;
     metaInFlight = new Map();
     constructor(store, config) {
         this.store = store;
@@ -244,8 +248,19 @@ export class GitHub {
         const url = `https://raw.githubusercontent.com/${repo}/${ref}/${path.split('/').map(encodeURIComponent).join('/')}`;
         for (let attempt = 0;; attempt++) {
             try {
-                const res = await this.rawSlots.run(() => fetchText(url, { maxBytes, timeoutMs: 15_000 }));
-                if ((res.status === 429 || res.status >= 500) && attempt < 2) {
+                const res = await this.rawSlots.run(async () => {
+                    const pause = this.rawPausedUntil - Date.now();
+                    if (pause > 0)
+                        await sleep(pause);
+                    return fetchText(url, { maxBytes, timeoutMs: 15_000 });
+                });
+                if (res.status === 429 && attempt < 3) {
+                    // Throttled: every download waits (1, 2, 4 s), not only this one. Retrying after 0.4 s still failed
+                    // while a repo with hundreds of skills was read (2026-11-03).
+                    this.rawPausedUntil = Math.max(this.rawPausedUntil, Date.now() + RAW_THROTTLE_WAIT_MS * 2 ** attempt);
+                    continue;
+                }
+                if (res.status >= 500 && attempt < 2) {
                     await sleep(400 * (attempt + 1));
                     continue;
                 }

package/server/src/tasks.js

@@ -1,7 +1,7 @@
 import { RISK_RULES_VERSION, riskLevel } from "./analyze/risk.js";
 import { skillMdPath } from "./discovery/tree.js";
 import { PluginUnavailableError, recheckRisk } from "./indexer.js";
-import { GRAPHQL_BATCH } from "./sources/github.js";
+import { GitHubError, GRAPHQL_BATCH } from "./sources/github.js";
 import { syncMarketplace } from "./sources/marketplace.js";
 import { collectPopularRepos, popularState } from "./sources/popularRepos.js";
 import { searchActivity } from "./search/activity.js";
@@ -41,18 +41,25 @@ export function startSync(ctx, marketplace, force = false) {
             .listMarketplaces()
             .filter((m) => (marketplace ? m.name === marketplace : m.enabled === 1));
         ctl.setTotal(targets.length);
+        const eager = [];
         for (const m of targets) {
             await ctl.checkpoint();
             try {
                 await syncMarketplace(ctx.store, ctx.github, m.name, force);
                 ctl.tick();
+                if (m.index_mode === 'eager')
+                    eager.push(m.name);
             }
             catch (err) {
                 ctl.tick(`${m.name}: ${String(err.message ?? err).slice(0, 200)}`);
-                continue;
             }
-            if (m.index_mode === 'eager')
-                await indexPending(ctx, m.name, ctl);
+        }
+        // Every list first, then the skills of the marketplace with the fewest plugins to read: Anthropic's own skills
+        // (5 plugins) used to wait about 100 s for the 300-plugin official list on a first run (2026-11-03).
+        const pending = new Map(eager.map((name) => [name, ctx.store.listPluginsNeedingIndex(name).length]));
+        for (const name of eager.sort((a, b) => pending.get(a) - pending.get(b))) {
+            await indexPending(ctx, name, ctl);
+            ctx.store.clearSearchCache();
         }
         ctx.store.clearSearchCache();
         if (!marketplace && getSettings(ctx.store).autoCollect) {
@@ -68,6 +75,30 @@ const POPULAR_EVERY_MS = 24 * 60 * 60 * 1000;
 export const REPO_META_EVERY_MS = 6 * 60 * 60 * 1000;
 const REPO_META_STALE_MS = 18 * 60 * 60 * 1000;
 let repoMetaRunning = false;
+/** The smallest group asked again in halves; a group this small that still fails waits for the next run. */
+const REPO_META_MIN_SPLIT = 25;
+/** GitHub gave up on the query (502/503/504, usually when it takes too long) or our own 30 s limit ran out. */
+const gaveUp = (err) => (err instanceof GitHubError && [502, 503, 504].includes(err.status)) || err?.name === 'TimeoutError';
+/**
+ * One group of repos, asked again in halves when GitHub gives up on it: a hundred lookups in one query can take
+ * too long, and one 502 used to end the whole run, leaving the rest stale until the next one 6 hours later
+ * (2026-11-03). Other errors (limit reached, network) still end the run.
+ */
+async function refreshRepoGroup(ctx, repos) {
+    try {
+        return { stored: await ctx.github.refreshRepoMeta(repos), left: 0 };
+    }
+    catch (err) {
+        if (!gaveUp(err))
+            throw err;
+        if (repos.length <= REPO_META_MIN_SPLIT)
+            return { stored: 0, left: repos.length, error: err };
+        const half = Math.ceil(repos.length / 2);
+        const a = await refreshRepoGroup(ctx, repos.slice(0, half));
+        const b = await refreshRepoGroup(ctx, repos.slice(half));
+        return { stored: a.stored + b.stored, left: a.left + b.left, error: b.error ?? a.error };
+    }
+}
 /**
  * Keeps star counts of the repos skills come from current (GraphQL, 100 per call), so a search reads them
  * from the database instead of asking GitHub per repo, and fills repos never fetched (indexed without a
@@ -80,11 +111,20 @@ export async function refreshRepoMeta(ctx) {
     let stored = 0;
     try {
         const repos = ctx.store.listReposToRefresh(new Date(Date.now() - REPO_META_STALE_MS).toISOString(), 10_000);
+        let left = 0;
         for (let i = 0; i < repos.length; i += GRAPHQL_BATCH) {
             while (searchActivity.busy())
                 await sleep(500);
-            stored += await ctx.github.refreshRepoMeta(repos.slice(i, i + GRAPHQL_BATCH));
+            const group = repos.slice(i, i + GRAPHQL_BATCH);
+            const r = await refreshRepoGroup(ctx, group);
+            stored += r.stored;
+            left += r.left;
+            // not one piece answered: GitHub is down rather than slow, so stop asking until the next run
+            if (r.left === group.length)
+                throw r.error;
         }
+        if (left)
+            console.warn('[repo-meta]', `GitHub가 일부 요청에 답하지 않아 저장소 ${left}개는 다음 갱신 때 다시 묻습니다`);
     }
     catch (err) {
         console.warn('[repo-meta]', String(err.message ?? err).slice(0, 200));

package/server/src/translate/models.js

@@ -0,0 +1,140 @@
+import { createHash } from 'node:crypto';
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { mkdir, rename, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+import { zstdDecompress } from 'node:zlib';
+export const MODEL_LANGS = ['ko', 'ja'];
+const CDN = 'https://firefox-settings-attachments.cdn.mozilla.net/';
+/**
+ * Pinned from Firefox's Remote Settings collection `translations-models-v2` (desktop records, version 3.0), the
+ * versions tried with the engine. A newer model may need a newer engine, so they are changed together.
+ */
+export const MODEL_FILES = {
+    ko: [
+        { part: 'lex', name: 'lex.50.50.enko.s2t.bin', location: 'main-workspace/translations-models-v2/493336e3-8e7b-4223-86c5-e88846a85d46.zst', size: 2675843, sha256: '7216367b7f524c0ae1f39353e5c4622b230d1c2e47924182a5853418c1e69f64', unpackedSize: 6676356, unpackedSha256: '7734265822ce315c72334ad538e698ab3900ab8e8f04bd43768d0aecad6b6df7' },
+        { part: 'model', name: 'model.enko.intgemm.alphas.bin', location: 'main-workspace/translations-models-v2/e4e7e781-bd8e-4dae-9f8e-d0c8a8a5b07b.zst', size: 44942689, sha256: 'ebe3a360c2a007e31df8595bc01a856376c195718ed3c0a27b336eb9b349d313', unpackedSize: 59504955, unpackedSha256: '1c310a79b61b8824b2eb26b045db043d92722f3e66ea06998f7c89f48da9f6bc' },
+        { part: 'vocab', name: 'vocab.enko.spm', location: 'main-workspace/translations-models-v2/ac6a46cf-1cf0-4ea0-b1ad-9fa40d6f6513.zst', size: 595440, sha256: 'dc5b8d46f6dff08950bf36db741f52431a41466cf33f5a105d1c5154586559d2', unpackedSize: 1406926, unpackedSha256: '709bf0e425345e0e92ce41ed84348a1296a2107c92bdb6624404450ffecbd1f9' },
+    ],
+    ja: [
+        { part: 'lex', name: 'lex.50.50.enja.s2t.bin', location: 'main-workspace/translations-models-v2/83e22dc9-40c4-4ac1-8c25-5d3877b5f628.zst', size: 1751977, sha256: '8954d5b63432366992b22ad2aaa11895e2643014835049fe677655540a705915', unpackedSize: 4128360, unpackedSha256: 'edfb7eb47b98a2689b804ab3614c31b24f1257aa54b1154da3d85e1ae8152d9f' },
+        { part: 'trgvocab', name: 'trgvocab.enja.spm', location: 'main-workspace/translations-models-v2/e54d177c-1131-4fbf-90ea-6741a98d6675.zst', size: 361180, sha256: '5384abf34f0e1298f795bd14909a577272059041e60f91a5eb10500e35d0ee98', unpackedSize: 827144, unpackedSha256: '3b3d9f8f3a034d98d0a476f1794fa79c01e4e98a967ceb6777a66ba2d03ec1e1' },
+        { part: 'srcvocab', name: 'srcvocab.enja.spm', location: 'main-workspace/translations-models-v2/d10e4482-b9ba-4565-ab44-5ba3cebcf94b.zst', size: 345747, sha256: 'dd1e93d625d9a8d8a42f75255343f302ea37527933eda1a490652b66a7408a64', unpackedSize: 796275, unpackedSha256: '970c98d174fc01e0339fbabbf45af36a4be3f26f819ec1a5ea1189f71e091889' },
+        { part: 'model', name: 'model.enja.intgemm.alphas.bin', location: 'main-workspace/translations-models-v2/a01a5318-41c7-4266-8fe1-66c105d6d711.zst', size: 32431590, sha256: 'f92259b9904812350b71f440c3328f3e2e856f31c1a3ab6002756380dd908e00', unpackedSize: 43849787, unpackedSha256: '59ae659f9bb63e4f81f474fe3c03d3f4499434b5f9e779fab7c12a45f31fd562' },
+    ],
+};
+/** The address the page reads the files from (outside /api: the engine's fetch sends no X-Skill-Finder header). */
+export const MODEL_URL_PATH = '/translate-models';
+const FILE_TIMEOUT_MS = 5 * 60 * 1000;
+const sha256 = (b) => createHash('sha256').update(b).digest('hex');
+const pairDir = (lang) => `en${lang}`;
+/** Written last, so a half-finished download is never taken for a model. Its content names the pinned files. */
+const READY = 'ready.json';
+export class TranslateModels {
+    dir;
+    fetchImpl;
+    files;
+    running = new Map();
+    /** `dir`: where the unpacked files live (`<dataDir>/translate-models`), served as {@link MODEL_URL_PATH}. `files`: tests. */
+    constructor(dir, fetchImpl = (...a) => fetch(...a), files = MODEL_FILES) {
+        this.dir = dir;
+        this.fetchImpl = fetchImpl;
+        this.files = files;
+    }
+    get root() {
+        return this.dir;
+    }
+    readyMark(lang) {
+        return JSON.stringify(this.files[lang].map((f) => f.unpackedSha256));
+    }
+    /** The mark, and every file still there at its size (one deleted by hand or quarantined means download again). */
+    ready(lang) {
+        try {
+            const dir = join(this.dir, pairDir(lang));
+            return readFileSync(join(dir, READY), 'utf8') === this.readyMark(lang) && this.files[lang].every((f) => statSync(join(dir, f.name)).size === f.unpackedSize);
+        }
+        catch {
+            return false;
+        }
+    }
+    status(lang) {
+        const run = this.running.get(lang);
+        if (run && run.state !== 'ready')
+            return { ...run };
+        const total = this.files[lang].reduce((n, f) => n + f.size, 0);
+        return this.ready(lang) ? { lang, state: 'ready', received: total, total, error: null } : { lang, state: 'absent', received: 0, total, error: null };
+    }
+    /** Starts the download unless it is there or under way; the returned status is polled with {@link status}. */
+    start(lang) {
+        const now = this.status(lang);
+        if (now.state === 'ready' || now.state === 'downloading')
+            return now;
+        const run = { lang, state: 'downloading', received: 0, total: now.total, error: null };
+        this.running.set(lang, run);
+        this.download(lang, run).then(() => (run.state = 'ready'), (err) => {
+            run.state = 'error';
+            run.error = String(err?.message ?? err).slice(0, 300);
+            console.warn('[translate]', run.error);
+        });
+        return { ...run };
+    }
+    async download(lang, run) {
+        const dir = join(this.dir, pairDir(lang));
+        await mkdir(dir, { recursive: true });
+        await rm(join(dir, READY), { force: true });
+        for (const f of this.files[lang]) {
+            const target = join(dir, f.name);
+            if (existsSync(target) && sha256(readFileSync(target)) === f.unpackedSha256) {
+                run.received += f.size;
+                continue;
+            }
+            // network failures and the time limit in the app's words (they used to show as English runtime messages)
+            const failed = (err) => new Error(`${f.name}: 내려받지 못했습니다 (${err?.name === 'TimeoutError' ? '시간 초과' : '연결 실패'})`);
+            const res = await this.fetchImpl(CDN + f.location, { signal: AbortSignal.timeout(FILE_TIMEOUT_MS) }).catch((err) => {
+                throw failed(err);
+            });
+            if (!res.ok || !res.body)
+                throw new Error(`${f.name}: 내려받지 못했습니다 (${res.status})`);
+            const chunks = [];
+            let size = 0;
+            try {
+                for await (const chunk of res.body) {
+                    size += chunk.byteLength;
+                    if (size > f.size)
+                        throw new Error(`${f.name}: 크기가 맞지 않습니다`);
+                    chunks.push(chunk);
+                    run.received += chunk.byteLength;
+                }
+            }
+            catch (err) {
+                throw size > f.size ? err : failed(err);
+            }
+            const packed = Buffer.concat(chunks);
+            if (sha256(packed) !== f.sha256)
+                throw new Error(`${f.name}: SHA-256이 맞지 않습니다`);
+            const unpacked = await promisify(zstdDecompress)(packed);
+            if (unpacked.length !== f.unpackedSize || sha256(unpacked) !== f.unpackedSha256)
+                throw new Error(`${f.name}: 압축을 푼 파일이 맞지 않습니다`);
+            await writeFile(`${target}.part`, unpacked);
+            await rename(`${target}.part`, target);
+        }
+        await writeFile(join(dir, READY), this.readyMark(lang));
+    }
+    /**
+     * The engine's model registry for the languages that are ready: per pair, its files with their addresses and
+     * SHA-256 (the engine checks them again with subresource integrity). The address carries the file's hash, so a
+     * browser that kept an older model (cached for good) asks for the new one when the pins change.
+     */
+    registry() {
+        const out = {};
+        for (const lang of MODEL_LANGS) {
+            if (!this.ready(lang))
+                continue;
+            out[pairDir(lang)] = Object.fromEntries(this.files[lang].map((f) => [
+                f.part,
+                { name: `${MODEL_URL_PATH}/${pairDir(lang)}/${f.name}?v=${f.unpackedSha256.slice(0, 16)}`, size: f.unpackedSize, expectedSha256Hash: f.unpackedSha256 },
+            ]));
+        }
+        return out;
+    }
+}

package/web/dist/THIRD-PARTY-LICENSES.txt

@@ -73,6 +73,8 @@ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+## @mkljczk/bergamot-translator - 0.4.16 (MPL-2.0)
+
 ## @phosphor-icons/react - 2.1.10 (MIT)
 
 MIT License
@@ -2265,3 +2267,7 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+## @mkljczk/bergamot-translator - 0.4.16 (MPL-2.0)
+
+The translation engine in bergamot/ is shipped unmodified. Its source (version 0.4.16): https://codeberg.org/mkljczk/bergamot-translator (MPL-2.0, https://mozilla.org/MPL/2.0/). Its WebAssembly is built from bergamot-translator with the libraries it includes, such as Marian NMT (MIT), SentencePiece (Apache-2.0) and intgemm (MIT); their notices are in that source. The translation models are Mozilla's (MPL-2.0), downloaded from Mozilla when turned on.