@seojin08/skill-finder 0.0.0-stage → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -0
- package/CHANGELOG.md +39 -0
- package/LICENSE +21 -0
- package/README.md +86 -2
- package/bin/skill-finder.mjs +389 -0
- package/package.json +39 -4
- package/server/src/analyze/frontmatter.js +130 -0
- package/server/src/analyze/quality.js +34 -0
- package/server/src/analyze/risk.js +123 -0
- package/server/src/app.js +57 -0
- package/server/src/bootstrap.js +46 -0
- package/server/src/claude/cli.js +89 -0
- package/server/src/config.js +70 -0
- package/server/src/context.js +1 -0
- package/server/src/db/index.js +128 -0
- package/server/src/db/schema.js +95 -0
- package/server/src/db/store.js +353 -0
- package/server/src/discovery/classifyRepo.js +81 -0
- package/server/src/discovery/genericRules.js +180 -0
- package/server/src/discovery/pluginRules.js +59 -0
- package/server/src/discovery/tree.js +79 -0
- package/server/src/index.js +55 -0
- package/server/src/indexer.js +502 -0
- package/server/src/install/installed.js +206 -0
- package/server/src/install/plan.js +118 -0
- package/server/src/install/run.js +97 -0
- package/server/src/install/scan.js +70 -0
- package/server/src/install/summary.js +14 -0
- package/server/src/install/uninstall.js +59 -0
- package/server/src/install/validate.js +24 -0
- package/server/src/jobs.js +107 -0
- package/server/src/recommend/readme.js +508 -0
- package/server/src/release.js +41 -0
- package/server/src/routes/admin.js +203 -0
- package/server/src/routes/favorites.js +61 -0
- package/server/src/routes/health.js +27 -0
- package/server/src/routes/install.js +283 -0
- package/server/src/routes/presets.js +66 -0
- package/server/src/routes/recommend.js +36 -0
- package/server/src/routes/search.js +53 -0
- package/server/src/routes/skills.js +214 -0
- package/server/src/search/activity.js +13 -0
- package/server/src/search/category.js +109 -0
- package/server/src/search/koreanTerms.js +284 -0
- package/server/src/search/pipeline.js +975 -0
- package/server/src/search/rank.js +138 -0
- package/server/src/search/result.js +129 -0
- package/server/src/search/searchLog.js +53 -0
- package/server/src/security.js +53 -0
- package/server/src/settings.js +56 -0
- package/server/src/sources/gitTree.js +101 -0
- package/server/src/sources/github.js +503 -0
- package/server/src/sources/marketplace.js +78 -0
- package/server/src/sources/pluginSource.js +98 -0
- package/server/src/sources/popularRepos.js +74 -0
- package/server/src/sources/skillsSh.js +74 -0
- package/server/src/tasks.js +216 -0
- package/server/src/update.js +53 -0
- package/server/src/util/folderPicker.js +208 -0
- package/server/src/util/http.js +41 -0
- package/server/src/util/limiter.js +110 -0
- package/server/src/util/log.js +94 -0
- package/server/src/util/ndjson.js +27 -0
- package/server/src/util/proc.js +80 -0
- package/server/src/util/resolveFolder.js +78 -0
- package/server/src/util/sanitize.js +45 -0
- package/server/src/util/sizedCache.js +74 -0
- package/server/src/util/textFiles.js +9 -0
- package/server/src/util/tokens.js +21 -0
- package/server/src/version.js +12 -0
- package/web/dist/LICENSE.txt +21 -0
- package/web/dist/THIRD-PARTY-LICENSES.txt +2267 -0
- package/web/dist/assets/ArasakaGame-B0V_LZLl.css +1 -0
- package/web/dist/assets/ArasakaGame-C1Wkmqik.js +36 -0
- package/web/dist/assets/index-BgL1OaU4.css +2 -0
- package/web/dist/assets/index-Ec2XBK4-.js +66 -0
- package/web/dist/assets/jetbrains-mono-cyrillic-wght-normal-D73BlboJ.woff2 +0 -0
- package/web/dist/assets/jetbrains-mono-greek-wght-normal-Bw9x6K1M.woff2 +0 -0
- package/web/dist/assets/jetbrains-mono-latin-ext-wght-normal-DBQx-q_a.woff2 +0 -0
- package/web/dist/assets/jetbrains-mono-latin-wght-normal-B9CIFXIH.woff2 +0 -0
- package/web/dist/assets/jetbrains-mono-vietnamese-wght-normal-Bt-aOZkq.woff2 +0 -0
- package/web/dist/index.html +15 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { searchActivity } from "../search/activity.js";
|
|
2
|
+
import { getSettings } from "../settings.js";
|
|
3
|
+
import { pool, sleep } from "../util/limiter.js";
|
|
4
|
+
/**
|
|
5
|
+
* Popular-repo collection: the search's minimum-stars filter (default 5,000) left few results because most
|
|
6
|
+
* candidates come from small repos. This finds well-starred repos that hold skills via GitHub repository
|
|
7
|
+
* search (which, unlike code search, filters by stars) and indexes their skills locally, so local search
|
|
8
|
+
* has plenty of candidates that pass the filter.
|
|
9
|
+
*/
|
|
10
|
+
/** Repository-search terms; each runs with `stars:>=<minimum>`. Topic and keyword searches cover different repos. */
|
|
11
|
+
const QUERIES = [
|
|
12
|
+
'topic:claude-skills',
|
|
13
|
+
'topic:agent-skills',
|
|
14
|
+
'topic:claude-code-skills',
|
|
15
|
+
'topic:claude-code',
|
|
16
|
+
'topic:anthropic-claude',
|
|
17
|
+
'"claude skills"',
|
|
18
|
+
'"agent skills"',
|
|
19
|
+
'claude code skills',
|
|
20
|
+
'SKILL.md in:readme',
|
|
21
|
+
];
|
|
22
|
+
const STATE_KEY = 'popularRepos';
|
|
23
|
+
const RECHECK_MS = 7 * 24 * 60 * 60 * 1000;
|
|
24
|
+
const MAX_REPOS = 150;
|
|
25
|
+
/** Without a star filter the search would be every repo with a matching topic; keep a floor. */
|
|
26
|
+
const MIN_STARS_FLOOR = 1000;
|
|
27
|
+
export function popularState(ctx) {
|
|
28
|
+
return ctx.store.getSetting(STATE_KEY) ?? { lastRunAt: null, repos: {} };
|
|
29
|
+
}
|
|
30
|
+
export async function collectPopularRepos(ctx, ctl, background, force) {
|
|
31
|
+
const minStars = Math.max(MIN_STARS_FLOOR, getSettings(ctx.store).minStars);
|
|
32
|
+
const state = popularState(ctx);
|
|
33
|
+
// 1. find repos (sequential: the search API allows 30 requests a minute)
|
|
34
|
+
const found = new Map();
|
|
35
|
+
for (const q of QUERIES) {
|
|
36
|
+
await ctl.checkpoint();
|
|
37
|
+
try {
|
|
38
|
+
// fresh: the point is to find repos that are new since the last run (and to store their current stars)
|
|
39
|
+
for (const r of await ctx.github.searchRepos(`${q} stars:>=${minStars}`, 100, { fresh: true }))
|
|
40
|
+
found.set(r.repo, Math.max(found.get(r.repo) ?? 0, r.stars));
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
ctl.tick(`검색 "${q}": ${String(err.message ?? err).slice(0, 160)}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
const now = Date.now();
|
|
47
|
+
const top = [...found]
|
|
48
|
+
.sort((a, b) => b[1] - a[1])
|
|
49
|
+
.slice(0, MAX_REPOS)
|
|
50
|
+
.map(([repo]) => repo);
|
|
51
|
+
// Repos with enough stars that only a lookup or two put into the index (a skills.sh or code-search hit):
|
|
52
|
+
// index them whole once, so their other skills can be found too.
|
|
53
|
+
const known = new Set([...top, ...Object.keys(state.repos)].map((r) => r.toLowerCase()));
|
|
54
|
+
const partial = ctx.store.listReposWithGithubSkills(minStars).filter((r) => !known.has(r.toLowerCase()));
|
|
55
|
+
const due = [...top, ...partial].filter((repo) => force || !state.repos[repo] || now - Date.parse(state.repos[repo].checkedAt) > RECHECK_MS);
|
|
56
|
+
ctl.addTotal(due.length);
|
|
57
|
+
// 2. index their skills (yields to searches when running in the background)
|
|
58
|
+
await pool(due, background ? 3 : 4, async (repo) => {
|
|
59
|
+
await ctl.checkpoint();
|
|
60
|
+
while (background && searchActivity.busy() && !ctl.cancelled)
|
|
61
|
+
await sleep(500);
|
|
62
|
+
try {
|
|
63
|
+
const prev = state.repos[repo];
|
|
64
|
+
const r = await ctx.indexer.indexRepo(repo, { unchangedSince: force ? null : prev?.sha });
|
|
65
|
+
state.repos[repo] = { checkedAt: new Date().toISOString(), skills: r.unchanged ? (prev?.skills ?? 0) : r.skills, ...(r.sha ? { sha: r.sha } : {}) };
|
|
66
|
+
ctl.tick();
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
ctl.tick(`${repo}: ${String(err.message ?? err).slice(0, 200)}`);
|
|
70
|
+
}
|
|
71
|
+
}, () => ctl.cancelled);
|
|
72
|
+
state.lastRunAt = new Date().toISOString();
|
|
73
|
+
ctx.store.setSetting(STATE_KEY, state);
|
|
74
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { USER_AGENT } from "../util/http.js";
|
|
3
|
+
import { sanitizeLine } from "../util/sanitize.js";
|
|
4
|
+
const ResponseSchema = z.object({
|
|
5
|
+
skills: z.array(z.looseObject({
|
|
6
|
+
id: z.string(),
|
|
7
|
+
name: z.string(),
|
|
8
|
+
source: z.string().optional().default(''),
|
|
9
|
+
skillId: z.string().optional(),
|
|
10
|
+
installs: z.number().optional().default(0),
|
|
11
|
+
})),
|
|
12
|
+
});
|
|
13
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
14
|
+
export const SKILLS_SH_BASE = 'https://skills.sh';
|
|
15
|
+
/**
|
|
16
|
+
* Undocumented endpoint used by the official skills CLI (`skills find`). Only called on a user search,
|
|
17
|
+
* at most a few times per search, with results cached for 24h (robots.txt disallows crawling this path).
|
|
18
|
+
*/
|
|
19
|
+
export async function searchSkillsSh(store, query, limit = 20) {
|
|
20
|
+
const q = query.trim().replace(/\s+/g, ' ').slice(0, 200);
|
|
21
|
+
if (!q)
|
|
22
|
+
return [];
|
|
23
|
+
const key = `skillssh:${limit}:${q.toLowerCase()}`;
|
|
24
|
+
const cached = store.getCached(key, DAY_MS);
|
|
25
|
+
if (cached)
|
|
26
|
+
return cached;
|
|
27
|
+
// one request per query at a time (a re-search while the first is still out reuses it)
|
|
28
|
+
let p = inFlight.get(key);
|
|
29
|
+
if (!p) {
|
|
30
|
+
p = fetchSkillsSh(store, key, q, limit)
|
|
31
|
+
.catch((err) => {
|
|
32
|
+
// down or slow: an answer from the last week beats none (installs are the first sort key)
|
|
33
|
+
const stale = store.getCached(key, STALE_OK_MS);
|
|
34
|
+
if (stale)
|
|
35
|
+
return stale;
|
|
36
|
+
throw err;
|
|
37
|
+
})
|
|
38
|
+
.finally(() => inFlight.delete(key));
|
|
39
|
+
inFlight.set(key, p);
|
|
40
|
+
}
|
|
41
|
+
return p;
|
|
42
|
+
}
|
|
43
|
+
const inFlight = new Map();
|
|
44
|
+
/** How old an earlier answer may be to stand in when skills.sh fails. */
|
|
45
|
+
const STALE_OK_MS = 7 * DAY_MS;
|
|
46
|
+
async function fetchSkillsSh(store, key, q, limit) {
|
|
47
|
+
const params = new URLSearchParams({ q, limit: String(limit) });
|
|
48
|
+
const res = await fetch(`${SKILLS_SH_BASE}/api/search?${params}`, {
|
|
49
|
+
headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' },
|
|
50
|
+
signal: AbortSignal.timeout(5000),
|
|
51
|
+
});
|
|
52
|
+
if (!res.ok)
|
|
53
|
+
throw new Error(`skills.sh 응답 ${res.status}`);
|
|
54
|
+
const body = ResponseSchema.parse(await res.json());
|
|
55
|
+
const hits = body.skills.map((s) => ({
|
|
56
|
+
id: sanitizeLine(s.id),
|
|
57
|
+
source: sanitizeLine(s.source),
|
|
58
|
+
skillId: s.skillId ? sanitizeLine(s.skillId) : null,
|
|
59
|
+
name: sanitizeLine(s.name),
|
|
60
|
+
installs: Math.max(0, Math.floor(s.installs)),
|
|
61
|
+
}));
|
|
62
|
+
store.putCached(key, hits);
|
|
63
|
+
return hits;
|
|
64
|
+
}
|
|
65
|
+
const OWNER_REPO_RE = /^[A-Za-z0-9-]{1,39}\/[A-Za-z0-9._-]{1,100}$/;
|
|
66
|
+
const DOMAIN_RE = /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/i;
|
|
67
|
+
export function classifySkillsShSource(source) {
|
|
68
|
+
if (OWNER_REPO_RE.test(source))
|
|
69
|
+
return { kind: 'github', repo: source };
|
|
70
|
+
const host = source.replace(/^https?:\/\//, '').replace(/\/.*$/, '');
|
|
71
|
+
if (DOMAIN_RE.test(host))
|
|
72
|
+
return { kind: 'domain', domain: host.toLowerCase() };
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { RISK_RULES_VERSION, riskLevel } from "./analyze/risk.js";
|
|
2
|
+
import { skillMdPath } from "./discovery/tree.js";
|
|
3
|
+
import { PluginUnavailableError, recheckRisk } from "./indexer.js";
|
|
4
|
+
import { GRAPHQL_BATCH } from "./sources/github.js";
|
|
5
|
+
import { syncMarketplace } from "./sources/marketplace.js";
|
|
6
|
+
import { collectPopularRepos, popularState } from "./sources/popularRepos.js";
|
|
7
|
+
import { searchActivity } from "./search/activity.js";
|
|
8
|
+
import { getSettings, SEARCH_LOG_KEEP_MS } from "./settings.js";
|
|
9
|
+
import { pool, sleep } from "./util/limiter.js";
|
|
10
|
+
const COLLECT_CONCURRENCY = 4;
|
|
11
|
+
const BACKGROUND_CONCURRENCY = 3;
|
|
12
|
+
async function indexPending(ctx, marketplace, ctl, background = false) {
|
|
13
|
+
const pending = ctx.store.listPluginsNeedingIndex(marketplace);
|
|
14
|
+
ctl.addTotal(pending.length);
|
|
15
|
+
await pool(pending, background ? BACKGROUND_CONCURRENCY : COLLECT_CONCURRENCY, async (p) => {
|
|
16
|
+
await ctl.checkpoint();
|
|
17
|
+
// Background collection steps aside while a search is running so searches stay fast.
|
|
18
|
+
while (background && searchActivity.busy() && !ctl.cancelled)
|
|
19
|
+
await sleep(500);
|
|
20
|
+
try {
|
|
21
|
+
await ctx.indexer.indexPlugin(p.id);
|
|
22
|
+
ctl.tick();
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
const line = `${p.id}: ${String(err.message ?? err).slice(0, 200)}`;
|
|
26
|
+
if (err instanceof PluginUnavailableError)
|
|
27
|
+
ctl.skip(line);
|
|
28
|
+
else
|
|
29
|
+
ctl.tick(line);
|
|
30
|
+
}
|
|
31
|
+
}, () => ctl.cancelled);
|
|
32
|
+
}
|
|
33
|
+
/** Syncs marketplace.json for one or all enabled marketplaces; eager ones also index their plugins' skills. */
|
|
34
|
+
export function startSync(ctx, marketplace, force = false) {
|
|
35
|
+
const label = marketplace ?? '*';
|
|
36
|
+
const running = ctx.jobs.active('sync', label);
|
|
37
|
+
if (running)
|
|
38
|
+
return running;
|
|
39
|
+
return ctx.jobs.start('sync', label, async (ctl) => {
|
|
40
|
+
const targets = ctx.store
|
|
41
|
+
.listMarketplaces()
|
|
42
|
+
.filter((m) => (marketplace ? m.name === marketplace : m.enabled === 1));
|
|
43
|
+
ctl.setTotal(targets.length);
|
|
44
|
+
for (const m of targets) {
|
|
45
|
+
await ctl.checkpoint();
|
|
46
|
+
try {
|
|
47
|
+
await syncMarketplace(ctx.store, ctx.github, m.name, force);
|
|
48
|
+
ctl.tick();
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
ctl.tick(`${m.name}: ${String(err.message ?? err).slice(0, 200)}`);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (m.index_mode === 'eager')
|
|
55
|
+
await indexPending(ctx, m.name, ctl);
|
|
56
|
+
}
|
|
57
|
+
ctx.store.clearSearchCache();
|
|
58
|
+
if (!marketplace && getSettings(ctx.store).autoCollect) {
|
|
59
|
+
startAutoCollect(ctx);
|
|
60
|
+
const last = popularState(ctx).lastRunAt;
|
|
61
|
+
if (!last || Date.now() - Date.parse(last) > POPULAR_EVERY_MS)
|
|
62
|
+
startPopularCollect(ctx, true);
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
const POPULAR_EVERY_MS = 24 * 60 * 60 * 1000;
|
|
67
|
+
/** How often repo metadata is refreshed, and what counts as old enough to refresh. */
|
|
68
|
+
export const REPO_META_EVERY_MS = 6 * 60 * 60 * 1000;
|
|
69
|
+
const REPO_META_STALE_MS = 18 * 60 * 60 * 1000;
|
|
70
|
+
let repoMetaRunning = false;
|
|
71
|
+
/**
|
|
72
|
+
* Keeps star counts of the repos skills come from current (GraphQL, 100 per call), so a search reads them
|
|
73
|
+
* from the database instead of asking GitHub per repo, and fills repos never fetched (indexed without a
|
|
74
|
+
* token). Steps aside while a search runs; does nothing without a token.
|
|
75
|
+
*/
|
|
76
|
+
export async function refreshRepoMeta(ctx) {
|
|
77
|
+
if (repoMetaRunning || !ctx.github.hasToken)
|
|
78
|
+
return 0;
|
|
79
|
+
repoMetaRunning = true;
|
|
80
|
+
let stored = 0;
|
|
81
|
+
try {
|
|
82
|
+
const repos = ctx.store.listReposToRefresh(new Date(Date.now() - REPO_META_STALE_MS).toISOString(), 10_000);
|
|
83
|
+
for (let i = 0; i < repos.length; i += GRAPHQL_BATCH) {
|
|
84
|
+
while (searchActivity.busy())
|
|
85
|
+
await sleep(500);
|
|
86
|
+
stored += await ctx.github.refreshRepoMeta(repos.slice(i, i + GRAPHQL_BATCH));
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
console.warn('[repo-meta]', String(err.message ?? err).slice(0, 200));
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
repoMetaRunning = false;
|
|
94
|
+
}
|
|
95
|
+
return stored;
|
|
96
|
+
}
|
|
97
|
+
const RISK_RECHECK_BATCH = 50;
|
|
98
|
+
const RISK_RECHECK_EVERY_MS = 20 * 60 * 60 * 1000;
|
|
99
|
+
/** Setting: when the last risk re-check pass finished. */
|
|
100
|
+
const RISK_RECHECK_KEY = 'riskRecheckAt';
|
|
101
|
+
let riskRecheckRunning = false;
|
|
102
|
+
/**
|
|
103
|
+
* Checks stored skills again after the risk rules changed (RISK_RULES_VERSION): reads each SKILL.md at the
|
|
104
|
+
* commit it was indexed from (raw.githubusercontent.com, not the API quota) and recomputes the flags that come
|
|
105
|
+
* from the text. One pass in id order, 50 rows per transaction, stepping aside while a search runs. Rows that
|
|
106
|
+
* could not be read (gone, or a different file) keep their old version and are tried on the next pass, at most
|
|
107
|
+
* once a day (RISK_RECHECK_EVERY_MS) however often the app is restarted. `changed`: rows whose flags changed.
|
|
108
|
+
*/
|
|
109
|
+
export async function recheckRiskRules(ctx, now = Date.now()) {
|
|
110
|
+
const last = ctx.store.getSetting(RISK_RECHECK_KEY);
|
|
111
|
+
if (riskRecheckRunning || (last && now - Date.parse(last) < RISK_RECHECK_EVERY_MS))
|
|
112
|
+
return { checked: 0, changed: 0 };
|
|
113
|
+
riskRecheckRunning = true;
|
|
114
|
+
let checked = 0;
|
|
115
|
+
let changed = 0;
|
|
116
|
+
try {
|
|
117
|
+
for (let after = '';;) {
|
|
118
|
+
const rows = ctx.store.listSkillsForRiskRecheck(RISK_RULES_VERSION, after, RISK_RECHECK_BATCH);
|
|
119
|
+
if (rows.length === 0)
|
|
120
|
+
break;
|
|
121
|
+
after = rows.at(-1).id;
|
|
122
|
+
const results = await pool(rows, 8, async (row) => {
|
|
123
|
+
while (searchActivity.busy())
|
|
124
|
+
await sleep(500); // raw downloads share their slots with searches
|
|
125
|
+
const r = await ctx.github.getRawText(row.repo, row.ref, skillMdPath(row.dir_path));
|
|
126
|
+
return r.text == null ? null : recheckRisk(row, r.text);
|
|
127
|
+
});
|
|
128
|
+
ctx.store.transaction(() => {
|
|
129
|
+
rows.forEach((row, i) => {
|
|
130
|
+
const res = results[i];
|
|
131
|
+
const next = res?.ok ? res.value : null;
|
|
132
|
+
// only onto the same SKILL.md: a row indexed again meanwhile (new commit) already has current flags
|
|
133
|
+
if (!next || !ctx.store.setSkillRisk(row.id, row.skill_md_sha256, next.risk_json, next.risk_level, RISK_RULES_VERSION))
|
|
134
|
+
return;
|
|
135
|
+
checked++;
|
|
136
|
+
if (next.risk_json !== row.risk_json || next.risk_level !== row.risk_level)
|
|
137
|
+
changed++;
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
ctx.store.setSetting(RISK_RECHECK_KEY, new Date(now).toISOString());
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
console.warn('[risk recheck]', String(err.message ?? err).slice(0, 200));
|
|
145
|
+
}
|
|
146
|
+
finally {
|
|
147
|
+
riskRecheckRunning = false;
|
|
148
|
+
}
|
|
149
|
+
if (changed > 0)
|
|
150
|
+
ctx.store.clearSearchCache();
|
|
151
|
+
if (checked > 0)
|
|
152
|
+
console.log(`[risk recheck] ${checked} skills checked, ${changed} changed`);
|
|
153
|
+
return { checked, changed };
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Derives every stored level from its stored flags again (at startup). A change to how flags add up to a level,
|
|
157
|
+
* such as a flag becoming information only (UI debate D-3), needs no download. Returns the rows that moved.
|
|
158
|
+
*/
|
|
159
|
+
export function relevelRisk(ctx) {
|
|
160
|
+
const moved = ctx.store.listSkillRiskLevels().flatMap((r) => {
|
|
161
|
+
const level = riskLevel(r.risk_json ? JSON.parse(r.risk_json) : []);
|
|
162
|
+
return level === r.risk_level ? [] : [{ id: r.id, level }];
|
|
163
|
+
});
|
|
164
|
+
for (let i = 0; i < moved.length; i += 500) {
|
|
165
|
+
ctx.store.transaction(() => {
|
|
166
|
+
for (const m of moved.slice(i, i + 500))
|
|
167
|
+
ctx.store.setSkillRiskLevel(m.id, m.level);
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
if (moved.length > 0) {
|
|
171
|
+
ctx.store.clearSearchCache();
|
|
172
|
+
console.log(`[risk] ${moved.length} skills got a new level from their stored signals`);
|
|
173
|
+
}
|
|
174
|
+
return moved.length;
|
|
175
|
+
}
|
|
176
|
+
/** Deletes search-log entries older than 90 days (U5-2; at startup and daily). */
|
|
177
|
+
export function pruneSearchLog(ctx, now = Date.now()) {
|
|
178
|
+
try {
|
|
179
|
+
return ctx.store.clearSearchLog(new Date(now - SEARCH_LOG_KEEP_MS).toISOString());
|
|
180
|
+
}
|
|
181
|
+
catch (err) {
|
|
182
|
+
console.warn('[search log]', String(err.message ?? err).slice(0, 200));
|
|
183
|
+
return 0;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
export const POPULAR_JOB_LABEL = 'popular-repos';
|
|
187
|
+
/** Finds GitHub repos with many stars that hold skills and indexes them (daily after sync, or on demand). */
|
|
188
|
+
export function startPopularCollect(ctx, background = false, force = false) {
|
|
189
|
+
const running = ctx.jobs.active('collect', POPULAR_JOB_LABEL);
|
|
190
|
+
if (running)
|
|
191
|
+
return running;
|
|
192
|
+
return ctx.jobs.start('collect', POPULAR_JOB_LABEL, async (ctl) => {
|
|
193
|
+
await collectPopularRepos(ctx, ctl, background, force);
|
|
194
|
+
ctx.store.clearSearchCache();
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
/** After a full sync: collect skills of every enabled lazy marketplace in the background (yields to searches). */
|
|
198
|
+
export function startAutoCollect(ctx) {
|
|
199
|
+
for (const m of ctx.store.listMarketplaces()) {
|
|
200
|
+
if (m.enabled !== 1 || m.index_mode !== 'lazy')
|
|
201
|
+
continue;
|
|
202
|
+
if (ctx.store.listPluginsNeedingIndex(m.name).length === 0)
|
|
203
|
+
continue;
|
|
204
|
+
startCollect(ctx, m.name, true);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
/** Background skill collection for a lazy marketplace; pause/resume via the job controls, restart-safe. */
|
|
208
|
+
export function startCollect(ctx, marketplace, background = false) {
|
|
209
|
+
const running = ctx.jobs.active('collect', marketplace);
|
|
210
|
+
if (running)
|
|
211
|
+
return running;
|
|
212
|
+
return ctx.jobs.start('collect', marketplace, async (ctl) => {
|
|
213
|
+
await indexPending(ctx, marketplace, ctl, background);
|
|
214
|
+
ctx.store.clearSearchCache();
|
|
215
|
+
});
|
|
216
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { releaseMessage, signedNotes, verifyText } from "./release.js";
|
|
2
|
+
const DEFAULT_BASE = 'https://skill-finder.a01075011775.workers.dev/download/';
|
|
3
|
+
const CHECK_EVERY_MS = 6 * 60 * 60 * 1000;
|
|
4
|
+
let latest = null;
|
|
5
|
+
const parse = (v) => v.split('.').map((n) => Number(n) || 0);
|
|
6
|
+
/** a > b for "1.2.3" versions. */
|
|
7
|
+
export function newerVersion(a, b) {
|
|
8
|
+
const [x, y] = [parse(a), parse(b)];
|
|
9
|
+
for (let i = 0; i < 3; i++)
|
|
10
|
+
if ((x[i] ?? 0) !== (y[i] ?? 0))
|
|
11
|
+
return (x[i] ?? 0) > (y[i] ?? 0);
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* The newest signed version the download page offers (meta.json); kept quietly as it was when the page cannot be
|
|
16
|
+
* read or the signature does not match. `publicKey`: tests sign with their own key.
|
|
17
|
+
*/
|
|
18
|
+
export async function checkForUpdate(fetchImpl = fetch, publicKey) {
|
|
19
|
+
if (process.env.SKILL_FINDER_UPDATES !== '1')
|
|
20
|
+
return null;
|
|
21
|
+
let base = process.env.SKILL_FINDER_UPDATE_URL || DEFAULT_BASE;
|
|
22
|
+
if (!base.endsWith('/'))
|
|
23
|
+
base += '/';
|
|
24
|
+
try {
|
|
25
|
+
const res = await fetchImpl(`${base}meta.json`, { signal: AbortSignal.timeout(5000) });
|
|
26
|
+
const meta = res.ok ? (await res.json()) : {};
|
|
27
|
+
const { version, bytes, sha256 } = meta;
|
|
28
|
+
if (typeof version === 'string' &&
|
|
29
|
+
/^\d{1,4}\.\d{1,4}\.\d{1,6}$/.test(version) &&
|
|
30
|
+
typeof bytes === 'number' &&
|
|
31
|
+
typeof sha256 === 'string' &&
|
|
32
|
+
/^[0-9a-f]{64}$/.test(sha256) &&
|
|
33
|
+
verifyText(releaseMessage({ version, bytes, sha256 }), meta.signature, publicKey) &&
|
|
34
|
+
// an older signed release served again does not hide the newer one already seen
|
|
35
|
+
(!latest || !newerVersion(latest.version, version))) {
|
|
36
|
+
latest = { version, notes: signedNotes(meta, version, publicKey) };
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
// offline: try again later
|
|
41
|
+
}
|
|
42
|
+
return latest?.version ?? null;
|
|
43
|
+
}
|
|
44
|
+
export function startUpdateChecks() {
|
|
45
|
+
if (process.env.SKILL_FINDER_UPDATES !== '1')
|
|
46
|
+
return;
|
|
47
|
+
void checkForUpdate();
|
|
48
|
+
setInterval(() => void checkForUpdate(), CHECK_EVERY_MS).unref();
|
|
49
|
+
}
|
|
50
|
+
/** `{ version, notes }` when the download page has a newer signed version than `current`. */
|
|
51
|
+
export function updateInfo(current) {
|
|
52
|
+
return latest && current && newerVersion(latest.version, current) ? { ...latest } : null;
|
|
53
|
+
}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import { statSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute } from 'node:path';
|
|
3
|
+
import { run } from "./proc.js";
|
|
4
|
+
// Browsers never reveal a folder's absolute path to a page, so the local server opens the OS folder dialog instead.
|
|
5
|
+
// The initial folder is passed through an environment variable, never spliced into the script text.
|
|
6
|
+
const TITLE = '프로젝트 폴더 선택';
|
|
7
|
+
const TIMEOUT_MS = 10 * 60 * 1000;
|
|
8
|
+
// Explorer-style dialog (IFileOpenDialog with FOS_PICKFOLDERS) through COM interop, owned by an invisible
|
|
9
|
+
// topmost window so it opens in front of the browser. Falls back to FolderBrowserDialog if the COM path fails.
|
|
10
|
+
const WINDOWS_PICKER_CS = `
|
|
11
|
+
using System;
|
|
12
|
+
using System.Runtime.InteropServices;
|
|
13
|
+
public static class SfFolderPicker {
|
|
14
|
+
[ComImport, Guid("DC1C5A9C-E88A-4dde-A5A1-60F82A20AEF7")] class FileOpenDialogRcw {}
|
|
15
|
+
[ComImport, Guid("42f85136-db7e-439c-85f1-e4075d135fc8"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
|
|
16
|
+
interface IFileOpenDialog {
|
|
17
|
+
[PreserveSig] int Show(IntPtr hwndOwner);
|
|
18
|
+
void SetFileTypes(uint cFileTypes, IntPtr rgFilterSpec);
|
|
19
|
+
void SetFileTypeIndex(uint iFileType);
|
|
20
|
+
void GetFileTypeIndex(out uint piFileType);
|
|
21
|
+
void Advise(IntPtr pfde, out uint pdwCookie);
|
|
22
|
+
void Unadvise(uint dwCookie);
|
|
23
|
+
void SetOptions(uint fos);
|
|
24
|
+
void GetOptions(out uint pfos);
|
|
25
|
+
void SetDefaultFolder(IShellItem psi);
|
|
26
|
+
void SetFolder(IShellItem psi);
|
|
27
|
+
void GetFolder(out IShellItem ppsi);
|
|
28
|
+
void GetCurrentSelection(out IShellItem ppsi);
|
|
29
|
+
void SetFileName([MarshalAs(UnmanagedType.LPWStr)] string pszName);
|
|
30
|
+
void GetFileName([MarshalAs(UnmanagedType.LPWStr)] out string pszName);
|
|
31
|
+
void SetTitle([MarshalAs(UnmanagedType.LPWStr)] string pszTitle);
|
|
32
|
+
void SetOkButtonLabel([MarshalAs(UnmanagedType.LPWStr)] string pszText);
|
|
33
|
+
void SetFileNameLabel([MarshalAs(UnmanagedType.LPWStr)] string pszLabel);
|
|
34
|
+
void GetResult(out IShellItem ppsi);
|
|
35
|
+
void AddPlace(IShellItem psi, int fdap);
|
|
36
|
+
void SetDefaultExtension([MarshalAs(UnmanagedType.LPWStr)] string pszDefaultExtension);
|
|
37
|
+
void Close(int hr);
|
|
38
|
+
void SetClientGuid(ref Guid guid);
|
|
39
|
+
void ClearClientData();
|
|
40
|
+
void SetFilter(IntPtr pFilter);
|
|
41
|
+
void GetResults(out IntPtr ppenum);
|
|
42
|
+
void GetSelectedItems(out IntPtr ppsai);
|
|
43
|
+
}
|
|
44
|
+
[ComImport, Guid("43826D1E-E718-42EE-BC55-A1E261C37BFE"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
|
|
45
|
+
interface IShellItem {
|
|
46
|
+
void BindToHandler(IntPtr pbc, ref Guid bhid, ref Guid riid, out IntPtr ppv);
|
|
47
|
+
void GetParent(out IShellItem ppsi);
|
|
48
|
+
void GetDisplayName(uint sigdnName, [MarshalAs(UnmanagedType.LPWStr)] out string ppszName);
|
|
49
|
+
void GetAttributes(uint sfgaoMask, out uint psfgaoAttribs);
|
|
50
|
+
void Compare(IShellItem psi, uint hint, out int piOrder);
|
|
51
|
+
}
|
|
52
|
+
[DllImport("shell32.dll", CharSet = CharSet.Unicode, PreserveSig = false)]
|
|
53
|
+
static extern void SHCreateItemFromParsingName(string pszPath, IntPtr pbc, ref Guid riid, out IShellItem ppv);
|
|
54
|
+
|
|
55
|
+
// Bringing the dialog forward: a background process may not take the foreground on its own
|
|
56
|
+
// (foreground lock), so a synthetic Alt key press unlocks SetForegroundWindow, and the dialog
|
|
57
|
+
// window is made topmost once it exists.
|
|
58
|
+
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern IntPtr FindWindow(string cls, string title);
|
|
59
|
+
[DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint pid);
|
|
60
|
+
[DllImport("user32.dll")] static extern bool SetWindowPos(IntPtr hWnd, IntPtr after, int x, int y, int cx, int cy, uint flags);
|
|
61
|
+
[DllImport("user32.dll")] static extern bool SetForegroundWindow(IntPtr hWnd);
|
|
62
|
+
[DllImport("user32.dll")] static extern bool BringWindowToTop(IntPtr hWnd);
|
|
63
|
+
[DllImport("user32.dll")] static extern void keybd_event(byte vk, byte scan, uint flags, UIntPtr extra);
|
|
64
|
+
[DllImport("user32.dll")] static extern bool FlashWindow(IntPtr hWnd, bool invert);
|
|
65
|
+
[DllImport("user32.dll")] static extern IntPtr GetForegroundWindow();
|
|
66
|
+
|
|
67
|
+
static void Raise(string title) {
|
|
68
|
+
uint me = (uint)System.Diagnostics.Process.GetCurrentProcess().Id;
|
|
69
|
+
for (int i = 0; i < 100; i++) {
|
|
70
|
+
System.Threading.Thread.Sleep(50);
|
|
71
|
+
IntPtr h = FindWindow("#32770", title);
|
|
72
|
+
if (h == IntPtr.Zero) continue;
|
|
73
|
+
uint pid;
|
|
74
|
+
GetWindowThreadProcessId(h, out pid);
|
|
75
|
+
if (pid != me) continue;
|
|
76
|
+
SetWindowPos(h, new IntPtr(-1), 0, 0, 0, 0, 0x0001 | 0x0002 | 0x0040); // HWND_TOPMOST, NOSIZE|NOMOVE|SHOWWINDOW
|
|
77
|
+
FlashWindow(h, true);
|
|
78
|
+
// The browser can take focus back right after the click that opened us; keep asking for ~1.5s.
|
|
79
|
+
for (int k = 0; k < 10 && GetForegroundWindow() != h; k++) {
|
|
80
|
+
keybd_event(0x12, 0, 0, UIntPtr.Zero); // Alt down
|
|
81
|
+
keybd_event(0x12, 0, 0x0002, UIntPtr.Zero); // Alt up
|
|
82
|
+
BringWindowToTop(h);
|
|
83
|
+
SetForegroundWindow(h);
|
|
84
|
+
System.Threading.Thread.Sleep(150);
|
|
85
|
+
}
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const uint FOS_PICKFOLDERS = 0x20, FOS_FORCEFILESYSTEM = 0x40, FOS_PATHMUSTEXIST = 0x800;
|
|
91
|
+
const uint SIGDN_FILESYSPATH = 0x80058000;
|
|
92
|
+
|
|
93
|
+
public static string Pick(IntPtr owner, string title, string initial) {
|
|
94
|
+
IFileOpenDialog dlg = (IFileOpenDialog)new FileOpenDialogRcw();
|
|
95
|
+
System.Threading.Thread raiser = new System.Threading.Thread(() => Raise(title));
|
|
96
|
+
raiser.IsBackground = true;
|
|
97
|
+
raiser.Start();
|
|
98
|
+
try {
|
|
99
|
+
uint opts;
|
|
100
|
+
dlg.GetOptions(out opts);
|
|
101
|
+
dlg.SetOptions(opts | FOS_PICKFOLDERS | FOS_FORCEFILESYSTEM | FOS_PATHMUSTEXIST);
|
|
102
|
+
dlg.SetTitle(title);
|
|
103
|
+
if (!String.IsNullOrEmpty(initial)) {
|
|
104
|
+
try {
|
|
105
|
+
Guid iid = typeof(IShellItem).GUID;
|
|
106
|
+
IShellItem folder;
|
|
107
|
+
SHCreateItemFromParsingName(initial, IntPtr.Zero, ref iid, out folder);
|
|
108
|
+
dlg.SetFolder(folder);
|
|
109
|
+
} catch (Exception) { }
|
|
110
|
+
}
|
|
111
|
+
if (dlg.Show(owner) != 0) return null;
|
|
112
|
+
IShellItem result;
|
|
113
|
+
dlg.GetResult(out result);
|
|
114
|
+
string path;
|
|
115
|
+
result.GetDisplayName(SIGDN_FILESYSPATH, out path);
|
|
116
|
+
return path;
|
|
117
|
+
} finally {
|
|
118
|
+
Marshal.ReleaseComObject(dlg);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
`;
|
|
123
|
+
const WINDOWS_SCRIPT = `
|
|
124
|
+
[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
|
|
125
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
126
|
+
$initial = ''
|
|
127
|
+
if ($env:SF_INITIAL_DIR -and (Test-Path -LiteralPath $env:SF_INITIAL_DIR -PathType Container)) { $initial = $env:SF_INITIAL_DIR }
|
|
128
|
+
# Invisible (fully transparent) topmost owner at the screen centre, so the dialog opens centred and above other windows.
|
|
129
|
+
$owner = New-Object System.Windows.Forms.Form
|
|
130
|
+
$owner.TopMost = $true
|
|
131
|
+
$owner.ShowInTaskbar = $false
|
|
132
|
+
$owner.FormBorderStyle = 'None'
|
|
133
|
+
$owner.Opacity = 0
|
|
134
|
+
$owner.StartPosition = 'CenterScreen'
|
|
135
|
+
$owner.Size = New-Object System.Drawing.Size(1, 1)
|
|
136
|
+
$owner.Show()
|
|
137
|
+
$owner.Activate()
|
|
138
|
+
try {
|
|
139
|
+
Add-Type -TypeDefinition @'
|
|
140
|
+
${WINDOWS_PICKER_CS}
|
|
141
|
+
'@
|
|
142
|
+
$picked = [SfFolderPicker]::Pick($owner.Handle, '${TITLE}', $initial)
|
|
143
|
+
} catch {
|
|
144
|
+
$dialog = New-Object System.Windows.Forms.FolderBrowserDialog
|
|
145
|
+
$dialog.Description = '${TITLE}'
|
|
146
|
+
$dialog.ShowNewFolderButton = $true
|
|
147
|
+
if ($initial) { $dialog.SelectedPath = $initial }
|
|
148
|
+
$picked = $null
|
|
149
|
+
if ($dialog.ShowDialog($owner) -eq [System.Windows.Forms.DialogResult]::OK) { $picked = $dialog.SelectedPath }
|
|
150
|
+
}
|
|
151
|
+
$owner.Close()
|
|
152
|
+
if ($picked) { [Console]::Out.Write($picked) }
|
|
153
|
+
`;
|
|
154
|
+
function encodePowerShell(script) {
|
|
155
|
+
return Buffer.from(script, 'utf16le').toString('base64');
|
|
156
|
+
}
|
|
157
|
+
function validDir(p) {
|
|
158
|
+
if (!p || !isAbsolute(p) || /[\0\r\n]/.test(p))
|
|
159
|
+
return undefined;
|
|
160
|
+
try {
|
|
161
|
+
return statSync(p).isDirectory() ? p : undefined;
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
export class FolderPickerUnavailable extends Error {
|
|
168
|
+
}
|
|
169
|
+
/** Opens the native folder dialog; resolves the chosen absolute path, or null when cancelled. */
|
|
170
|
+
export async function pickFolder(initial) {
|
|
171
|
+
const env = { ...process.env, SF_INITIAL_DIR: validDir(initial) ?? '' };
|
|
172
|
+
let result;
|
|
173
|
+
if (process.platform === 'win32') {
|
|
174
|
+
result = await run('powershell.exe', ['-NoProfile', '-NonInteractive', '-STA', '-ExecutionPolicy', 'Bypass', '-EncodedCommand', encodePowerShell(WINDOWS_SCRIPT)], {
|
|
175
|
+
env,
|
|
176
|
+
timeoutMs: TIMEOUT_MS,
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
else if (process.platform === 'darwin') {
|
|
180
|
+
const script = `set p to system attribute "SF_INITIAL_DIR"
|
|
181
|
+
if p is not "" then
|
|
182
|
+
POSIX path of (choose folder with prompt "${TITLE}" default location (POSIX file p))
|
|
183
|
+
else
|
|
184
|
+
POSIX path of (choose folder with prompt "${TITLE}")
|
|
185
|
+
end if`;
|
|
186
|
+
result = await run('osascript', ['-e', script], { env, timeoutMs: TIMEOUT_MS });
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
const start = env.SF_INITIAL_DIR ? `${env.SF_INITIAL_DIR}/` : '';
|
|
190
|
+
try {
|
|
191
|
+
result = await run('zenity', ['--file-selection', '--directory', `--title=${TITLE}`, ...(start ? [`--filename=${start}`] : [])], { env, timeoutMs: TIMEOUT_MS });
|
|
192
|
+
}
|
|
193
|
+
catch {
|
|
194
|
+
try {
|
|
195
|
+
result = await run('kdialog', ['--getexistingdirectory', env.SF_INITIAL_DIR || '.', '--title', TITLE], { env, timeoutMs: TIMEOUT_MS });
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
throw new FolderPickerUnavailable('zenity 또는 kdialog가 없어 폴더 선택 창을 열 수 없습니다');
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
const raw = result.stdout.trim();
|
|
203
|
+
if (!raw)
|
|
204
|
+
return null;
|
|
205
|
+
// macOS returns "/path/"; keep drive roots ("C:\") and "/" intact.
|
|
206
|
+
const chosen = /^([A-Za-z]:[\\/]|\/)$/.test(raw) ? raw : raw.replace(/[\\/]+$/, '');
|
|
207
|
+
return validDir(chosen) ?? null;
|
|
208
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export const USER_AGENT = 'skill-finder/0.1 (+local)';
|
|
2
|
+
/** GET with a timeout and a byte cap; `text` is null on non-2xx. */
|
|
3
|
+
export async function fetchText(url, opts = {}) {
|
|
4
|
+
const res = await fetch(url, {
|
|
5
|
+
headers: { 'User-Agent': USER_AGENT, ...opts.headers },
|
|
6
|
+
signal: AbortSignal.timeout(opts.timeoutMs ?? 15_000),
|
|
7
|
+
redirect: 'follow',
|
|
8
|
+
});
|
|
9
|
+
const etag = res.headers.get('etag');
|
|
10
|
+
if (!res.ok || !res.body) {
|
|
11
|
+
await res.body?.cancel().catch(() => { });
|
|
12
|
+
return { status: res.status, text: null, truncated: false, etag };
|
|
13
|
+
}
|
|
14
|
+
const max = opts.maxBytes ?? 2 * 1024 * 1024;
|
|
15
|
+
const reader = res.body.getReader();
|
|
16
|
+
const chunks = [];
|
|
17
|
+
let size = 0;
|
|
18
|
+
let truncated = false;
|
|
19
|
+
for (;;) {
|
|
20
|
+
const { done, value } = await reader.read();
|
|
21
|
+
if (done)
|
|
22
|
+
break;
|
|
23
|
+
size += value.byteLength;
|
|
24
|
+
if (size > max) {
|
|
25
|
+
truncated = true;
|
|
26
|
+
chunks.push(value.subarray(0, value.byteLength - (size - max)));
|
|
27
|
+
await reader.cancel().catch(() => { });
|
|
28
|
+
break;
|
|
29
|
+
}
|
|
30
|
+
chunks.push(value);
|
|
31
|
+
}
|
|
32
|
+
return { status: res.status, text: Buffer.concat(chunks).toString('utf8'), truncated, etag };
|
|
33
|
+
}
|
|
34
|
+
/** Heuristic binary check: NUL bytes or a high share of replacement characters. */
|
|
35
|
+
export function looksBinary(text) {
|
|
36
|
+
if (text.includes('\u0000'))
|
|
37
|
+
return true;
|
|
38
|
+
const sample = text.slice(0, 4096);
|
|
39
|
+
const bad = (sample.match(/�/g) ?? []).length;
|
|
40
|
+
return sample.length > 0 && bad / sample.length > 0.02;
|
|
41
|
+
}
|