dsh-tool-folder 0.2.5 → 0.3.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 hfhh32
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -149,6 +149,47 @@ were never unregistered, so nothing else changes.
 
 ## Changelog
 
+### v0.3.0 — 语义腿真正可用 + 融合修正（2026-10-17）
+
+**动机（实测发现的真问题，不是推测）**：本机没装 Ollama（`127.0.0.1:11434` 返回
+HTTP 000），`embedText` 恒为 null → `tools_search` 实际一直是**纯 BM25**。在 43 个
+真实工具（`tools-real.json`）、18 条真实中文意图上测：**Hit@1 只有 6/18（33%）**。
+
+1. **远程 embedding 腿（`lib/embed-remote.js` 新增）**：OpenAI 兼容端点，默认火山
+   `doubao-embedding-vision`（2048 维），key 读 `DSH_EMBED_KEY` / `OPENVIKING_ARK_KEY`。
+   带熔断（连续 3 次失败 → 5 分钟冷却）+ 批量（4 条/批、2 并发、429 退避 0.8/2/5s）。
+   纯语义 Hit@1 **11/18（61%）**。
+2. **加权 RRF + 自适应 BM25 权重**：语义腿默认权重 3，BM25 默认 0.25；
+   若查询里的 ASCII 词命中 BM25 首条的名称/描述（真实词匹配、工具名、server 名），
+   BM25 权重回升到 1。实测 1:1 融合只有 8/18，加权后 **11/18**。
+3. **重复 id 修复（关键）**：语料里同名工具出现两次时，旧 RRF 会把同一 id 的分累加
+   两次——`list_sessions` / `get_config` 各两份，直接把正确项从第一顶到第三以外
+   （纯语义 11/18 被拉到 2/18）。现在 bm25/语义两腿都按 id 去重，组装侧也先按名去重。
+4. **索引可用性修复**：旧实现要求**全部** doc 成功才落缓存（一次失败就永远重建），
+   且 60s 硬截止串行调用；改为批量并发 + **部分成功也落盘** + 覆盖率 ≥80% 即可用。
+5. **缓存路径修正**：`~/.dsh/state/` → 优先 `$DSH_HOME/state/`（DSH_HOME 指向
+   `.dsh-community` 时旧实现会把缓存写到另一个目录）。缓存身份键带 provider，
+   远程/本地切换不会复用错向量。
+6. **索引不再阻塞**：在 assemble 后台构建，`tools_search` 绝不等待冷启动（旧实现
+   首次搜索可能卡 30s+）；查询向量有内存缓存 + 15s 节流后台预热。
+7. **正文精排（`lib/rerank.js` 新增，默认关）**：借鉴 SkillRouter 的两阶段检索，
+   在候选集内用「完整描述 + 参数名 + 参数描述 + 枚举」重排。**实测在本语料上
+   Hit@1 持平、Hit@3 反而 -1**（15→14），所以默认关闭，`bodyRerank: true` 可开。
+8. **语义腿参与自动注入**（`semanticDynamic`，默认开）：只用已缓存的查询向量，
+   零额外延迟；命中就把结果并进动态段。
+9. `debugDump`：把当前工具语料 dump 成 JSON，供离线评测复现。
+10. 测试 114 → **123 条**，全绿；新增 `scripts/eval-tools.mjs` 离线评测脚本。
+
+**实测汇总（43 工具 / 18 中文意图）**
+
+| 方案 | Hit@1 | Hit@3 |
+|---|---|---|
+| BM25（v0.2.5 实际状态） | 6/18 | 8/18 |
+| RRF 1:1 | 8/18 | 12/18 |
+| 语义单腿（参考上限） | 11/18 | 15/18 |
+| **v0.3.0 默认（语义3 : BM25 0.25 自适应）** | **11/18** | **15/18** |
+| 再加正文精排 0.35 | 11/18 | 14/18 |
+
 ### v0.2.5 — red-team firewall fixes + retrieval hardening (2026-09-02)
 1. **Exec-tool recognition (red-team F1)** — bare-name matching let MCP-prefixed
    exec tools (`mcp__windows__Cmd`) bypass the high-risk gate; `isExecTool` now

package/lib/bm25.js

@@ -1,127 +1,127 @@
-/**
- * bm25.js — zero-dependency BM25 lexical search for tool retrieval.
- *
- * VENDOR SOURCE (2026-09-02): dsh-skill-folder/lib/bm25.js vendors this file
- * verbatim (comments may diverge per repo). Modify here, then mirror the code
- * change into the skill repo. Sync gate: `node E:\DSH-Data\scripts\check-vendor-sync.mjs`.
- * semantic.js is intentionally DIVERGED (tool has incremental cache/sha1) and
- * is NOT covered by the gate.
- *
- * Design (ported from the xiaowan agent's ToolBM25Index idea; the algorithm is
- * the reference, this code is a clean-room JS implementation):
- *   - BM25 with standard k1=1.2, b=0.75 (RAG-MCP / BoR validated the
- *     retrieval-first approach; BM25 is the cheap lexical leg).
- *   - Tokenizer handles mixed CN/EN: ASCII words + CJK bigrams. No jieba, no
- *     network, no dependencies — must run inside the DSH plugin runtime.
- *   - Synchronous and fast: indexing ~50 tools is <1ms, a query <1ms.
- *
- * Cordis sandbox note: this module only does string/number math. It never
- * touches fs/net/process, so it is safe in any host or sandbox context.
- */
-
-const K1 = 1.2;
-const B = 0.75;
-
-const EN_STOP = new Set([
-  "the", "a", "an", "and", "or", "of", "to", "in", "on", "for", "with",
-  "at", "by", "from", "as", "is", "are", "was", "were", "be", "been",
-  "it", "this", "that", "these", "those", "i", "you", "he", "she", "we",
-  "they", "do", "does", "did", "have", "has", "had", "not", "no", "yes",
-  "will", "would", "can", "could", "should", "may", "might", "must",
-  "your", "my", "our", "their", "its", "all", "any", "some", "each",
-]);
-
-/** Tokenize mixed CN/EN text into lowercase tokens. */
-function tokenize(text) {
-  const out = [];
-  const s = String(text || "").toLowerCase();
-  for (const m of s.matchAll(/[a-z0-9][a-z0-9_+-]{1,}/g)) {
-    const t = m[0];
-    if (!EN_STOP.has(t)) out.push(t);
-    // Sub-words of hyphen/underscore compounds. Without this, a tool name like
-    // "mcp__open-design__start_run" is ONE term, so querying "design" scores
-    // zero. The whole token is kept too, so exact-name matches and existing
-    // IDF are unchanged; sub-words only ADD recall.
-    if (t.length > 2 && (t.includes("-") || t.includes("_"))) {
-      for (const part of t.split(/[-_]+/)) {
-        if (part.length > 1 && part !== t && !EN_STOP.has(part)) out.push(part);
-      }
-    }
-  }
-  // CJK bigrams: "开源框架" -> "开源","源框","框架". Bigrams beat single
-  // chars for short tool descriptions and are dependency-free.
-  const cjk = s.replace(/[^\u4e00-\u9fff]/g, "");
-  if (cjk.length >= 2) {
-    for (let i = 0; i + 2 <= cjk.length; i++) out.push(cjk.slice(i, i + 2));
-  }
-  return out;
-}
-
-/** Build a BM25 index over an array of docs. Each doc: {id, text, ...}. */
-function buildIndex(docs) {
-  const n = docs.length;
-  const docTokens = [];
-  const df = new Map(); // term -> #docs containing it
-  let totalLen = 0;
-
-  for (const doc of docs) {
-    const tokens = tokenize(doc.text);
-    docTokens.push(tokens);
-    totalLen += tokens.length;
-    const seen = new Set();
-    for (const t of tokens) {
-      if (!seen.has(t)) {
-        seen.add(t);
-        df.set(t, (df.get(t) || 0) + 1);
-      }
-    }
-  }
-  const avgdl = n > 0 ? totalLen / n : 1;
-
-  return { n, docTokens, df, avgdl };
-}
-
-/** Score one doc's token list against the query's IDF map. */
-function score(tokens, queryIdf, n, avgdl) {
-  const tf = new Map();
-  for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1);
-  let acc = 0;
-  const dl = tokens.length;
-  for (const [term, idf] of queryIdf) {
-    const f = tf.get(term);
-    if (!f) continue;
-    acc += idf * ((f * (K1 + 1)) / (f + K1 * (1 - B + (B * dl) / avgdl)));
-  }
-  return acc;
-}
-
-/**
- * Search an index. Returns up to topK doc ids sorted by BM25 score desc.
- * @param index   result of buildIndex
- * @param query   raw query string (mixed CN/EN ok)
- * @param topK    max results (default 10)
- * @param opts    {filter: (docId) => boolean}
- */
-function search(index, query, topK = 10, opts = {}) {
-  const { n, docTokens, df, avgdl } = index;
-  if (!n || !query) return [];
-  const qTerms = tokenize(query);
-  if (qTerms.length === 0) return [];
-
-  const queryIdf = new Map();
-  for (const t of qTerms) {
-    const d = df.get(t) || 0;
-    // BM25+ style: add 1 inside the log to keep IDF positive for unseen terms.
-    queryIdf.set(t, Math.log((n - d + 0.5) / (d + 0.5) + 1));
-  }
-
-  const scored = [];
-  for (let i = 0; i < n; i++) {
-    const s = score(docTokens[i], queryIdf, n, avgdl);
-    if (s > 0 && (!opts.filter || opts.filter(i))) scored.push([i, s]);
-  }
-  scored.sort((a, b) => b[1] - a[1]);
-  return scored.slice(0, topK).map(([i]) => i);
-}
-
-export { tokenize, buildIndex, search, score };
+/**
+ * bm25.js — zero-dependency BM25 lexical search for tool retrieval.
+ *
+ * VENDOR SOURCE (2026-09-02): dsh-skill-folder/lib/bm25.js vendors this file
+ * verbatim (comments may diverge per repo). Modify here, then mirror the code
+ * change into the skill repo. Both copies are kept in sync by the maintainer.
+ * semantic.js is intentionally DIVERGED (tool has incremental cache/sha1) and
+ * is NOT covered by the gate.
+ *
+ * Design (ported from the xiaowan agent's ToolBM25Index idea; the algorithm is
+ * the reference, this code is a clean-room JS implementation):
+ *   - BM25 with standard k1=1.2, b=0.75 (RAG-MCP / BoR validated the
+ *     retrieval-first approach; BM25 is the cheap lexical leg).
+ *   - Tokenizer handles mixed CN/EN: ASCII words + CJK bigrams. No jieba, no
+ *     network, no dependencies — must run inside the DSH plugin runtime.
+ *   - Synchronous and fast: indexing ~50 tools is <1ms, a query <1ms.
+ *
+ * Cordis sandbox note: this module only does string/number math. It never
+ * touches fs/net/process, so it is safe in any host or sandbox context.
+ */
+
+const K1 = 1.2;
+const B = 0.75;
+
+const EN_STOP = new Set([
+  "the", "a", "an", "and", "or", "of", "to", "in", "on", "for", "with",
+  "at", "by", "from", "as", "is", "are", "was", "were", "be", "been",
+  "it", "this", "that", "these", "those", "i", "you", "he", "she", "we",
+  "they", "do", "does", "did", "have", "has", "had", "not", "no", "yes",
+  "will", "would", "can", "could", "should", "may", "might", "must",
+  "your", "my", "our", "their", "its", "all", "any", "some", "each",
+]);
+
+/** Tokenize mixed CN/EN text into lowercase tokens. */
+function tokenize(text) {
+  const out = [];
+  const s = String(text || "").toLowerCase();
+  for (const m of s.matchAll(/[a-z0-9][a-z0-9_+-]{1,}/g)) {
+    const t = m[0];
+    if (!EN_STOP.has(t)) out.push(t);
+    // Sub-words of hyphen/underscore compounds. Without this, a tool name like
+    // "mcp__open-design__start_run" is ONE term, so querying "design" scores
+    // zero. The whole token is kept too, so exact-name matches and existing
+    // IDF are unchanged; sub-words only ADD recall.
+    if (t.length > 2 && (t.includes("-") || t.includes("_"))) {
+      for (const part of t.split(/[-_]+/)) {
+        if (part.length > 1 && part !== t && !EN_STOP.has(part)) out.push(part);
+      }
+    }
+  }
+  // CJK bigrams: "开源框架" -> "开源","源框","框架". Bigrams beat single
+  // chars for short tool descriptions and are dependency-free.
+  const cjk = s.replace(/[^\u4e00-\u9fff]/g, "");
+  if (cjk.length >= 2) {
+    for (let i = 0; i + 2 <= cjk.length; i++) out.push(cjk.slice(i, i + 2));
+  }
+  return out;
+}
+
+/** Build a BM25 index over an array of docs. Each doc: {id, text, ...}. */
+function buildIndex(docs) {
+  const n = docs.length;
+  const docTokens = [];
+  const df = new Map(); // term -> #docs containing it
+  let totalLen = 0;
+
+  for (const doc of docs) {
+    const tokens = tokenize(doc.text);
+    docTokens.push(tokens);
+    totalLen += tokens.length;
+    const seen = new Set();
+    for (const t of tokens) {
+      if (!seen.has(t)) {
+        seen.add(t);
+        df.set(t, (df.get(t) || 0) + 1);
+      }
+    }
+  }
+  const avgdl = n > 0 ? totalLen / n : 1;
+
+  return { n, docTokens, df, avgdl };
+}
+
+/** Score one doc's token list against the query's IDF map. */
+function score(tokens, queryIdf, n, avgdl) {
+  const tf = new Map();
+  for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1);
+  let acc = 0;
+  const dl = tokens.length;
+  for (const [term, idf] of queryIdf) {
+    const f = tf.get(term);
+    if (!f) continue;
+    acc += idf * ((f * (K1 + 1)) / (f + K1 * (1 - B + (B * dl) / avgdl)));
+  }
+  return acc;
+}
+
+/**
+ * Search an index. Returns up to topK doc ids sorted by BM25 score desc.
+ * @param index   result of buildIndex
+ * @param query   raw query string (mixed CN/EN ok)
+ * @param topK    max results (default 10)
+ * @param opts    {filter: (docId) => boolean}
+ */
+function search(index, query, topK = 10, opts = {}) {
+  const { n, docTokens, df, avgdl } = index;
+  if (!n || !query) return [];
+  const qTerms = tokenize(query);
+  if (qTerms.length === 0) return [];
+
+  const queryIdf = new Map();
+  for (const t of qTerms) {
+    const d = df.get(t) || 0;
+    // BM25+ style: add 1 inside the log to keep IDF positive for unseen terms.
+    queryIdf.set(t, Math.log((n - d + 0.5) / (d + 0.5) + 1));
+  }
+
+  const scored = [];
+  for (let i = 0; i < n; i++) {
+    const s = score(docTokens[i], queryIdf, n, avgdl);
+    if (s > 0 && (!opts.filter || opts.filter(i))) scored.push([i, s]);
+  }
+  scored.sort((a, b) => b[1] - a[1]);
+  return scored.slice(0, topK).map(([i]) => i);
+}
+
+export { tokenize, buildIndex, search, score };

package/lib/embed-remote.js

@@ -0,0 +1,193 @@
+/**
+ * embed-remote.js — 远程 OpenAI 兼容 embedding（默认火山方舟 doubao-embedding-vision）。
+ *
+ * 为什么需要（2026-10-17 实测）：v0.2.x 的语义腿只走本地 Ollama bge-m3，
+ * 而本机 `127.0.0.1:11434` 返回 HTTP 000（Ollama 未安装）→ `embedText` 恒为 null
+ * → `tools_search` 实际退化为纯 BM25。中文意图打英文工具描述时经常零命中。
+ *
+ * 设计要点（与 dsh-skill-folder v0.4.0 的同名模块同源）：
+ *   - 批量：OpenAI 兼容端点支持 `input: string[]`，一次 4 条（火山限流下的稳定值）
+ *   - 并发：2 路并发，批间 250ms 节流；429/5xx 退避重试（0.8s / 2s / 5s）
+ *   - 熔断：连续 3 次失败 → 冷却 5 分钟，期间 `remoteEmbedAvailable()` 为 false，
+ *     调用方自动回落到 Ollama / 纯 BM25
+ *   - 任何失败都返回 null/数组中的 null，绝不抛
+ */
+
+const DEFAULT_BASE = "https://ark.cn-beijing.volces.com/api/plan/v3";
+const DEFAULT_MODEL = "doubao-embedding-vision";
+const DEFAULT_TIMEOUT = 15000;
+const FAIL_THRESHOLD = 3;
+const COOLDOWN_MS = 5 * 60 * 1000;
+const DEFAULT_BATCH = 4;
+const DEFAULT_CONCURRENCY = 2;
+const BATCH_GAP_MS = 250;
+const BACKOFF_MS = [800, 2000, 5000];
+
+let _base = process.env.DSH_EMBED_BASE || DEFAULT_BASE;
+let _model = process.env.DSH_EMBED_MODEL || DEFAULT_MODEL;
+let _key = process.env.DSH_EMBED_KEY || process.env.OPENVIKING_ARK_KEY || "";
+let _timeout = DEFAULT_TIMEOUT;
+let _fails = 0;
+let _disabledUntil = 0;
+
+export function configureRemoteEmbed(cfg = {}) {
+  if (!cfg || typeof cfg !== "object") return;
+  if (typeof cfg.apiBase === "string" && cfg.apiBase) _base = cfg.apiBase;
+  if (typeof cfg.model === "string" && cfg.model) _model = cfg.model;
+  if (typeof cfg.apiKey === "string") _key = cfg.apiKey; // 允许显式空串禁用（测试用）
+  if (Number.isFinite(Number(cfg.timeoutMs)) && Number(cfg.timeoutMs) >= 100) _timeout = Number(cfg.timeoutMs);
+}
+
+export function remoteEmbedAvailable() {
+  return Boolean(_key) && Date.now() >= _disabledUntil;
+}
+
+export function remoteEmbedState() {
+  return {
+    available: remoteEmbedAvailable(),
+    base: _base,
+    model: _model,
+    fails: _fails,
+    disabledUntil: _disabledUntil,
+    hasKey: Boolean(_key),
+  };
+}
+
+function trip(reason) {
+  _fails += 1;
+  if (_fails >= FAIL_THRESHOLD && Date.now() >= _disabledUntil) {
+    _disabledUntil = Date.now() + COOLDOWN_MS;
+    try {
+      console.log(`[tool-folder] remote embedding 熔断（${reason}），${COOLDOWN_MS / 1000}s 内只用本地/词法腿`);
+    } catch {
+      /* 日志失败不影响功能 */
+    }
+  }
+}
+
+function reset() {
+  _fails = 0;
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+/** 一次 HTTP 调用，input 可以是单条字符串或字符串数组。 */
+async function callOnce(input, timeoutMs) {
+  const ctrl = new AbortController();
+  const timer = setTimeout(() => ctrl.abort(), timeoutMs);
+  try {
+    const res = await fetch(`${_base.replace(/\/+$/, "")}/embeddings`, {
+      method: "POST",
+      headers: { "content-type": "application/json", authorization: `Bearer ${_key}` },
+      body: JSON.stringify({
+        model: _model,
+        input,
+        encoding_format: "float",
+      }),
+      signal: ctrl.signal,
+    });
+    if (!res.ok) {
+      const body = await res.text().catch(() => "");
+      const err = new Error(`HTTP ${res.status} ${body.slice(0, 120)}`);
+      err.retryable = res.status === 429 || res.status >= 500;
+      err.status = res.status;
+      throw err;
+    }
+    const j = await res.json();
+    const rows = Array.isArray(j?.data) ? j.data : [];
+    if (!rows.length) throw new Error("empty embedding response");
+    // 按 index 对齐（端点不保证顺序）
+    const out = rows
+      .slice()
+      .sort((a, b) => (a.index ?? 0) - (b.index ?? 0))
+      .map((r) => (Array.isArray(r?.embedding) && r.embedding.length ? r.embedding.map(Number) : null));
+    if (!out.some(Boolean)) throw new Error("no usable embedding");
+    return out;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** 单条文本 → 向量；失败返回 null（绝不抛）。 */
+export async function remoteEmbed(text, timeoutMs = _timeout) {
+  const s = String(text || "").trim();
+  if (!s || !remoteEmbedAvailable()) return null;
+  for (let attempt = 0; attempt <= BACKOFF_MS.length; attempt++) {
+    try {
+      const [v] = await callOnce(s.slice(0, 6000) || " ", timeoutMs);
+      if (Array.isArray(v) && v.length) {
+        reset();
+        return v;
+      }
+      trip("empty embedding");
+      return null;
+    } catch (e) {
+      const retryable = e?.retryable || e?.name === "AbortError";
+      if (retryable && attempt < BACKOFF_MS.length) {
+        await sleep(BACKOFF_MS[attempt]);
+        continue;
+      }
+      trip(`${e?.status || e?.name || "error"}`);
+      return null;
+    }
+  }
+  return null;
+}
+
+/**
+ * 批量 embedding。返回与 texts 等长的数组（失败的槽位是 null）。
+ * 批内失败会自动拆成单条重试，尽力保住能拿到的向量。
+ *
+ * @param {string[]} texts
+ * @param {{timeoutMs?: number, batchSize?: number, concurrency?: number, onProgress?: Function}} [opts]
+ * @returns {Promise<Array<number[]|null>>}
+ */
+export async function remoteEmbedBatch(texts, opts = {}) {
+  const list = Array.isArray(texts) ? texts : [];
+  const out = new Array(list.length).fill(null);
+  if (!list.length || !remoteEmbedAvailable()) return out;
+
+  const timeoutMs = Number(opts.timeoutMs) || _timeout;
+  const batchSize = Math.min(Math.max(Number(opts.batchSize) || DEFAULT_BATCH, 1), 16);
+  const concurrency = Math.min(Math.max(Number(opts.concurrency) || DEFAULT_CONCURRENCY, 1), 4);
+
+  const batches = [];
+  for (let i = 0; i < list.length; i += batchSize) {
+    batches.push({ start: i, items: list.slice(i, i + batchSize) });
+  }
+
+  let cursor = 0;
+  const worker = async () => {
+    while (cursor < batches.length) {
+      const b = batches[cursor++];
+      if (!remoteEmbedAvailable()) return; // 熔断后立刻停手
+      const payload = b.items.map((t) => String(t || "").trim().slice(0, 6000) || " ");
+      try {
+        const vecs = await callOnce(payload, timeoutMs);
+        reset();
+        for (let k = 0; k < vecs.length && k < b.items.length; k++) {
+          if (vecs[k]) out[b.start + k] = vecs[k];
+        }
+      } catch (e) {
+        trip(`${e?.status || e?.name || "error"}`);
+        // 批失败拆单条，尽量保住部分向量
+        for (let k = 0; k < payload.length; k++) {
+          if (!remoteEmbedAvailable()) break;
+          const v = await remoteEmbed(payload[k], timeoutMs);
+          if (v) out[b.start + k] = v;
+        }
+      }
+      if (typeof opts.onProgress === "function") {
+        try {
+          opts.onProgress(out.filter(Boolean).length, list.length);
+        } catch {
+          /* 进度回调失败不影响索引 */
+        }
+      }
+      await sleep(BATCH_GAP_MS);
+    }
+  };
+
+  await Promise.all(Array.from({ length: concurrency }, () => worker()));
+  return out;
+}