nodebench-mcp 1.1.0 → 1.4.0

package/dist/__tests__/fixtures/generateSwebenchVerifiedFixture.js

@@ -0,0 +1,189 @@
+#!/usr/bin/env npx tsx
+/**
+ * Refreshes a local fixture from SWE-bench Verified (open-source GitHub issue tasks).
+ *
+ * Source:
+ *   - https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified
+ *   - rows API: https://datasets-server.huggingface.co/rows
+ *
+ * Usage:
+ *   npx tsx src/__tests__/fixtures/generateSwebenchVerifiedFixture.ts
+ *   npx tsx src/__tests__/fixtures/generateSwebenchVerifiedFixture.ts --limit 12 --min-statement-length 700 --min-fail-to-pass 2
+ *   npx tsx src/__tests__/fixtures/generateSwebenchVerifiedFixture.ts --output ./tmp/swebench.sample.json
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+const DATASET = "princeton-nlp/SWE-bench_Verified";
+const CONFIG = "default";
+const SPLIT = "test";
+const ROWS_API = "https://datasets-server.huggingface.co/rows";
+const DATASET_PAGE = "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified";
+function getArg(flag) {
+    const index = process.argv.indexOf(flag);
+    if (index < 0)
+        return undefined;
+    return process.argv[index + 1];
+}
+function parseIntArg(flag, fallback) {
+    const raw = getArg(flag);
+    if (!raw)
+        return fallback;
+    const value = Number.parseInt(raw, 10);
+    if (!Number.isFinite(value))
+        return fallback;
+    return value;
+}
+function compactWhitespace(input) {
+    return input.replace(/\s+/g, " ").trim();
+}
+function parseStringArray(value) {
+    if (Array.isArray(value)) {
+        return value.filter((item) => typeof item === "string");
+    }
+    if (typeof value !== "string" || value.trim().length === 0)
+        return [];
+    try {
+        const parsed = JSON.parse(value);
+        if (!Array.isArray(parsed))
+            return [];
+        return parsed.filter((item) => typeof item === "string");
+    }
+    catch {
+        return [];
+    }
+}
+function difficultyWeight(difficulty) {
+    const normalized = difficulty.toLowerCase();
+    if (normalized.includes("1-4 hours"))
+        return 50;
+    if (normalized.includes("15 min - 1 hour"))
+        return 30;
+    if (normalized.includes("<15 min"))
+        return 10;
+    return 20;
+}
+function toTask(raw, index) {
+    const id = compactWhitespace(String(raw.instance_id ?? ""));
+    const repo = compactWhitespace(String(raw.repo ?? ""));
+    const problemStatement = compactWhitespace(String(raw.problem_statement ?? ""));
+    if (!id || !repo || !problemStatement)
+        return null;
+    const hints = compactWhitespace(String(raw.hints_text ?? ""));
+    const difficulty = compactWhitespace(String(raw.difficulty ?? "unspecified"));
+    const failToPassCount = parseStringArray(raw.FAIL_TO_PASS).length;
+    const passToPassCount = parseStringArray(raw.PASS_TO_PASS).length;
+    const statementLength = problemStatement.length;
+    const hintLength = hints.length;
+    const complexityScore = Math.ceil(statementLength / 25) +
+        Math.ceil(hintLength / 40) +
+        failToPassCount * 14 +
+        Math.ceil(passToPassCount / 8) +
+        difficultyWeight(difficulty);
+    const promptSections = [
+        `Repository: ${repo}`,
+        `Instance: ${id}`,
+        `Difficulty: ${difficulty}`,
+        `Problem: ${problemStatement}`,
+    ];
+    if (hints.length > 0) {
+        promptSections.push(`Hints: ${hints}`);
+    }
+    return {
+        id,
+        title: `SWE-bench ${id}`,
+        prompt: promptSections.join("\n"),
+        repo,
+        difficulty,
+        statementLength,
+        hintLength,
+        failToPassCount,
+        passToPassCount,
+        complexityScore,
+    };
+}
+async function fetchRows(offset, length) {
+    const url = new URL(ROWS_API);
+    url.searchParams.set("dataset", DATASET);
+    url.searchParams.set("config", CONFIG);
+    url.searchParams.set("split", SPLIT);
+    url.searchParams.set("offset", String(offset));
+    url.searchParams.set("length", String(length));
+    const response = await fetch(url.toString());
+    if (!response.ok) {
+        throw new Error(`Failed to fetch SWE-bench rows: ${response.status} ${response.statusText}`);
+    }
+    return (await response.json());
+}
+async function loadRows(maxRows, pageSize) {
+    const rows = [];
+    for (let offset = 0; offset < maxRows; offset += pageSize) {
+        const remaining = maxRows - offset;
+        const length = Math.max(1, Math.min(pageSize, remaining));
+        const page = await fetchRows(offset, length);
+        const pageRows = Array.isArray(page.rows) ? page.rows : [];
+        if (pageRows.length === 0)
+            break;
+        for (const entry of pageRows) {
+            if (!entry || typeof entry !== "object")
+                continue;
+            const row = entry.row;
+            if (row && typeof row === "object")
+                rows.push(row);
+        }
+        if (pageRows.length < length)
+            break;
+    }
+    return rows;
+}
+async function main() {
+    const limit = Math.max(1, parseIntArg("--limit", 12));
+    const minStatementLength = Math.max(100, parseIntArg("--min-statement-length", 700));
+    const minFailToPass = Math.max(1, parseIntArg("--min-fail-to-pass", 2));
+    const maxRowsScanned = Math.max(50, parseIntArg("--max-rows", 500));
+    const pageSize = Math.max(10, parseIntArg("--page-size", 100));
+    const here = dirname(fileURLToPath(import.meta.url));
+    const outputPath = getArg("--output") ?? join(here, "swebench_verified.sample.json");
+    const rawRows = await loadRows(maxRowsScanned, pageSize);
+    const tasks = rawRows
+        .map((row, index) => toTask(row, index))
+        .filter((task) => task !== null);
+    const candidates = tasks
+        .filter((task) => task.statementLength >= minStatementLength && task.failToPassCount >= minFailToPass)
+        .sort((a, b) => {
+        if (b.complexityScore !== a.complexityScore)
+            return b.complexityScore - a.complexityScore;
+        return a.id.localeCompare(b.id);
+    });
+    const selected = candidates.slice(0, limit);
+    if (selected.length === 0) {
+        throw new Error(`No SWE-bench tasks matched filters (minStatementLength=${minStatementLength}, minFailToPass=${minFailToPass}).`);
+    }
+    const fixture = {
+        dataset: DATASET,
+        split: SPLIT,
+        sourceUrl: DATASET_PAGE,
+        rowsApi: ROWS_API,
+        generatedAt: new Date().toISOString(),
+        selection: {
+            requestedLimit: limit,
+            minStatementLength,
+            minFailToPass,
+            maxRowsScanned,
+            pageSize,
+            totalRecords: tasks.length,
+            candidateRecords: candidates.length,
+        },
+        tasks: selected,
+    };
+    mkdirSync(dirname(outputPath), { recursive: true });
+    writeFileSync(outputPath, JSON.stringify(fixture, null, 2) + "\n", "utf8");
+    console.log(`[swebench-fixture] wrote ${selected.length} tasks to ${outputPath}`);
+    console.log(`[swebench-fixture] total=${tasks.length}, candidates=${candidates.length}, filters=statement>=${minStatementLength}, failToPass>=${minFailToPass}`);
+}
+main().catch((error) => {
+    const message = error instanceof Error ? error.stack ?? error.message : String(error);
+    process.stderr.write(`${message}\n`);
+    process.exit(1);
+});
+//# sourceMappingURL=generateSwebenchVerifiedFixture.js.map

package/dist/__tests__/fixtures/generateSwebenchVerifiedFixture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generateSwebenchVerifiedFixture.js","sourceRoot":"","sources":["../../../src/__tests__/fixtures/generateSwebenchVerifiedFixture.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,OAAO,GAAG,kCAAkC,CAAC;AACnD,MAAM,MAAM,GAAG,SAAS,CAAC;AACzB,MAAM,KAAK,GAAG,MAAM,CAAC;AACrB,MAAM,QAAQ,GAAG,6CAA6C,CAAC;AAC/D,MAAM,YAAY,GAAG,kEAAkE,CAAC;AAgDxF,SAAS,MAAM,CAAC,IAAY;IAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAChC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,QAAgB;IACjD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,CAAC,GAAG;QAAE,OAAO,QAAQ,CAAC;IAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACtE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,EAAE,CAAC;QACtC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,UAAkB;IAC1C,MAAM,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAO,EAAE,CAAC;IAChD,IAAI,UAAU,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAAE,OAAO,EAAE,CAAC;IACtD,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IAC9C,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,MAAM,CAAC,GAAmB,EAAE,KAAa;IAChD,MAAM,EAAE,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;IACvD,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;IAChF,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,gBAAgB;QAAE,OAAO,IAAI,CAAC;IAEnD,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,aAAa,CAAC,CAAC,CAAC;IAC9E,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,eAAe,GAAG,gBAAgB,CAAC,MAAM,CAAC;IAChD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;IAEhC,MAAM,eAAe,GACnB,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QAC1B,eAAe,GAAG,EAAE;QACpB,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QAC9B,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAE/B,MAAM,cAAc,GAAG;QACrB,eAAe,IAAI,EAAE;QACrB,aAAa,EAAE,EAAE;QACjB,eAAe,UAAU,EAAE;QAC3B,YAAY,gBAAgB,EAAE;KAC/B,CAAC;IACF,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,cAAc,CAAC,IAAI,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,OAAO;QACL,EAAE;QACF,KAAK,EAAE,aAAa,EAAE,EAAE;QACxB,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC;QACjC,IAAI;QACJ,UAAU;QACV,eAAe;QACf,UAAU;QACV,eAAe;QACf,eAAe;QACf,eAAe;KAChB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,MAAc;IACrD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACzC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACrC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/C,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAE/C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IAC/F,CAAC;IAED,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAgB,CAAC;AAChD,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,OAAe,EAAE,QAAgB;IACvD,MAAM,IAAI,GAAqB,EAAE,CAAC;IAClC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC1D,MAAM,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAEjC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,SAAS;YAClD,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;YACtB,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrD,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,MAAM;YAAE,MAAM;IACtC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;IACtD,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC,CAAC;IACrF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,WAAW,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC;IACpE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,WAAW,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,CAAC;IAE/D,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,+BAA+B,CAAC,CAAC;IAErF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,OAAO;SAClB,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;SACvC,MAAM,CAAC,CAAC,IAAI,EAA+B,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IAEhE,MAAM,UAAU,GAAG,KAAK;SACrB,MAAM,CACL,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,IAAI,kBAAkB,IAAI,IAAI,CAAC,eAAe,IAAI,aAAa,CAC9F;SACA,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,IAAI,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,eAAe;YAAE,OAAO,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC;QAC1F,OAAO,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEL,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC5C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,0DAA0D,kBAAkB,mBAAmB,aAAa,IAAI,CACjH,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAoB;QAC/B,OAAO,EAAE,OAAO;QAChB,KAAK,EAAE,KAAK;QACZ,SAAS,EAAE,YAAY;QACvB,OAAO,EAAE,QAAQ;QACjB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,SAAS,EAAE;YACT,cAAc,EAAE,KAAK;YACrB,kBAAkB;YAClB,aAAa;YACb,cAAc;YACd,QAAQ;YACR,YAAY,EAAE,KAAK,CAAC,MAAM;YAC1B,gBAAgB,EAAE,UAAU,CAAC,MAAM;SACpC;QACD,KAAK,EAAE,QAAQ;KAChB,CAAC;IAEF,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;IAE3E,OAAO,CAAC,GAAG,CAAC,4BAA4B,QAAQ,CAAC,MAAM,aAAa,UAAU,EAAE,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,CACT,4BAA4B,KAAK,CAAC,MAAM,gBAAgB,UAAU,CAAC,MAAM,wBAAwB,kBAAkB,iBAAiB,aAAa,EAAE,CACpJ,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/__tests__/fixtures/generateToolbenchInstructionFixture.d.ts

@@ -0,0 +1,16 @@
+#!/usr/bin/env npx tsx
+/**
+ * Refreshes a local fixture from the open-source ToolBench instruction dataset.
+ *
+ * Source:
+ *   - https://github.com/OpenBMB/ToolBench
+ *   - data_example/instruction/G1_query.json
+ *   - data_example/instruction/G2_query.json
+ *   - data_example/instruction/G3_query.json
+ *
+ * Usage:
+ *   npx tsx src/__tests__/fixtures/generateToolbenchInstructionFixture.ts
+ *   npx tsx src/__tests__/fixtures/generateToolbenchInstructionFixture.ts --limit 10 --min-api 6
+ *   npx tsx src/__tests__/fixtures/generateToolbenchInstructionFixture.ts --output ./tmp/toolbench.sample.json
+ */
+export {};

package/dist/__tests__/fixtures/generateToolbenchInstructionFixture.js

@@ -0,0 +1,154 @@
+#!/usr/bin/env npx tsx
+/**
+ * Refreshes a local fixture from the open-source ToolBench instruction dataset.
+ *
+ * Source:
+ *   - https://github.com/OpenBMB/ToolBench
+ *   - data_example/instruction/G1_query.json
+ *   - data_example/instruction/G2_query.json
+ *   - data_example/instruction/G3_query.json
+ *
+ * Usage:
+ *   npx tsx src/__tests__/fixtures/generateToolbenchInstructionFixture.ts
+ *   npx tsx src/__tests__/fixtures/generateToolbenchInstructionFixture.ts --limit 10 --min-api 6
+ *   npx tsx src/__tests__/fixtures/generateToolbenchInstructionFixture.ts --output ./tmp/toolbench.sample.json
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+const SOURCES = [
+    {
+        group: "G1",
+        url: "https://raw.githubusercontent.com/OpenBMB/ToolBench/master/data_example/instruction/G1_query.json",
+    },
+    {
+        group: "G2",
+        url: "https://raw.githubusercontent.com/OpenBMB/ToolBench/master/data_example/instruction/G2_query.json",
+    },
+    {
+        group: "G3",
+        url: "https://raw.githubusercontent.com/OpenBMB/ToolBench/master/data_example/instruction/G3_query.json",
+    },
+];
+function getArg(flag) {
+    const index = process.argv.indexOf(flag);
+    if (index < 0)
+        return undefined;
+    return process.argv[index + 1];
+}
+function parseIntArg(flag, fallback) {
+    const raw = getArg(flag);
+    if (!raw)
+        return fallback;
+    const value = Number.parseInt(raw, 10);
+    if (!Number.isFinite(value))
+        return fallback;
+    return value;
+}
+function compactWhitespace(input) {
+    return input.replace(/\s+/g, " ").trim();
+}
+function asArray(value) {
+    return Array.isArray(value) ? value : [];
+}
+function toTask(record, group, index) {
+    const query = compactWhitespace(String(record.query ?? ""));
+    if (!query)
+        return null;
+    const apiList = asArray(record.api_list).filter((entry) => typeof entry === "object" && entry !== null);
+    const requiredParameterCount = apiList.reduce((count, api) => {
+        return count + asArray(api.required_parameters).length;
+    }, 0);
+    const optionalParameterCount = apiList.reduce((count, api) => {
+        return count + asArray(api.optional_parameters).length;
+    }, 0);
+    const apiCategories = Array.from(new Set(apiList
+        .map((api) => String(api.category_name ?? "").trim())
+        .filter((name) => name.length > 0)));
+    const apiCount = apiList.length;
+    const relevantApiCount = asArray(record["relevant APIs"]).length;
+    const rawId = String(record.query_id ?? `${group}_${index + 1}`).trim();
+    const id = `${group}_${rawId}`;
+    const title = `ToolBench ${group} query ${rawId}`;
+    const complexityScore = apiCount * 12 +
+        requiredParameterCount * 2 +
+        optionalParameterCount +
+        relevantApiCount * 3 +
+        Math.ceil(query.length / 40);
+    return {
+        id,
+        title,
+        prompt: query,
+        group,
+        apiCount,
+        relevantApiCount,
+        requiredParameterCount,
+        optionalParameterCount,
+        apiCategories,
+        complexityScore,
+    };
+}
+async function fetchSource(group, url) {
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`Failed to fetch ${group} dataset: ${response.status} ${response.statusText}`);
+    }
+    const raw = await response.json();
+    if (!Array.isArray(raw)) {
+        throw new Error(`Unexpected payload for ${group}: expected array`);
+    }
+    return raw;
+}
+async function main() {
+    const limit = Math.max(1, parseIntArg("--limit", 10));
+    const minApiCount = Math.max(1, parseIntArg("--min-api", 6));
+    const minQueryLength = Math.max(1, parseIntArg("--min-query-length", 180));
+    const here = dirname(fileURLToPath(import.meta.url));
+    const outputPath = getArg("--output") ?? join(here, "toolbench_instruction.sample.json");
+    const fetched = await Promise.all(SOURCES.map(async (source) => {
+        const records = await fetchSource(source.group, source.url);
+        return { ...source, records };
+    }));
+    const tasks = fetched.flatMap((entry) => entry.records
+        .map((record, index) => toTask(record, entry.group, index))
+        .filter((task) => task !== null));
+    const candidates = tasks
+        .filter((task) => task.apiCount >= minApiCount && task.prompt.length >= minQueryLength)
+        .sort((a, b) => {
+        if (b.complexityScore !== a.complexityScore)
+            return b.complexityScore - a.complexityScore;
+        return a.id.localeCompare(b.id);
+    });
+    const selected = candidates.slice(0, limit);
+    if (selected.length === 0) {
+        throw new Error(`No ToolBench tasks matched filters (minApiCount=${minApiCount}, minQueryLength=${minQueryLength}).`);
+    }
+    const sourceUrls = SOURCES.reduce((acc, source) => {
+        acc[source.group] = source.url;
+        return acc;
+    }, {});
+    const fixture = {
+        dataset: "OpenBMB/ToolBench",
+        split: "data_example/instruction (G1,G2,G3)",
+        sourceUrls,
+        generatedAt: new Date().toISOString(),
+        selection: {
+            requestedLimit: limit,
+            minApiCount,
+            minQueryLength,
+            totalRecords: tasks.length,
+            candidateRecords: candidates.length,
+        },
+        tasks: selected,
+    };
+    mkdirSync(dirname(outputPath), { recursive: true });
+    writeFileSync(outputPath, JSON.stringify(fixture, null, 2) + "\n", "utf8");
+    console.log(`[toolbench-fixture] wrote ${selected.length} tasks to ${outputPath}`);
+    console.log(`[toolbench-fixture] total=${tasks.length}, candidates=${candidates.length}, filters=api>=${minApiCount}, queryLength>=${minQueryLength}`);
+}
+main().catch((error) => {
+    const message = error instanceof Error ? error.stack ?? error.message : String(error);
+    process.stderr.write(`${message}\n`);
+    process.exit(1);
+});
+//# sourceMappingURL=generateToolbenchInstructionFixture.js.map

package/dist/__tests__/fixtures/generateToolbenchInstructionFixture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generateToolbenchInstructionFixture.js","sourceRoot":"","sources":["../../../src/__tests__/fixtures/generateToolbenchInstructionFixture.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,OAAO,GAAG;IACd;QACE,KAAK,EAAE,IAAI;QACX,GAAG,EAAE,mGAAmG;KACzG;IACD;QACE,KAAK,EAAE,IAAI;QACX,GAAG,EAAE,mGAAmG;KACzG;IACD;QACE,KAAK,EAAE,IAAI;QACX,GAAG,EAAE,mGAAmG;KACzG;CACO,CAAC;AA+CX,SAAS,MAAM,CAAC,IAAY;IAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAChC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,QAAgB;IACjD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,IAAI,CAAC,GAAG;QAAE,OAAO,QAAQ,CAAC;IAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,OAAO,CAAI,KAAc;IAChC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,KAAa,CAAC,CAAC,CAAC,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,MAAM,CAAC,MAA0B,EAAE,KAAkB,EAAE,KAAa;IAC3E,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,OAAO,GAAG,OAAO,CAAe,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAC3D,CAAC,KAAK,EAAyB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAC9E,CAAC;IAEF,MAAM,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC3D,OAAO,KAAK,GAAG,OAAO,CAAU,GAAG,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAC;IAClE,CAAC,EAAE,CAAC,CAAC,CAAC;IACN,MAAM,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC3D,OAAO,KAAK,GAAG,OAAO,CAAU,GAAG,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAC;IAClE,CAAC,EAAE,CAAC,CAAC,CAAC;IACN,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAC9B,IAAI,GAAG,CACL,OAAO;SACJ,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;SACpD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CACrC,CACF,CAAC;IAEF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,MAAM,gBAAgB,GAAG,OAAO,CAAU,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC;IAE1E,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,IAAI,GAAG,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACxE,MAAM,EAAE,GAAG,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAG,aAAa,KAAK,UAAU,KAAK,EAAE,CAAC;IAElD,MAAM,eAAe,GACnB,QAAQ,GAAG,EAAE;QACb,sBAAsB,GAAG,CAAC;QAC1B,sBAAsB;QACtB,gBAAgB,GAAG,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;IAE/B,OAAO;QACL,EAAE;QACF,KAAK;QACL,MAAM,EAAE,KAAK;QACb,KAAK;QACL,QAAQ;QACR,gBAAgB;QAChB,sBAAsB;QACtB,sBAAsB;QACtB,aAAa;QACb,eAAe;KAChB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,KAAkB,EAAE,GAAW;IACxD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,aAAa,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAClC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,kBAAkB,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,GAA2B,CAAC;AACrC,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;IACtD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC,CAAC;IAE3E,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,MAAM,UAAU,GACd,MAAM,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,mCAAmC,CAAC,CAAC;IAExE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC3B,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5D,OAAO,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,CAAC;IAChC,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,OAAO;SACV,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;SAC1D,MAAM,CAAC,CAAC,IAAI,EAAgC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CACjE,CAAC;IAEF,MAAM,UAAU,GAAG,KAAK;SACrB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,IAAI,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,cAAc,CAAC;SACtF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,IAAI,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,eAAe;YAAE,OAAO,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC;QAC1F,OAAO,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEL,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC5C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,mDAAmD,WAAW,oBAAoB,cAAc,IAAI,CACrG,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAC/B,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;QACd,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC;QAC/B,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAAiC,CAClC,CAAC;IAEF,MAAM,OAAO,GAAqB;QAChC,OAAO,EAAE,mBAAmB;QAC5B,KAAK,EAAE,qCAAqC;QAC5C,UAAU;QACV,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,SAAS,EAAE;YACT,cAAc,EAAE,KAAK;YACrB,WAAW;YACX,cAAc;YACd,YAAY,EAAE,KAAK,CAAC,MAAM;YAC1B,gBAAgB,EAAE,UAAU,CAAC,MAAM;SACpC;QACD,KAAK,EAAE,QAAQ;KAChB,CAAC;IAEF,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;IAE3E,OAAO,CAAC,GAAG,CAAC,6BAA6B,QAAQ,CAAC,MAAM,aAAa,UAAU,EAAE,CAAC,CAAC;IACnF,OAAO,CAAC,GAAG,CACT,6BAA6B,KAAK,CAAC,MAAM,gBAAgB,UAAU,CAAC,MAAM,kBAAkB,WAAW,kBAAkB,cAAc,EAAE,CAC1I,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACrB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}