@aexol/spectral 0.9.254 → 0.9.255

package/dist/extensions/dataset-uploader/analyzer.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Heuristic quality/coverage analyzer for sharegpt dataset JSONL files
+ * (Dataset-uploader.md §3.1 — the `dataset_analyze` tool's core).
+ *
+ * Pure read-only analysis: no files are modified. The report is a plain
+ * object the extension renders as markdown so the agent can decide whether
+ * the dataset is worth cleaning/uploading.
+ *
+ * Report contents:
+ *   - total samples + malformed line count
+ *   - length distribution (p50/p95 chars per sample, rough token est chars/4)
+ *   - per-tool coverage: how many samples use each function_call tool name,
+ *     flagged when below LOW_TOOL_COVERAGE_THRESHOLD samples
+ *   - local duplicates: canonicalized-JSON hash (sorted keys), dup count
+ *   - dry-run scrub residue: per-kind redaction counts over all samples —
+ *     this is the "residue red flag": > 0 means the dataset MUST go through
+ *     `dataset_clean` before upload
+ *   - quality flags: samples with 0 tool calls, samples with an observation
+ *     longer than LONG_OBSERVATION_CHARS
+ */
+/** Tools with fewer samples than this get flagged in the coverage table. */
+export declare const LOW_TOOL_COVERAGE_THRESHOLD = 20;
+/** Observations longer than this are flagged as oversized. */
+export declare const LONG_OBSERVATION_CHARS = 8000;
+/** Rough token estimate divisor (chars / 4). */
+export declare const TOKEN_ESTIMATE_DIVISOR = 4;
+export interface ToolCoverageEntry {
+    tool: string;
+    /** Number of samples containing at least one function_call for this tool. */
+    samples: number;
+    /** True when samples < LOW_TOOL_COVERAGE_THRESHOLD. */
+    low: boolean;
+}
+export interface DatasetAnalysisReport {
+    inputPath: string;
+    totalSamples: number;
+    malformedLines: number;
+    lengthDistribution: {
+        p50Chars: number;
+        p95Chars: number;
+        p50EstTokens: number;
+        p95EstTokens: number;
+    };
+    /** Per-tool sample coverage, sorted by samples desc, then name. */
+    toolCoverage: ToolCoverageEntry[];
+    duplicates: {
+        total: number;
+        unique: number;
+        /** total - unique (samples that are exact canonicalized duplicates). */
+        duplicateSamples: number;
+    };
+    /**
+     * Dry-run scrub over every sample: how many redactions the scrubber WOULD
+     * make. Any total > 0 is the residue red flag — run dataset_clean first.
+     */
+    scrubResidue: {
+        totalRedactions: number;
+        byKind: Record<string, number>;
+    };
+    qualityFlags: {
+        /** Samples with zero function_call entries. */
+        zeroToolCalls: number;
+        /** Samples containing an observation longer than LONG_OBSERVATION_CHARS. */
+        longObservations: number;
+    };
+}
+export interface LoadedJsonl {
+    samples: unknown[];
+    malformedLines: number;
+}
+/**
+ * Load a JSONL file into parsed values, skipping (and counting) malformed
+ * lines. Non-object JSON (e.g. a bare string) counts as malformed too.
+ */
+export declare function loadJsonl(inputPath: string): LoadedJsonl;
+/**
+ * Total character count of a sharegpt sample: all conversation entry values
+ * plus the system prompt (when present). Used for length distribution,
+ * maxSampleChars filtering, and token estimates.
+ */
+export declare function sampleChars(sample: Record<string, unknown>): number;
+/** Tool names used by a sample's function_call entries (unique, in order). */
+export declare function sampleToolNames(sample: Record<string, unknown>): string[];
+/**
+ * Canonicalize a JSON value: recursively sort object keys, drop undefined,
+ * then stringify. Deterministic for duplicate detection.
+ */
+export declare function canonicalizeJson(value: unknown): string;
+/**
+ * Analyze a sharegpt dataset JSONL file. Read-only; never throws on
+ * malformed lines (they are counted instead). Throws only when the input
+ * file itself cannot be read.
+ *
+ * `options.extraValues` (same opaque values `dataset_clean` scrubs with —
+ * secret-store + .env values) extends the dry-run scrub residue counts;
+ * values are never printed.
+ */
+export declare function analyzeDataset(inputPath: string, options?: {
+    extraValues?: string[];
+}): DatasetAnalysisReport;
+/** Render an analysis report as a markdown table the agent can read. */
+export declare function renderAnalysisMarkdown(report: DatasetAnalysisReport): string;
+//# sourceMappingURL=analyzer.d.ts.map

package/dist/extensions/dataset-uploader/analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyzer.d.ts","sourceRoot":"","sources":["../../../src/extensions/dataset-uploader/analyzer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAOH,4EAA4E;AAC5E,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAE9C,8DAA8D;AAC9D,eAAO,MAAM,sBAAsB,OAAO,CAAC;AAE3C,gDAAgD;AAChD,eAAO,MAAM,sBAAsB,IAAI,CAAC;AAExC,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,OAAO,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,GAAG,EAAE,OAAO,CAAC;CACd;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,mEAAmE;IACnE,YAAY,EAAE,iBAAiB,EAAE,CAAC;IAClC,UAAU,EAAE;QACV,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,wEAAwE;QACxE,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF;;;OAGG;IACH,YAAY,EAAE;QACZ,eAAe,EAAE,MAAM,CAAC;QACxB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAChC,CAAC;IACF,YAAY,EAAE;QACZ,+CAA+C;QAC/C,aAAa,EAAE,MAAM,CAAC;QACtB,4EAA4E;QAC5E,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH;AAMD,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,CAmBxD;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAYnE;AAUD,8EAA8E;AAC9E,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,EAAE,CAyBzE;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAWvD;AAcD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IAAE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAO,GACvC,qBAAqB,CAyEvB;AAMD,wEAAwE;AACxE,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,qBAAqB,GAAG,MAAM,CAmD5E"}

package/dist/extensions/dataset-uploader/analyzer.js

@@ -0,0 +1,266 @@
+/**
+ * Heuristic quality/coverage analyzer for sharegpt dataset JSONL files
+ * (Dataset-uploader.md §3.1 — the `dataset_analyze` tool's core).
+ *
+ * Pure read-only analysis: no files are modified. The report is a plain
+ * object the extension renders as markdown so the agent can decide whether
+ * the dataset is worth cleaning/uploading.
+ *
+ * Report contents:
+ *   - total samples + malformed line count
+ *   - length distribution (p50/p95 chars per sample, rough token est chars/4)
+ *   - per-tool coverage: how many samples use each function_call tool name,
+ *     flagged when below LOW_TOOL_COVERAGE_THRESHOLD samples
+ *   - local duplicates: canonicalized-JSON hash (sorted keys), dup count
+ *   - dry-run scrub residue: per-kind redaction counts over all samples —
+ *     this is the "residue red flag": > 0 means the dataset MUST go through
+ *     `dataset_clean` before upload
+ *   - quality flags: samples with 0 tool calls, samples with an observation
+ *     longer than LONG_OBSERVATION_CHARS
+ */
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { scrub } from "./scrubber.js";
+/** Tools with fewer samples than this get flagged in the coverage table. */
+export const LOW_TOOL_COVERAGE_THRESHOLD = 20;
+/** Observations longer than this are flagged as oversized. */
+export const LONG_OBSERVATION_CHARS = 8000;
+/** Rough token estimate divisor (chars / 4). */
+export const TOKEN_ESTIMATE_DIVISOR = 4;
+/**
+ * Load a JSONL file into parsed values, skipping (and counting) malformed
+ * lines. Non-object JSON (e.g. a bare string) counts as malformed too.
+ */
+export function loadJsonl(inputPath) {
+    const raw = readFileSync(inputPath, "utf8");
+    const samples = [];
+    let malformedLines = 0;
+    for (const line of raw.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        try {
+            const value = JSON.parse(trimmed);
+            if (value === null || typeof value !== "object" || Array.isArray(value)) {
+                malformedLines++;
+                continue;
+            }
+            samples.push(value);
+        }
+        catch {
+            malformedLines++;
+        }
+    }
+    return { samples, malformedLines };
+}
+/**
+ * Total character count of a sharegpt sample: all conversation entry values
+ * plus the system prompt (when present). Used for length distribution,
+ * maxSampleChars filtering, and token estimates.
+ */
+export function sampleChars(sample) {
+    let chars = 0;
+    const conversations = sample.conversations;
+    if (Array.isArray(conversations)) {
+        for (const entry of conversations) {
+            if (entry && typeof entry === "object" && typeof entry.value === "string") {
+                chars += entry.value.length;
+            }
+        }
+    }
+    if (typeof sample.system === "string")
+        chars += sample.system.length;
+    return chars;
+}
+function entryValue(entry) {
+    if (entry && typeof entry === "object") {
+        const value = entry.value;
+        if (typeof value === "string")
+            return value;
+    }
+    return null;
+}
+/** Tool names used by a sample's function_call entries (unique, in order). */
+export function sampleToolNames(sample) {
+    const names = [];
+    const seen = new Set();
+    const conversations = sample.conversations;
+    if (!Array.isArray(conversations))
+        return names;
+    for (const entry of conversations) {
+        if (!(entry && typeof entry === "object") ||
+            entry.from !== "function_call") {
+            continue;
+        }
+        const value = entryValue(entry);
+        if (value === null)
+            continue;
+        try {
+            const parsed = JSON.parse(value);
+            if (typeof parsed.name === "string" && parsed.name.length > 0 && !seen.has(parsed.name)) {
+                seen.add(parsed.name);
+                names.push(parsed.name);
+            }
+        }
+        catch {
+            // Malformed function_call payload — ignore for coverage.
+        }
+    }
+    return names;
+}
+/**
+ * Canonicalize a JSON value: recursively sort object keys, drop undefined,
+ * then stringify. Deterministic for duplicate detection.
+ */
+export function canonicalizeJson(value) {
+    if (value === null || typeof value !== "object")
+        return JSON.stringify(value) ?? "null";
+    if (Array.isArray(value))
+        return `[${value.map((v) => canonicalizeJson(v)).join(",")}]`;
+    const keys = Object.keys(value)
+        .filter((k) => value[k] !== undefined)
+        .sort();
+    const parts = keys.map((k) => {
+        const v = value[k];
+        return `${JSON.stringify(k)}:${canonicalizeJson(v)}`;
+    });
+    return `{${parts.join(",")}}`;
+}
+/** Nearest-rank percentile of a non-negative metric array (unsorted ok). */
+function percentile(values, p) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const idx = Math.max(0, Math.ceil(p * sorted.length) - 1);
+    return sorted[idx];
+}
+// ---------------------------------------------------------------------------
+// Analysis
+// ---------------------------------------------------------------------------
+/**
+ * Analyze a sharegpt dataset JSONL file. Read-only; never throws on
+ * malformed lines (they are counted instead). Throws only when the input
+ * file itself cannot be read.
+ *
+ * `options.extraValues` (same opaque values `dataset_clean` scrubs with —
+ * secret-store + .env values) extends the dry-run scrub residue counts;
+ * values are never printed.
+ */
+export function analyzeDataset(inputPath, options = {}) {
+    const { samples, malformedLines } = loadJsonl(inputPath);
+    const typed = samples;
+    const lengths = typed.map((s) => sampleChars(s));
+    const p50 = percentile(lengths, 0.5);
+    const p95 = percentile(lengths, 0.95);
+    // ---- per-tool coverage (samples using each function_call tool) --------
+    const toolSampleCounts = {};
+    let zeroToolCalls = 0;
+    let longObservations = 0;
+    for (const sample of typed) {
+        const names = sampleToolNames(sample);
+        if (names.length === 0)
+            zeroToolCalls++;
+        for (const name of names)
+            toolSampleCounts[name] = (toolSampleCounts[name] ?? 0) + 1;
+        const conversations = sample.conversations;
+        if (Array.isArray(conversations)) {
+            for (const entry of conversations) {
+                if (entry?.from === "observation") {
+                    const value = entryValue(entry);
+                    if (value !== null && value.length > LONG_OBSERVATION_CHARS) {
+                        longObservations++;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    const toolCoverage = Object.entries(toolSampleCounts)
+        .map(([tool, samples_]) => ({
+        tool,
+        samples: samples_,
+        low: samples_ < LOW_TOOL_COVERAGE_THRESHOLD,
+    }))
+        .sort((a, b) => b.samples - a.samples || a.tool.localeCompare(b.tool));
+    // ---- local duplicates (canonicalized JSON hash) ------------------------
+    const seen = new Set();
+    for (const sample of typed) {
+        seen.add(createHash("sha256").update(canonicalizeJson(sample)).digest("hex"));
+    }
+    // ---- dry-run scrub residue (the red flag) -------------------------------
+    const residue = {};
+    for (const sample of typed) {
+        const result = scrub(JSON.stringify(sample), options.extraValues);
+        for (const r of result.redactions)
+            residue[r.kind] = (residue[r.kind] ?? 0) + r.count;
+    }
+    const totalRedactions = Object.values(residue).reduce((sum, n) => sum + n, 0);
+    return {
+        inputPath,
+        totalSamples: typed.length,
+        malformedLines,
+        lengthDistribution: {
+            p50Chars: p50,
+            p95Chars: p95,
+            p50EstTokens: Math.round(p50 / TOKEN_ESTIMATE_DIVISOR),
+            p95EstTokens: Math.round(p95 / TOKEN_ESTIMATE_DIVISOR),
+        },
+        toolCoverage,
+        duplicates: {
+            total: typed.length,
+            unique: seen.size,
+            duplicateSamples: typed.length - seen.size,
+        },
+        scrubResidue: {
+            totalRedactions,
+            byKind: Object.fromEntries(Object.entries(residue).sort((a, b) => a[0].localeCompare(b[0]))),
+        },
+        qualityFlags: { zeroToolCalls, longObservations },
+    };
+}
+// ---------------------------------------------------------------------------
+// Markdown rendering
+// ---------------------------------------------------------------------------
+/** Render an analysis report as a markdown table the agent can read. */
+export function renderAnalysisMarkdown(report) {
+    const lines = [];
+    lines.push(`# Dataset analysis — ${report.inputPath}`);
+    lines.push("");
+    lines.push(`- Samples: **${report.totalSamples}**${report.malformedLines > 0 ? ` (${report.malformedLines} malformed line(s) skipped)` : ""}`);
+    lines.push(`- Length: p50 ≈ ${report.lengthDistribution.p50Chars} chars (~${report.lengthDistribution.p50EstTokens} tokens), ` +
+        `p95 ≈ ${report.lengthDistribution.p95Chars} chars (~${report.lengthDistribution.p95EstTokens} tokens)`);
+    lines.push(`- Local duplicates: ${report.duplicates.duplicateSamples} of ${report.duplicates.total} ` +
+        `(${report.duplicates.unique} unique)`);
+    lines.push(`- Quality flags: ${report.qualityFlags.zeroToolCalls} sample(s) without tool calls, ` +
+        `${report.qualityFlags.longObservations} sample(s) with oversized observations (> ${LONG_OBSERVATION_CHARS} chars)`);
+    lines.push("");
+    lines.push("## Tool coverage");
+    lines.push("");
+    if (report.toolCoverage.length === 0) {
+        lines.push("_No tool calls found in any sample._");
+    }
+    else {
+        lines.push("| Tool | Samples | < 20 samples |");
+        lines.push("| --- | ---: | --- |");
+        for (const entry of report.toolCoverage) {
+            lines.push(`| ${entry.tool} | ${entry.samples} | ${entry.low ? "⚠️ yes" : ""} |`);
+        }
+    }
+    lines.push("");
+    lines.push("## Scrub residue (dry run)");
+    lines.push("");
+    if (report.scrubResidue.totalRedactions === 0) {
+        lines.push("✅ No secret-like patterns detected — dataset is clean by heuristic.");
+    }
+    else {
+        lines.push(`🚩 **Residue red flag: ${report.scrubResidue.totalRedactions} redaction(s) detected. ` +
+            "Run `dataset_clean` before uploading.**");
+        lines.push("");
+        lines.push("| Kind | Count |");
+        lines.push("| --- | ---: |");
+        for (const [kind, count] of Object.entries(report.scrubResidue.byKind)) {
+            lines.push(`| ${kind} | ${count} |`);
+        }
+    }
+    return lines.join("\n");
+}

package/dist/extensions/dataset-uploader/cleaner.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Dataset cleaner (Dataset-uploader.md §2.3 — the `dataset_clean` tool's core).
+ *
+ * Loads a sharegpt dataset JSONL, scrubs EVERY string field of every sample
+ * (the JSON tree is walked; structure is never changed), applies optional
+ * quality filters, and writes `<name>.clean.jsonl` next to the input.
+ *
+ *   - minTurns          — drop samples with fewer assistant ("gpt") turns
+ *   - maxSampleChars    — drop samples longer than this many chars
+ *   - keepOnlyToolCalls — drop samples without any function_call entry
+ *
+ * The scrubber receives `extraValues` collected by the caller (secret-store
+ * values, backend/.env.local values). This module never reads any store and
+ * never prints values — only per-kind redaction counts.
+ */
+export interface CleanDatasetOptions {
+    /** Output path. Default: `<input-without>.clean.jsonl` next to the input. */
+    outputPath?: string;
+    /** Minimum number of assistant ("gpt") turns per sample. */
+    minTurns?: number;
+    /** Maximum total characters per sample (conversation values + system). */
+    maxSampleChars?: number;
+    /** When true, drop samples that contain no function_call entries. */
+    keepOnlyToolCalls?: boolean;
+    /**
+     * Opaque secret values to redact literally (secret-store values, .env
+     * values). The cleaner never reads any store itself.
+     */
+    extraValues?: string[];
+}
+export interface CleanDatasetResult {
+    inputPath: string;
+    outputPath: string;
+    total: number;
+    kept: number;
+    removed: number;
+    malformedLines: number;
+    removedByFilter: {
+        minTurns: number;
+        maxSampleChars: number;
+        keepOnlyToolCalls: number;
+    };
+    /** Per-kind redaction counts (only kinds with count > 0, stable order). */
+    redactions: {
+        kind: string;
+        count: number;
+    }[];
+}
+/** Derive the default clean output path: `export-x.jsonl` → `export-x.clean.jsonl`. */
+export declare function defaultCleanPath(inputPath: string): string;
+/**
+ * Clean a dataset JSONL file: scrub all string fields, apply optional
+ * filters, write `<name>.clean.jsonl` next to the input.
+ *
+ * Throws when the input file cannot be read. Malformed lines are counted,
+ * not fatal.
+ */
+export declare function cleanDataset(inputPath: string, options?: CleanDatasetOptions): CleanDatasetResult;
+//# sourceMappingURL=cleaner.d.ts.map

package/dist/extensions/dataset-uploader/cleaner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleaner.d.ts","sourceRoot":"","sources":["../../../src/extensions/dataset-uploader/cleaner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAQH,MAAM,WAAW,mBAAmB;IAClC,6EAA6E;IAC7E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,cAAc,EAAE,MAAM,CAAC;QACvB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;IACF,2EAA2E;IAC3E,UAAU,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC/C;AA6CD,uFAAuF;AACvF,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAK1D;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,mBAAwB,GAChC,kBAAkB,CAyDpB"}

package/dist/extensions/dataset-uploader/cleaner.js

@@ -0,0 +1,118 @@
+/**
+ * Dataset cleaner (Dataset-uploader.md §2.3 — the `dataset_clean` tool's core).
+ *
+ * Loads a sharegpt dataset JSONL, scrubs EVERY string field of every sample
+ * (the JSON tree is walked; structure is never changed), applies optional
+ * quality filters, and writes `<name>.clean.jsonl` next to the input.
+ *
+ *   - minTurns          — drop samples with fewer assistant ("gpt") turns
+ *   - maxSampleChars    — drop samples longer than this many chars
+ *   - keepOnlyToolCalls — drop samples without any function_call entry
+ *
+ * The scrubber receives `extraValues` collected by the caller (secret-store
+ * values, backend/.env.local values). This module never reads any store and
+ * never prints values — only per-kind redaction counts.
+ */
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { loadJsonl, sampleChars } from "./analyzer.js";
+import { scrub } from "./scrubber.js";
+const REDACTION_KINDS = ["api_key", "jwt", "bearer", "home_path", "secret_value"];
+/**
+ * Walk a JSON tree and scrub every string leaf. Objects/arrays are rebuilt
+ * (structure and key order preserved); non-string leaves pass through.
+ */
+function scrubDeep(value, extraValues, counts) {
+    if (typeof value === "string") {
+        const result = scrub(value, extraValues);
+        for (const r of result.redactions)
+            counts[r.kind] = (counts[r.kind] ?? 0) + r.count;
+        return result.text;
+    }
+    if (Array.isArray(value))
+        return value.map((v) => scrubDeep(v, extraValues, counts));
+    if (value !== null && typeof value === "object") {
+        const out = {};
+        for (const [key, v] of Object.entries(value)) {
+            out[key] = scrubDeep(v, extraValues, counts);
+        }
+        return out;
+    }
+    return value;
+}
+function countGptTurns(sample) {
+    const conversations = sample.conversations;
+    if (!Array.isArray(conversations))
+        return 0;
+    return conversations.filter((e) => e && typeof e === "object" && e.from === "gpt").length;
+}
+function hasToolCall(sample) {
+    const conversations = sample.conversations;
+    if (!Array.isArray(conversations))
+        return false;
+    return conversations.some((e) => e && typeof e === "object" && e.from === "function_call");
+}
+/** Derive the default clean output path: `export-x.jsonl` → `export-x.clean.jsonl`. */
+export function defaultCleanPath(inputPath) {
+    if (inputPath.endsWith(".jsonl")) {
+        return `${inputPath.slice(0, -".jsonl".length)}.clean.jsonl`;
+    }
+    return `${inputPath}.clean.jsonl`;
+}
+/**
+ * Clean a dataset JSONL file: scrub all string fields, apply optional
+ * filters, write `<name>.clean.jsonl` next to the input.
+ *
+ * Throws when the input file cannot be read. Malformed lines are counted,
+ * not fatal.
+ */
+export function cleanDataset(inputPath, options = {}) {
+    if (!existsSync(inputPath)) {
+        throw new Error(`Dataset file not found: ${inputPath}`);
+    }
+    const { samples, malformedLines } = loadJsonl(inputPath);
+    const typed = samples;
+    const counts = {};
+    const kept = [];
+    const removedByFilter = { minTurns: 0, maxSampleChars: 0, keepOnlyToolCalls: 0 };
+    for (const sample of typed) {
+        // Filters first (on the original sample — cheaper and keeps counts
+        // independent of scrubbing).
+        if (typeof options.minTurns === "number" &&
+            options.minTurns > 0 &&
+            countGptTurns(sample) < options.minTurns) {
+            removedByFilter.minTurns++;
+            continue;
+        }
+        if (typeof options.maxSampleChars === "number" &&
+            options.maxSampleChars > 0 &&
+            sampleChars(sample) > options.maxSampleChars) {
+            removedByFilter.maxSampleChars++;
+            continue;
+        }
+        if (options.keepOnlyToolCalls === true && !hasToolCall(sample)) {
+            removedByFilter.keepOnlyToolCalls++;
+            continue;
+        }
+        kept.push(scrubDeep(sample, options.extraValues, counts));
+    }
+    const outputPath = options.outputPath ?? defaultCleanPath(inputPath);
+    mkdirSync(dirname(outputPath), { recursive: true });
+    const content = kept.length > 0
+        ? kept.map((s) => JSON.stringify(s)).join("\n") + "\n"
+        : "";
+    writeFileSync(outputPath, content, "utf8");
+    const redactions = REDACTION_KINDS
+        .filter((kind) => (counts[kind] ?? 0) > 0)
+        .map((kind) => ({ kind, count: counts[kind] }));
+    return {
+        inputPath,
+        outputPath,
+        total: typed.length,
+        kept: kept.length,
+        removed: typed.length - kept.length,
+        malformedLines,
+        removedByFilter,
+        redactions,
+    };
+}