@aexol/spectral 0.9.251 → 0.9.255

package/dist/extensions/dataset-uploader/exporter.d.ts

@@ -0,0 +1,147 @@
+/**
+ * Session → sharegpt SFT dataset exporter (Dataset-uploader.md §2.1).
+ *
+ * Reads the local sessions DB read-only (default `~/.spectral/sessions.db`)
+ * and maps stored messages to sharegpt conversation samples:
+ *
+ *   user row      → { from: "human", value: content }
+ *   assistant row → events_jsonl parsed in event order:
+ *                    concatenated text_delta  → { from: "gpt" }
+ *                    each tool_call           → { from: "function_call",
+ *                                                value: JSON.stringify({name, arguments}) }
+ *                    matching tool_result     → { from: "observation",
+ *                                                value: text of result.content,
+ *                                                "[ERROR] " prefix when isError }
+ *                    thinking_* events        → DISCARDED (non-thinking target)
+ *
+ * Per-turn expansion: each assistant message row that produces at least one
+ * entry becomes one sample — `{ conversations: [all entries up to and
+ * including this turn], system?, tools? }`. `system`/`tools` are plain
+ * parameters (the assembling index.ts passes `ctx.getSystemPrompt()` and
+ * the runtime tool registry). `system` is attached ONLY to samples from
+ * sessions whose `active_agent` equals the MOST COMMON active_agent among
+ * the exported sessions (the ExtensionContext exposes no current-agent
+ * accessor, so this heuristic is the documented fallback); sessions with a
+ * different — or null — active_agent get no system field.
+ *
+ * Subagent handling: when `includeSubagents` is false (default) every
+ * observation is capped at MAX_OBSERVATION_CHARS with a "...[truncated]"
+ * marker (subagent transcripts are the dominant source of oversized
+ * observations).
+ */
+interface ReadOnlyStatement {
+    all(...params: unknown[]): unknown[];
+}
+interface ReadOnlyDb {
+    prepare(sql: string): ReadOnlyStatement;
+    close(): void;
+}
+/** Open a SQLite database file read-only (file must already exist). */
+export declare function openReadOnlySqlite(path: string): ReadOnlyDb;
+export interface SharegptEntry {
+    from: "human" | "gpt" | "function_call" | "observation";
+    value: string;
+}
+/** OpenAI-style tool definition, as stored in the sample `tools` field. */
+export interface ToolDefinition {
+    type: "function";
+    function: {
+        name: string;
+        description?: string;
+        parameters?: unknown;
+    };
+}
+export interface SharegptSample {
+    conversations: SharegptEntry[];
+    system?: string;
+    tools?: ToolDefinition[];
+}
+export interface ExportSessionsOptions {
+    /** Path to the sessions DB. Default: `<configDir>/sessions.db`. */
+    dbPath?: string;
+    /** Restrict the export to these session ids. */
+    sessionIds?: string[];
+    /** Max number of sessions (newest first by updated_at). */
+    limit?: number;
+    /** Only sessions with updated_at >= since (ISO 8601 string). */
+    since?: string;
+    /**
+     * When true, observations are NOT capped (subagent transcripts kept
+     * intact). Default false → observations capped at MAX_OBSERVATION_CHARS.
+     */
+    includeSubagents?: boolean;
+    /** Output JSONL path. Default: `<configDir>/datasets/export-<timestamp>.jsonl`. */
+    outputPath?: string;
+    /**
+     * System prompt to attach to samples of sessions whose `active_agent`
+     * matches the most common active_agent among the exported sessions;
+     * omitted when null/empty.
+     */
+    system?: string | null;
+    /** Tool registry to attach to every sample; omitted when null/empty. */
+    tools?: ToolDefinition[] | undefined;
+}
+export interface ExportSessionsResult {
+    /** Number of sessions processed. */
+    sessions: number;
+    /** Number of sharegpt samples written. */
+    samples: number;
+    /** Total bytes written to the output file(s). */
+    bytes: number;
+    /** Count of tool calls per tool name (per call event, not per sample). */
+    toolCoverage: Record<string, number>;
+    /** Output file paths (always a single JSONL in v1). */
+    files: string[];
+    /**
+     * The active_agent value that received the system prompt (the most
+     * common active_agent among the exported sessions), when a system
+     * prompt was provided. null when no system prompt was attached.
+     */
+    systemAgent: string | null;
+    /** Number of sessions whose samples had the system prompt attached. */
+    systemSessionCount: number;
+}
+/**
+ * Cap for observation text when `includeSubagents` is false. history-
+ * projection's 256 KB field cap is too large for training data, so this
+ * module defines its own tighter limit.
+ */
+export declare const MAX_OBSERVATION_CHARS = 4000;
+export declare const OBSERVATION_TRUNCATION_MARKER = "...[truncated]";
+interface ToolResultContentPart {
+    type?: string;
+    text?: string;
+}
+interface AssistantEvent {
+    type: string;
+    delta?: string;
+    id?: string;
+    name?: string;
+    args?: unknown;
+    result?: {
+        content?: ToolResultContentPart[];
+    } | null;
+    isError?: boolean;
+}
+export interface ParsedAssistantTurn {
+    entries: SharegptEntry[];
+    /** Tool names invoked by this turn's tool_call events (in order). */
+    toolNames: string[];
+}
+/** Extract the plain text of a tool_result payload (concatenated text parts). */
+export declare function extractResultText(result: AssistantEvent["result"]): string;
+/**
+ * Parse an assistant row's events_jsonl into sharegpt entries, in event
+ * order. Thinking events are discarded. When `capChars` is non-null,
+ * observation values are truncated to it + OBSERVATION_TRUNCATION_MARKER.
+ */
+export declare function parseAssistantEvents(eventsJsonl: string, capChars?: number | null): ParsedAssistantTurn;
+/**
+ * Export sessions from the local sessions DB to a sharegpt JSONL file.
+ *
+ * The DB is opened strictly read-only; the output file is written next to
+ * (or at) `outputPath`. One JSON sample per line.
+ */
+export declare function exportSessions(options?: ExportSessionsOptions): ExportSessionsResult;
+export {};
+//# sourceMappingURL=exporter.d.ts.map

package/dist/extensions/dataset-uploader/exporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"exporter.d.ts","sourceRoot":"","sources":["../../../src/extensions/dataset-uploader/exporter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAkBH,UAAU,iBAAiB;IACzB,GAAG,CAAC,GAAG,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;CACtC;AAED,UAAU,UAAU;IAClB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,iBAAiB,CAAC;IACxC,KAAK,IAAI,IAAI,CAAC;CACf;AAiCD,uEAAuE;AACvE,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAiB3D;AAMD,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,GAAG,KAAK,GAAG,eAAe,GAAG,aAAa,CAAC;IACxD,KAAK,EAAE,MAAM,CAAC;CACf;AAED,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,aAAa,EAAE,CAAC;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,qBAAqB;IACpC,mEAAmE;IACnE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,2DAA2D;IAC3D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mFAAmF;IACnF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,wEAAwE;IACxE,KAAK,CAAC,EAAE,cAAc,EAAE,GAAG,SAAS,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB;IACnC,oCAAoC;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,KAAK,EAAE,MAAM,CAAC;IACd,0EAA0E;IAC1E,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,uDAAuD;IACvD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB;;;;OAIG;IACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,uEAAuE;IACvE,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAMD;;;;GAIG;AACH,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAE1C,eAAO,MAAM,6BAA6B,mBAAmB,CAAC;AAW9D,UAAU,qBAAqB;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,UAAU,cAAc;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,qBAAqB,EAAE,CAAA;KAAE,GAAG,IAAI,CAAC;IACtD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,qEAAqE;IACrE,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,iFAAiF;AACjF,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,cAAc,CAAC,QAAQ,CAAC,GAAG,MAAM,CAO1E;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,MAAM,EACnB,QAAQ,GAAE,MAAM,GAAG,IAAW,GAC7B,mBAAmB,CAyDrB;AA8CD;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,OAAO,GAAE,qBAA0B,GAAG,oBAAoB,CA4GxF"}

package/dist/extensions/dataset-uploader/exporter.js

@@ -0,0 +1,313 @@
+/**
+ * Session → sharegpt SFT dataset exporter (Dataset-uploader.md §2.1).
+ *
+ * Reads the local sessions DB read-only (default `~/.spectral/sessions.db`)
+ * and maps stored messages to sharegpt conversation samples:
+ *
+ *   user row      → { from: "human", value: content }
+ *   assistant row → events_jsonl parsed in event order:
+ *                    concatenated text_delta  → { from: "gpt" }
+ *                    each tool_call           → { from: "function_call",
+ *                                                value: JSON.stringify({name, arguments}) }
+ *                    matching tool_result     → { from: "observation",
+ *                                                value: text of result.content,
+ *                                                "[ERROR] " prefix when isError }
+ *                    thinking_* events        → DISCARDED (non-thinking target)
+ *
+ * Per-turn expansion: each assistant message row that produces at least one
+ * entry becomes one sample — `{ conversations: [all entries up to and
+ * including this turn], system?, tools? }`. `system`/`tools` are plain
+ * parameters (the assembling index.ts passes `ctx.getSystemPrompt()` and
+ * the runtime tool registry). `system` is attached ONLY to samples from
+ * sessions whose `active_agent` equals the MOST COMMON active_agent among
+ * the exported sessions (the ExtensionContext exposes no current-agent
+ * accessor, so this heuristic is the documented fallback); sessions with a
+ * different — or null — active_agent get no system field.
+ *
+ * Subagent handling: when `includeSubagents` is false (default) every
+ * observation is capped at MAX_OBSERVATION_CHARS with a "...[truncated]"
+ * marker (subagent transcripts are the dominant source of oversized
+ * observations).
+ */
+import { createRequire } from "node:module";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { getConfigDir } from "../../config.js";
+const _require = createRequire(import.meta.url);
+let nodeSqliteModule;
+let nodeSqliteProbed = false;
+function loadNodeSqlite() {
+    if (nodeSqliteProbed)
+        return nodeSqliteModule;
+    nodeSqliteProbed = true;
+    // Same warning suppression as sqlite-adapter.ts: node:sqlite emits an
+    // ExperimentalWarning on first load; we opt into it deliberately.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const proc = process;
+    const originalEmitWarning = proc.emitWarning;
+    try {
+        proc.emitWarning = (warning, ...rest) => {
+            const message = typeof warning === "string"
+                ? warning
+                : warning instanceof Error
+                    ? warning.message
+                    : String(warning);
+            if (message.includes("SQLite is an experimental feature"))
+                return;
+            return originalEmitWarning.call(process, warning, ...rest);
+        };
+        nodeSqliteModule = _require("node:sqlite");
+    }
+    catch {
+        nodeSqliteModule = null;
+    }
+    finally {
+        proc.emitWarning = originalEmitWarning;
+    }
+    return nodeSqliteModule;
+}
+/** Open a SQLite database file read-only (file must already exist). */
+export function openReadOnlySqlite(path) {
+    if (typeof globalThis.Bun !== "undefined") {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const { Database } = _require("bun:sqlite");
+        return new Database(path, { readonly: true, create: false });
+    }
+    const mod = loadNodeSqlite();
+    if (typeof mod?.DatabaseSync === "function") {
+        return new mod.DatabaseSync(path, { readOnly: true });
+    }
+    const Db = _require("better-sqlite3");
+    return new Db(path, { readonly: true });
+}
+// ---------------------------------------------------------------------------
+// Observation cap
+// ---------------------------------------------------------------------------
+/**
+ * Cap for observation text when `includeSubagents` is false. history-
+ * projection's 256 KB field cap is too large for training data, so this
+ * module defines its own tighter limit.
+ */
+export const MAX_OBSERVATION_CHARS = 4000;
+export const OBSERVATION_TRUNCATION_MARKER = "...[truncated]";
+function capObservation(limit, text) {
+    if (limit === null || text.length <= limit)
+        return text;
+    return text.slice(0, limit) + OBSERVATION_TRUNCATION_MARKER;
+}
+/** Extract the plain text of a tool_result payload (concatenated text parts). */
+export function extractResultText(result) {
+    const parts = result?.content;
+    if (!Array.isArray(parts))
+        return "";
+    return parts
+        .filter((p) => p && p.type === "text" && typeof p.text === "string")
+        .map((p) => p.text)
+        .join("\n");
+}
+/**
+ * Parse an assistant row's events_jsonl into sharegpt entries, in event
+ * order. Thinking events are discarded. When `capChars` is non-null,
+ * observation values are truncated to it + OBSERVATION_TRUNCATION_MARKER.
+ */
+export function parseAssistantEvents(eventsJsonl, capChars = null) {
+    const entries = [];
+    const toolNames = [];
+    const pendingCallIds = new Set();
+    let textBuffer = "";
+    const flushText = () => {
+        if (textBuffer.length > 0) {
+            entries.push({ from: "gpt", value: textBuffer });
+            textBuffer = "";
+        }
+    };
+    for (const line of eventsJsonl.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        let event;
+        try {
+            event = JSON.parse(trimmed);
+        }
+        catch {
+            continue; // skip malformed lines
+        }
+        switch (event.type) {
+            case "text_delta":
+                textBuffer += event.delta ?? "";
+                break;
+            case "tool_call":
+                flushText();
+                if (typeof event.name === "string" && event.name.length > 0) {
+                    entries.push({
+                        from: "function_call",
+                        value: JSON.stringify({ name: event.name, arguments: event.args ?? {} }),
+                    });
+                    toolNames.push(event.name);
+                    if (typeof event.id === "string")
+                        pendingCallIds.add(event.id);
+                }
+                break;
+            case "tool_result": {
+                flushText();
+                // Only emit observations that match a tool_call from this turn.
+                if (typeof event.id !== "string" || !pendingCallIds.has(event.id))
+                    break;
+                pendingCallIds.delete(event.id);
+                const text = extractResultText(event.result);
+                if (text.length === 0)
+                    break;
+                entries.push({
+                    from: "observation",
+                    value: (event.isError ? "[ERROR] " : "") + capObservation(capChars, text),
+                });
+                break;
+            }
+            default:
+                // thinking_*, message_*, token_usage, subagent_* → discarded
+                break;
+        }
+    }
+    flushText();
+    return { entries, toolNames };
+}
+// ---------------------------------------------------------------------------
+// Export
+// ---------------------------------------------------------------------------
+function makeSample(entries, system, tools) {
+    const sample = { conversations: [...entries] };
+    if (typeof system === "string" && system.length > 0)
+        sample.system = system;
+    if (Array.isArray(tools) && tools.length > 0)
+        sample.tools = tools;
+    return sample;
+}
+/**
+ * Most common non-null active_agent among the exported sessions (ties
+ * broken by first occurrence in newest-first order). null when no session
+ * has a non-null active_agent.
+ */
+function mostCommonActiveAgent(sessions) {
+    const counts = new Map();
+    for (const s of sessions) {
+        if (typeof s.activeAgent === "string" && s.activeAgent.length > 0) {
+            counts.set(s.activeAgent, (counts.get(s.activeAgent) ?? 0) + 1);
+        }
+    }
+    let best = null;
+    let bestCount = 0;
+    for (const [agent, count] of counts) {
+        if (count > bestCount) {
+            best = agent;
+            bestCount = count;
+        }
+    }
+    return best;
+}
+function defaultTimestamp() {
+    return new Date().toISOString().replace(/[:.]/g, "-");
+}
+/**
+ * Export sessions from the local sessions DB to a sharegpt JSONL file.
+ *
+ * The DB is opened strictly read-only; the output file is written next to
+ * (or at) `outputPath`. One JSON sample per line.
+ */
+export function exportSessions(options = {}) {
+    const dbPath = options.dbPath ?? join(getConfigDir(), "sessions.db");
+    if (!existsSync(dbPath)) {
+        throw new Error(`sessions DB not found at ${dbPath}`);
+    }
+    const db = openReadOnlySqlite(dbPath);
+    try {
+        // ---- session selection (newest first) --------------------------------
+        const where = [];
+        const params = [];
+        if (options.sessionIds !== undefined && options.sessionIds.length > 0) {
+            where.push(`id IN (${options.sessionIds.map(() => "?").join(",")})`);
+            params.push(...options.sessionIds);
+        }
+        if (options.since !== undefined) {
+            const sinceMs = Date.parse(options.since);
+            if (Number.isNaN(sinceMs)) {
+                throw new Error(`invalid "since" value: ${JSON.stringify(options.since)}`);
+            }
+            where.push("updated_at >= ?");
+            params.push(sinceMs);
+        }
+        let sql = "SELECT id, active_agent FROM sessions";
+        if (where.length > 0)
+            sql += ` WHERE ${where.join(" AND ")}`;
+        sql += " ORDER BY updated_at DESC";
+        if (typeof options.limit === "number" && options.limit > 0) {
+            sql += " LIMIT ?";
+            params.push(options.limit);
+        }
+        const selectedSessions = db.prepare(sql).all(...params).map((r) => ({
+            id: r.id,
+            activeAgent: r.active_agent ?? null,
+        }));
+        // System prompt heuristic: attach it only to sessions whose
+        // active_agent is the most common one among the exported sessions
+        // (no current-agent accessor exists on ExtensionContext).
+        const hasSystem = typeof options.system === "string" && options.system.length > 0;
+        const systemAgent = hasSystem
+            ? mostCommonActiveAgent(selectedSessions)
+            : null;
+        let systemSessionCount = 0;
+        // ---- per-session mapping ---------------------------------------------
+        const cap = options.includeSubagents ? null : MAX_OBSERVATION_CHARS;
+        const toolCoverage = {};
+        const samples = [];
+        const messageStmt = db.prepare("SELECT role, content, events_jsonl FROM messages WHERE session_id = ? ORDER BY seq ASC, id ASC");
+        for (const session of selectedSessions) {
+            const sessionId = session.id;
+            const sessionSystem = systemAgent !== null && session.activeAgent === systemAgent
+                ? options.system
+                : null;
+            const rows = messageStmt.all(sessionId);
+            const entries = [];
+            for (const row of rows) {
+                if (row.role === "user") {
+                    if (row.content.trim().length > 0) {
+                        entries.push({ from: "human", value: row.content });
+                    }
+                    continue;
+                }
+                if (row.role !== "assistant")
+                    continue; // system rows are not part of SFT samples
+                const turn = parseAssistantEvents(row.events_jsonl, cap);
+                if (turn.entries.length === 0)
+                    continue;
+                for (const name of turn.toolNames) {
+                    toolCoverage[name] = (toolCoverage[name] ?? 0) + 1;
+                }
+                entries.push(...turn.entries);
+                // Per-turn expansion: snapshot everything up to and including this turn.
+                samples.push(makeSample(entries, sessionSystem, options.tools));
+            }
+            if (sessionSystem !== null)
+                systemSessionCount++;
+        }
+        // ---- write JSONL --------------------------------------------------------
+        const outputPath = options.outputPath
+            ?? join(getConfigDir(), "datasets", `export-${defaultTimestamp()}.jsonl`);
+        mkdirSync(dirname(outputPath), { recursive: true });
+        const content = samples.length > 0
+            ? samples.map((s) => JSON.stringify(s)).join("\n") + "\n"
+            : "";
+        writeFileSync(outputPath, content, "utf8");
+        return {
+            sessions: selectedSessions.length,
+            samples: samples.length,
+            bytes: Buffer.byteLength(content, "utf8"),
+            toolCoverage,
+            files: [outputPath],
+            systemAgent,
+            systemSessionCount,
+        };
+    }
+    finally {
+        db.close();
+    }
+}

package/dist/extensions/dataset-uploader/extra-values.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Shared opaque-secret collector for the dataset tools (used by BOTH
+ * `dataset_clean` and `dataset_analyze` so their scrub runs agree).
+ *
+ * Collects:
+ *   (a) secret-store values (listSecrets + getSecretValue)
+ *   (b) values from backend/.env.local under cwd (KEY=VALUE lines)
+ *
+ * Values are NEVER printed — they are only passed to the scrubber.
+ * Failures are non-fatal (the pattern-based rules still apply).
+ */
+/**
+ * Collect opaque secret values for the scrubber (see module doc). Returns
+ * an empty array when no values are available.
+ */
+export declare function collectExtraValues(cwd: string): string[];
+//# sourceMappingURL=extra-values.d.ts.map

package/dist/extensions/dataset-uploader/extra-values.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extra-values.d.ts","sourceRoot":"","sources":["../../../src/extensions/dataset-uploader/extra-values.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAOH;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CA2BxD"}

package/dist/extensions/dataset-uploader/extra-values.js

@@ -0,0 +1,51 @@
+/**
+ * Shared opaque-secret collector for the dataset tools (used by BOTH
+ * `dataset_clean` and `dataset_analyze` so their scrub runs agree).
+ *
+ * Collects:
+ *   (a) secret-store values (listSecrets + getSecretValue)
+ *   (b) values from backend/.env.local under cwd (KEY=VALUE lines)
+ *
+ * Values are NEVER printed — they are only passed to the scrubber.
+ * Failures are non-fatal (the pattern-based rules still apply).
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { listSecrets, getSecretValue } from "../../secrets/store.js";
+/**
+ * Collect opaque secret values for the scrubber (see module doc). Returns
+ * an empty array when no values are available.
+ */
+export function collectExtraValues(cwd) {
+    const values = [];
+    try {
+        for (const meta of listSecrets()) {
+            const value = getSecretValue(meta.name);
+            if (typeof value === "string" && value.length > 0)
+                values.push(value);
+        }
+    }
+    catch {
+        // Store unreadable — non-fatal, pattern rules still apply.
+    }
+    try {
+        const envPath = join(cwd, "backend", ".env.local");
+        if (existsSync(envPath)) {
+            for (const line of readFileSync(envPath, "utf8").split("\n")) {
+                const trimmed = line.trim();
+                if (!trimmed || trimmed.startsWith("#"))
+                    continue;
+                const eq = trimmed.indexOf("=");
+                if (eq > 0) {
+                    const value = trimmed.slice(eq + 1).trim();
+                    if (value.length > 0)
+                        values.push(value);
+                }
+            }
+        }
+    }
+    catch {
+        // .env unreadable — non-fatal.
+    }
+    return values;
+}

package/dist/extensions/dataset-uploader/index.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Dataset Uploader — native spectral extension (Dataset-uploader.md, Phase 2–3).
+ *
+ * Turns real harness sessions into SFT (sharegpt) training data and uploads
+ * it to the team dataset vault on the Aexol backend. Registers four tools:
+ *
+ *   - dataset_export   sessions DB → sharegpt JSONL (system + runtime tool
+ *                      registry attached from the live agent context)
+ *   - dataset_analyze  heuristic quality report (coverage, duplicates,
+ *                      dry-run scrub residue, quality flags)
+ *   - dataset_clean    scrub every string field (secret-store + .env values
+ *                      as extraValues) + quality filters → *.clean.jsonl
+ *   - dataset_upload   chunked upload to the backend vault (dry-run aware)
+ *
+ * USAGE POLICY (encoded in every tool description): these tools exist for
+ * model-training dataset building and must be used ONLY when the user
+ * explicitly asks to export/analyze/clean/upload session data for a
+ * training dataset. The agent must never run them proactively.
+ */
+import type { ExtensionAPI } from "../../sdk/coding-agent/index.js";
+export default function datasetUploaderExtension(ext: ExtensionAPI): Promise<void>;
+//# sourceMappingURL=index.d.ts.map

package/dist/extensions/dataset-uploader/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extensions/dataset-uploader/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,KAAK,EAEV,YAAY,EAGb,MAAM,iCAAiC,CAAC;AA+KzC,wBAA8B,wBAAwB,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA4GvF"}