@aexol/spectral 0.9.254 → 0.9.256

package/dist/extensions/dataset-uploader/upload.js

@@ -0,0 +1,274 @@
+/**
+ * Chunked dataset upload to the team dataset vault (Dataset-uploader.md §3.2 —
+ * the `dataset_upload` tool's core).
+ *
+ * Loads a (cleaned) sharegpt JSONL, validates each line, groups it into
+ * batches of `batchSize` and calls the backend MCP tool
+ * `upload_dataset_samples` (JSON-RPC tools/call, Bearer auth) per batch,
+ * then sums the per-batch `DatasetUploadResult` ({accepted, duplicates,
+ * rejected}).
+ *
+ * Auth/URL resolution mirrors `aexol-mcp.ts`: `~/.spectral/config.json`
+ * (`readConfig()`), token = `teamApiKey || userJwt`, URL via `getApiUrl()`,
+ * and a single `ensureAuthenticated()` OAuth retry on 401/403.
+ *
+ * projectId: the Studio binding (`.aexol/aexol.jsonc`) supplies a single
+ * batch-level projectId for ALL batches — the exporter does not persist
+ * per-sample project provenance in the JSONL, so per-project grouping is not
+ * possible without exporter rework (documented in the result notes).
+ *
+ * Testability: the backend call lives behind `callBackendTool`, overridable
+ * per-call (`options.callBackendTool`) or via `setCallBackendToolForTesting`
+ * (module-level), so tests can stub the network entirely.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { getApiUrl, readConfig } from "../../config.js";
+import { AexolMcpClient, AexolMcpError } from "../../mcp-client.js";
+import { ensureAuthenticated } from "../../auth-helper.js";
+import { readStudioBinding } from "../../studio-binding.js";
+import { renderContentToString } from "../aexol-backend-client.js";
+import { loadJsonl } from "./analyzer.js";
+import { scrub } from "./scrubber.js";
+export const DEFAULT_BATCH_SIZE = 50;
+/** Backend MCP tool that ingests one batch of samples. */
+export const BACKEND_UPLOAD_TOOL = "upload_dataset_samples";
+/** Backend MCP tool returning vault stats (best-effort, post-upload). */
+export const BACKEND_VAULT_STATS_TOOL = "dataset_vault_stats";
+export const DATASET_FORMAT = "sharegpt";
+// ---------------------------------------------------------------------------
+// Default backend call (real network path)
+// ---------------------------------------------------------------------------
+let testOverride = null;
+/**
+ * Test hook: override the backend call for the whole module. Pass null to
+ * restore the real implementation. Prefer `options.callBackendTool` for
+ * per-call injection; this exists so tests that drive code through other
+ * entry points (e.g. the extension index) can stub the network too.
+ */
+export function setCallBackendToolForTesting(fn) {
+    testOverride = fn;
+}
+function resolveCallBackendTool(options) {
+    return options.callBackendTool ?? testOverride ?? defaultCallBackendTool;
+}
+function firstJsonContent(content) {
+    if (content.length === 0)
+        return null;
+    const first = content[0];
+    if (first.json !== undefined)
+        return first.json;
+    if (typeof first.text === "string") {
+        try {
+            return JSON.parse(first.text);
+        }
+        catch {
+            return first.text;
+        }
+    }
+    return null;
+}
+/**
+ * Real backend call: config → token → AexolMcpClient.tools/call, with a
+ * single OAuth re-auth retry on 401/403 (same flow as aexol-mcp.ts).
+ */
+async function defaultCallBackendTool(tool, args) {
+    const cfg = await readConfig();
+    if (!cfg) {
+        throw new Error("No Spectral config found — run `spectral login` before uploading. No data was sent.");
+    }
+    const apiUrl = getApiUrl(cfg.apiUrl);
+    let token = cfg.teamApiKey || cfg.userJwt || "";
+    if (!token) {
+        throw new Error("No auth token in Spectral config — run `spectral login` before uploading. No data was sent.");
+    }
+    let client = new AexolMcpClient(apiUrl, token);
+    try {
+        const res = await client.callTool(tool, args);
+        if (res.isError === true) {
+            throw new Error(`Backend tool "${tool}" reported an error: ${renderContentToString(res.content)}`);
+        }
+        return firstJsonContent(res.content);
+    }
+    catch (err) {
+        if (err instanceof AexolMcpError && (err.status === 401 || err.status === 403)) {
+            const auth = await ensureAuthenticated();
+            if (auth.ok && auth.config) {
+                const newToken = auth.config.teamApiKey || auth.config.userJwt || "";
+                if (newToken && newToken !== token) {
+                    client = new AexolMcpClient(apiUrl, newToken);
+                    const res = await client.callTool(tool, args);
+                    if (res.isError === true) {
+                        throw new Error(`Backend tool "${tool}" reported an error after re-auth: ${renderContentToString(res.content)}`);
+                    }
+                    return firstJsonContent(res.content);
+                }
+            }
+            throw new Error(`Authentication failed (HTTP ${err.status}) and re-authentication did not produce a working token. ` +
+                `Run \`spectral login\`. Original error: ${err.message}`);
+        }
+        throw err;
+    }
+}
+// ---------------------------------------------------------------------------
+// Upload
+// ---------------------------------------------------------------------------
+function toInt(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) ? Math.trunc(value) : fallback;
+}
+/**
+ * Upload a dataset JSONL to the team vault in chunks.
+ *
+ * dryRun: parse + validate only — no backend call, `wouldUpload` carries the
+ * count, totals stay zero.
+ */
+export async function uploadDataset(options) {
+    if (!existsSync(options.inputPath)) {
+        throw new Error(`Dataset file not found: ${options.inputPath}`);
+    }
+    const batchSize = typeof options.batchSize === "number" && options.batchSize > 0
+        ? Math.trunc(options.batchSize)
+        : DEFAULT_BATCH_SIZE;
+    const { samples, malformedLines } = loadJsonl(options.inputPath);
+    const notes = [];
+    // ---- team / project resolution (single projectId for all batches) ------
+    const cwd = options.cwd ?? process.cwd();
+    let binding = null;
+    try {
+        binding = await readStudioBinding(cwd);
+    }
+    catch {
+        binding = null;
+    }
+    const teamId = options.teamId ?? binding?.teamId ?? null;
+    const projectId = binding?.projectId ?? null;
+    if (projectId) {
+        notes.push(`All batches use Studio binding projectId ${projectId} (batch-level provenance).`);
+    }
+    else {
+        notes.push("No Studio binding found — batches are sent without projectId (sessions without a bound project).");
+    }
+    notes.push("Per-project batching is not supported by the exporter JSONL (no per-sample project provenance); " +
+        "a single projectId from the binding is used for all batches.");
+    const validSamples = samples.map((s) => JSON.stringify(s));
+    const invalidLines = malformedLines;
+    // ---- pre-upload residue warning (pattern-only dry-run scrub) ------------
+    // Runs in BOTH dry-run and real uploads, before the first network batch.
+    // Pattern rules only (no extraValues) — counts only, never values.
+    let residueWarning = null;
+    try {
+        const rawContent = readFileSync(options.inputPath, "utf8");
+        const scrubResult = scrub(rawContent);
+        const totalRedactions = scrubResult.redactions.reduce((sum, r) => sum + r.count, 0);
+        if (totalRedactions > 0) {
+            residueWarning = {
+                totalRedactions,
+                kinds: scrubResult.redactions.map((r) => r.kind),
+            };
+        }
+    }
+    catch {
+        // Non-fatal — the warning is advisory.
+    }
+    const chunkCount = Math.ceil(validSamples.length / batchSize);
+    const batches = [];
+    const totals = { accepted: 0, duplicates: 0, rejected: 0 };
+    let vaultStats;
+    if (options.dryRun === true) {
+        for (let i = 0; i < chunkCount; i++) {
+            const chunk = validSamples.slice(i * batchSize, (i + 1) * batchSize);
+            batches.push({ projectId, sent: chunk.length, accepted: 0, duplicates: 0, rejected: 0 });
+        }
+    }
+    else {
+        const call = resolveCallBackendTool(options);
+        for (let i = 0; i < chunkCount; i++) {
+            const chunk = validSamples.slice(i * batchSize, (i + 1) * batchSize);
+            const args = {
+                format: DATASET_FORMAT,
+                samples: chunk,
+            };
+            if (teamId)
+                args.teamId = teamId;
+            if (projectId)
+                args.projectId = projectId;
+            const payload = await call(BACKEND_UPLOAD_TOOL, args);
+            const result = (payload && typeof payload === "object" ? payload : {});
+            const accepted = toInt(result.accepted, 0);
+            const duplicates = toInt(result.duplicates, 0);
+            const rejected = toInt(result.rejected, 0);
+            batches.push({ projectId, sent: chunk.length, accepted, duplicates, rejected });
+            totals.accepted += accepted;
+            totals.duplicates += duplicates;
+            totals.rejected += rejected;
+        }
+        // Best-effort vault stats — a failure here never fails the upload.
+        try {
+            const statsArgs = {};
+            if (teamId)
+                statsArgs.teamId = teamId;
+            vaultStats = await call(BACKEND_VAULT_STATS_TOOL, statsArgs);
+        }
+        catch {
+            vaultStats = undefined;
+            notes.push("Vault stats could not be fetched (non-fatal).");
+        }
+    }
+    return {
+        inputPath: options.inputPath,
+        dryRun: options.dryRun === true,
+        totalLines: validSamples.length + invalidLines,
+        validSamples: validSamples.length,
+        invalidLines,
+        batchSize,
+        teamId,
+        projectId,
+        batches,
+        totals,
+        wouldUpload: validSamples.length,
+        ...(vaultStats !== undefined ? { vaultStats } : {}),
+        ...(residueWarning !== null ? { residueWarning } : {}),
+        notes,
+    };
+}
+// ---------------------------------------------------------------------------
+// Markdown rendering
+// ---------------------------------------------------------------------------
+/** Render an upload result as a markdown summary for the agent. */
+export function renderUploadMarkdown(result) {
+    const lines = [];
+    lines.push(`# Dataset upload ${result.dryRun ? "(dry run)" : ""} — ${result.inputPath}`);
+    if (result.residueWarning) {
+        lines.push("");
+        lines.push(`⚠️ residue detected (${result.residueWarning.totalRedactions} redactions across kinds: ${result.residueWarning.kinds.join(", ")}) — run dataset_clean before uploading`);
+    }
+    lines.push("");
+    lines.push(`- Samples: **${result.validSamples}** valid` +
+        (result.invalidLines > 0 ? ` (${result.invalidLines} invalid line(s) skipped)` : "") +
+        `, batch size ${result.batchSize}`);
+    lines.push(`- Scope: team ${result.teamId ?? "(resolved by backend)"} / project ${result.projectId ?? "none"}`);
+    if (result.dryRun) {
+        lines.push(`- Dry run: **${result.wouldUpload} sample(s) would be uploaded in ${result.batches.length} batch(es)** — no data was sent.`);
+    }
+    else {
+        lines.push(`- Totals: accepted **${result.totals.accepted}**, duplicates **${result.totals.duplicates}**, rejected **${result.totals.rejected}**`);
+        lines.push("");
+        lines.push("| Batch | Sent | Accepted | Duplicates | Rejected |");
+        lines.push("| ---: | ---: | ---: | ---: | ---: |");
+        result.batches.forEach((b, i) => {
+            lines.push(`| ${i + 1} | ${b.sent} | ${b.accepted} | ${b.duplicates} | ${b.rejected} |`);
+        });
+    }
+    if (result.vaultStats !== undefined) {
+        lines.push("");
+        lines.push("## Vault stats");
+        lines.push("");
+        lines.push("```json");
+        lines.push(JSON.stringify(result.vaultStats, null, 2));
+        lines.push("```");
+    }
+    lines.push("");
+    lines.push("## Notes");
+    for (const note of result.notes)
+        lines.push(`- ${note}`);
+    return lines.join("\n");
+}

package/dist/generated/zeus/const.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"const.d.ts","sourceRoot":"","sources":["../../../src/generated/zeus/const.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAC,GAAG,CAy6B5C,CAAA;AAED,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAC,GAAG,CA+1C1C,CAAA;AAED,eAAO,MAAM,GAAG;;;;CAIf,CAAA"}
+{"version":3,"file":"const.d.ts","sourceRoot":"","sources":["../../../src/generated/zeus/const.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAC,GAAG,CA47B5C,CAAA;AAED,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAC,GAAG,CA24C1C,CAAA;AAED,eAAO,MAAM,GAAG;;;;CAIf,CAAA"}

package/dist/generated/zeus/const.js

@@ -178,6 +178,10 @@ export const AllTypesProps = {
         adminCreditPackCount: {},
         adminTeamPayments: {},
         adminTeamPaymentCount: {},
+        adminUsageByModel: {
+            from: "DateTime",
+            to: "DateTime"
+        },
         adminCreditGrants: {},
         adminLeads: {
             status: "LeadStatus",
@@ -207,6 +211,8 @@ export const AllTypesProps = {
         syntheticDataset: {},
         fineTuneJobs: {},
         fineTuneJob: {},
+        datasetSamples: {},
+        datasetVaultStats: {},
         dedicatedUsage: {},
         seoSearchAnalytics: {
             input: "SeoSearchAnalyticsInput"
@@ -480,6 +486,10 @@ export const AllTypesProps = {
         cancelFineTuneJob: {},
         deleteFineTuneJob: {},
         deleteSyntheticDataset: {},
+        uploadDatasetSamples: {
+            input: "UploadDatasetSamplesInput"
+        },
+        deleteDatasetSample: {},
         deployDedicated: {
             input: "DeployDedicatedInput"
         },
@@ -538,6 +548,7 @@ export const AllTypesProps = {
     GenerateSyntheticDatasetInput: {},
     ImportSyntheticDatasetInput: {},
     StartFineTuneJobInput: {},
+    UploadDatasetSamplesInput: {},
     AdminCreateBillingPlanInput: {},
     AdminUpdateBillingPlanInput: {},
     AdminCreateCreditPackInput: {},
@@ -1096,6 +1107,7 @@ export const ReturnTypes = {
         adminCreditPackCount: "Int",
         adminTeamPayments: "AdminTeamPayment",
         adminTeamPaymentCount: "Int",
+        adminUsageByModel: "AdminModelUsagePage",
         adminCreditGrants: "CreditGrant",
         adminLeads: "Lead",
         adminLeadCount: "Int",
@@ -1108,6 +1120,8 @@ export const ReturnTypes = {
         syntheticDataset: "SyntheticDataset",
         fineTuneJobs: "FineTuneJob",
         fineTuneJob: "FineTuneJob",
+        datasetSamples: "DatasetSample",
+        datasetVaultStats: "DatasetVaultStats",
         dedicatedHardwareOptions: "DedicatedHardware",
         dedicatedUsage: "DedicatedUsageSummary",
         latestSpectralRelease: "SpectralRelease",
@@ -1271,6 +1285,8 @@ export const ReturnTypes = {
         cancelFineTuneJob: "Boolean",
         deleteFineTuneJob: "Boolean",
         deleteSyntheticDataset: "Boolean",
+        uploadDatasetSamples: "DatasetUploadResult",
+        deleteDatasetSample: "Boolean",
         deployDedicated: "UserModel",
         updateDedicated: "UserModel",
         destroyDedicated: "UserModel",
@@ -1470,6 +1486,27 @@ export const ReturnTypes = {
         message: "String",
         outputModelName: "String"
     },
+    DatasetSample: {
+        id: "ID",
+        teamId: "ID",
+        projectId: "ID",
+        contentHash: "String",
+        format: "String",
+        sampleCount: "Int",
+        sourceSessionId: "String",
+        createdAt: "DateTime"
+    },
+    DatasetUploadResult: {
+        accepted: "Int",
+        duplicates: "Int",
+        rejected: "Int",
+        sampleIds: "ID"
+    },
+    DatasetVaultStats: {
+        count: "Int",
+        totalBytes: "Int",
+        topTools: "String"
+    },
     DedicatedHardware: {
         id: "String",
         name: "String",
@@ -1563,6 +1600,24 @@ export const ReturnTypes = {
         completedAt: "DateTime",
         expiresAt: "DateTime"
     },
+    AdminModelUsageEntry: {
+        date: "String",
+        model: "String",
+        inputTokens: "Float",
+        outputTokens: "Float",
+        cacheReadTokens: "Float",
+        cacheWriteTokens: "Float",
+        totalTokens: "Float",
+        creditsUsed: "Float",
+        costUsd: "Float",
+        eventCount: "Int",
+        teamCount: "Int"
+    },
+    AdminModelUsagePage: {
+        rows: "AdminModelUsageEntry",
+        page: "Int",
+        pageSize: "Int"
+    },
     CreditGrant: {
         id: "ID",
         teamId: "ID",