@aexol/spectral 0.9.254 → 0.9.256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dataset-uploader/analyzer.d.ts +103 -0
- package/dist/extensions/dataset-uploader/analyzer.d.ts.map +1 -0
- package/dist/extensions/dataset-uploader/analyzer.js +266 -0
- package/dist/extensions/dataset-uploader/cleaner.d.ts +59 -0
- package/dist/extensions/dataset-uploader/cleaner.d.ts.map +1 -0
- package/dist/extensions/dataset-uploader/cleaner.js +118 -0
- package/dist/extensions/dataset-uploader/exporter.d.ts +147 -0
- package/dist/extensions/dataset-uploader/exporter.d.ts.map +1 -0
- package/dist/extensions/dataset-uploader/exporter.js +313 -0
- package/dist/extensions/dataset-uploader/extra-values.d.ts +17 -0
- package/dist/extensions/dataset-uploader/extra-values.d.ts.map +1 -0
- package/dist/extensions/dataset-uploader/extra-values.js +51 -0
- package/dist/extensions/dataset-uploader/index.d.ts +22 -0
- package/dist/extensions/dataset-uploader/index.d.ts.map +1 -0
- package/dist/extensions/dataset-uploader/index.js +247 -0
- package/dist/extensions/dataset-uploader/scrubber.d.ts +34 -0
- package/dist/extensions/dataset-uploader/scrubber.d.ts.map +1 -0
- package/dist/extensions/dataset-uploader/scrubber.js +91 -0
- package/dist/extensions/dataset-uploader/upload.d.ts +105 -0
- package/dist/extensions/dataset-uploader/upload.d.ts.map +1 -0
- package/dist/extensions/dataset-uploader/upload.js +274 -0
- package/dist/generated/zeus/const.d.ts.map +1 -1
- package/dist/generated/zeus/const.js +55 -0
- package/dist/generated/zeus/index.d.ts +264 -0
- package/dist/generated/zeus/index.d.ts.map +1 -1
- package/dist/mcp/commands.d.ts.map +1 -1
- package/dist/mcp/commands.js +3 -0
- package/dist/mcp/direct-tools.d.ts.map +1 -1
- package/dist/mcp/direct-tools.js +4 -3
- package/dist/mcp/index.d.ts.map +1 -1
- package/dist/mcp/index.js +5 -1
- package/dist/mcp/mcp-auth-flow.d.ts +20 -2
- package/dist/mcp/mcp-auth-flow.d.ts.map +1 -1
- package/dist/mcp/mcp-auth-flow.js +87 -7
- package/dist/mcp/mcp-auth-start.d.ts +63 -0
- package/dist/mcp/mcp-auth-start.d.ts.map +1 -0
- package/dist/mcp/mcp-auth-start.js +131 -0
- package/dist/mcp/mcp-oauth-provider.d.ts +24 -0
- package/dist/mcp/mcp-oauth-provider.d.ts.map +1 -1
- package/dist/mcp/mcp-oauth-provider.js +20 -0
- package/dist/mcp/proxy-modes.d.ts +7 -0
- package/dist/mcp/proxy-modes.d.ts.map +1 -1
- package/dist/mcp/proxy-modes.js +71 -11
- package/dist/mcp/server-manager.d.ts +2 -1
- package/dist/mcp/server-manager.d.ts.map +1 -1
- package/dist/mcp/server-manager.js +9 -4
- package/dist/mcp/types.d.ts +9 -0
- package/dist/mcp/types.d.ts.map +1 -1
- package/dist/sdk/coding-agent/core/extensions/native-extensions.d.ts.map +1 -1
- package/dist/sdk/coding-agent/core/extensions/native-extensions.js +10 -0
- package/dist/server/handlers/mcp-auth.d.ts +7 -4
- package/dist/server/handlers/mcp-auth.d.ts.map +1 -1
- package/dist/server/handlers/mcp-auth.js +15 -80
- package/package.json +1 -1
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dataset Uploader — native spectral extension (Dataset-uploader.md, Phase 2–3).
|
|
3
|
+
*
|
|
4
|
+
* Turns real harness sessions into SFT (sharegpt) training data and uploads
|
|
5
|
+
* it to the team dataset vault on the Aexol backend. Registers four tools:
|
|
6
|
+
*
|
|
7
|
+
* - dataset_export sessions DB → sharegpt JSONL (system + runtime tool
|
|
8
|
+
* registry attached from the live agent context)
|
|
9
|
+
* - dataset_analyze heuristic quality report (coverage, duplicates,
|
|
10
|
+
* dry-run scrub residue, quality flags)
|
|
11
|
+
* - dataset_clean scrub every string field (secret-store + .env values
|
|
12
|
+
* as extraValues) + quality filters → *.clean.jsonl
|
|
13
|
+
* - dataset_upload chunked upload to the backend vault (dry-run aware)
|
|
14
|
+
*
|
|
15
|
+
* USAGE POLICY (encoded in every tool description): these tools exist for
|
|
16
|
+
* model-training dataset building and must be used ONLY when the user
|
|
17
|
+
* explicitly asks to export/analyze/clean/upload session data for a
|
|
18
|
+
* training dataset. The agent must never run them proactively.
|
|
19
|
+
*/
|
|
20
|
+
import { Type } from "typebox";
|
|
21
|
+
import { createAllToolDefinitions } from "../../sdk/coding-agent/core/tools/index.js";
|
|
22
|
+
import { exportSessions, } from "./exporter.js";
|
|
23
|
+
import { analyzeDataset, renderAnalysisMarkdown } from "./analyzer.js";
|
|
24
|
+
import { cleanDataset } from "./cleaner.js";
|
|
25
|
+
import { collectExtraValues } from "./extra-values.js";
|
|
26
|
+
import { uploadDataset, renderUploadMarkdown, } from "./upload.js";
|
|
27
|
+
const EXPORT_DESCRIPTION = "Export local harness sessions (from the local sessions DB) into a sharegpt SFT JSONL file, " +
|
|
28
|
+
"for building a MODEL-TRAINING dataset. " +
|
|
29
|
+
"Use ONLY when the user EXPLICITLY asks to export session data for a training dataset — " +
|
|
30
|
+
"never run this proactively. Params: sessionIds, limit, since, includeSubagents, outputPath.";
|
|
31
|
+
const ANALYZE_DESCRIPTION = "Analyze a sharegpt dataset JSONL file for MODEL-TRAINING dataset building: sample count, " +
|
|
32
|
+
"length distribution (p50/p95), per-tool coverage (tools with < 20 samples flagged), local " +
|
|
33
|
+
"duplicates, dry-run scrub residue (secrets that would be redacted), and quality flags. " +
|
|
34
|
+
"Use ONLY when the user EXPLICITLY asks to analyze session data for a training dataset — " +
|
|
35
|
+
"never run this proactively. Read-only: the file is not modified.";
|
|
36
|
+
const CLEAN_DESCRIPTION = "Clean a sharegpt dataset JSONL file for MODEL-TRAINING dataset building: scrubs every string " +
|
|
37
|
+
"field (API keys, JWTs, bearer tokens, home paths, plus secret-store and backend/.env.local " +
|
|
38
|
+
"values) and applies optional quality filters (minTurns, maxSampleChars, keepOnlyToolCalls), " +
|
|
39
|
+
"writing <name>.clean.jsonl next to the input. Use ONLY when the user EXPLICITLY asks to " +
|
|
40
|
+
"clean session data for a training dataset — never run this proactively. Secret values are " +
|
|
41
|
+
"never printed; only per-kind redaction counts are returned.";
|
|
42
|
+
const UPLOAD_DESCRIPTION = "Upload a (cleaned) sharegpt dataset JSONL to the team dataset vault on the Aexol backend in " +
|
|
43
|
+
"chunks, for MODEL-TRAINING dataset building. Use ONLY when the user EXPLICITLY asks to upload " +
|
|
44
|
+
"session data for a training dataset — never run this proactively. Supports dryRun (validate " +
|
|
45
|
+
"only, no network) and reports accepted/duplicates/rejected totals plus vault stats.";
|
|
46
|
+
function errorResult(msg) {
|
|
47
|
+
return {
|
|
48
|
+
content: [{ type: "text", text: `❌ ${msg}` }],
|
|
49
|
+
details: { isError: true, error: msg },
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
function textResult(text) {
|
|
53
|
+
return { content: [{ type: "text", text }], details: { isError: false } };
|
|
54
|
+
}
|
|
55
|
+
// ---------------------------------------------------------------------------
|
|
56
|
+
// dataset_export
|
|
57
|
+
// ---------------------------------------------------------------------------
|
|
58
|
+
const exportSchema = Type.Object({
|
|
59
|
+
sessionIds: Type.Optional(Type.Array(Type.String(), { description: "Restrict the export to these session ids." })),
|
|
60
|
+
limit: Type.Optional(Type.Number({ description: "Max number of sessions (newest first)." })),
|
|
61
|
+
since: Type.Optional(Type.String({ description: "Only sessions updated since this ISO 8601 timestamp." })),
|
|
62
|
+
includeSubagents: Type.Optional(Type.Boolean({
|
|
63
|
+
description: "Keep subagent transcripts intact (observations uncapped). Default false (capped).",
|
|
64
|
+
})),
|
|
65
|
+
outputPath: Type.Optional(Type.String({ description: "Output JSONL path. Default: <configDir>/datasets/export-<ts>.jsonl" })),
|
|
66
|
+
});
|
|
67
|
+
function renderExportSummary(result, toolsIncluded) {
|
|
68
|
+
const lines = [];
|
|
69
|
+
lines.push(`# Dataset export`);
|
|
70
|
+
lines.push("");
|
|
71
|
+
lines.push(`- Sessions: **${result.sessions}**`);
|
|
72
|
+
lines.push(`- Samples: **${result.samples}**`);
|
|
73
|
+
lines.push(`- Bytes: ${result.bytes}`);
|
|
74
|
+
lines.push(`- Output: ${result.files.join(", ")}`);
|
|
75
|
+
lines.push(`- Tools field: ${toolsIncluded ? "attached (runtime registry)" : "omitted (registry unavailable)"}`);
|
|
76
|
+
if (result.systemAgent !== null) {
|
|
77
|
+
lines.push(`- System prompt: attached to ${result.systemSessionCount}/${result.sessions} session(s) ` +
|
|
78
|
+
`with active_agent "${result.systemAgent}" (most common agent among exported sessions)`);
|
|
79
|
+
}
|
|
80
|
+
if (Object.keys(result.toolCoverage).length > 0) {
|
|
81
|
+
lines.push("");
|
|
82
|
+
lines.push("Tool call coverage (per call event):");
|
|
83
|
+
lines.push("");
|
|
84
|
+
for (const [tool, count] of Object.entries(result.toolCoverage).sort((a, b) => b[1] - a[1])) {
|
|
85
|
+
lines.push(`- ${tool}: ${count}`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return lines.join("\n");
|
|
89
|
+
}
|
|
90
|
+
/** Reconstruct the OpenAI-style tool registry from the runtime tool definitions. */
|
|
91
|
+
function buildToolRegistry(cwd) {
|
|
92
|
+
// NOTE: extension/MCP tools are intentionally excluded from the exported
|
|
93
|
+
// tools registry (v1) — only the base tool definitions are included.
|
|
94
|
+
try {
|
|
95
|
+
const registry = createAllToolDefinitions(cwd);
|
|
96
|
+
return Object.values(registry).map((t) => ({
|
|
97
|
+
type: "function",
|
|
98
|
+
function: {
|
|
99
|
+
name: t.name,
|
|
100
|
+
description: t.description,
|
|
101
|
+
parameters: t.parameters,
|
|
102
|
+
},
|
|
103
|
+
}));
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
// ---------------------------------------------------------------------------
|
|
110
|
+
// dataset_clean
|
|
111
|
+
// ---------------------------------------------------------------------------
|
|
112
|
+
const cleanSchema = Type.Object({
|
|
113
|
+
inputPath: Type.String({ description: "Path to the dataset JSONL file to clean." }),
|
|
114
|
+
minTurns: Type.Optional(Type.Number({ description: "Drop samples with fewer assistant (gpt) turns than this." })),
|
|
115
|
+
maxSampleChars: Type.Optional(Type.Number({ description: "Drop samples longer than this many total characters." })),
|
|
116
|
+
keepOnlyToolCalls: Type.Optional(Type.Boolean({ description: "When true, drop samples without any function_call entry." })),
|
|
117
|
+
});
|
|
118
|
+
function renderCleanSummary(result) {
|
|
119
|
+
const lines = [];
|
|
120
|
+
lines.push(`# Dataset clean — ${result.inputPath}`);
|
|
121
|
+
lines.push("");
|
|
122
|
+
lines.push(`- Samples: ${result.total} total → **${result.kept} kept**, ${result.removed} removed` +
|
|
123
|
+
(result.malformedLines > 0 ? ` (${result.malformedLines} malformed line(s) skipped)` : ""));
|
|
124
|
+
if (result.removed > 0) {
|
|
125
|
+
lines.push(`- Removed by filter: minTurns ${result.removedByFilter.minTurns}, ` +
|
|
126
|
+
`maxSampleChars ${result.removedByFilter.maxSampleChars}, ` +
|
|
127
|
+
`keepOnlyToolCalls ${result.removedByFilter.keepOnlyToolCalls}`);
|
|
128
|
+
}
|
|
129
|
+
lines.push(`- Output: ${result.outputPath}`);
|
|
130
|
+
lines.push("");
|
|
131
|
+
lines.push("## Redactions (values never shown — counts only)");
|
|
132
|
+
lines.push("");
|
|
133
|
+
if (result.redactions.length === 0) {
|
|
134
|
+
lines.push("✅ No redactions — no secret-like patterns found.");
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
lines.push("| Kind | Count |");
|
|
138
|
+
lines.push("| --- | ---: |");
|
|
139
|
+
for (const r of result.redactions)
|
|
140
|
+
lines.push(`| ${r.kind} | ${r.count} |`);
|
|
141
|
+
}
|
|
142
|
+
return lines.join("\n");
|
|
143
|
+
}
|
|
144
|
+
// ---------------------------------------------------------------------------
|
|
145
|
+
// Extension entry
|
|
146
|
+
// ---------------------------------------------------------------------------
|
|
147
|
+
export default async function datasetUploaderExtension(ext) {
|
|
148
|
+
// ---- dataset_export -----------------------------------------------------
|
|
149
|
+
ext.registerTool({
|
|
150
|
+
name: "dataset_export",
|
|
151
|
+
label: "Dataset Export",
|
|
152
|
+
description: EXPORT_DESCRIPTION,
|
|
153
|
+
parameters: exportSchema,
|
|
154
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
155
|
+
try {
|
|
156
|
+
// System prompt: attach the live session's system prompt (optional field).
|
|
157
|
+
const system = typeof ctx.getSystemPrompt === "function" ? ctx.getSystemPrompt() : undefined;
|
|
158
|
+
// Tools: reconstruct the runtime tool registry for this cwd.
|
|
159
|
+
const tools = buildToolRegistry(ctx.cwd);
|
|
160
|
+
const result = await exportSessions({
|
|
161
|
+
sessionIds: params.sessionIds,
|
|
162
|
+
limit: params.limit,
|
|
163
|
+
since: params.since,
|
|
164
|
+
includeSubagents: params.includeSubagents,
|
|
165
|
+
outputPath: params.outputPath,
|
|
166
|
+
system: system || null,
|
|
167
|
+
tools,
|
|
168
|
+
});
|
|
169
|
+
return textResult(renderExportSummary(result, tools !== undefined));
|
|
170
|
+
}
|
|
171
|
+
catch (err) {
|
|
172
|
+
return errorResult(err instanceof Error ? err.message : String(err));
|
|
173
|
+
}
|
|
174
|
+
},
|
|
175
|
+
});
|
|
176
|
+
// ---- dataset_analyze ------------------------------------------------------
|
|
177
|
+
ext.registerTool({
|
|
178
|
+
name: "dataset_analyze",
|
|
179
|
+
label: "Dataset Analyze",
|
|
180
|
+
description: ANALYZE_DESCRIPTION,
|
|
181
|
+
parameters: Type.Object({
|
|
182
|
+
inputPath: Type.String({ description: "Path to the dataset JSONL file to analyze." }),
|
|
183
|
+
}),
|
|
184
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
185
|
+
try {
|
|
186
|
+
// Same opaque values as dataset_clean (never printed — counts only).
|
|
187
|
+
const extraValues = collectExtraValues(ctx.cwd);
|
|
188
|
+
const report = analyzeDataset(params.inputPath, { extraValues });
|
|
189
|
+
return textResult(renderAnalysisMarkdown(report));
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
return errorResult(err instanceof Error ? err.message : String(err));
|
|
193
|
+
}
|
|
194
|
+
},
|
|
195
|
+
});
|
|
196
|
+
// ---- dataset_clean --------------------------------------------------------
|
|
197
|
+
ext.registerTool({
|
|
198
|
+
name: "dataset_clean",
|
|
199
|
+
label: "Dataset Clean",
|
|
200
|
+
description: CLEAN_DESCRIPTION,
|
|
201
|
+
parameters: cleanSchema,
|
|
202
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
203
|
+
try {
|
|
204
|
+
// Collect opaque secret values BEFORE scrubbing (never printed).
|
|
205
|
+
const extraValues = collectExtraValues(ctx.cwd);
|
|
206
|
+
const result = cleanDataset(params.inputPath, {
|
|
207
|
+
minTurns: params.minTurns,
|
|
208
|
+
maxSampleChars: params.maxSampleChars,
|
|
209
|
+
keepOnlyToolCalls: params.keepOnlyToolCalls,
|
|
210
|
+
extraValues,
|
|
211
|
+
});
|
|
212
|
+
return textResult(renderCleanSummary(result));
|
|
213
|
+
}
|
|
214
|
+
catch (err) {
|
|
215
|
+
return errorResult(err instanceof Error ? err.message : String(err));
|
|
216
|
+
}
|
|
217
|
+
},
|
|
218
|
+
});
|
|
219
|
+
// ---- dataset_upload -------------------------------------------------------
|
|
220
|
+
ext.registerTool({
|
|
221
|
+
name: "dataset_upload",
|
|
222
|
+
label: "Dataset Upload",
|
|
223
|
+
description: UPLOAD_DESCRIPTION,
|
|
224
|
+
parameters: Type.Object({
|
|
225
|
+
inputPath: Type.String({ description: "Path to the (cleaned) dataset JSONL file." }),
|
|
226
|
+
teamId: Type.Optional(Type.String({ description: "Team id. Default: Studio binding teamId, then backend resolves from the token." })),
|
|
227
|
+
batchSize: Type.Optional(Type.Number({ description: "Samples per backend call. Default 50." })),
|
|
228
|
+
dryRun: Type.Optional(Type.Boolean({ description: "When true, only parse + validate — no data is sent." })),
|
|
229
|
+
}),
|
|
230
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
231
|
+
try {
|
|
232
|
+
const result = await uploadDataset({
|
|
233
|
+
inputPath: params.inputPath,
|
|
234
|
+
teamId: params.teamId,
|
|
235
|
+
batchSize: params.batchSize,
|
|
236
|
+
dryRun: params.dryRun,
|
|
237
|
+
cwd: ctx.cwd,
|
|
238
|
+
});
|
|
239
|
+
return textResult(renderUploadMarkdown(result));
|
|
240
|
+
}
|
|
241
|
+
catch (err) {
|
|
242
|
+
return errorResult(err instanceof Error ? err.message : String(err));
|
|
243
|
+
}
|
|
244
|
+
},
|
|
245
|
+
});
|
|
246
|
+
process.stderr.write("[dataset-uploader] Registered 4 dataset tools (export, analyze, clean, upload).\n");
|
|
247
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure text scrubber for dataset export (see Dataset-uploader.md §2.2).
|
|
3
|
+
*
|
|
4
|
+
* Replaces secret-looking patterns with `[REDACTED:<kind>]` markers:
|
|
5
|
+
* - api_key — Anthropic / OpenRouter / OpenAI / GitHub / Slack / AWS keys
|
|
6
|
+
* - jwt — bare JWTs (eyJ... payload ... signature)
|
|
7
|
+
* - bearer — "Bearer <token>" and Authorization header values
|
|
8
|
+
* - home_path — /Users/<name>/ → /home/
|
|
9
|
+
* - secret_value — opaque values passed in by the caller (secret-store
|
|
10
|
+
* values, .env values). The scrubber NEVER reads any
|
|
11
|
+
* store itself; callers decide what to pass.
|
|
12
|
+
*
|
|
13
|
+
* Pure function — no I/O, no global state. Safe to run per-sample.
|
|
14
|
+
*/
|
|
15
|
+
export interface ScrubRedaction {
|
|
16
|
+
kind: string;
|
|
17
|
+
count: number;
|
|
18
|
+
}
|
|
19
|
+
export interface ScrubResult {
|
|
20
|
+
text: string;
|
|
21
|
+
redactions: ScrubRedaction[];
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Scrub `text` for secrets.
|
|
25
|
+
*
|
|
26
|
+
* @param text Raw text (sample JSON, observation, args, ...)
|
|
27
|
+
* @param extraValues Optional opaque secret values to redact literally
|
|
28
|
+
* (e.g. secret-store values, .env values). The scrubber
|
|
29
|
+
* never reads any store itself.
|
|
30
|
+
* @returns Redacted text plus per-kind redaction counts (only kinds with
|
|
31
|
+
* count > 0, in stable order).
|
|
32
|
+
*/
|
|
33
|
+
export declare function scrub(text: string, extraValues?: string[]): ScrubResult;
|
|
34
|
+
//# sourceMappingURL=scrubber.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scrubber.d.ts","sourceRoot":"","sources":["../../../src/extensions/dataset-uploader/scrubber.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,cAAc,EAAE,CAAC;CAC9B;AAiCD;;;;;;;;;GASG;AACH,wBAAgB,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,WAAW,CA6CvE"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure text scrubber for dataset export (see Dataset-uploader.md §2.2).
|
|
3
|
+
*
|
|
4
|
+
* Replaces secret-looking patterns with `[REDACTED:<kind>]` markers:
|
|
5
|
+
* - api_key — Anthropic / OpenRouter / OpenAI / GitHub / Slack / AWS keys
|
|
6
|
+
* - jwt — bare JWTs (eyJ... payload ... signature)
|
|
7
|
+
* - bearer — "Bearer <token>" and Authorization header values
|
|
8
|
+
* - home_path — /Users/<name>/ → /home/
|
|
9
|
+
* - secret_value — opaque values passed in by the caller (secret-store
|
|
10
|
+
* values, .env values). The scrubber NEVER reads any
|
|
11
|
+
* store itself; callers decide what to pass.
|
|
12
|
+
*
|
|
13
|
+
* Pure function — no I/O, no global state. Safe to run per-sample.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* API key / token prefixes per the dataset plan §2.2:
|
|
17
|
+
* sk-ant-*, sk-or-*, sk-proj-*, ghp_*, gho_*, github_pat_*, xox[bp]-*, AKIA*.
|
|
18
|
+
*/
|
|
19
|
+
const API_KEY_PATTERN = /\b(?:sk-(?:ant|or|proj)-[A-Za-z0-9._-]{8,}|ghp_[A-Za-z0-9_]{8,}|gho_[A-Za-z0-9_]{8,}|github_pat_[A-Za-z0-9_]{8,}|xox[bp]-[A-Za-z0-9-]{8,}|AKIA[A-Z0-9]{12,})\b/g;
|
|
20
|
+
/**
|
|
21
|
+
* Authorization header values, with or without a "Bearer " prefix. Run
|
|
22
|
+
* BEFORE the bare Bearer pattern so "Authorization: Bearer <token>" is
|
|
23
|
+
* redacted as ONE unit (not double-counted). The header name is preserved
|
|
24
|
+
* (capture group 1); only the value is replaced.
|
|
25
|
+
*/
|
|
26
|
+
const AUTH_HEADER_PATTERN = /\b(authorization\s*[:=]\s*)(?:bearer\s+)?[A-Za-z0-9._~+/=-]+/gi;
|
|
27
|
+
/** Standalone "Bearer <token>" (no Authorization prefix). */
|
|
28
|
+
const BEARER_PATTERN = /\bbearer\s+[A-Za-z0-9._~+/=-]+/gi;
|
|
29
|
+
/** Bare JWT: three base64url segments, the first starting with "eyJ". */
|
|
30
|
+
const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g;
|
|
31
|
+
/**
|
|
32
|
+
* Home directory segment: /Users/<name> followed by "/" or end-of-text.
|
|
33
|
+
* Normalized to /home (the marker would break the surrounding path).
|
|
34
|
+
*/
|
|
35
|
+
const HOME_PATH_PATTERN = /\/Users\/[A-Za-z0-9._-]+(?=\/|$)/g;
|
|
36
|
+
/** Stable ordering of kinds in the returned redactions array. */
|
|
37
|
+
const REDACTION_KINDS = ["api_key", "jwt", "bearer", "home_path", "secret_value"];
|
|
38
|
+
/**
|
|
39
|
+
* Scrub `text` for secrets.
|
|
40
|
+
*
|
|
41
|
+
* @param text Raw text (sample JSON, observation, args, ...)
|
|
42
|
+
* @param extraValues Optional opaque secret values to redact literally
|
|
43
|
+
* (e.g. secret-store values, .env values). The scrubber
|
|
44
|
+
* never reads any store itself.
|
|
45
|
+
* @returns Redacted text plus per-kind redaction counts (only kinds with
|
|
46
|
+
* count > 0, in stable order).
|
|
47
|
+
*/
|
|
48
|
+
export function scrub(text, extraValues) {
|
|
49
|
+
let out = text;
|
|
50
|
+
const counts = {};
|
|
51
|
+
// 1) Opaque caller-provided values first (longest first so overlapping
|
|
52
|
+
// values are consumed whole, not partially).
|
|
53
|
+
const values = (extraValues ?? [])
|
|
54
|
+
.filter((v) => typeof v === "string" && v.length > 0)
|
|
55
|
+
.sort((a, b) => b.length - a.length);
|
|
56
|
+
for (const value of values) {
|
|
57
|
+
const parts = out.split(value);
|
|
58
|
+
if (parts.length > 1) {
|
|
59
|
+
counts.secret_value = (counts.secret_value ?? 0) + (parts.length - 1);
|
|
60
|
+
out = parts.join("[REDACTED:secret_value]");
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
// 2) Pattern-based redactions.
|
|
64
|
+
out = out.replace(API_KEY_PATTERN, () => {
|
|
65
|
+
counts.api_key = (counts.api_key ?? 0) + 1;
|
|
66
|
+
return "[REDACTED:api_key]";
|
|
67
|
+
});
|
|
68
|
+
out = out.replace(AUTH_HEADER_PATTERN, (match, prefix) => {
|
|
69
|
+
counts.bearer = (counts.bearer ?? 0) + 1;
|
|
70
|
+
return `${prefix}[REDACTED:bearer]`;
|
|
71
|
+
});
|
|
72
|
+
out = out.replace(BEARER_PATTERN, () => {
|
|
73
|
+
counts.bearer = (counts.bearer ?? 0) + 1;
|
|
74
|
+
return "[REDACTED:bearer]";
|
|
75
|
+
});
|
|
76
|
+
out = out.replace(JWT_PATTERN, () => {
|
|
77
|
+
counts.jwt = (counts.jwt ?? 0) + 1;
|
|
78
|
+
return "[REDACTED:jwt]";
|
|
79
|
+
});
|
|
80
|
+
out = out.replace(HOME_PATH_PATTERN, () => {
|
|
81
|
+
counts.home_path = (counts.home_path ?? 0) + 1;
|
|
82
|
+
return "/home";
|
|
83
|
+
});
|
|
84
|
+
const redactions = [];
|
|
85
|
+
for (const kind of REDACTION_KINDS) {
|
|
86
|
+
const count = counts[kind];
|
|
87
|
+
if (count > 0)
|
|
88
|
+
redactions.push({ kind, count });
|
|
89
|
+
}
|
|
90
|
+
return { text: out, redactions };
|
|
91
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chunked dataset upload to the team dataset vault (Dataset-uploader.md §3.2 —
|
|
3
|
+
* the `dataset_upload` tool's core).
|
|
4
|
+
*
|
|
5
|
+
* Loads a (cleaned) sharegpt JSONL, validates each line, groups it into
|
|
6
|
+
* batches of `batchSize` and calls the backend MCP tool
|
|
7
|
+
* `upload_dataset_samples` (JSON-RPC tools/call, Bearer auth) per batch,
|
|
8
|
+
* then sums the per-batch `DatasetUploadResult` ({accepted, duplicates,
|
|
9
|
+
* rejected}).
|
|
10
|
+
*
|
|
11
|
+
* Auth/URL resolution mirrors `aexol-mcp.ts`: `~/.spectral/config.json`
|
|
12
|
+
* (`readConfig()`), token = `teamApiKey || userJwt`, URL via `getApiUrl()`,
|
|
13
|
+
* and a single `ensureAuthenticated()` OAuth retry on 401/403.
|
|
14
|
+
*
|
|
15
|
+
* projectId: the Studio binding (`.aexol/aexol.jsonc`) supplies a single
|
|
16
|
+
* batch-level projectId for ALL batches — the exporter does not persist
|
|
17
|
+
* per-sample project provenance in the JSONL, so per-project grouping is not
|
|
18
|
+
* possible without exporter rework (documented in the result notes).
|
|
19
|
+
*
|
|
20
|
+
* Testability: the backend call lives behind `callBackendTool`, overridable
|
|
21
|
+
* per-call (`options.callBackendTool`) or via `setCallBackendToolForTesting`
|
|
22
|
+
* (module-level), so tests can stub the network entirely.
|
|
23
|
+
*/
|
|
24
|
+
export declare const DEFAULT_BATCH_SIZE = 50;
|
|
25
|
+
/** Backend MCP tool that ingests one batch of samples. */
|
|
26
|
+
export declare const BACKEND_UPLOAD_TOOL = "upload_dataset_samples";
|
|
27
|
+
/** Backend MCP tool returning vault stats (best-effort, post-upload). */
|
|
28
|
+
export declare const BACKEND_VAULT_STATS_TOOL = "dataset_vault_stats";
|
|
29
|
+
export declare const DATASET_FORMAT = "sharegpt";
|
|
30
|
+
/**
|
|
31
|
+
* Backend tool call abstraction. Returns the parsed JSON payload of the
|
|
32
|
+
* tool's first content item (the raw content items when only text was
|
|
33
|
+
* returned).
|
|
34
|
+
*/
|
|
35
|
+
export type CallBackendTool = (tool: string, args: Record<string, unknown>) => Promise<unknown>;
|
|
36
|
+
export interface UploadDatasetOptions {
|
|
37
|
+
/** Path to the (cleaned) sharegpt JSONL file. */
|
|
38
|
+
inputPath: string;
|
|
39
|
+
/** Team id; defaults to the Studio binding's teamId, then omitted (backend resolves from the token). */
|
|
40
|
+
teamId?: string | null;
|
|
41
|
+
/** Samples per backend call. Default 50. */
|
|
42
|
+
batchSize?: number;
|
|
43
|
+
/** When true, only parse + validate; no network call is made. */
|
|
44
|
+
dryRun?: boolean;
|
|
45
|
+
/** Working directory for Studio binding lookup. Default: process.cwd(). */
|
|
46
|
+
cwd?: string;
|
|
47
|
+
/** Test injection: overrides the module-level and default backend call. */
|
|
48
|
+
callBackendTool?: CallBackendTool;
|
|
49
|
+
}
|
|
50
|
+
export interface UploadBatchResult {
|
|
51
|
+
projectId: string | null;
|
|
52
|
+
/** Number of samples in this batch. */
|
|
53
|
+
sent: number;
|
|
54
|
+
accepted: number;
|
|
55
|
+
duplicates: number;
|
|
56
|
+
rejected: number;
|
|
57
|
+
}
|
|
58
|
+
export interface UploadDatasetResult {
|
|
59
|
+
inputPath: string;
|
|
60
|
+
dryRun: boolean;
|
|
61
|
+
totalLines: number;
|
|
62
|
+
validSamples: number;
|
|
63
|
+
invalidLines: number;
|
|
64
|
+
batchSize: number;
|
|
65
|
+
teamId: string | null;
|
|
66
|
+
projectId: string | null;
|
|
67
|
+
batches: UploadBatchResult[];
|
|
68
|
+
totals: {
|
|
69
|
+
accepted: number;
|
|
70
|
+
duplicates: number;
|
|
71
|
+
rejected: number;
|
|
72
|
+
};
|
|
73
|
+
/** Valid sample count that WOULD be uploaded (equals validSamples in dry-run). */
|
|
74
|
+
wouldUpload: number;
|
|
75
|
+
/** Vault stats returned by the backend (absent on failure or dry-run). */
|
|
76
|
+
vaultStats?: unknown;
|
|
77
|
+
/**
|
|
78
|
+
* Pre-upload residue warning: pattern-only dry-run scrub (NO extraValues)
|
|
79
|
+
* over the file contents. Present when any redactions would be made —
|
|
80
|
+
* counts only, never values.
|
|
81
|
+
*/
|
|
82
|
+
residueWarning?: {
|
|
83
|
+
totalRedactions: number;
|
|
84
|
+
kinds: string[];
|
|
85
|
+
};
|
|
86
|
+
/** Human-readable notes (e.g. binding state, per-project grouping). */
|
|
87
|
+
notes: string[];
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Test hook: override the backend call for the whole module. Pass null to
|
|
91
|
+
* restore the real implementation. Prefer `options.callBackendTool` for
|
|
92
|
+
* per-call injection; this exists so tests that drive code through other
|
|
93
|
+
* entry points (e.g. the extension index) can stub the network too.
|
|
94
|
+
*/
|
|
95
|
+
export declare function setCallBackendToolForTesting(fn: CallBackendTool | null): void;
|
|
96
|
+
/**
|
|
97
|
+
* Upload a dataset JSONL to the team vault in chunks.
|
|
98
|
+
*
|
|
99
|
+
* dryRun: parse + validate only — no backend call, `wouldUpload` carries the
|
|
100
|
+
* count, totals stay zero.
|
|
101
|
+
*/
|
|
102
|
+
export declare function uploadDataset(options: UploadDatasetOptions): Promise<UploadDatasetResult>;
|
|
103
|
+
/** Render an upload result as a markdown summary for the agent. */
|
|
104
|
+
export declare function renderUploadMarkdown(result: UploadDatasetResult): string;
|
|
105
|
+
//# sourceMappingURL=upload.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../../src/extensions/dataset-uploader/upload.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAYH,eAAO,MAAM,kBAAkB,KAAK,CAAC;AAErC,0DAA0D;AAC1D,eAAO,MAAM,mBAAmB,2BAA2B,CAAC;AAE5D,yEAAyE;AACzE,eAAO,MAAM,wBAAwB,wBAAwB,CAAC;AAE9D,eAAO,MAAM,cAAc,aAAa,CAAC;AAEzC;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,CAC5B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC1B,OAAO,CAAC,OAAO,CAAC,CAAC;AAEtB,MAAM,WAAW,oBAAoB;IACnC,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC;IAClB,wGAAwG;IACxG,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,2EAA2E;IAC3E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,2EAA2E;IAC3E,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACnE,kFAAkF;IAClF,WAAW,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;OAIG;IACH,cAAc,CAAC,EAAE;QAAE,eAAe,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC9D,uEAAuE;IACvE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAQD;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAAC,EAAE,EAAE,eAAe,GAAG,IAAI,GAAG,IAAI,CAE7E;AAmFD;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,mBAAmB,CAAC,CAkH9B;AAMD,mEAAmE;AACnE,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,CA2CxE"}
|