bitfab-cli 0.2.380 → 0.2.382
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +178 -132
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -9176,12 +9176,12 @@ var require_dist = __commonJS({
|
|
|
9176
9176
|
throw new Error(`Unknown format "${name}"`);
|
|
9177
9177
|
return f;
|
|
9178
9178
|
};
|
|
9179
|
-
function addFormats(ajv, list,
|
|
9179
|
+
function addFormats(ajv, list, fs18, exportName) {
|
|
9180
9180
|
var _a3;
|
|
9181
9181
|
var _b;
|
|
9182
9182
|
(_a3 = (_b = ajv.opts.code).formats) !== null && _a3 !== void 0 ? _a3 : _b.formats = (0, codegen_1._)`require("ajv-formats/dist/formats").${exportName}`;
|
|
9183
9183
|
for (const f of list)
|
|
9184
|
-
ajv.addFormat(f,
|
|
9184
|
+
ajv.addFormat(f, fs18[f]);
|
|
9185
9185
|
}
|
|
9186
9186
|
module.exports = exports = formatsPlugin;
|
|
9187
9187
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -23783,7 +23783,7 @@ var listTraceFunctions = {
|
|
|
23783
23783
|
var searchTraces = {
|
|
23784
23784
|
name: "search_traces",
|
|
23785
23785
|
title: "Search Traces",
|
|
23786
|
-
description: "Search and filter traces for a traced function. Returns matching traces with IDs, status, timestamps, total duration (ms), total token usage (in/out/cached/total), model, and output preview (all when available). Replay traces (from
|
|
23786
|
+
description: "Search and filter traces for a traced function. Returns matching traces with IDs, status, timestamps, total duration (ms), total token usage (in/out/cached/total), model, and output preview (all when available). Replay traces (from experiments) carry the same totals. Use get_traces for per-span duration, tokens, time-to-first-token, and model. Supports keyword search, date range, status filter, and regex matching. Use drillDown to refine previous results.",
|
|
23787
23787
|
inputSchema: {
|
|
23788
23788
|
traceFunctionKey: external_exports.string().describe("The trace function key to search"),
|
|
23789
23789
|
searchQuery: external_exports.string().optional().describe("Full-text keyword search"),
|
|
@@ -23804,8 +23804,8 @@ var searchTraces = {
|
|
|
23804
23804
|
labelSource: external_exports.enum(["human", "agent"]).optional().describe('Filter by who authored the label: "human" for human-authored labels only, "agent" for agent-authored labels only. Omit to include any source. This is a pure authorship filter - for trust/validation, use the separate `validated` parameter.'),
|
|
23805
23805
|
validated: external_exports.preprocess(parseJsonString, external_exports.boolean()).optional().describe('If true, only return traces with a validated label - a label that is either human-authored OR an agent-authored label that a human has approved (`approvedAt` is set). Use this when you want trustworthy/reviewed labels. Combine with `labelSource: "agent"` to get only approved-agent labels (excluding pure-human labels). Omit (or pass false) to include unapproved labels too.'),
|
|
23806
23806
|
labelResult: external_exports.preprocess(parseJsonString, external_exports.boolean()).optional().describe("Filter by label result: true for passing labels, false for failing labels"),
|
|
23807
|
-
includeReplays: external_exports.preprocess(parseJsonString, external_exports.boolean()).optional().describe("Include replayed traces (traces from
|
|
23808
|
-
|
|
23807
|
+
includeReplays: external_exports.preprocess(parseJsonString, external_exports.boolean()).optional().describe("Include replayed traces (traces from experiments) in results. Defaults to false because replays have new inputs, not the original production inputs, so they're not useful when selecting traces to label or when sampling real user behavior. Set to true only when you specifically want to inspect replay results."),
|
|
23808
|
+
experimentId: external_exports.uuid().optional().describe("Filter to traces from a specific experiment ID. Automatically sets includeReplays to true."),
|
|
23809
23809
|
datasetId: external_exports.uuid().optional().describe("Filter to traces belonging to a specific dataset. Combine with other filters (status, labelResult, etc.) to narrow within the dataset."),
|
|
23810
23810
|
hasDbSnapshot: external_exports.preprocess(parseJsonString, external_exports.boolean()).optional().describe("Filter by whether the trace captured a database snapshot reference. true returns only traces that can be replayed against a pinned historical DB branch (the 'Snapshot captured' traces); false returns only traces with no snapshot. Omit to include both.")
|
|
23811
23811
|
}
|
|
@@ -23854,13 +23854,13 @@ var labelEvidenceShape = external_exports.preprocess(parseJsonString, external_e
|
|
|
23854
23854
|
var saveAgentLabels = {
|
|
23855
23855
|
name: "save_agent_labels",
|
|
23856
23856
|
title: "Save Agent Labels",
|
|
23857
|
-
description: "Set, skip, or archive the agent's pass/fail verdict on one or more traces (`labelSource=\"agent\"`). Use this AFTER you have read the traces with get_traces and decided yourself whether each one looks like a pass, a fail, or genuinely cannot be judged. To set a verdict, pass `label` (true=PASS, false=FAIL) and `annotation` (your reasoning, shown to the human reviewer in the labeling UI). Add `evidence` as `{ spanId, text }` entries to ground that reasoning in spans on the evaluated trace. Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`) to record how confident you are - surfaced in the labeling UI so reviewers can prioritize low-confidence verdicts. To explicitly skip a trace you cannot decide on (instead of leaving it unlabeled), pass `skip: true` and omit label/annotation - this records an explicit skip so coverage checks know you intentionally did not verdict it. To clear a previously-set agent verdict (e.g., you changed your mind or labeled the wrong trace), pass `archive: true` and omit label/annotation. New verdicts start unapproved (`approvedAt=null`); once a human approves one, it joins the validated dataset (queryable via `search_traces` with `validated: true`). Archiving is non-destructive - the row is hidden from all reads but kept for audit, and you can immediately re-label the trace from scratch. For replay results, key each label by the replay item's `originalTraceId` (the original trace it was replayed from; `sourceTraceId` is accepted as a deprecated alias) plus the top-level `
|
|
23857
|
+
description: "Set, skip, or archive the agent's pass/fail verdict on one or more traces (`labelSource=\"agent\"`). Use this AFTER you have read the traces with get_traces and decided yourself whether each one looks like a pass, a fail, or genuinely cannot be judged. To set a verdict, pass `label` (true=PASS, false=FAIL) and `annotation` (your reasoning, shown to the human reviewer in the labeling UI). Add `evidence` as `{ spanId, text }` entries to ground that reasoning in spans on the evaluated trace. Optionally pass `confidence` (`VeryLow|Low|Medium|High|VeryHigh`) to record how confident you are - surfaced in the labeling UI so reviewers can prioritize low-confidence verdicts. To explicitly skip a trace you cannot decide on (instead of leaving it unlabeled), pass `skip: true` and omit label/annotation - this records an explicit skip so coverage checks know you intentionally did not verdict it. To clear a previously-set agent verdict (e.g., you changed your mind or labeled the wrong trace), pass `archive: true` and omit label/annotation. New verdicts start unapproved (`approvedAt=null`); once a human approves one, it joins the validated dataset (queryable via `search_traces` with `validated: true`). Archiving is non-destructive - the row is hidden from all reads but kept for audit, and you can immediately re-label the trace from scratch. For replay results, key each label by the replay item's `originalTraceId` (the original trace it was replayed from; `sourceTraceId` is accepted as a deprecated alias) plus the top-level `experimentId` instead of a `traceId`: the server resolves it to the replay trace via lineage, so you never need a server-generated replay trace id. When the experiment ran each trace more than once, also pass the item's `attempt` so each attempt gets its own verdict. Returns an agent-readable summary with one parseable effective label line per updated trace, keyed by the id you supplied (`originalTraceId` for replay verdicts, otherwise `traceId`), so command callers can verify persistence. Before judging a replay, call get_trace_assertions on the original trace ids: an assertion says what the user asked this specific case to do, so the verdict is measured against that rather than a guess. When an assertion's target cannot be found on the trace you are judging, the check never ran, so pass `skip: true` for that trace rather than recording a FAIL. Pass `assertionId` to score ONE assertion on the trace (the `[ID: ...]` get_trace_assertions prints), one entry per assertion, all in the same call. Omit it for the trace's single whole-trace verdict. get_trace_labels reads these back one line per assertion, keyed by the same assertionId, so a per-assertion write is verifiable per assertion.",
|
|
23858
23858
|
inputSchema: {
|
|
23859
|
-
|
|
23859
|
+
experimentId: external_exports.uuid().optional().describe("The experiment ID. Required when any label targets a trace by originalTraceId (replay verdicts); ignored otherwise."),
|
|
23860
23860
|
labels: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.object({
|
|
23861
23861
|
traceId: external_exports.uuid().optional().describe("The trace ID to update. Provide this OR originalTraceId, not both."),
|
|
23862
23862
|
assertionId: external_exports.uuid().optional().describe("The assertion this verdict scores, taken from the `[ID: ...]` get_trace_assertions prints. Omit it to set the trace's whole-trace verdict, of which there is one. A replay may name an assertion it inherited from the original trace it was replayed from."),
|
|
23863
|
-
originalTraceId: external_exports.uuid().optional().describe("For replay verdicts: the original trace ID the replay item was replayed from (the replay item's originalTraceId). The server resolves it to the replay trace via lineage, so you never need a server replay trace id. Requires the top-level
|
|
23863
|
+
originalTraceId: external_exports.uuid().optional().describe("For replay verdicts: the original trace ID the replay item was replayed from (the replay item's originalTraceId). The server resolves it to the replay trace via lineage, so you never need a server replay trace id. Requires the top-level experimentId. Provide this OR traceId, not both."),
|
|
23864
23864
|
sourceTraceId: external_exports.uuid().optional().meta({ deprecated: true }).describe("Deprecated alias for originalTraceId. Prefer originalTraceId; accepted for back-compat."),
|
|
23865
23865
|
attempt: external_exports.number().int().min(0).optional().describe("For replay verdicts on an experiment that ran each trace more than once: which attempt of originalTraceId this verdict is for (0-based, the replay item's attempt). Defaults to 0, so single-attempt callers never need it."),
|
|
23866
23866
|
archive: external_exports.boolean().optional().describe("If true, archive the existing agent label for this trace and ignore label/annotation/skip/confidence. Default false."),
|
|
@@ -24087,32 +24087,32 @@ var rerunGradersOnExperiment = {
|
|
|
24087
24087
|
title: "Re-run Graders on Experiment",
|
|
24088
24088
|
description: "Re-score an experiment's completed replay traces with its attached graders, overwriting their previous verdicts. Use this after attaching a grader to an already-completed experiment (which persists the assignment but does not grade the existing traces) or after editing a grader's criteria. Only the replays belonging to this experiment are graded, never the originals they were compared against. Omit graderIds to re-run every grader attached to the experiment, or pass a subset; ids that are not attached are rejected. Waits up to 90 seconds and reports how many traces were graded; if it is still running, call again with the same arguments to check progress. Read the resulting verdicts with get_grader_labels.",
|
|
24089
24089
|
inputSchema: {
|
|
24090
|
-
|
|
24090
|
+
experimentId: external_exports.uuid().describe("The experiment ID, from list_experiments"),
|
|
24091
24091
|
graderIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(100)).optional().describe("Grader ids to re-run (1-100). Omit to re-run every grader attached to the experiment.")
|
|
24092
24092
|
}
|
|
24093
24093
|
};
|
|
24094
24094
|
var addGradersToExperiment = {
|
|
24095
24095
|
name: "add_graders_to_experiment",
|
|
24096
24096
|
title: "Add Graders to Experiment",
|
|
24097
|
-
description: "Attach one or more graders directly to an experiment
|
|
24097
|
+
description: "Attach one or more graders directly to an experiment so they run against its replay traces. Idempotent: attaching a grader already assigned is a no-op. Graders must be active (or live) and belong to this organization and the experiment's trace function; ids that are archived, not runnable, or mismatched are skipped and reported. The effective grader set at completion is the union of these direct attachments and the dataset's current runnable graders. Attaching to an already-completed experiment persists the assignment but does not immediately grade its existing replay traces; those traces are graded only the next time the experiment runs completion again (for example an SDK replay retry that re-runs complete) or a fresh replay runs. Use list_experiments to find experiment ids and list_graders to find grader ids.",
|
|
24098
24098
|
inputSchema: {
|
|
24099
|
-
|
|
24099
|
+
experimentId: external_exports.uuid().describe("The experiment ID, from list_experiments"),
|
|
24100
24100
|
graderIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(100)).describe("Grader ids to attach (1-100)")
|
|
24101
24101
|
}
|
|
24102
24102
|
};
|
|
24103
24103
|
var removeGradersFromExperiment = {
|
|
24104
24104
|
name: "remove_graders_from_experiment",
|
|
24105
24105
|
title: "Remove Graders from Experiment",
|
|
24106
|
-
description: "Detach one or more graders from an experiment
|
|
24106
|
+
description: "Detach one or more graders from an experiment. Grader ids that are not currently attached are ignored and reported. The graders themselves are not deleted. Effect depends on run state: for an in-progress experiment, a detached grader that is also assigned to the experiment's dataset is re-added from the dataset when the experiment completes (so it still runs); for an already-completed experiment the grader set is a finalized snapshot with no dataset re-union, so detaching permanently drops that grader from the run's results.",
|
|
24107
24107
|
inputSchema: {
|
|
24108
|
-
|
|
24108
|
+
experimentId: external_exports.uuid().describe("The experiment ID"),
|
|
24109
24109
|
graderIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(100)).describe("Grader ids to detach (1-100)")
|
|
24110
24110
|
}
|
|
24111
24111
|
};
|
|
24112
24112
|
var addGradersToExperimentGroup = {
|
|
24113
24113
|
name: "add_graders_to_experiment_group",
|
|
24114
24114
|
title: "Add Graders to Experiment Group",
|
|
24115
|
-
description: "Assign one or more graders to every experiment
|
|
24115
|
+
description: "Assign one or more graders to every experiment currently in an experiment group. Grader assignments are stored on each experiment, not on the group, so experiments added later do not inherit them automatically. Completed experiments immediately queue any missing grader evaluations; pending experiments use the graders when they complete. Idempotent: assigning a grader already attached to an experiment is a no-op.",
|
|
24116
24116
|
inputSchema: {
|
|
24117
24117
|
experimentGroupId: external_exports.uuid().describe("The experiment group id"),
|
|
24118
24118
|
graderIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(100)).describe("Grader ids to assign (1-100)")
|
|
@@ -24121,7 +24121,7 @@ var addGradersToExperimentGroup = {
|
|
|
24121
24121
|
var removeGradersFromExperimentGroup = {
|
|
24122
24122
|
name: "remove_graders_from_experiment_group",
|
|
24123
24123
|
title: "Remove Graders from Experiment Group",
|
|
24124
|
-
description: "Detach one or more graders from every experiment
|
|
24124
|
+
description: "Detach one or more graders from every experiment currently in an experiment group, undoing add_graders_to_experiment_group. Grader ids not assigned to any experiment in the group are ignored and reported. The graders themselves are not deleted, and archived graders can still be detached. Because assignments live on each experiment rather than on the group, this only affects experiments currently in the group. Effect depends on run state, matching remove_graders_from_experiment: an in-progress experiment re-adds a detached grader from its dataset when it completes if the dataset still assigns it, while an already-completed experiment has a finalized grader set, so detaching permanently drops that grader from its results.",
|
|
24125
24125
|
inputSchema: {
|
|
24126
24126
|
experimentGroupId: external_exports.uuid().describe("The experiment group id"),
|
|
24127
24127
|
graderIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(100)).describe("Grader ids to detach (1-100)")
|
|
@@ -24130,7 +24130,7 @@ var removeGradersFromExperimentGroup = {
|
|
|
24130
24130
|
var listExperiments = {
|
|
24131
24131
|
name: "list_experiments",
|
|
24132
24132
|
title: "List Experiments",
|
|
24133
|
-
description: "List experiments (
|
|
24133
|
+
description: "List experiments (replays against changed code) newest first, with status, pass/fail totals, delta (fixed/regressed/still-passing/still-failing), and the git state each ran from. Every filter is optional and they combine: narrow by traced function, git branch, dataset, or any mix; with no filters it lists the organization's most recent experiments. Call this to see past experiment results, or at the start of experiment mode to understand what's been tried before.",
|
|
24134
24134
|
inputSchema: {
|
|
24135
24135
|
traceFunctionKey: external_exports.string().min(1).optional().describe("Only experiments for this trace function key"),
|
|
24136
24136
|
gitBranch: external_exports.string().min(1).optional().describe("Only experiments run from this git branch (exact branch name, e.g. feature/shorter-prompt)"),
|
|
@@ -24141,17 +24141,17 @@ var listExperiments = {
|
|
|
24141
24141
|
var listExperimentTraces = {
|
|
24142
24142
|
name: "list_experiment_traces",
|
|
24143
24143
|
title: "List Experiment Traces",
|
|
24144
|
-
description: "Get individual trace results for an experiment
|
|
24144
|
+
description: "Get individual trace results for an experiment, including each replay trace's verdict (fixed, regressed, still-passing, still-failing, unpaired) by comparing against the original trace's label, plus token usage (input, output, cached, total) for the replay and the paired original so you can reason about cost and cache-read deltas. Use after list_experiments to drill into a specific experiment's results.",
|
|
24145
24145
|
inputSchema: {
|
|
24146
|
-
|
|
24146
|
+
experimentId: external_exports.uuid().describe("The experiment ID to get traces for")
|
|
24147
24147
|
}
|
|
24148
24148
|
};
|
|
24149
24149
|
var getReplayStatus = {
|
|
24150
24150
|
name: "get_replay_status",
|
|
24151
24151
|
title: "Get Replay Status",
|
|
24152
|
-
description: "Read the current
|
|
24152
|
+
description: "Read the current status of an experiment's replay, including local replay trace ID to server replay trace ID mapping. Safe to poll while replay is still running.",
|
|
24153
24153
|
inputSchema: {
|
|
24154
|
-
|
|
24154
|
+
experimentId: external_exports.uuid().describe("The experiment ID to get status for")
|
|
24155
24155
|
}
|
|
24156
24156
|
};
|
|
24157
24157
|
var getTemplateReference = {
|
|
@@ -24184,10 +24184,10 @@ var saveTemplate = {
|
|
|
24184
24184
|
var saveExperimentGroup = {
|
|
24185
24185
|
name: "save_experiment_group",
|
|
24186
24186
|
title: "Save Experiment Group",
|
|
24187
|
-
description: "Create or edit an experiment group. To create, omit experimentGroupId and provide
|
|
24187
|
+
description: "Create or edit an experiment group. To create, omit experimentGroupId and provide experimentIds plus an optional name and notes; the supplied experiments are grouped together. To edit, provide experimentGroupId and at least one of name or notes, but omit experimentIds; editing metadata does not change membership. Creation is idempotent when all supplied experiments already share one group. All resources must belong to the active organization.",
|
|
24188
24188
|
inputSchema: {
|
|
24189
24189
|
experimentGroupId: external_exports.uuid().optional().describe("Existing experiment group id to rename. Omit when creating."),
|
|
24190
|
-
|
|
24190
|
+
experimentIds: external_exports.preprocess(parseJsonString, external_exports.array(external_exports.uuid()).min(1).max(100)).optional().describe("Experiment IDs to group (1-100). Required when creating; omit when editing."),
|
|
24191
24191
|
name: external_exports.string().trim().min(1).max(120).optional().describe("Human-readable group name. Optional when creating or editing."),
|
|
24192
24192
|
notes: external_exports.string().max(1e4).optional().describe("Optional group notes. Omit to keep existing notes when editing; pass an empty string to clear them.")
|
|
24193
24193
|
}
|
|
@@ -24211,9 +24211,9 @@ var getExperimentGroup = {
|
|
|
24211
24211
|
var saveExperiment = {
|
|
24212
24212
|
name: "save_experiment",
|
|
24213
24213
|
title: "Save Experiment",
|
|
24214
|
-
description: "Update an existing experiment
|
|
24214
|
+
description: "Update an existing experiment. Provide the experiment ID and at least one of name, notes, or experimentGroupId. Omitted fields keep their current values; pass an empty notes string to clear notes, a group id to move the experiment to that existing group, or null to remove it from its group.",
|
|
24215
24215
|
inputSchema: {
|
|
24216
|
-
|
|
24216
|
+
experimentId: external_exports.uuid().describe("The experiment ID to update"),
|
|
24217
24217
|
name: external_exports.string().trim().min(1).max(120).optional().describe("Optional human-readable experiment name"),
|
|
24218
24218
|
notes: external_exports.string().max(1e4).optional().describe("Optional experiment notes. Omit to keep existing notes; pass an empty string to clear them."),
|
|
24219
24219
|
experimentGroupId: external_exports.uuid().nullable().optional().describe("Existing experiment group id to move this experiment into. Omit to keep its current group; pass null to remove it from its group.")
|
|
@@ -24222,9 +24222,9 @@ var saveExperiment = {
|
|
|
24222
24222
|
var getExperiment = {
|
|
24223
24223
|
name: "get_experiment",
|
|
24224
24224
|
title: "Get Experiment",
|
|
24225
|
-
description: "Get a single experiment
|
|
24225
|
+
description: "Get a single experiment by experiment ID, with its name, notes, status, pass/fail totals, delta, experiment group, and grader results: the aggregate passing/failing checks and pass rate, plus a per-grader breakdown (each assigned grader's own passing/failing and pass rate on the run). Use this when you already have an experiment ID and want just that run's results, instead of paging the whole function's history with list_experiments.",
|
|
24226
24226
|
inputSchema: {
|
|
24227
|
-
|
|
24227
|
+
experimentId: external_exports.uuid().describe("The experiment ID, from list_experiments")
|
|
24228
24228
|
}
|
|
24229
24229
|
};
|
|
24230
24230
|
var getGraderLabels = {
|
|
@@ -24698,44 +24698,51 @@ var McpProxy = class {
|
|
|
24698
24698
|
};
|
|
24699
24699
|
|
|
24700
24700
|
// ../bitfab-plugin-lib/dist/bitfabToolHandlers.js
|
|
24701
|
+
function withLegacyExperimentKeys(args) {
|
|
24702
|
+
return {
|
|
24703
|
+
...args,
|
|
24704
|
+
...args.experimentId === void 0 ? {} : { testRunId: args.experimentId },
|
|
24705
|
+
...args.experimentIds === void 0 ? {} : { testRunIds: args.experimentIds }
|
|
24706
|
+
};
|
|
24707
|
+
}
|
|
24701
24708
|
function createBitfabToolHandlers(platform2, getConfig2, pluginVersion) {
|
|
24702
24709
|
const proxy = new McpProxy(platform2, pluginVersion);
|
|
24703
24710
|
function proxyToolCall(toolName, args) {
|
|
24704
|
-
return proxy.toolCall(getConfig2(), toolName, args);
|
|
24711
|
+
return proxy.toolCall(getConfig2(), toolName, withLegacyExperimentKeys(args));
|
|
24705
24712
|
}
|
|
24706
24713
|
async function handleSaveAgentLabels(args) {
|
|
24707
|
-
const {
|
|
24714
|
+
const { experimentId, labels } = args;
|
|
24708
24715
|
return proxyToolCall("save_agent_labels", {
|
|
24709
|
-
|
|
24716
|
+
experimentId,
|
|
24710
24717
|
labels: labels.map((l) => l.originalTraceId !== void 0 && l.sourceTraceId === void 0 ? { ...l, sourceTraceId: l.originalTraceId } : l)
|
|
24711
24718
|
});
|
|
24712
24719
|
}
|
|
24713
24720
|
async function handleSaveExperiment(args) {
|
|
24714
|
-
const {
|
|
24721
|
+
const { experimentId, name, notes, experimentGroupId } = args;
|
|
24715
24722
|
if (name === void 0 && notes === void 0 && experimentGroupId === void 0) {
|
|
24716
24723
|
return errorResult("Error saving experiment: provide name, notes, and/or experimentGroupId.");
|
|
24717
24724
|
}
|
|
24718
24725
|
return proxyToolCall("save_experiment", {
|
|
24719
|
-
|
|
24726
|
+
experimentId,
|
|
24720
24727
|
name,
|
|
24721
24728
|
notes,
|
|
24722
24729
|
experimentGroupId
|
|
24723
24730
|
});
|
|
24724
24731
|
}
|
|
24725
24732
|
async function handleSaveExperimentGroup(args) {
|
|
24726
|
-
const { experimentGroupId,
|
|
24727
|
-
if (experimentGroupId === void 0 &&
|
|
24728
|
-
return errorResult("Error saving experiment group:
|
|
24733
|
+
const { experimentGroupId, experimentIds, name, notes } = args;
|
|
24734
|
+
if (experimentGroupId === void 0 && experimentIds === void 0) {
|
|
24735
|
+
return errorResult("Error saving experiment group: experimentIds is required when creating an experiment group.");
|
|
24729
24736
|
}
|
|
24730
|
-
if (experimentGroupId !== void 0 &&
|
|
24731
|
-
return errorResult("Error saving experiment group:
|
|
24737
|
+
if (experimentGroupId !== void 0 && experimentIds !== void 0) {
|
|
24738
|
+
return errorResult("Error saving experiment group: experimentIds cannot be provided when editing an experiment group; editing does not change membership.");
|
|
24732
24739
|
}
|
|
24733
24740
|
if (experimentGroupId !== void 0 && name === void 0 && notes === void 0) {
|
|
24734
24741
|
return errorResult("Error saving experiment group: provide name and/or notes when editing an experiment group.");
|
|
24735
24742
|
}
|
|
24736
24743
|
return proxyToolCall("save_experiment_group", {
|
|
24737
24744
|
experimentGroupId,
|
|
24738
|
-
|
|
24745
|
+
experimentIds,
|
|
24739
24746
|
name,
|
|
24740
24747
|
notes
|
|
24741
24748
|
});
|
|
@@ -24889,7 +24896,7 @@ import path4 from "path";
|
|
|
24889
24896
|
var BITFAB_SKILLS = ["setup", "assistant", "update"];
|
|
24890
24897
|
|
|
24891
24898
|
// ../bitfab-plugin-lib/dist/hooks/skillDetection.js
|
|
24892
|
-
var SKILL_PATTERN = new RegExp(`^\\s
|
|
24899
|
+
var SKILL_PATTERN = new RegExp(`^\\s*[/$]?bitfab[:-](?<skill>${BITFAB_SKILLS.join("|")})(?=\\s|$)`, "i");
|
|
24893
24900
|
|
|
24894
24901
|
// ../bitfab-plugin-lib/dist/chatSessions/arming.js
|
|
24895
24902
|
import fs3 from "fs";
|
|
@@ -24899,6 +24906,9 @@ import path3 from "path";
|
|
|
24899
24906
|
// ../bitfab-plugin-lib/dist/chatSessions/transcriptReader.js
|
|
24900
24907
|
import fs4 from "fs";
|
|
24901
24908
|
|
|
24909
|
+
// ../bitfab-plugin-lib/dist/chatSessions/index.js
|
|
24910
|
+
var AMP_SESSION_RETENTION_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
24911
|
+
|
|
24902
24912
|
// ../bitfab-plugin-lib/dist/replayCapabilities.js
|
|
24903
24913
|
var semver = __toESM(require_semver2(), 1);
|
|
24904
24914
|
|
|
@@ -24907,10 +24917,10 @@ var semver3 = __toESM(require_semver2(), 1);
|
|
|
24907
24917
|
|
|
24908
24918
|
// ../bitfab-plugin-lib/dist/bakedSdkVersions.js
|
|
24909
24919
|
var BAKED_SDK_VERSIONS = {
|
|
24910
|
-
typescript: "0.59.
|
|
24911
|
-
python: "0.59.
|
|
24912
|
-
ruby: "0.59.
|
|
24913
|
-
go: "0.59.
|
|
24920
|
+
typescript: "0.59.1",
|
|
24921
|
+
python: "0.59.1",
|
|
24922
|
+
ruby: "0.59.1",
|
|
24923
|
+
go: "0.59.1"
|
|
24914
24924
|
};
|
|
24915
24925
|
|
|
24916
24926
|
// ../bitfab-plugin-lib/dist/installedSdk.js
|
|
@@ -25987,13 +25997,15 @@ function emitJson(event) {
|
|
|
25987
25997
|
// ../bitfab-plugin-lib/dist/commands/persistReplayLabels.js
|
|
25988
25998
|
import { readFileSync, unlinkSync } from "fs";
|
|
25989
25999
|
var lineageFileSchema = external_exports.object({
|
|
25990
|
-
|
|
25991
|
-
|
|
25992
|
-
testRunId: external_exports.uuid(),
|
|
26000
|
+
experimentId: external_exports.uuid().optional(),
|
|
26001
|
+
testRunId: external_exports.uuid().optional(),
|
|
25993
26002
|
// Canonical; older item files carry the deprecated expectedSourceTraceIds.
|
|
25994
26003
|
expectedOriginalTraceIds: external_exports.array(external_exports.uuid()).min(1).optional(),
|
|
25995
26004
|
expectedSourceTraceIds: external_exports.array(external_exports.uuid()).min(1).optional(),
|
|
25996
26005
|
verdicts: external_exports.array(external_exports.unknown()).min(1)
|
|
26006
|
+
}).refine((d) => d.experimentId !== void 0 || d.testRunId !== void 0, {
|
|
26007
|
+
message: "lineage file must carry experimentId (or the deprecated testRunId)",
|
|
26008
|
+
path: ["experimentId"]
|
|
25997
26009
|
}).refine((d) => d.expectedOriginalTraceIds !== void 0 || d.expectedSourceTraceIds !== void 0, {
|
|
25998
26010
|
message: "lineage file must carry expectedOriginalTraceIds (or the deprecated expectedSourceTraceIds)",
|
|
25999
26011
|
path: ["expectedOriginalTraceIds"]
|
|
@@ -26016,28 +26028,54 @@ import { dirname as dirname3, resolve as resolve3 } from "path";
|
|
|
26016
26028
|
import { createInterface } from "readline";
|
|
26017
26029
|
|
|
26018
26030
|
// ../bitfab-plugin-lib/dist/replayItemWriter.js
|
|
26019
|
-
import { mkdirSync as
|
|
26020
|
-
import { join as
|
|
26031
|
+
import { mkdirSync as mkdirSync3, renameSync, writeFileSync as writeFileSync3 } from "fs";
|
|
26032
|
+
import { join as join3 } from "path";
|
|
26021
26033
|
|
|
26022
26034
|
// ../bitfab-plugin-lib/dist/bestEffortFs.js
|
|
26023
26035
|
import { mkdirSync, writeFileSync as writeFileSync2 } from "fs";
|
|
26024
26036
|
import { dirname } from "path";
|
|
26025
26037
|
|
|
26038
|
+
// ../bitfab-plugin-lib/dist/replayRunDir.js
|
|
26039
|
+
import { randomUUID } from "crypto";
|
|
26040
|
+
import { appendFileSync, existsSync, mkdirSync as mkdirSync2, readFileSync as readFileSync2, statSync } from "fs";
|
|
26041
|
+
import { dirname as dirname2, join as join2, resolve } from "path";
|
|
26042
|
+
var EXPERIMENT_ID_KEYS = [
|
|
26043
|
+
"experimentId",
|
|
26044
|
+
"experiment_id",
|
|
26045
|
+
"testRunId",
|
|
26046
|
+
"test_run_id"
|
|
26047
|
+
];
|
|
26048
|
+
function firstString(result, keys) {
|
|
26049
|
+
if (!result || typeof result !== "object") {
|
|
26050
|
+
return null;
|
|
26051
|
+
}
|
|
26052
|
+
const rec = result;
|
|
26053
|
+
for (const key of keys) {
|
|
26054
|
+
const value = rec[key];
|
|
26055
|
+
if (typeof value === "string") {
|
|
26056
|
+
return value;
|
|
26057
|
+
}
|
|
26058
|
+
}
|
|
26059
|
+
return null;
|
|
26060
|
+
}
|
|
26061
|
+
function extractExperimentId(result) {
|
|
26062
|
+
return firstString(result, EXPERIMENT_ID_KEYS);
|
|
26063
|
+
}
|
|
26064
|
+
|
|
26026
26065
|
// ../bitfab-plugin-lib/dist/replayResultResolver.js
|
|
26027
|
-
import { readFileSync as
|
|
26028
|
-
import { resolve } from "path";
|
|
26066
|
+
import { readFileSync as readFileSync3, rmSync } from "fs";
|
|
26067
|
+
import { resolve as resolve2 } from "path";
|
|
26029
26068
|
var replayResultSchema = external_exports.object({
|
|
26030
26069
|
items: external_exports.array(external_exports.unknown()),
|
|
26070
|
+
experimentId: external_exports.string().optional(),
|
|
26071
|
+
experiment_id: external_exports.string().optional(),
|
|
26072
|
+
experimentUrl: external_exports.string().optional(),
|
|
26073
|
+
experiment_url: external_exports.string().optional(),
|
|
26031
26074
|
testRunId: external_exports.string().optional(),
|
|
26032
26075
|
test_run_id: external_exports.string().optional(),
|
|
26033
26076
|
testRunUrl: external_exports.string().optional(),
|
|
26034
26077
|
test_run_url: external_exports.string().optional()
|
|
26035
|
-
}).passthrough().refine((
|
|
26036
|
-
|
|
26037
|
-
// ../bitfab-plugin-lib/dist/replayRunDir.js
|
|
26038
|
-
import { randomUUID } from "crypto";
|
|
26039
|
-
import { appendFileSync, existsSync, mkdirSync as mkdirSync3, readFileSync as readFileSync3, statSync } from "fs";
|
|
26040
|
-
import { dirname as dirname2, join as join3, resolve as resolve2 } from "path";
|
|
26078
|
+
}).passthrough().refine((result) => extractExperimentId(result) !== null);
|
|
26041
26079
|
|
|
26042
26080
|
// ../bitfab-plugin-lib/dist/commands/replayProgress.js
|
|
26043
26081
|
var HEARTBEAT_MS = Number(process.env.BITFAB_REPLAY_HEARTBEAT_MS) || 12e3;
|
|
@@ -26160,7 +26198,7 @@ async function checkForUpdate(currentVersion, platform2, localBuildSha) {
|
|
|
26160
26198
|
current: currentVersion,
|
|
26161
26199
|
latest,
|
|
26162
26200
|
updateAvailable: versionNewer || staleSameVersion,
|
|
26163
|
-
autoUpdateEnabled: platform2.supportsAutoUpdate ? isAutoUpdateEnabled(platform2) : false,
|
|
26201
|
+
autoUpdateEnabled: platform2.supportsAutoUpdate && platform2.enableAutoUpdateHint !== void 0 ? isAutoUpdateEnabled(platform2) : false,
|
|
26164
26202
|
staleSameVersion
|
|
26165
26203
|
};
|
|
26166
26204
|
}
|
|
@@ -28824,7 +28862,7 @@ var assistantFlow = Flow.parse({
|
|
|
28824
28862
|
},
|
|
28825
28863
|
openExperiments: {
|
|
28826
28864
|
description: "Print a link to experiments.",
|
|
28827
|
-
args: "<
|
|
28865
|
+
args: "<experimentIds>"
|
|
28828
28866
|
},
|
|
28829
28867
|
startDataset: {
|
|
28830
28868
|
description: "Print a dataset link.",
|
|
@@ -28915,7 +28953,7 @@ This skill has nine invocation modes, each a different entry point into the same
|
|
|
28915
28953
|
|
|
28916
28954
|
When you genuinely can't tell, default \`all\`. State the basis in the same one-line confirmation as \`costRun\`, e.g. "Starting experiment for \`generate-email\`, tracking token cost (uncached)." As with \`costRun\`, deciding this once here is what lets \`<tokensSuffix>\` resolve deterministically downstream instead of each open re-guessing the basis. \`costBasis\` is meaningless when \`costRun\` is false (no lens shows at all), so only carry it on a cost run.
|
|
28917
28955
|
|
|
28918
|
-
**Reset \`costRun\` if the user changes their mind after entry** (set it true on any later cost signal, not just an explicit "show me tokens": asking what a run costs, noting token counts look high, worrying about spend; set it false when they say to stop). The same intent-not-keywords reading applies here as at entry. The entry value is the default, not a lock; resetting on an explicit request is not the autonomous step-to-step re-guessing you must avoid. On a reset, re-open whatever token-capable surface is currently open so the live view matches: the experiments page in any form (\`?experimentGroupId=\`, \`?
|
|
28956
|
+
**Reset \`costRun\` if the user changes their mind after entry** (set it true on any later cost signal, not just an explicit "show me tokens": asking what a run costs, noting token counts look high, worrying about spend; set it false when they say to stop). The same intent-not-keywords reading applies here as at entry. The entry value is the default, not a lock; resetting on an explicit request is not the autonomous step-to-step re-guessing you must avoid. On a reset, re-open whatever token-capable surface is currently open so the live view matches: the experiments page in any form (\`?experimentGroupId=\`, \`?experimentIds=\`, or \`?datasetId=\`), re-navigated with or without \`&tokens=1\`. If nothing relevant is open, just hold the new value and the next open picks it up. One asymmetry to know: re-navigating with \`&tokens=1\` turns the lens on, but a param-less re-nav cannot turn it off (the page keeps the token columns sticky once shown); resetting to false still correctly drops the chat-side token-delta reporting in \`evaluate-results\` / \`share-results\` / the scorecard. **\`costBasis\` resets the same way.** A later signal can flip the basis without touching \`costRun\` (e.g. a token-count run where the user then says "what's the bill" flips \`costBasis\` to \`uncached\`; "I just mean total tokens" flips it back to \`all\`). On a basis change, re-open the open experiments surface with the matching suffix (\`&tokens=1&tokenType=all\` for \`all\`, \`&tokens=1&tokenType=uncached\` for \`uncached\`) so the live trend recolors, and the chat-side reports pick up the new basis on the next report. The toggle in the page header is the user's own live override of the same basis.
|
|
28919
28957
|
|
|
28920
28958
|
**Disambiguating \`benchmark\` from \`experiment\`** (both replay a dataset, so free-form text is easy to misroute):
|
|
28921
28959
|
|
|
@@ -30182,7 +30220,7 @@ Hold the chosen mode in working context. Every iteration below (\`make-change\`,
|
|
|
30182
30220
|
branches: [
|
|
30183
30221
|
{
|
|
30184
30222
|
when: "{{#claude}}bash output is `parallel` (bypass found in committed or user-global settings){{/claude}}{{^claude}}(unreachable on this editor){{/claude}}",
|
|
30185
|
-
description: `**Parallel mode.** For each independent experiment, fork to a subagent using the Agent tool with \`isolation: "worktree"\` and \`subagent_type: "general-purpose"\`. The subagent edits its worktree, runs replay, returns its scored items + \`
|
|
30223
|
+
description: `**Parallel mode.** For each independent experiment, fork to a subagent using the Agent tool with \`isolation: "worktree"\` and \`subagent_type: "general-purpose"\`. The subagent edits its worktree, runs replay, returns its scored items + \`experimentId\` to this main agent`,
|
|
30186
30224
|
next: "iterate/detect-replay-capabilities"
|
|
30187
30225
|
},
|
|
30188
30226
|
{
|
|
@@ -30281,7 +30319,7 @@ If \`renameFrom\` identifies the legacy TypeScript \`bitfab\` package, replace i
|
|
|
30281
30319
|
kind: "action",
|
|
30282
30320
|
toolCalls: ["bash"],
|
|
30283
30321
|
title: "Prepare experiment group before replay",
|
|
30284
|
-
body: "Generate one experimentGroupId for this iteration before making changes or running replay. Pass that same ID through the replay registry using --experiment-group-id. Do not emit or share an experiment link yet: generating a UUID does not create the group on the server. The first persisted replay creates it. Keep reporting replay progress in chat, then share the verified group or
|
|
30322
|
+
body: "Generate one experimentGroupId for this iteration before making changes or running replay. Pass that same ID through the replay registry using --experiment-group-id. Do not emit or share an experiment link yet: generating a UUID does not create the group on the server. The first persisted replay creates it. Keep reporting replay progress in chat, then share the verified group or experiment link in the open-experiments step after replay.",
|
|
30285
30323
|
next: {
|
|
30286
30324
|
byMode: {
|
|
30287
30325
|
wizard: "iterate/make-change",
|
|
@@ -30343,16 +30381,16 @@ The schema is flat, every file object is exactly \`{ path, before, after }\`. Do
|
|
|
30343
30381
|
|
|
30344
30382
|
**Automatic fallback capture.** If you do not pass \`--code-change\`, a recent SDK auto-captures the branch's diff **against trunk** (the merge-base with \`origin/main\`) inside \`replay()\` itself and attaches it, so even a run where you did not hand-write a payload still shows a diff, as long as the SDK is new enough (\`supportsCodeChanges\`), you are in a git repo, and the working tree carries the change. This fallback is **cumulative** (whole branch vs trunk), not per-experiment. The precise per-experiment before/after still comes from writing the \`--code-change\` payload above, which always wins over the fallback, so keep doing that in \`make-change\`; the trunk diff is a last resort, not a reason to skip it.
|
|
30345
30383
|
|
|
30346
|
-
**Check the \`supportsExperimentGroups\` flag** (from \`detect-replay-capabilities\`). If true and an \`experimentGroupId\` exists, pass \`--experiment-group-id <experimentGroupId>\` (from \`open-experiments-before-replay\`) so the
|
|
30384
|
+
**Check the \`supportsExperimentGroups\` flag** (from \`detect-replay-capabilities\`). If true and an \`experimentGroupId\` exists, pass \`--experiment-group-id <experimentGroupId>\` (from \`open-experiments-before-replay\`) so the experiment is tagged with the group. This includes the initial \`fixReplayScope = "single-trace"\` pass. If false, skip the flag; the post-replay Bitfab inspection can fall back to \`experimentId\` when available.
|
|
30347
30385
|
|
|
30348
|
-
**Check the \`supportsExperimentNames\` flag** (from \`detect-replay-capabilities\`). If true, pass \`--name "<experimentName>"\` so the resulting experiment
|
|
30386
|
+
**Check the \`supportsExperimentNames\` flag** (from \`detect-replay-capabilities\`). If true, pass \`--name "<experimentName>"\` so the resulting experiment is readable in the UI. Use the one-line change description from \`make-change\` as \`<experimentName>\`; in \`benchmark\` mode use \`Benchmark: current code baseline\`. Keep it 120 characters or fewer. If false, omit \`--name\`; this is cosmetic and the replay still runs.
|
|
30349
30387
|
|
|
30350
30388
|
**Choose trace selection.**
|
|
30351
30389
|
|
|
30352
30390
|
- **Initial \`fix\` pass (\`fixReplayScope = "single-trace"\`)**: pass \`--trace-ids <targetTraceId>\` (the trace ID held from Phase Fix \`resolve\`) and do NOT pass \`--dataset-id\`. This is the user's requested targeted replay. It proves the bug turned green before anything is added to a dataset or the rest of the dataset is run.
|
|
30353
30391
|
- **Full dataset runs**: check the \`supportsDatasetId\` flag (from \`detect-replay-capabilities\`). When true, this is the **preferred way to replay the dataset**: pass \`--dataset-id <datasetId>\` (the dataset id held in working context: from \`pick-dataset\` in \`experiment\` / \`cost-optimize\` / \`benchmark\` modes, or from \`fix-add-to-dataset\` in \`fix\` mode, which skips \`pick-dataset\`) and **omit \`--trace-ids\` entirely**. The server replays exactly the dataset's traces and durably attributes the experiment to the dataset, so it shows under the dataset's experiments even when trace lineage can't be reconstructed, and you don't have to enumerate the dataset's trace IDs by hand. Only when \`supportsDatasetId\` is false do you fall back to \`--trace-ids <the dataset's resolved trace ids>\` (attribution then relies on the derived trace-lineage join). If the installed SDK lacks \`--dataset-id\`, prefer upgrading it (see \`upgrade-replay-script\`) over the trace-ids fallback.
|
|
30354
30392
|
|
|
30355
|
-
**Run the replay through \`{{command:replayProgress}}\` in the background, then relay the progress lines it prints.** Before starting, choose a unique run directory for this replay, for example \`.bitfab/replays/<experimentN>-<timestamp>-<short-random>\`, pass it with \`--run-dir\`, and hold that path in context; do not use a shared \`--events-log\` path. A foreground run blocks you for the whole replay and the user just sees "a shell is running" with no detail. \`{{command:replayProgress}}\` runs the replay, turns the SDK's per-trace \`@@bitfab:progress\` events into one self-contained line per trace on its stdout (a header, then for each trace as it finishes a pass/fail glyph, the running \`n/total\`, and that trace's duration, with the error reason inline on failure, plus a liveness heartbeat line when a slow trace goes quiet so the run never looks frozen, then a final summary with total + average time), and writes a small JSONL event log at \`<run-dir>/events.jsonl\`: \`type: "progress"\` rows as items finish, then a final \`type: "complete"\` row. Large per-item payloads are written atomically under \`<run-dir>/items/\` and referenced by \`item.itemPath\` / \`items[].itemPath\`, so outputs are not duplicated in the event log. It also tees every human line to \`<run-dir>/progress.log\`, so the user can \`tail -f\` it in a separate terminal for a live view that never collapses the way a tool card does. The wrapper writes \`<run-dir>/run.json\`; once the server
|
|
30393
|
+
**Run the replay through \`{{command:replayProgress}}\` in the background, then relay the progress lines it prints.** Before starting, choose a unique run directory for this replay, for example \`.bitfab/replays/<experimentN>-<timestamp>-<short-random>\`, pass it with \`--run-dir\`, and hold that path in context; do not use a shared \`--events-log\` path. A foreground run blocks you for the whole replay and the user just sees "a shell is running" with no detail. \`{{command:replayProgress}}\` runs the replay, turns the SDK's per-trace \`@@bitfab:progress\` events into one self-contained line per trace on its stdout (a header, then for each trace as it finishes a pass/fail glyph, the running \`n/total\`, and that trace's duration, with the error reason inline on failure, plus a liveness heartbeat line when a slow trace goes quiet so the run never looks frozen, then a final summary with total + average time), and writes a small JSONL event log at \`<run-dir>/events.jsonl\`: \`type: "progress"\` rows as items finish, then a final \`type: "complete"\` row. Large per-item payloads are written atomically under \`<run-dir>/items/\` and referenced by \`item.itemPath\` / \`items[].itemPath\`, so outputs are not duplicated in the event log. It also tees every human line to \`<run-dir>/progress.log\`, so the user can \`tail -f\` it in a separate terminal for a live view that never collapses the way a tool card does. The wrapper writes \`<run-dir>/run.json\`; once the server experiment ID is known it also writes \`<parent-of-run-dir>/by-experiment/<experimentId>/run.json\` pointing back to the run directory. If the replay finishes no traces at all (an \`@bitfab/sdk\` too old to carry the lifecycle reporter, or an SDK too old to emit installed-command lifecycle events), it closes with a \`\u26A0 done \xB7 replay finished but reported no progress\` line instead of ending silently, so a missing stream of per-trace lines never reads as a hang, fix it via \`upgrade-replay-script\` and re-run. If the replay command exits 0 but its local \`ReplayResult\` capture is unavailable, the wrapper also exits 0, emits a \`\u26A0 unverified\` capture warning, and records \`status: "unverified"\` with \`resultCaptured: false\` in \`run.json\`: this prevents the host and metadata consumers from falsely reporting either failure or completion. Progress lines alone do not verify the result. Treat the outcome as unverified until the server experiment status resolves it; then explicitly report the server-confirmed outcome. You do NOT decide what or when to print: it does the formatting; you just show each new line.
|
|
30356
30394
|
|
|
30357
30395
|
With a current replay-capable SDK, the event log also contains \`type: "started"\` lifecycle rows. These identify historical trace IDs that have entered a worker but have not necessarily settled; they are diagnostic only and must not be evaluated as completed items.
|
|
30358
30396
|
|
|
@@ -30372,13 +30410,13 @@ cd <project-dir> && {{command:replayProgress}} --label <pipeline-name> --run-dir
|
|
|
30372
30410
|
|
|
30373
30411
|
3. **Run the live replay/evaluation loop while the replay is still running.** Use the run directory you passed with \`--run-dir\` and poll \`<run-dir>/events.jsonl\` alongside the human output. Each \`type: "progress"\` JSONL row is a normalized SDK item-finish event; \`item.originalTraceId\` is the **original trace id, the key verdicts are persisted by** (every item file carries it under the same \`originalTraceId\` field, plus a deprecated \`sourceTraceId\` alias). On modern SDKs, \`item.traceId\` may already hold the server replay trace ID after the per-item flush; it is null only when that flush/readback could not confirm the ID, and the final persistence barrier/result fills it when possible. \`item.replayTraceId\` is only set by older SDKs. Treat a new successful progress row as "this original trace's replay item finished" and start evaluation for that item as soon as the row carries \`item.itemPath\` and an \`item.originalTraceId\`: read \`item.itemPath\` for the full input, replay output, original output, and metadata. Failed progress rows become unreplayable candidates immediately and should be carried forward with their error string. If the row is progress-only (old SDK/current basic reporter: no \`item.itemPath\`), do the cheap prep only and defer judging to the final \`type: "complete"\` row's item refs. If the JSONL file is missing or empty (old SDK without installed-command lifecycle events, or an unwritable log path), do not block or treat it as a replay failure: continue from the final complete event from the final result artifact.
|
|
30374
30412
|
|
|
30375
|
-
For every item you can judge during the run, key the verdict by its **original trace id** (the \`originalTraceId\` field in each item file, the same \`item.originalTraceId\` on progress rows), the original trace the item was replayed from, which the server resolves to this run's replay trace, no local-to-server id mapping step. You need the run's \`
|
|
30413
|
+
For every item you can judge during the run, key the verdict by its **original trace id** (the \`originalTraceId\` field in each item file, the same \`item.originalTraceId\` on progress rows), the original trace the item was replayed from, which the server resolves to this run's replay trace, no local-to-server id mapping step. You need the run's \`experimentId\` to persist (enriched progress rows may carry \`event.experimentId\`; older scripts may only reveal it in the final \`ReplayResult\`). As soon as a small batch of judged items has its original trace ids and you know the \`experimentId\`, persist that batch with \`{{command:persistReplayLabels}}\`. Keep a set of original trace IDs already persisted so the final reconciliation never double-writes a verdict. A verdict judged live was judged before the item's outlines existed (\`originalTraceOutline\` and \`traceOutline\` are \`null\` on progress item files), so also keep the set of original trace ids judged without outlines; step 4 re-checks those against the final item files.
|
|
30376
30414
|
|
|
30377
|
-
4. **When the background command finishes, read the final \`type: "complete"\` row from this run's \`events.jsonl\`**. Its \`result\` carries run metadata (\`
|
|
30415
|
+
4. **When the background command finishes, read the final \`type: "complete"\` row from this run's \`events.jsonl\`**. Its \`result\` carries run metadata (\`experimentId\`, \`experimentUrl\`, \`itemCount\`) and its \`items\` array carries item refs. Read each needed \`items[].itemPath\` for the full replay item (trace ID, duration, tokens, model, full original/new outputs, and the two trace outlines). Read from the **files**, not from the captured command output, which the harness truncates in the middle. On current SDKs each completed item also carries \`originalTraceOutline\` and \`traceOutline\`: the original and the replayed trace's span tree (span names, types, nesting, order, durations, tokens, model, errors, and whether each span was mocked) with no inputs or outputs. The SDK fills both in at completion from the server, so they are \`null\` on live progress rows and on older SDKs. Use them to judge execution shape (did the replay call the same tools in the same order, did a span that used to run get mocked or skipped, did a child span error) straight from the item file, instead of reading both traces back from the server for that comparison. **Re-check every item you judged live:** its verdict was derived before the outlines existed, so re-read its final item file (the complete row's write fills both outlines) and run the execution-shape comparison now. When that comparison changes the verdict, include the corrected \`{ originalTraceId, label, annotation }\` in the next \`{{command:persistReplayLabels}}\` batch: a new agent verdict for the same original trace replaces the earlier one in place, so this correction is the one case where re-persisting an already-persisted id is right. When it does not change the verdict, leave the persisted verdict alone.
|
|
30378
30416
|
|
|
30379
30417
|
**Before running: verify the installed replay command returns the full original and new output values AND at least one verdict persist key (\`item.originalTraceId\` or \`item.traceId\`) for every item** (not just lengths, counts, hashes, or truncated previews) so the run's \`items/*.json\` files carry them. Modern items may carry both: prefer lineage persistence by \`item.originalTraceId\` (older SDKs may print it under the deprecated \`sourceTraceId\` alias), and fall back to the server replay \`item.traceId\` only when original lineage is absent. After an item's replay trace is flushed, its \`item.traceId\` is available to lifecycle callbacks and in the final result. The oldest SDKs have no \`item.originalTraceId\` and persist by \`item.traceId\`. If the installed command returns neither, upgrade the SDK first; the Replay Output Contract and registry examples live in the SDK reference at \`https://docs.bitfab.ai/<language>-sdk.md\`. Subagents can't evaluate an improvement from \`5 \u2192 7 (+2)\`, and an item that carries no persist key blocks verdict persistence for that item.
|
|
30380
30418
|
|
|
30381
|
-
**Capture the \`
|
|
30419
|
+
**Capture the \`experimentId\` from the replay complete event**: read the final \`type: "complete"\` row in this run's \`events.jsonl\`; it carries \`experimentId\` and \`experimentUrl\` when the SDK returned them. Track every \`experimentId\` produced across all iterations of this phase for the \`open-experiments\` fallback.
|
|
30382
30420
|
|
|
30383
30421
|
**If an external child span fails during replay, verify it is actually mockable before tagging it.** When a non-root span throws because a paid call, external service, or stored dependency is unavailable, it can block the whole trace even though the failure is outside the function you're iterating on. The short-term fix is recorded-output mocking, but only after the wrapper and execution-context checks below clear:
|
|
30384
30422
|
|
|
@@ -30404,7 +30442,7 @@ cd <project-dir> && {{command:replayProgress}} --label <pipeline-name> --run-dir
|
|
|
30404
30442
|
Use this when the goal is to unblock iteration on the root function, not when the child itself is what you're trying to improve.
|
|
30405
30443
|
|
|
30406
30444
|
**After the run, check whether verdicts can be persisted, and by which key.** For every completed item, confirm it carries a persist key and note which keying the run uses:
|
|
30407
|
-
- **Lineage keying (modern SDKs):** the item carries \`item.originalTraceId\` (the original trace it was replayed from; older SDKs print it under the deprecated \`sourceTraceId\` alias). This is the preferred path, verdicts persist by \`{
|
|
30445
|
+
- **Lineage keying (modern SDKs):** the item carries \`item.originalTraceId\` (the original trace it was replayed from; older SDKs print it under the deprecated \`sourceTraceId\` alias). This is the preferred path, verdicts persist by \`{ experimentId, originalTraceId }\`.
|
|
30408
30446
|
- **Direct keying:** the item has no \`originalTraceId\`; \`item.traceId\` is the trace's own id, so verdicts persist by \`traceId\` directly. This is the general way any trace is labeled by its id. In the replay flow the label targets the replay trace, so direct keying applies when the replay trace's own id is known client-side (which older SDKs surface, writing the server id into \`item.traceId\` after the run).
|
|
30409
30447
|
|
|
30410
30448
|
Hold a boolean flag (\`canPersistVerdicts\`) for the \`check-verdict-persistence\` step, true when every completed item carries one of those keys, and remember which keying the run uses (they don't mix within a run: \`item.originalTraceId\` when present, else the direct \`item.traceId\`). If some completed items carry neither, the SDK is too old to persist those verdicts to the server. Do NOT stop here, just flag it.
|
|
@@ -30503,9 +30541,9 @@ After whichever workaround the user picks, re-run \`replay-against-dataset\` and
|
|
|
30503
30541
|
kind: "branch",
|
|
30504
30542
|
toolCalls: ["getReplayStatus", "listExperimentTraces", "getTraces"],
|
|
30505
30543
|
title: "Resolve unverified replay from the server",
|
|
30506
|
-
body: `**Run only when the replay command exited 0 but local ReplayResult capture was unavailable.** Extract the \`
|
|
30544
|
+
body: `**Run only when the replay command exited 0 but local ReplayResult capture was unavailable.** Extract the \`experimentId\` from \`run.json\` or the command output (older SDKs print it as \`testRunId\`). If it exists, poll {{tool:getReplayStatus}} every 5 seconds while the server status is \`pending\`, for at most 30 seconds total. If it is still pending after 30 seconds, stop polling and report that the outcome remains unverified.
|
|
30507
30545
|
|
|
30508
|
-
When the server reports \`completed\` with \`traceCount > 0\`, call {{tool:listExperimentTraces}} for the same
|
|
30546
|
+
When the server reports \`completed\` with \`traceCount > 0\`, call {{tool:listExperimentTraces}} for the same experiment to enumerate every server replay trace ID and its \`original\` trace ID, then call {{tool:getTraces}} with the server replay IDs and \`scope: "full"\` to recover their inputs, outputs, and errors. Use each experiment trace's \`original\` field as its \`originalTraceId\` lineage key. Do not use the keys of {{tool:getReplayStatus}}'s \`traceIds\` mapping as original trace IDs: those keys are local client \`sourceTraceId\` values. Recompute \`completed\`, \`shapeErrored\`, \`mockErrored\`, \`setupErrored\`, \`dependencyErrored\`, \`codeErrored\`, and \`canPersistVerdicts\` from those server traces exactly as \`replay-against-dataset\` does, set \`serverRecovered = true\`, then continue normally. Do not infer success from the child exit code or local progress lines.
|
|
30509
30547
|
|
|
30510
30548
|
{{whens}}`,
|
|
30511
30549
|
branches: [
|
|
@@ -30520,7 +30558,7 @@ When the server reports \`completed\` with \`traceCount > 0\`, call {{tool:listE
|
|
|
30520
30558
|
next: "iterate/replay-against-dataset"
|
|
30521
30559
|
},
|
|
30522
30560
|
{
|
|
30523
|
-
when: "no
|
|
30561
|
+
when: "no experiment ID can be resolved, server status cannot be read, or server status remains `pending` after 30 seconds",
|
|
30524
30562
|
description: "the outcome cannot be verified. Upgrade or fix the installed SDK's reporter/result-artifact contract, then retry",
|
|
30525
30563
|
next: "iterate/replay-against-dataset"
|
|
30526
30564
|
}
|
|
@@ -30667,14 +30705,14 @@ ${FAN_OUT_JUDGING}
|
|
|
30667
30705
|
|
|
30668
30706
|
**Persist via \`persistReplayLabels.js\`.** Write verdicts to a tmp JSON file then run the script. You may do this incrementally for complete batches as replay results arrive, and again at the end for any remaining original trace IDs that were not already persisted. Each call is one batched MCP write; the script parses \`save_agent_labels\`'s agent-readable effective label lines internally to verify each verdict actually persisted, then deletes the file on success. Its JSON status is the signal you route on:
|
|
30669
30707
|
|
|
30670
|
-
1. Pick an **absolute** tmp path. The script reads the file relative to its own process cwd, which in parallel-worktree mode is NOT the project root, so a relative path can resolve to a different directory than where you wrote the file (\`ENOENT\`). Recommended: \`<repoRoot>/.bitfab/tmp/verdicts-<
|
|
30708
|
+
1. Pick an **absolute** tmp path. The script reads the file relative to its own process cwd, which in parallel-worktree mode is NOT the project root, so a relative path can resolve to a different directory than where you wrote the file (\`ENOENT\`). Recommended: \`<repoRoot>/.bitfab/tmp/verdicts-<experimentId>.json\` where \`<repoRoot>\` is the output of \`git rev-parse --show-toplevel\` (create the dir if missing). Falls back to an absolute path under \`os.tmpdir()\` if the project root isn't writable.
|
|
30671
30709
|
2. Use the {{tool:write}} tool to write the verdicts file. **The script accepts two shapes; use the one that matches this run's keying (from \`check-verdict-persistence\`).**
|
|
30672
30710
|
|
|
30673
30711
|
**Lineage keying (modern SDKs, items carry \`originalTraceId\`), preferred:**
|
|
30674
30712
|
|
|
30675
30713
|
\`\`\`json
|
|
30676
30714
|
{
|
|
30677
|
-
"
|
|
30715
|
+
"experimentId": "<experimentId>",
|
|
30678
30716
|
"expectedOriginalTraceIds": ["<originalTraceId1>", "<originalTraceId2>", "..."],
|
|
30679
30717
|
"verdicts": [
|
|
30680
30718
|
{ "originalTraceId": "<originalTraceId1>", "label": true, "annotation": "Now returns the missing field; original annotation said it was empty.", "confidence": "High" },
|
|
@@ -30684,7 +30722,7 @@ ${FAN_OUT_JUDGING}
|
|
|
30684
30722
|
}
|
|
30685
30723
|
\`\`\`
|
|
30686
30724
|
|
|
30687
|
-
**Direct keying (the item has no \`originalTraceId\`; it carries the trace's own id in \`item.traceId\`):** same shape but keyed by \`traceId\`, and **omit \`
|
|
30725
|
+
**Direct keying (the item has no \`originalTraceId\`; it carries the trace's own id in \`item.traceId\`):** same shape but keyed by \`traceId\`, and **omit \`experimentId\`** (the trace id already identifies the trace to label):
|
|
30688
30726
|
|
|
30689
30727
|
\`\`\`json
|
|
30690
30728
|
{
|
|
@@ -30696,7 +30734,7 @@ ${FAN_OUT_JUDGING}
|
|
|
30696
30734
|
}
|
|
30697
30735
|
\`\`\`
|
|
30698
30736
|
|
|
30699
|
-
In lineage keying, \`
|
|
30737
|
+
In lineage keying, \`experimentId\` is this replay run's experiment ID (from the progress rows or the final \`ReplayResult\`). The server uses it to resolve each original trace to its replay trace within this run. The \`expected*\` list MUST be the full set of ids covered by this call's batch (and across all batches, every completed \`item.error\`-unset replay item must be persisted exactly once, no fewer, per the mandatory-coverage rule above). For the final end-of-run call, use only the ids not already successfully persisted by an earlier batch. \`verdicts\` MUST cover every id in the \`expected*\` list, keyed by the same id field as that list. A trace with no assertions gets exactly one entry, either a \`{label, annotation, confidence?}\` verdict or a \`{skip: true}\` explicit skip (skips allowed only for the three enumerated skip cases above, never for an environmental doubt). A trace that has assertions gets one entry per assertion instead, in the shapes the per-assertion block below fixes. \`confidence\` is optional but recommended (\`VeryLow|Low|Medium|High|VeryHigh\`). It surfaces in the labeling UI so reviewers can prioritize low-confidence verdicts. If any expected id gets no entry at all, the script returns \`status: "missing-coverage"\` and the verify step routes you back to fill the gaps.
|
|
30700
30738
|
|
|
30701
30739
|
**A replay inherits the original trace's assertions, so score them one at a time.** Before you judge this batch, call {{tool:getTraceAssertions}} once with the batch's **original** trace ids (one call covers up to 100 ids, so make it once and before you judge anything). Each assertion comes back under its trace as \`[ID: <uuid>] checks <target>: <assertion>\`, and that \`[ID: <uuid>]\` value is the \`assertionId\` its verdict carries. Only assertions a person has approved are listed, since only those are checked on a replay. A trace that comes back "no approved assertions" has nothing for you to score one at a time.
|
|
30702
30740
|
|
|
@@ -30710,7 +30748,7 @@ Per-assertion and whole-trace entries travel in the same file and the same call,
|
|
|
30710
30748
|
|
|
30711
30749
|
\`\`\`json
|
|
30712
30750
|
{
|
|
30713
|
-
"
|
|
30751
|
+
"experimentId": "<experimentId>",
|
|
30714
30752
|
"expectedOriginalTraceIds": ["<originalTraceId1>", "<originalTraceId2>"],
|
|
30715
30753
|
"verdicts": [
|
|
30716
30754
|
{ "originalTraceId": "<originalTraceId1>", "assertionId": "<assertionId1>", "label": true, "annotation": "The rebooked leg now names the carrier this assertion asks for.", "confidence": "High" },
|
|
@@ -30726,14 +30764,14 @@ Per-assertion and whole-trace entries travel in the same file and the same call,
|
|
|
30726
30764
|
3. Run the script:
|
|
30727
30765
|
|
|
30728
30766
|
\`\`\`bash
|
|
30729
|
-
{{command:persistReplayLabels}} <repoRoot>/.bitfab/tmp/verdicts-<
|
|
30767
|
+
{{command:persistReplayLabels}} <repoRoot>/.bitfab/tmp/verdicts-<experimentId>.json
|
|
30730
30768
|
\`\`\`
|
|
30731
30769
|
|
|
30732
30770
|
4. Read its single JSON line on stdout. Hold the parsed result for the next step.
|
|
30733
30771
|
|
|
30734
30772
|
**Spill working notes to a separate tmp file if context gets big.** Don't conflate working notes with the verdicts file, the script deletes the verdicts file on success.{{#claude}}
|
|
30735
30773
|
|
|
30736
|
-
**If you're a worktree subagent** (parallel mode from \`pick-execution-mode\`): after the script returns, hand the parsed result + \`
|
|
30774
|
+
**If you're a worktree subagent** (parallel mode from \`pick-execution-mode\`): after the script returns, hand the parsed result + \`experimentId\` + unreplayable list back to the main agent and exit. The main agent collects results from all parallel experiments before \`open-experiments\`.{{/claude}}`,
|
|
30737
30775
|
next: "iterate/verify-replay-labels"
|
|
30738
30776
|
},
|
|
30739
30777
|
{
|
|
@@ -30777,12 +30815,12 @@ Per-assertion and whole-trace entries travel in the same file and the same call,
|
|
|
30777
30815
|
},
|
|
30778
30816
|
{
|
|
30779
30817
|
when: '`status: "invalid-input"` (malformed verdicts JSON or missing fields)',
|
|
30780
|
-
description: "the verdicts file you wrote doesn't match either accepted shape. Read the script's `message` field, fix the JSON (most common: missing annotation on a non-skip entry; a lineage file missing `
|
|
30818
|
+
description: "the verdicts file you wrote doesn't match either accepted shape. Read the script's `message` field, fix the JSON (most common: missing annotation on a non-skip entry; a lineage file missing `experimentId` or with an empty `expectedOriginalTraceIds`; a verdict keyed by the wrong id field for the file's mode), and re-run the script. Loop back here",
|
|
30781
30819
|
next: "iterate/verify-replay-labels"
|
|
30782
30820
|
},
|
|
30783
30821
|
{
|
|
30784
30822
|
when: '`status: "mcp-error"` (MCP call to save_agent_labels failed mid-batch)',
|
|
30785
|
-
description: "Read the script's `message` field. In lineage mode, if it says an original trace has no replay trace in this run or is not in this organization, confirm the `
|
|
30823
|
+
description: "Read the script's `message` field. In lineage mode, if it says an original trace has no replay trace in this run or is not in this organization, confirm the `experimentId` matches the run these items came from and that each `originalTraceId` is the original trace it was replayed from (from the item files, not a local or replay trace id). In direct mode, if trace ids are invalid, confirm each verdicts-file `traceId` is the server replay id (`item.traceId`) from the item files. Fix the file and re-run. For network or auth errors, the script's `partialIds` lists which ids were already persisted; tell the user, recommend re-running the script (it's idempotent, already-persisted labels just upsert), and loop back here. If it keeps failing, stop and surface the error",
|
|
30786
30824
|
next: "iterate/verify-replay-labels"
|
|
30787
30825
|
},
|
|
30788
30826
|
{
|
|
@@ -30798,7 +30836,7 @@ Per-assertion and whole-trace entries travel in the same file and the same call,
|
|
|
30798
30836
|
toolCalls: ["bash", "listExperiments", "listExperimentTraces"],
|
|
30799
30837
|
commandCalls: ["pageLink", "openExperiments"],
|
|
30800
30838
|
title: "Share the experiments link",
|
|
30801
|
-
body: 'Skip the link for a benchmark without the `page` flag or a single-trace fix awaiting its verdict, or a full-dataset fix with fixDatasetBitfab false. Otherwise call {{tool:listExperiments}} for the trace function and verify the returned
|
|
30839
|
+
body: 'Skip the link for a benchmark without the `page` flag or a single-trace fix awaiting its verdict, or a full-dataset fix with fixDatasetBitfab false. Otherwise call {{tool:listExperiments}} for the trace function and verify the returned experiment IDs exist on the server. If a persisted run belongs to the experimentGroupId prepared before replay, run {{command:pageLink}} "/experiments?experimentGroupId=<experimentGroupId><tokensSuffix>" with that same group ID and relay the URL. Use an empty tokensSuffix for quality runs and &tokens=1&tokenType=<costBasis> for cost runs, preserving the selected all or uncached basis. If the registry did not persist the group, run {{command:openExperiments}} <experimentIds> with the verified IDs instead; include --show-tokens for cost runs, plus --uncached when costBasis is uncached. If no run was persisted, report that fact without emitting a broken link. Both commands exit immediately. Continue to evaluating and persisting replay labels.',
|
|
30802
30840
|
next: "iterate/evaluate-results"
|
|
30803
30841
|
},
|
|
30804
30842
|
{
|
|
@@ -30878,7 +30916,7 @@ Then ask what to do next. One option always lets the user **show this fix in Bit
|
|
|
30878
30916
|
- **The dataset has sibling traces:** the headline choice is whether to re-run the entire dataset now to check the fix against every trace in it (you'll then ask Bitfab vs terminal-only). Present all four options below and recommend **"Re-run the entire dataset"**.
|
|
30879
30917
|
- **No siblings** (the added trace is the dataset's only trace, or the user continued without saving so nothing was added): do **NOT** offer "Re-run the entire dataset", there is no other dataset trace to run, and re-running a one-trace dataset would just replay the trace you already proved green. Present only "Show in Bitfab", "Keep iterating", and "Stop and wrap up", and recommend **"Stop and wrap up"**.
|
|
30880
30918
|
|
|
30881
|
-
When the user picks **Show in Bitfab**, open it with \`fixSingleTraceExperimentGroupId\` (or the \`
|
|
30919
|
+
When the user picks **Show in Bitfab**, open it with \`fixSingleTraceExperimentGroupId\` (or the \`experimentIds\` fallback), then ask this question again. When they pick **Keep iterating**, clear \`fixDatasetSkipped\` if it was set (this is a fresh attempt, so a later green should re-offer the save prompt they declined only on the previous pass), set \`fixReplayScope = "single-trace"\` and \`fixSkipMakeChange = false\`. When they pick **Stop and wrap up**, the saved trace stays if one was added (\`fixAddedToDataset\`); nothing is saved if they continued without adding (\`fixDatasetSkipped\`).
|
|
30882
30920
|
|
|
30883
30921
|
{{branches}}`,
|
|
30884
30922
|
branches: [
|
|
@@ -30957,7 +30995,7 @@ Report the failed target result first. Say whether the replay produced a real FA
|
|
|
30957
30995
|
|
|
30958
30996
|
Then ask what to do next. Offer inspection, another fix attempt, saving the trace as a failing test to revisit later, or stopping. Do not offer a full-dataset run here: there is no dataset trace to run unless it was already added earlier, and running one before the target passes would defeat the test-first add-to-dataset flow:
|
|
30959
30997
|
|
|
30960
|
-
When the user picks **Show in Bitfab**, open it with \`fixSingleTraceExperimentGroupId\` (or the \`
|
|
30998
|
+
When the user picks **Show in Bitfab**, open it with \`fixSingleTraceExperimentGroupId\` (or the \`experimentIds\` fallback), then ask this question again. Recommend **Keep iterating** only when the replay produced a real FAIL; for an unreplayable or shape-incompatible result, recommend making the trace replayable or stopping instead. When they pick it, set \`fixReplayScope = "single-trace"\` and \`fixSkipMakeChange = false\`.
|
|
30961
30999
|
|
|
30962
31000
|
{{branches}}`,
|
|
30963
31001
|
branches: [
|
|
@@ -31026,7 +31064,7 @@ Set \`fixAddedToDataset = true\` only once the attach above is confirmed. Tell t
|
|
|
31026
31064
|
toolCalls: ["bash"],
|
|
31027
31065
|
commandCalls: ["pageLink"],
|
|
31028
31066
|
title: "Share the targeted before/after link",
|
|
31029
|
-
body: "Run {{command:pageLink}} /experiments?
|
|
31067
|
+
body: "Run {{command:pageLink}} /experiments?experimentIds=<experimentId>&autoOpenFirst=1 (quote the entire path) and relay the returned URL so the user can inspect the before/after. Return to the fix decision in chat.",
|
|
31030
31068
|
next: "iterate/fix-target-replay-status"
|
|
31031
31069
|
},
|
|
31032
31070
|
{
|
|
@@ -31083,7 +31121,7 @@ Then route by the approved plan, **without asking the user** (they already confi
|
|
|
31083
31121
|
{
|
|
31084
31122
|
id: "quick-replay",
|
|
31085
31123
|
title: "Phase Replay: Single-Trace Quick Replay",
|
|
31086
|
-
intro: "Reached only from `replay` mode. The user already has a trace ID and (usually) already made a fix; they just want to replay that one trace and hear whether it worked. This is the **minimal, atomic** path: no Bitfab/browser, no dataset, no experiment groups. Locate the replay registry, read the trace, run replay against the single trace ID, compare the new output to the original, and report a one-line verdict in chat. **Whenever you derive a pass/fail verdict, persist it onto the replay trace** (the same local label you show in chat, saved via {{command:persistReplayLabels}}) so it isn't silently thrown away. The one exception is an SDK too old to expose replay trace IDs: persistence is then impossible, so the verdict stays in-chat only with an upgrade nudge. The replay itself creates
|
|
31124
|
+
intro: "Reached only from `replay` mode. The user already has a trace ID and (usually) already made a fix; they just want to replay that one trace and hear whether it worked. This is the **minimal, atomic** path: no Bitfab/browser, no dataset, no experiment groups. Locate the replay registry, read the trace, run replay against the single trace ID, compare the new output to the original, and report a one-line verdict in chat. **Whenever you derive a pass/fail verdict, persist it onto the replay trace** (the same local label you show in chat, saved via {{command:persistReplayLabels}}) so it isn't silently thrown away. The one exception is an SDK too old to expose replay trace IDs: persistence is then impossible, so the verdict stays in-chat only with an upgrade nudge. The replay itself creates an experiment intrinsically (the SDK does this); persistence just adds the agent verdict on top.",
|
|
31087
31125
|
stepStyle: "list",
|
|
31088
31126
|
steps: [
|
|
31089
31127
|
{
|
|
@@ -31143,7 +31181,7 @@ ${REPLAY_SAFETY_CHECK}
|
|
|
31143
31181
|
# Ruby: cd <project-dir> && bundle exec bitfab-replay --registry <registry-path> <pipeline> --trace-ids <trace-id>
|
|
31144
31182
|
\`\`\`
|
|
31145
31183
|
|
|
31146
|
-
This is a single-trace, in-chat path: run the replay directly, no progress-bar wrapper (one item has nothing to track). Do **not** pass \`--code-change\` or \`--experiment-group-id\`, this minimal path skips code-change payloads and experiment groups (persisting the verdict in the next step needs neither). Capture the full replay-result JSON and exit code, and from it hold the run's
|
|
31184
|
+
This is a single-trace, in-chat path: run the replay directly, no progress-bar wrapper (one item has nothing to track). Do **not** pass \`--code-change\` or \`--experiment-group-id\`, this minimal path skips code-change payloads and experiment groups (persisting the verdict in the next step needs neither). Capture the full replay-result JSON and exit code, and from it hold the run's experiment ID (\`experimentId\` in TS, \`experiment_id\` in Python/Ruby; older SDKs print \`testRunId\` / \`test_run_id\`) and the completed item's trace id (\`traceId\` in TS, \`trace_id\` in Python/Ruby). **In the final replay result this trace id is already the SERVER replay trace id** (the SDK's \`completeReplay\` overwrites the local id with the server row id before returning), so the verdict step persists against it directly, no \`get_replay_status\` mapping. **If it is \`null\`, persistence is impossible this run** (an old server/SDK that returns no server-trace-id mapping), note that so the verdict step falls back to an in-chat-only verdict.
|
|
31147
31185
|
|
|
31148
31186
|
**Quick health check.** If the replay crashed (non-zero exit, no items) or the single item has \`item.error\` set, hold the error for the verdict step. Otherwise hold the completed item's new output alongside the original output you read in \`setup\`.`,
|
|
31149
31187
|
next: "quick-replay/verdict"
|
|
@@ -31202,7 +31240,7 @@ Then compare the new output against the original trace's assertions, label, and
|
|
|
31202
31240
|
\`\`\`
|
|
31203
31241
|
|
|
31204
31242
|
\`\`\`bash
|
|
31205
|
-
{{command:persistReplayLabels}} <repoRoot>/.bitfab/tmp/verdicts-<
|
|
31243
|
+
{{command:persistReplayLabels}} <repoRoot>/.bitfab/tmp/verdicts-<experiment-id>.json
|
|
31206
31244
|
\`\`\`
|
|
31207
31245
|
|
|
31208
31246
|
\`label\` is \`true\` for Pass, \`false\` for Still-failing / Regressed. Read the script's single JSON status line: \`ok\` means the verdict is now on the replay trace, add "\xB7 saved" to your one-line report.
|
|
@@ -31243,7 +31281,7 @@ Then compare the new output against the original trace's assertions, label, and
|
|
|
31243
31281
|
toolCalls: ["bash", "grep"],
|
|
31244
31282
|
title: "Re-seed the trace in place",
|
|
31245
31283
|
emit: "Re-seeding",
|
|
31246
|
-
body: `**Run the trace again for real and record the result under the same id.** A re-seed is a seed, not a replay: nothing is mocked, no
|
|
31284
|
+
body: `**Run the trace again for real and record the result under the same id.** A re-seed is a seed, not a replay: nothing is mocked, no experiment is created, and the trace keeps its id, labels, assertions, dataset membership, name, and metadata. The previous run is kept as its own trace, linked back to this one, so nothing is deleted.
|
|
31247
31285
|
|
|
31248
31286
|
**It runs the function exactly as production does, side effects included.** If the safety check in \`setup\` found any unsafe action (an email, a payment, a write to a live system), say exactly which call would run for real and get an explicit go-ahead before continuing; a re-seed has no mocking to hide behind.
|
|
31249
31287
|
|
|
@@ -31379,7 +31417,7 @@ var setupFlow = Flow.parse({
|
|
|
31379
31417
|
},
|
|
31380
31418
|
openExperiments: {
|
|
31381
31419
|
description: "Print a link to experiments.",
|
|
31382
|
-
args: "<
|
|
31420
|
+
args: "<experimentIds>"
|
|
31383
31421
|
},
|
|
31384
31422
|
startDataset: {
|
|
31385
31423
|
description: "Print a dataset link.",
|
|
@@ -32778,7 +32816,7 @@ Leave the live request path untouched: only the replayed function reads the bran
|
|
|
32778
32816
|
1. Resolve the exact fresh trace ID. Before triggering the instrumented function, call {{tool:searchTraces}} with \`{ traceFunctionKey: "<key>", limit: 10 }\` and retain the returned IDs. Record the current timestamp, then run the instrumented function once (or have the user trigger it). Poll the same search once or twice and compare the returned IDs with the before set. Select the ID only when exactly one new root trace has a timestamp after the recorded time. If there are zero new IDs, stop and report that no fresh trace arrived; if concurrent traffic produces more than one candidate, stop and ask the user for the intended trace ID. \`search_traces\` does not return trace inputs, so never guess based on an assumed input match. Confirm the exact ID is eligible by calling {{tool:searchTraces}} with \`{ traceFunctionKey: "<key>", traceIds: ["<fresh-trace-id>"], hasDbSnapshot: true, limit: 1 }\`. If it is absent, do not replay it: report that the fresh trace did not capture a snapshot and diagnose SDK freshness/database connection first.
|
|
32779
32817
|
2. **Mandatory replay-safety check, before any replay command.** Read the replay registry module, the real production root it imports, and every reachable external-action span. Inventory database writes, outbound mutations, queue publishes, email, payments, filesystem writes, and similar unsafe actions. Confirm each one is behind a manual replay-mockable descendant selected by the registry entry's actual strategy and has a serializable recorded output; \`mock: "marked"\` requires \`mockOnReplay\` on that boundary. Confirm the boundary runs in the same replay context: Python worker threads require \`trace_across_threads=True\`; Python async-generator spans cannot be mocked; Ruby child threads, pre-created consumers, and processes do not inherit replay interception; TypeScript synchronous selected spans cannot use lazy \`marked\` output and may use \`all\` only when freezing every matched child is acceptable. Also reject unsafe import-time/module initialization, because it runs before replay interception. Confirm the replay call requests \`dbBranch\` / \`db_branch\`. If any unsafe action is unselected, unmockable, outside context, or uncertain, **do not run the smoke test**; report the exact blocker and required boundary change. Mocking, not the app environment, is the safety boundary.
|
|
32780
32818
|
3. Run the SDK-installed replay command against only the verified ID using \`--trace-ids <fresh-trace-id>\` (for example, \`pnpm with-env bitfab-replay --registry scripts/replayRegistry.ts <pipeline> --trace-ids <fresh-trace-id>\`, \`poetry run bitfab-replay --registry scripts/replay_registry.py <pipeline> --trace-ids <fresh-trace-id>\`, or \`bundle exec bitfab-replay --registry scripts/replay_registry.rb <pipeline> --trace-ids <fresh-trace-id>\`, with the app's normal environment loader). Never substitute \`--limit 1\`.
|
|
32781
|
-
4. Confirm the branch was injected: inside the replayed function, \`getCurrentReplayBranch()\` (TypeScript), \`get_current_replay_branch()\` (Python), or \`Bitfab.current_replay_branch\` (Ruby) must be non-null. Compare its \`databaseUrl\` / \`database_url\` host and database with the app's normal \`DATABASE_URL\`; they should differ. Print the
|
|
32819
|
+
4. Confirm the branch was injected: inside the replayed function, \`getCurrentReplayBranch()\` (TypeScript), \`get_current_replay_branch()\` (Python), or \`Bitfab.current_replay_branch\` (Ruby) must be non-null. Compare its \`databaseUrl\` / \`database_url\` host and database with the app's normal \`DATABASE_URL\`; they should differ. Print the experiment URL from the replay output (\`experimentUrl\` in TypeScript, \`experiment_url\` in Python/Ruby; older SDKs print \`testRunUrl\` / \`test_run_url\`) so the user can open the experiment.
|
|
32782
32820
|
|
|
32783
32821
|
If the branch accessor is null for a freshly captured trace, check that the source database is connected, that the trace actually carries a snapshot reference, and that the SDK supports always-on capture (upgrade with \`{{cmd}}update\` when needed). Re-check the dashboard Database section in step {{step:db-snapshot/connect-db}}; there is no separate replay-environment active flag.
|
|
32784
32822
|
|
|
@@ -33575,14 +33613,17 @@ var setupV2Flow = Flow.parse({
|
|
|
33575
33613
|
}
|
|
33576
33614
|
});
|
|
33577
33615
|
|
|
33578
|
-
// ../bitfab-plugin-lib/dist/
|
|
33616
|
+
// ../bitfab-plugin-lib/dist/chatSessions/ampTranscript.js
|
|
33579
33617
|
import fs9 from "fs";
|
|
33580
33618
|
|
|
33581
|
-
// ../bitfab-plugin-lib/dist/hooks/
|
|
33619
|
+
// ../bitfab-plugin-lib/dist/hooks/captureHook.js
|
|
33582
33620
|
import fs10 from "fs";
|
|
33583
33621
|
|
|
33584
|
-
// ../bitfab-plugin-lib/dist/
|
|
33622
|
+
// ../bitfab-plugin-lib/dist/hooks/userPromptSubmit.js
|
|
33585
33623
|
import fs11 from "fs";
|
|
33624
|
+
|
|
33625
|
+
// ../bitfab-plugin-lib/dist/installScope.js
|
|
33626
|
+
import fs12 from "fs";
|
|
33586
33627
|
import os8 from "os";
|
|
33587
33628
|
import path9 from "path";
|
|
33588
33629
|
var INSTALL_SCOPES = ["user", "project", "local", "managed"];
|
|
@@ -33602,7 +33643,7 @@ function installScopeAppliesToProject(projectPath, projectDir) {
|
|
|
33602
33643
|
}
|
|
33603
33644
|
function canonicalPath(target) {
|
|
33604
33645
|
try {
|
|
33605
|
-
return
|
|
33646
|
+
return fs12.realpathSync(path9.resolve(target));
|
|
33606
33647
|
} catch {
|
|
33607
33648
|
return path9.resolve(target);
|
|
33608
33649
|
}
|
|
@@ -43200,7 +43241,7 @@ async function startMcpServer(platform2, getConfig2, pluginVersion) {
|
|
|
43200
43241
|
// src/amp.ts
|
|
43201
43242
|
import { spawnSync as spawnSync2 } from "child_process";
|
|
43202
43243
|
import crypto3 from "crypto";
|
|
43203
|
-
import
|
|
43244
|
+
import fs13 from "fs";
|
|
43204
43245
|
import os9 from "os";
|
|
43205
43246
|
import path10 from "path";
|
|
43206
43247
|
import * as p3 from "@clack/prompts";
|
|
@@ -43381,13 +43422,13 @@ function ampPluginDir(env = process.env) {
|
|
|
43381
43422
|
return path10.join(configHome, "amp", "plugins", PLUGIN_DIR_NAME);
|
|
43382
43423
|
}
|
|
43383
43424
|
function pluginDirState(dir) {
|
|
43384
|
-
if (!
|
|
43425
|
+
if (!fs13.existsSync(dir)) {
|
|
43385
43426
|
return "missing";
|
|
43386
43427
|
}
|
|
43387
|
-
if (
|
|
43428
|
+
if (fs13.existsSync(path10.join(dir, ".git"))) {
|
|
43388
43429
|
return "checkout";
|
|
43389
43430
|
}
|
|
43390
|
-
return
|
|
43431
|
+
return fs13.readdirSync(dir).length === 0 ? "empty" : "occupied";
|
|
43391
43432
|
}
|
|
43392
43433
|
function skillInvocation(skill, args) {
|
|
43393
43434
|
const trimmed = args?.trim();
|
|
@@ -43410,7 +43451,7 @@ function installOrUpdatePlugin() {
|
|
|
43410
43451
|
`${dir} already exists but is not a git checkout of the Bitfab plugin. Move it aside (for example \`mv ${dir} ${dir}.bak\`) and rerun.`
|
|
43411
43452
|
);
|
|
43412
43453
|
}
|
|
43413
|
-
|
|
43454
|
+
fs13.mkdirSync(path10.dirname(dir), { recursive: true });
|
|
43414
43455
|
runCli("git", ["clone", "--depth", "1", "--quiet", PLUGIN_REPO_URL, dir]);
|
|
43415
43456
|
return "installed";
|
|
43416
43457
|
}
|
|
@@ -43469,15 +43510,20 @@ function runAmpSetup(skipPermissions, prompt) {
|
|
|
43469
43510
|
warnSkipPermissions(skipPermissions);
|
|
43470
43511
|
handOffToAmp(skillInvocation(SETUP_SKILL, prompt));
|
|
43471
43512
|
}
|
|
43472
|
-
function runAmpAnalyzeRepo(
|
|
43513
|
+
function runAmpAnalyzeRepo(captureOverride, limit, prompt) {
|
|
43473
43514
|
const message = skillInvocation(
|
|
43474
43515
|
`Use the ${SETUP_SKILL} skill in analyze-repo mode.`,
|
|
43475
43516
|
buildAnalyzeRepoArgs({ limit, prompt })
|
|
43476
43517
|
);
|
|
43518
|
+
const env = { ...process.env };
|
|
43519
|
+
env.BITFAB_AGENT_RUN_ID = crypto3.randomUUID();
|
|
43520
|
+
if (captureOverride !== void 0) {
|
|
43521
|
+
env.BITFAB_CAPTURE_SESSIONS = captureOverride ? "1" : "0";
|
|
43522
|
+
}
|
|
43477
43523
|
p3.log.step("Analyzing the repo with Amp (headless)");
|
|
43478
43524
|
const result = spawnSync2(CLI, ["-x", message], {
|
|
43479
43525
|
stdio: "inherit",
|
|
43480
|
-
env
|
|
43526
|
+
env,
|
|
43481
43527
|
...SHELL_OPTS
|
|
43482
43528
|
});
|
|
43483
43529
|
if (result.error) {
|
|
@@ -43515,7 +43561,7 @@ var ampAdapter = {
|
|
|
43515
43561
|
label: "Amp",
|
|
43516
43562
|
runsAgentIn: "editor",
|
|
43517
43563
|
supportsSkipPermissions: false,
|
|
43518
|
-
capturesSessionLogs:
|
|
43564
|
+
capturesSessionLogs: true,
|
|
43519
43565
|
install: runAmpInstall,
|
|
43520
43566
|
ensureLatest: ensureLatestAmpPlugin,
|
|
43521
43567
|
setup: runAmpSetup,
|
|
@@ -43527,7 +43573,7 @@ var ampAdapter = {
|
|
|
43527
43573
|
// src/claude.ts
|
|
43528
43574
|
import { spawn as spawn3 } from "child_process";
|
|
43529
43575
|
import crypto4 from "crypto";
|
|
43530
|
-
import
|
|
43576
|
+
import fs14 from "fs";
|
|
43531
43577
|
import os10 from "os";
|
|
43532
43578
|
import path11 from "path";
|
|
43533
43579
|
import * as p4 from "@clack/prompts";
|
|
@@ -43717,12 +43763,12 @@ function activeLocalClaudeBitfabPlugin() {
|
|
|
43717
43763
|
path11.join(os10.homedir(), ".claude", "settings.json")
|
|
43718
43764
|
];
|
|
43719
43765
|
for (const settingsPath of settingsPaths) {
|
|
43720
|
-
if (!
|
|
43766
|
+
if (!fs14.existsSync(settingsPath)) {
|
|
43721
43767
|
continue;
|
|
43722
43768
|
}
|
|
43723
43769
|
let contents;
|
|
43724
43770
|
try {
|
|
43725
|
-
contents =
|
|
43771
|
+
contents = fs14.readFileSync(settingsPath, "utf-8");
|
|
43726
43772
|
} catch {
|
|
43727
43773
|
continue;
|
|
43728
43774
|
}
|
|
@@ -43778,7 +43824,7 @@ async function runClaudeAnalyzeRepo(captureOverride, limit, prompt) {
|
|
|
43778
43824
|
limit,
|
|
43779
43825
|
prompt
|
|
43780
43826
|
});
|
|
43781
|
-
const out =
|
|
43827
|
+
const out = fs14.createWriteStream(logPath);
|
|
43782
43828
|
const spinner7 = p4.spinner();
|
|
43783
43829
|
spinner7.start("Analyzing workflows to instrument");
|
|
43784
43830
|
const stdoutChunks = [];
|
|
@@ -44446,7 +44492,7 @@ function eventMessageContent(event) {
|
|
|
44446
44492
|
}
|
|
44447
44493
|
function analyzeRepoLogPath() {
|
|
44448
44494
|
const dir = path11.join(process.cwd(), ".bitfab", "logs");
|
|
44449
|
-
|
|
44495
|
+
fs14.mkdirSync(dir, { recursive: true });
|
|
44450
44496
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
44451
44497
|
return path11.join(dir, `analyze-repo-${stamp}.jsonl`);
|
|
44452
44498
|
}
|
|
@@ -44496,10 +44542,10 @@ function claudeProjectKeys(projectPath) {
|
|
|
44496
44542
|
function enableBitfabMcpServer(configPath, projectPath) {
|
|
44497
44543
|
let staging;
|
|
44498
44544
|
try {
|
|
44499
|
-
if (!
|
|
44545
|
+
if (!fs14.existsSync(configPath)) {
|
|
44500
44546
|
return false;
|
|
44501
44547
|
}
|
|
44502
|
-
const raw =
|
|
44548
|
+
const raw = fs14.readFileSync(configPath, "utf-8");
|
|
44503
44549
|
if (raw.trim() === "") {
|
|
44504
44550
|
return false;
|
|
44505
44551
|
}
|
|
@@ -44516,20 +44562,20 @@ function enableBitfabMcpServer(configPath, projectPath) {
|
|
|
44516
44562
|
project.disabledMcpServers = disabled.filter(
|
|
44517
44563
|
(name) => name !== MCP_SERVER_NAME
|
|
44518
44564
|
);
|
|
44519
|
-
const target =
|
|
44520
|
-
const mode =
|
|
44565
|
+
const target = fs14.realpathSync(configPath);
|
|
44566
|
+
const mode = fs14.statSync(target).mode & 511;
|
|
44521
44567
|
const stagingPath = `${target}.bitfab-${process.pid}.tmp`;
|
|
44522
|
-
|
|
44568
|
+
fs14.writeFileSync(stagingPath, JSON.stringify(config4, null, 2), {
|
|
44523
44569
|
mode,
|
|
44524
44570
|
flag: "wx"
|
|
44525
44571
|
});
|
|
44526
44572
|
staging = stagingPath;
|
|
44527
|
-
|
|
44573
|
+
fs14.renameSync(staging, target);
|
|
44528
44574
|
return true;
|
|
44529
44575
|
} catch {
|
|
44530
44576
|
if (staging !== void 0) {
|
|
44531
44577
|
try {
|
|
44532
|
-
|
|
44578
|
+
fs14.rmSync(staging, { force: true });
|
|
44533
44579
|
} catch {
|
|
44534
44580
|
}
|
|
44535
44581
|
}
|
|
@@ -44538,8 +44584,8 @@ function enableBitfabMcpServer(configPath, projectPath) {
|
|
|
44538
44584
|
}
|
|
44539
44585
|
function enableAutoUpdate(settingsPath) {
|
|
44540
44586
|
let settings = {};
|
|
44541
|
-
if (
|
|
44542
|
-
const raw =
|
|
44587
|
+
if (fs14.existsSync(settingsPath)) {
|
|
44588
|
+
const raw = fs14.readFileSync(settingsPath, "utf-8");
|
|
44543
44589
|
settings = raw.trim() === "" ? {} : JSON.parse(raw);
|
|
44544
44590
|
}
|
|
44545
44591
|
const marketplaces = settings.extraKnownMarketplaces ?? {};
|
|
@@ -44553,8 +44599,8 @@ function enableAutoUpdate(settingsPath) {
|
|
|
44553
44599
|
autoUpdate: true
|
|
44554
44600
|
};
|
|
44555
44601
|
settings.extraKnownMarketplaces = marketplaces;
|
|
44556
|
-
|
|
44557
|
-
|
|
44602
|
+
fs14.mkdirSync(path11.dirname(settingsPath), { recursive: true });
|
|
44603
|
+
fs14.writeFileSync(settingsPath, `${JSON.stringify(settings, null, 2)}
|
|
44558
44604
|
`);
|
|
44559
44605
|
return true;
|
|
44560
44606
|
}
|
|
@@ -44583,7 +44629,7 @@ var claudeAdapter = {
|
|
|
44583
44629
|
// src/codex.ts
|
|
44584
44630
|
import { spawn as spawn4 } from "child_process";
|
|
44585
44631
|
import crypto5 from "crypto";
|
|
44586
|
-
import
|
|
44632
|
+
import fs15 from "fs";
|
|
44587
44633
|
import os11 from "os";
|
|
44588
44634
|
import path12 from "path";
|
|
44589
44635
|
import * as p5 from "@clack/prompts";
|
|
@@ -44657,7 +44703,7 @@ async function runCodexAnalyzeRepo(captureOverride, limit, prompt) {
|
|
|
44657
44703
|
limit,
|
|
44658
44704
|
prompt
|
|
44659
44705
|
});
|
|
44660
|
-
const out =
|
|
44706
|
+
const out = fs15.createWriteStream(logPath);
|
|
44661
44707
|
const spinner7 = p5.spinner();
|
|
44662
44708
|
spinner7.start("Analyzing workflows to instrument");
|
|
44663
44709
|
const stdoutChunks = [];
|
|
@@ -44813,14 +44859,14 @@ function eventItem(event) {
|
|
|
44813
44859
|
}
|
|
44814
44860
|
function analyzeRepoLogPath2() {
|
|
44815
44861
|
const dir = path12.join(process.cwd(), ".bitfab", "logs");
|
|
44816
|
-
|
|
44862
|
+
fs15.mkdirSync(dir, { recursive: true });
|
|
44817
44863
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
44818
44864
|
return path12.join(dir, `analyze-repo-${stamp}.jsonl`);
|
|
44819
44865
|
}
|
|
44820
44866
|
function enableCodexPlugin(configPath) {
|
|
44821
44867
|
let content = "";
|
|
44822
|
-
if (
|
|
44823
|
-
content =
|
|
44868
|
+
if (fs15.existsSync(configPath)) {
|
|
44869
|
+
content = fs15.readFileSync(configPath, "utf-8");
|
|
44824
44870
|
}
|
|
44825
44871
|
const header = `[plugins."${PLUGIN_KEY2}"]`;
|
|
44826
44872
|
const desiredSection = `${header}
|
|
@@ -44843,15 +44889,15 @@ enabled = true
|
|
|
44843
44889
|
|
|
44844
44890
|
${desiredSection}`;
|
|
44845
44891
|
}
|
|
44846
|
-
|
|
44847
|
-
|
|
44892
|
+
fs15.mkdirSync(path12.dirname(configPath), { recursive: true });
|
|
44893
|
+
fs15.writeFileSync(configPath, next);
|
|
44848
44894
|
return true;
|
|
44849
44895
|
}
|
|
44850
44896
|
function isCodexPluginEnabled(configPath, pluginKey) {
|
|
44851
|
-
if (!
|
|
44897
|
+
if (!fs15.existsSync(configPath)) {
|
|
44852
44898
|
return false;
|
|
44853
44899
|
}
|
|
44854
|
-
const content =
|
|
44900
|
+
const content = fs15.readFileSync(configPath, "utf-8");
|
|
44855
44901
|
const sectionRegex = new RegExp(
|
|
44856
44902
|
`\\[plugins\\."${escapeRegex2(pluginKey)}"\\][^\\[]*`,
|
|
44857
44903
|
"m"
|
|
@@ -44890,7 +44936,7 @@ var codexAdapter = {
|
|
|
44890
44936
|
// src/cursor.ts
|
|
44891
44937
|
import { spawn as spawn5, spawnSync as spawnSync3 } from "child_process";
|
|
44892
44938
|
import crypto6 from "crypto";
|
|
44893
|
-
import
|
|
44939
|
+
import fs16 from "fs";
|
|
44894
44940
|
import os12 from "os";
|
|
44895
44941
|
import path13 from "path";
|
|
44896
44942
|
import * as p6 from "@clack/prompts";
|
|
@@ -44978,7 +45024,7 @@ async function runCursorAnalyzeRepo(captureOverride, limit, prompt) {
|
|
|
44978
45024
|
limit,
|
|
44979
45025
|
prompt
|
|
44980
45026
|
});
|
|
44981
|
-
const out =
|
|
45027
|
+
const out = fs16.createWriteStream(logPath);
|
|
44982
45028
|
const spinner7 = p6.spinner();
|
|
44983
45029
|
spinner7.start("Analyzing workflows to instrument");
|
|
44984
45030
|
const stdoutChunks = [];
|
|
@@ -45134,7 +45180,7 @@ function resolveCursorPluginDir() {
|
|
|
45134
45180
|
path13.join(os12.homedir(), ".cursor", "plugins", "local", "bitfab")
|
|
45135
45181
|
];
|
|
45136
45182
|
for (const candidate of candidates) {
|
|
45137
|
-
if (
|
|
45183
|
+
if (fs16.existsSync(path13.join(candidate, ".cursor-plugin", "plugin.json")) && fs16.existsSync(path13.join(candidate, "mcp.json"))) {
|
|
45138
45184
|
return candidate;
|
|
45139
45185
|
}
|
|
45140
45186
|
}
|
|
@@ -45142,7 +45188,7 @@ function resolveCursorPluginDir() {
|
|
|
45142
45188
|
}
|
|
45143
45189
|
function analyzeRepoLogPath3() {
|
|
45144
45190
|
const dir = path13.join(process.cwd(), ".bitfab", "logs");
|
|
45145
|
-
|
|
45191
|
+
fs16.mkdirSync(dir, { recursive: true });
|
|
45146
45192
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
45147
45193
|
return path13.join(dir, `analyze-repo-${stamp}.jsonl`);
|
|
45148
45194
|
}
|
|
@@ -45223,11 +45269,11 @@ var platform = {
|
|
|
45223
45269
|
};
|
|
45224
45270
|
|
|
45225
45271
|
// src/version.ts
|
|
45226
|
-
import
|
|
45272
|
+
import fs17 from "fs";
|
|
45227
45273
|
import path14 from "path";
|
|
45228
45274
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
45229
45275
|
var CLI_VERSION = JSON.parse(
|
|
45230
|
-
|
|
45276
|
+
fs17.readFileSync(
|
|
45231
45277
|
path14.join(path14.dirname(fileURLToPath2(import.meta.url)), "../package.json"),
|
|
45232
45278
|
"utf-8"
|
|
45233
45279
|
)
|
package/package.json
CHANGED