@grafana/create-plugin 7.12.3 → 7.14.0

package/src/codemods/agentic/index.test.ts

@@ -0,0 +1,342 @@
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { Addition, PromptOnlyAddition } from '../additions/additions.js';
+import { Context } from '../context.js';
+import { selectPrompt } from '../../utils/utils.console.js';
+import { AGENT_RUNS_DIR } from './handoff.js';
+import {
+  ABORT_CHOICE,
+  buildAdditionTodosWarning,
+  CONTINUE_CHOICE,
+  prepareAgenticAddition,
+  runAgenticStep,
+} from './index.js';
+import { resolveAgenticMode } from './resolve.js';
+import { runAgentSession } from './runner.js';
+import { AgentSessionResult, InstalledAgent } from './types.js';
+
+vi.mock('./resolve.js');
+vi.mock('./runner.js');
+vi.mock('../../utils/utils.console.js', () => ({
+  output: {
+    log: vi.fn(),
+    warning: vi.fn(),
+    logSingleLine: vi.fn(),
+    statusList: vi.fn(),
+  },
+  confirmPrompt: vi.fn(),
+  selectPrompt: vi.fn(),
+}));
+vi.mock('../../utils/utils.packageManager.js', () => ({
+  getPackageManagerWithFallback: () => ({ packageManagerName: 'npm', packageManagerVersion: '11.0.0' }),
+  getPackageManagerSilentInstallCmd: () => 'npm install --silent',
+  getPackageManagerExecCmd: () => 'npx -y',
+}));
+
+const resolveMock = vi.mocked(resolveAgenticMode);
+const runAgentSessionMock = vi.mocked(runAgentSession);
+const selectPromptMock = vi.mocked(selectPrompt);
+
+const fakeAgent: InstalledAgent = {
+  definition: {
+    id: 'claude-code',
+    displayName: 'Claude Code',
+    binaryNames: ['claude'],
+    buildInteractive: (invocationContext) => ({
+      args: ['--sys', invocationContext.systemPrompt, invocationContext.userPrompt],
+      env: { FAKE_AGENT: '1' },
+    }),
+  },
+  binaryPath: '/usr/local/bin/claude',
+};
+
+function handoffResult(status: 'success' | 'failed', summary: string): AgentSessionResult {
+  return { outcome: 'handoff', handoff: { status, summary } };
+}
+
+let basePath: string;
+let promptOnly: PromptOnlyAddition;
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  basePath = mkdtempSync(join(tmpdir(), 'cp-agentic-test-'));
+  const promptPath = join(basePath, 'rspack-overrides.md');
+  writeFileSync(promptPath, '# Port things');
+  promptOnly = {
+    name: 'rspack-overrides',
+    description: 'Port custom things.',
+    prompt: pathToFileURL(promptPath).href,
+  };
+});
+
+afterEach(() => {
+  rmSync(basePath, { recursive: true, force: true });
+});
+
+describe('prepareAgenticAddition', () => {
+  const scriptOnly: Addition = {
+    name: 'externalize-jsx-runtime',
+    description: 'No prompt here.',
+    scriptPath: 'file:///virtual/externalize-jsx-runtime.js',
+  };
+
+  it('should return undefined for an addition with no prompt', async () => {
+    const resolution = await prepareAgenticAddition(scriptOnly, undefined);
+
+    expect(resolution).toBeUndefined();
+    expect(resolveMock).not.toHaveBeenCalled();
+  });
+
+  it('should return the resolution when an agent is enabled', async () => {
+    resolveMock.mockResolvedValue({ mode: 'enabled', agent: fakeAgent });
+
+    await expect(prepareAgenticAddition(promptOnly, undefined)).resolves.toEqual({
+      mode: 'enabled',
+      agent: fakeAgent,
+    });
+  });
+
+  it.each([
+    ['opted-out via flag', { mode: 'opted-out', reason: 'flag' } as const],
+    ['opted-out via decline', { mode: 'opted-out', reason: 'declined' } as const],
+    ['inside an agent', { mode: 'inside-agent' } as const],
+  ])('should return the resolution when %s', async (_label, resolution) => {
+    resolveMock.mockResolvedValue(resolution);
+
+    await expect(prepareAgenticAddition(promptOnly, undefined)).resolves.toEqual(resolution);
+  });
+
+  it('should throw when no agent is installed', async () => {
+    resolveMock.mockResolvedValue({ mode: 'unavailable', reason: 'no-agents' });
+
+    await expect(prepareAgenticAddition(promptOnly, undefined)).rejects.toThrow(/needs an AI agent/);
+  });
+
+  it('should throw when there is no interactive terminal', async () => {
+    resolveMock.mockResolvedValue({ mode: 'unavailable', reason: 'no-tty' });
+
+    await expect(prepareAgenticAddition(promptOnly, undefined)).rejects.toThrow(/interactive terminal/);
+  });
+
+  it('should name every way forward when no agent is available', async () => {
+    resolveMock.mockResolvedValue({ mode: 'unavailable', reason: 'no-agents' });
+
+    await expect(prepareAgenticAddition(promptOnly, undefined)).rejects.toThrow(/--agent=<path>/);
+    await expect(prepareAgenticAddition(promptOnly, undefined)).rejects.toThrow(/--no-agent/);
+    await expect(prepareAgenticAddition(promptOnly, undefined)).rejects.toThrow(/rspack-overrides\.md/);
+  });
+
+  it('should pass the agent flag through to the resolver', async () => {
+    resolveMock.mockResolvedValue({ mode: 'enabled', agent: fakeAgent });
+
+    await prepareAgenticAddition(promptOnly, '/opt/homebrew/bin/claude');
+
+    expect(resolveMock).toHaveBeenCalledWith({ agentFlag: '/opt/homebrew/bin/claude' });
+  });
+});
+
+describe('runAgenticStep', () => {
+  function runStep(overrides: Partial<Parameters<typeof runAgenticStep>[0]> = {}) {
+    return runAgenticStep({
+      addition: promptOnly,
+      agent: fakeAgent,
+      basePath,
+      treeWasDirty: false,
+      ...overrides,
+    });
+  }
+
+  function createContextWithChanges() {
+    const context = new Context(basePath);
+    context.addFile('rspack.config.ts', 'export default {};');
+    return context;
+  }
+
+  it('should report the handoff summary when the agent succeeds', async () => {
+    runAgentSessionMock.mockResolvedValue(handoffResult('success', 'Ported the overrides.'));
+
+    await expect(runStep()).resolves.toEqual({ kind: 'applied', summary: 'Ported the overrides.' });
+  });
+
+  it('should throw when the agent hands off a failure', async () => {
+    runAgentSessionMock.mockResolvedValue(handoffResult('failed', 'Could not parse the config.'));
+
+    await expect(runStep()).rejects.toThrow(/Could not parse the config./);
+  });
+
+  it('should throw when the user aborts the session', async () => {
+    runAgentSessionMock.mockResolvedValue({ outcome: 'user-aborted' });
+
+    await expect(runStep()).rejects.toThrow(/aborted/);
+  });
+
+  it('should spawn the agent with its own invocation', async () => {
+    runAgentSessionMock.mockResolvedValue(handoffResult('success', 'done'));
+
+    await runStep();
+
+    expect(runAgentSessionMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        binaryPath: '/usr/local/bin/claude',
+        cwd: basePath,
+        env: { FAKE_AGENT: '1' },
+      })
+    );
+  });
+
+  it('should create the run directory under the agent runs cache', async () => {
+    runAgentSessionMock.mockResolvedValue(handoffResult('success', 'done'));
+
+    await runStep();
+
+    expect(existsSync(join(basePath, AGENT_RUNS_DIR))).toBe(true);
+    const handoffPath = runAgentSessionMock.mock.calls[0][0].handoffPath;
+    expect(handoffPath).toContain(AGENT_RUNS_DIR);
+    expect(handoffPath).toContain('rspack-overrides.json');
+  });
+
+  describe('prompt content', () => {
+    beforeEach(() => {
+      runAgentSessionMock.mockResolvedValue(handoffResult('success', 'done'));
+    });
+
+    function getUserPrompt() {
+      return runAgentSessionMock.mock.calls[0][0].args[2];
+    }
+
+    it('should omit the codemod changes section when there is no codemod half', async () => {
+      await runStep();
+
+      expect(getUserPrompt()).not.toContain('<codemod_changes>');
+    });
+
+    it('should point the agent at the git diff when the tree started clean', async () => {
+      await runStep({ context: createContextWithChanges(), treeWasDirty: false });
+
+      expect(getUserPrompt()).toContain('<codemod_changes>');
+      expect(getUserPrompt()).toContain('git diff');
+      expect(getUserPrompt()).not.toContain('<files_changed>');
+    });
+
+    it('should embed the changed file list when --force let a dirty tree through', async () => {
+      await runStep({ context: createContextWithChanges(), treeWasDirty: true });
+
+      expect(getUserPrompt()).toContain('<files_changed>');
+      expect(getUserPrompt()).toContain('rspack.config.ts');
+    });
+
+    it('should embed the instructions file contents', async () => {
+      await runStep();
+
+      expect(getUserPrompt()).toContain('# Port things');
+    });
+  });
+
+  async function getRejectionMessage(promise: Promise<unknown>): Promise<string> {
+    try {
+      await promise;
+    } catch (error) {
+      return error instanceof Error ? error.message : String(error);
+    }
+    throw new Error('Expected the agent step to throw');
+  }
+
+  describe('when a hybrid agent step does not finish', () => {
+    const unfinishedSessions = [
+      ['the agent hands off a failure', () => handoffResult('failed', 'Could not parse the config.')],
+      ['the user aborts the session', () => ({ outcome: 'user-aborted' as const })],
+    ] as const;
+
+    unfinishedSessions.forEach(([scenario, createResult]) => {
+      it(`should explain how to recover the automated changes when ${scenario}`, async () => {
+        runAgentSessionMock.mockResolvedValue(createResult());
+
+        const message = await getRejectionMessage(runStep({ context: createContextWithChanges() }));
+
+        expect(message).toMatch(/automated part of rspack-overrides has already changed your working tree/);
+        expect(message).toContain('git diff');
+        expect(message).toMatch(/rspack-overrides\.md/);
+        expect(message).toContain('git restore . && git clean -fd');
+      });
+    });
+
+    it('should explain recovery when the user aborts after an ambiguous exit', async () => {
+      runAgentSessionMock.mockResolvedValue({ outcome: 'ambiguous-exit', exitCode: 1, signal: null });
+      selectPromptMock.mockResolvedValue(ABORT_CHOICE);
+
+      await expect(runStep({ context: createContextWithChanges() })).rejects.toThrow(
+        /already changed your working tree/
+      );
+    });
+
+    it('should not suggest discarding changes when --force let a dirty tree through', async () => {
+      runAgentSessionMock.mockResolvedValue({ outcome: 'user-aborted' });
+
+      const message = await getRejectionMessage(runStep({ context: createContextWithChanges(), treeWasDirty: true }));
+
+      expect(message).toContain('git diff');
+      expect(message).not.toContain('git clean');
+    });
+
+    it('should add no recovery note for a prompt-only addition', async () => {
+      runAgentSessionMock.mockResolvedValue({ outcome: 'user-aborted' });
+
+      const message = await getRejectionMessage(runStep());
+
+      expect(message).not.toContain('working tree');
+    });
+  });
+
+  describe('ambiguous exit', () => {
+    beforeEach(() => {
+      runAgentSessionMock.mockResolvedValue({ outcome: 'ambiguous-exit', exitCode: 1, signal: null });
+    });
+
+    it('should assume the step was applied when the user chooses to continue', async () => {
+      selectPromptMock.mockResolvedValue(CONTINUE_CHOICE);
+
+      await expect(runStep()).resolves.toEqual({ kind: 'assumed-applied' });
+    });
+
+    it('should throw when the user chooses to abort', async () => {
+      selectPromptMock.mockResolvedValue(ABORT_CHOICE);
+
+      await expect(runStep()).rejects.toThrow(/aborted/);
+    });
+
+    it('should treat a cancelled prompt as an abort', async () => {
+      selectPromptMock.mockRejectedValue(new Error(''));
+
+      await expect(runStep()).rejects.toThrow(/aborted/);
+    });
+  });
+});
+
+describe('buildAdditionTodosWarning', () => {
+  it('should return nothing when the addition left no markers', () => {
+    expect(buildAdditionTodosWarning('rspack', [])).toBeUndefined();
+  });
+
+  it('should list each marker with its location', () => {
+    const warning = buildAdditionTodosWarning('rspack', [
+      { file: 'rspack.config.ts', line: 12, text: '// TODO(rspack): port CorsWorkerPlugin' },
+      { file: 'build/utils.ts', line: 3, text: '// TODO(rspack): swc plugin ABI mismatch' },
+    ]);
+
+    expect(warning).toEqual({
+      title: 'rspack left 2 items that need your attention.',
+      body: [
+        'rspack.config.ts:12  // TODO(rspack): port CorsWorkerPlugin',
+        'build/utils.ts:3  // TODO(rspack): swc plugin ABI mismatch',
+      ],
+    });
+  });
+
+  it('should use the singular for one marker', () => {
+    const warning = buildAdditionTodosWarning('rspack', [{ file: 'a.ts', line: 1, text: '// TODO(rspack): x' }]);
+
+    expect(warning?.title).toBe('rspack left 1 item that needs your attention.');
+  });
+});

package/src/codemods/agentic/index.ts

@@ -0,0 +1,221 @@
+import { output, selectPrompt } from '../../utils/utils.console.js';
+import {
+  getPackageManagerExecCmd,
+  getPackageManagerSilentInstallCmd,
+  getPackageManagerWithFallback,
+} from '../../utils/utils.packageManager.js';
+import { Addition, hasPromptStep } from '../additions/additions.js';
+import { Context } from '../context.js';
+import { createRunId, ensureRunDir, getHandoffPath, getRunDir, wipeAgentRuns } from './handoff.js';
+import {
+  buildHybridUserPrompt,
+  buildPromptOnlyUserPrompt,
+  buildSystemPrompt,
+  CodemodChanges,
+  getPromptPath,
+  loadPromptInstructions,
+} from './prompts.js';
+import { resolveAgenticMode } from './resolve.js';
+import { runAgentSession } from './runner.js';
+import { AgenticResolution, AgenticStepResult, AgentSessionResult, InstalledAgent } from './types.js';
+import type { AdditionTodo } from '../../utils/utils.git.js';
+
+export const ABORT_CHOICE = 'Abort the addition';
+export const CONTINUE_CHOICE = 'Treat the agent step as completed';
+
+export interface AgenticStepOptions {
+  addition: Addition & { prompt: string };
+  agent: InstalledAgent;
+  // present for hybrid additions, absent for prompt-only
+  context?: Context;
+  basePath: string;
+  // true when --force let a dirty tree through, so the working diff is not ours alone
+  treeWasDirty: boolean;
+}
+
+// Resolves the agent before any codemod work runs, so a missing agent can never leave an addition
+// half-applied. Returns undefined when the addition needs no agent, and throws when it needs one that
+// is not available.
+export async function prepareAgenticAddition(
+  addition: Addition,
+  agentFlag: string | boolean | undefined
+): Promise<AgenticResolution | undefined> {
+  if (!hasPromptStep(addition)) {
+    return undefined;
+  }
+
+  const resolution = await resolveAgenticMode({ agentFlag });
+  if (resolution.mode === 'unavailable') {
+    throw new Error(buildUnavailableMessage(addition, resolution.reason));
+  }
+
+  return resolution;
+}
+
+export async function runAgenticStep(options: AgenticStepOptions): Promise<AgenticStepResult> {
+  const { addition, agent, context, basePath, treeWasDirty } = options;
+  const instructionsPath = getPromptPath(addition.prompt);
+
+  // a unique dir per run means a stale handoff from an earlier run can never be read back
+  wipeAgentRuns(basePath);
+  const runDir = getRunDir(basePath, createRunId());
+  ensureRunDir(runDir);
+  const handoffPath = getHandoffPath(runDir, addition.name);
+
+  const { packageManagerName, packageManagerVersion } = getPackageManagerWithFallback();
+  const systemPrompt = buildSystemPrompt({
+    workspaceRoot: basePath,
+    packageManagerName,
+    packageManagerVersion,
+    installCmd: getPackageManagerSilentInstallCmd(packageManagerName, packageManagerVersion),
+    execCmd: getPackageManagerExecCmd(packageManagerName, packageManagerVersion),
+    handoffPath,
+  });
+
+  const userPromptOptions = {
+    addition: { name: addition.name, description: addition.description },
+    instructions: loadPromptInstructions(addition.prompt),
+    instructionsPath,
+    handoffPath,
+  };
+  const userPrompt = context
+    ? buildHybridUserPrompt({ ...userPromptOptions, codemodChanges: getCodemodChanges(context, treeWasDirty) })
+    : buildPromptOnlyUserPrompt(userPromptOptions);
+
+  output.log({
+    title: `Handing ${addition.name} to ${agent.definition.displayName} (${agent.binaryPath}).`,
+    body: [
+      'The session is interactive: you can watch and redirect the agent.',
+      'It ends automatically once the agent reports completion.',
+    ],
+  });
+
+  const invocation = agent.definition.buildInteractive({
+    systemPrompt,
+    userPrompt,
+    workspaceRoot: basePath,
+    handoffDir: runDir,
+  });
+
+  const sessionResult = await runAgentSession({
+    binaryPath: agent.binaryPath,
+    args: invocation.args,
+    env: invocation.env,
+    cwd: basePath,
+    handoffPath,
+  });
+
+  // a hybrid codemod has already written its changes, so an unfinished agent step leaves them behind
+  const withRecoveryNote = (message: string) =>
+    context ? [message, '', ...buildHybridRecoveryLines(addition, treeWasDirty)].join('\n') : message;
+
+  if (sessionResult.outcome === 'handoff') {
+    if (sessionResult.handoff.status === 'failed') {
+      throw new Error(
+        withRecoveryNote(`The agent reported failure for ${addition.name}: ${sessionResult.handoff.summary}`)
+      );
+    }
+    return { kind: 'applied', summary: sessionResult.handoff.summary };
+  }
+
+  if (sessionResult.outcome === 'user-aborted') {
+    throw new Error(withRecoveryNote(`The agent session for ${addition.name} was aborted.`));
+  }
+
+  return handleAmbiguousExit(addition.name, sessionResult, withRecoveryNote);
+}
+
+function buildHybridRecoveryLines(addition: Addition & { prompt: string }, treeWasDirty: boolean): string[] {
+  const lines = [
+    `The automated part of ${addition.name} has already changed your working tree.`,
+    `Review the changes with git diff, then apply the instructions in ${getPromptPath(addition.prompt)} yourself.`,
+  ];
+  // only safe to suggest when the tree was clean before the run, otherwise it discards the user's own work too
+  if (!treeWasDirty) {
+    lines.push('To undo the automated changes instead, run git restore . && git clean -fd');
+  }
+  return lines;
+}
+
+function buildUnavailableMessage(addition: Addition & { prompt: string }, reason: 'no-tty' | 'no-agents'): string {
+  const cause =
+    reason === 'no-tty' ? 'this is not an interactive terminal' : 'no supported agent CLI was found on your PATH';
+
+  return [
+    `${addition.name} needs an AI agent to apply its instructions, and ${cause}.`,
+    '',
+    'Ways forward:',
+    '  - install a supported agent CLI and run this again',
+    '  - pass --agent=<path> if an agent is installed outside your PATH',
+    `  - pass --no-agent to skip the agent, then apply ${getPromptPath(addition.prompt)} by hand`,
+  ].join('\n');
+}
+
+async function handleAmbiguousExit(
+  additionName: string,
+  sessionResult: Extract<AgentSessionResult, { outcome: 'ambiguous-exit' }>,
+  withRecoveryNote: (message: string) => string
+): Promise<AgenticStepResult> {
+  output.warning({
+    title: `The agent session for ${additionName} ended without reporting a result.`,
+    body: [
+      `Exit code: ${sessionResult.exitCode ?? 'none'}, signal: ${sessionResult.signal ?? 'none'}.`,
+      'Inspect the working tree before deciding how to proceed.',
+    ],
+  });
+
+  const choice = await safeSelect('How should the addition proceed?', [ABORT_CHOICE, CONTINUE_CHOICE]);
+  if (choice === CONTINUE_CHOICE) {
+    return { kind: 'assumed-applied' };
+  }
+
+  throw new Error(
+    withRecoveryNote(`The addition was aborted after the agent session for ${additionName} ended unexpectedly.`)
+  );
+}
+
+function getCodemodChanges(context: Context, treeWasDirty: boolean): CodemodChanges {
+  // add requires a clean tree, so after the codemod flushes the whole working tree diff is ours.
+  // --force can bypass that check, and then the diff includes changes we did not make.
+  if (!treeWasDirty) {
+    return { kind: 'git-diff' };
+  }
+
+  const files = Object.entries(context.listChanges()).map(([path, change]) => ({
+    path,
+    changeType: change.changeType,
+  }));
+  return { kind: 'file-list', files };
+}
+
+async function safeSelect(message: string, choices: string[]): Promise<string> {
+  try {
+    return await selectPrompt(message, choices);
+  } catch {
+    // enquirer rejects when the prompt is cancelled (ctrl+c) — treat as an abort
+    return ABORT_CHOICE;
+  }
+}
+
+export interface AdditionTodosWarning {
+  title: string;
+  body: string[];
+}
+
+// the handoff summary is the agent's own account. the markers are read from the tree, so anything
+// the agent could not apply is reported even when its summary leaves it out
+export function buildAdditionTodosWarning(
+  additionName: string,
+  todos: AdditionTodo[]
+): AdditionTodosWarning | undefined {
+  if (todos.length === 0) {
+    return undefined;
+  }
+
+  const itemsNeed = todos.length === 1 ? 'item that needs' : 'items that need';
+
+  return {
+    title: `${additionName} left ${todos.length} ${itemsNeed} your attention.`,
+    body: todos.map(({ file, line, text }) => `${file}:${line}  ${text}`),
+  };
+}