claude-dev-env 8.26.4 → 8.26.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/SKILL.md +3 -3
- package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/README.md +2 -2
- package/bin/ever-shipped-skills.mjs +1 -0
- package/bin/ever-shipped-skills.test.mjs +16 -2
- package/bin/install.cursor-rules.test.mjs +60 -0
- package/bin/install.mjs +12 -1
- package/bin/install.prune.test.mjs +24 -1
- package/bin/install.transaction.test.mjs +7 -1
- package/docs/codex-compatibility.md +11 -2
- package/package.json +1 -1
- /package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/baseline-responses.json +0 -0
- /package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/baseline.md +0 -0
- /package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/cases.json +0 -0
- /package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/review_eval_support/config/constants.py +0 -0
- /package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/review_eval_support/execution.py +0 -0
- /package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/review_eval_support/grading.py +0 -0
- /package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/run.py +0 -0
- /package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/test_run.py +0 -0
- /package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/reference/build-evaluation.md +0 -0
- /package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/reference/graders-and-commands.md +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
---
|
|
2
|
-
name:
|
|
2
|
+
name: build-eval
|
|
3
3
|
description: >-
|
|
4
4
|
Evaluate skills with a bounded direct Codex review suite or `claude plugin eval`.
|
|
5
5
|
Use labeled inputs and deterministic grading for output correctness; use a plugin
|
|
@@ -7,7 +7,7 @@ description: >-
|
|
|
7
7
|
Use when the user asks to eval or test a skill.
|
|
8
8
|
---
|
|
9
9
|
|
|
10
|
-
#
|
|
10
|
+
# Build eval
|
|
11
11
|
|
|
12
12
|
For building an evaluation of an AI workflow, follow [the design process](reference/build-evaluation.md).
|
|
13
13
|
For a direct Codex output-quality evaluation, use [the review suite](evals/review/README.md).
|
|
@@ -100,7 +100,7 @@ Otherwise work the steps in order.
|
|
|
100
100
|
and the `claude -p` recipe for flag-gated features.
|
|
101
101
|
|
|
102
102
|
```text
|
|
103
|
-
|
|
103
|
+
build-eval/
|
|
104
104
|
├── SKILL.md
|
|
105
105
|
└── reference/
|
|
106
106
|
└── graders-and-commands.md
|
|
@@ -21,9 +21,9 @@ Report finding precision and recall, clean-case specificity and false-positive r
|
|
|
21
21
|
From the repository root:
|
|
22
22
|
|
|
23
23
|
```powershell
|
|
24
|
-
$suite = 'packages/claude-dev-env/.agents/skills/
|
|
24
|
+
$suite = 'packages/claude-dev-env/.agents/skills/build-eval/evals/review/run.py'
|
|
25
25
|
python $suite validate
|
|
26
|
-
python -m pytest packages/claude-dev-env/.agents/skills/
|
|
26
|
+
python -m pytest packages/claude-dev-env/.agents/skills/build-eval/evals/review/test_run.py -q
|
|
27
27
|
python $suite live --limit 2 --timeout 90 --model gpt-6.1-sol --effort low --output review-smoke
|
|
28
28
|
python $suite live --split heldout --limit 6 --timeout 90 --model gpt-6.1-sol --effort low --output review-heldout
|
|
29
29
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { strict as assert } from 'node:assert';
|
|
2
2
|
import { test } from 'node:test';
|
|
3
|
-
import { existsSync, readdirSync } from 'node:fs';
|
|
3
|
+
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { fileURLToPath } from 'node:url';
|
|
6
6
|
import { EVER_SHIPPED_SKILL_NAMES } from './ever-shipped-skills.mjs';
|
|
@@ -41,7 +41,21 @@ test('EVER_SHIPPED_SKILL_NAMES includes the windows scheduled task skill', () =>
|
|
|
41
41
|
assert.equal(EVER_SHIPPED_SKILL_NAMES.has('windows-scheduled-task'), true);
|
|
42
42
|
});
|
|
43
43
|
|
|
44
|
-
test('
|
|
44
|
+
test('build-eval ships under its matching name without the retired active directory', () => {
|
|
45
|
+
const sourceSkillsDirectory = join(
|
|
46
|
+
PACKAGE_DIRECTORY,
|
|
47
|
+
PACKAGE_AGENTS_HOME_DIRECTORY_NAME,
|
|
48
|
+
MANAGED_SKILLS_DIRECTORY_NAME,
|
|
49
|
+
);
|
|
50
|
+
assert.ok(EVER_SHIPPED_SKILL_NAMES.has('build-eval'));
|
|
51
|
+
assert.match(
|
|
52
|
+
readFileSync(join(sourceSkillsDirectory, 'build-eval', 'SKILL.md'), 'utf8'),
|
|
53
|
+
/^---\r?\nname: build-eval\r?\n/,
|
|
54
|
+
);
|
|
55
|
+
assert.equal(existsSync(join(sourceSkillsDirectory, 'plugin-eval-standalone-skill')), false);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test('EVER_SHIPPED_SKILL_NAMES retains the retired plugin eval standalone skill', () => {
|
|
45
59
|
assert.equal(EVER_SHIPPED_SKILL_NAMES.has('plugin-eval-standalone-skill'), true);
|
|
46
60
|
});
|
|
47
61
|
|
|
@@ -239,6 +239,66 @@ test('reinstall moves the old routing hook and keeps user hooks', () => {
|
|
|
239
239
|
}
|
|
240
240
|
});
|
|
241
241
|
|
|
242
|
+
test('Codex reinstall removes omitted package hooks and preserves user configuration', () => {
|
|
243
|
+
const homeDirectory = mkdtempSync(join(tmpdir(), 'cdev-codex-migration-'));
|
|
244
|
+
try {
|
|
245
|
+
const codexHome = join(homeDirectory, 'custom-codex');
|
|
246
|
+
const codexHooksPath = join(codexHome, 'hooks.json');
|
|
247
|
+
const configPath = join(codexHome, 'config.toml');
|
|
248
|
+
const disabledConfig = '[features]\nhooks = false\n';
|
|
249
|
+
const allLegacyRegistrations = [
|
|
250
|
+
['PreToolUse', 'Edit', '.claude', 'blocking/pre_tool_use_dispatcher.py'],
|
|
251
|
+
['PreToolUse', 'Bash', '.agents', 'blocking/bash_pre_tool_use_dispatcher.py'],
|
|
252
|
+
['SessionStart', '*', 'custom-codex', 'session/issue_tracker_session_starter.py'],
|
|
253
|
+
['Stop', '*', '.claude', 'session/skill_loaded_reminder.py'],
|
|
254
|
+
['SessionEnd', '*', 'custom-codex', 'lifecycle/session_end_cleanup.py'],
|
|
255
|
+
];
|
|
256
|
+
const hooksByEvent = {};
|
|
257
|
+
for (const [event, matcher, root, script] of allLegacyRegistrations) {
|
|
258
|
+
hooksByEvent[event] ??= [];
|
|
259
|
+
hooksByEvent[event].push({
|
|
260
|
+
matcher,
|
|
261
|
+
metadata: 'keep-group',
|
|
262
|
+
hooks: [
|
|
263
|
+
{ type: 'command', command: `python "${join(homeDirectory, root, 'hooks', script)}"` },
|
|
264
|
+
{ type: 'command', command: `user-${event}-${matcher}`, timeout: 7 },
|
|
265
|
+
],
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
const foreignCommand = `python "${join(homeDirectory, 'foreign', '.claude', 'hooks', 'blocking', 'pre_tool_use_dispatcher.py')}"`;
|
|
269
|
+
hooksByEvent.PreToolUse.push({ matcher: 'foreign', hooks: [{ type: 'command', command: foreignCommand }] });
|
|
270
|
+
const suffixCommand = `python "${join(codexHome, 'hooks', 'blocking', 'pre_tool_use_dispatcher.py.backup')}"`;
|
|
271
|
+
hooksByEvent.PreToolUse.push({ matcher: 'suffix', hooks: [{ type: 'command', command: suffixCommand }] });
|
|
272
|
+
hooksByEvent.CustomEvent = { metadata: 'keep-event' };
|
|
273
|
+
const expectedUserHooks = JSON.parse(JSON.stringify(hooksByEvent));
|
|
274
|
+
for (const eachGroup of Object.values(expectedUserHooks).flat()) {
|
|
275
|
+
if (eachGroup.hooks && eachGroup.metadata) eachGroup.hooks.shift();
|
|
276
|
+
}
|
|
277
|
+
hooksByEvent.LegacyValidator = [{ hooks: [{ type: 'command', command: 'python -c "sys.path.insert(0, __claude_dev_env_managed_hooks__); from validators.run_all_validators import main; main()"' }] }];
|
|
278
|
+
mkdirSync(codexHome, { recursive: true });
|
|
279
|
+
writeFileSync(configPath, disabledConfig);
|
|
280
|
+
writeFileSync(codexHooksPath, JSON.stringify({ enabled: false, metadata: 'keep-file', hooks: hooksByEvent }));
|
|
281
|
+
|
|
282
|
+
runInstaller(homeDirectory, ['--only', 'core'], { CODEX_HOME: codexHome });
|
|
283
|
+
|
|
284
|
+
const migratedHooks = JSON.parse(readFileSync(codexHooksPath, 'utf8'));
|
|
285
|
+
const allSpawnGroups = migratedHooks.hooks.PreToolUse.filter(eachGroup =>
|
|
286
|
+
['Agent|Task', 'multi_agent_v1__spawn_agent'].includes(eachGroup.matcher));
|
|
287
|
+
assert.equal(allSpawnGroups.length, 2);
|
|
288
|
+
migratedHooks.hooks.PreToolUse = migratedHooks.hooks.PreToolUse.filter(eachGroup => !allSpawnGroups.includes(eachGroup));
|
|
289
|
+
assert.deepEqual(migratedHooks, { enabled: false, metadata: 'keep-file', hooks: expectedUserHooks });
|
|
290
|
+
assert.equal(readFileSync(configPath, 'utf8'), disabledConfig);
|
|
291
|
+
const firstInstall = readFileSync(codexHooksPath, 'utf8');
|
|
292
|
+
runInstaller(homeDirectory, ['--only', 'core'], { CODEX_HOME: codexHome });
|
|
293
|
+
assert.equal(readFileSync(codexHooksPath, 'utf8'), firstInstall);
|
|
294
|
+
const claudeSettings = JSON.parse(readFileSync(join(homeDirectory, '.claude', 'settings.json'), 'utf8'));
|
|
295
|
+
assert.ok(claudeSettings.hooks.SessionStart.length > 0);
|
|
296
|
+
assert.ok(claudeSettings.hooks.PreToolUse.some(eachGroup => eachGroup.matcher === 'Bash'));
|
|
297
|
+
} finally {
|
|
298
|
+
rmSync(homeDirectory, { recursive: true, force: true });
|
|
299
|
+
}
|
|
300
|
+
});
|
|
301
|
+
|
|
242
302
|
test('uninstall removes the routing hook and keeps a Codex user hook', () => {
|
|
243
303
|
const homeDirectory = mkdtempSync(join(tmpdir(), 'cdev-routing-uninstall-'));
|
|
244
304
|
try {
|
package/bin/install.mjs
CHANGED
|
@@ -2038,6 +2038,7 @@ function mergeHooksAtPath(
|
|
|
2038
2038
|
pythonCommand,
|
|
2039
2039
|
selectHooksConfig = hooksConfig => hooksConfig,
|
|
2040
2040
|
pluginRootDir = CLAUDE_HOME,
|
|
2041
|
+
shouldPruneSourceHooks = false,
|
|
2041
2042
|
) {
|
|
2042
2043
|
const hooksSource = resolvePackageManagedDirectory(
|
|
2043
2044
|
hooksSourceRoot,
|
|
@@ -2045,8 +2046,17 @@ function mergeHooksAtPath(
|
|
|
2045
2046
|
);
|
|
2046
2047
|
const hooksJsonPath = join(hooksSource, 'hooks.json');
|
|
2047
2048
|
if (!existsSync(hooksJsonPath)) return 0;
|
|
2048
|
-
const
|
|
2049
|
+
const sourceHooksConfig = JSON.parse(readFileSync(hooksJsonPath, 'utf8'));
|
|
2049
2050
|
const settings = loadClaudeSettingsObject(settingsPath);
|
|
2051
|
+
if (shouldPruneSourceHooks && settings.hooks && typeof settings.hooks === 'object') {
|
|
2052
|
+
stripRetiredHookEntries(
|
|
2053
|
+
settings,
|
|
2054
|
+
managedHookScriptRelativePaths(sourceHooksConfig),
|
|
2055
|
+
retiredHookOwnership(),
|
|
2056
|
+
);
|
|
2057
|
+
pruneManagedHooksFromSettings(settings, new Set());
|
|
2058
|
+
}
|
|
2059
|
+
const hooksConfig = selectHooksConfig(sourceHooksConfig);
|
|
2050
2060
|
const groupCount = mergeHooksIntoSettings(settings, hooksConfig, pluginRootDir, pythonCommand);
|
|
2051
2061
|
writeFileSync(settingsPath, JSON.stringify(settings, null, indentation) + '\n');
|
|
2052
2062
|
return groupCount;
|
|
@@ -2081,6 +2091,7 @@ function mergeCodexHooks(hooksSourceRoot, pythonCommand) {
|
|
|
2081
2091
|
pythonCommand,
|
|
2082
2092
|
selectCodexNativeHooksConfig,
|
|
2083
2093
|
dirname(CODEX_HOOKS_CONFIGURATION_PATH),
|
|
2094
|
+
true,
|
|
2084
2095
|
);
|
|
2085
2096
|
}
|
|
2086
2097
|
|
|
@@ -16,7 +16,7 @@ import { fileURLToPath } from 'node:url';
|
|
|
16
16
|
import { detectPython } from './install.mjs';
|
|
17
17
|
const THIS_DIRECTORY = dirname(fileURLToPath(import.meta.url));
|
|
18
18
|
const INSTALLER_PATH = join(THIS_DIRECTORY, 'install.mjs');
|
|
19
|
-
const INSTALLER_PROCESS_TIMEOUT_MS =
|
|
19
|
+
const INSTALLER_PROCESS_TIMEOUT_MS = 120_000;
|
|
20
20
|
const EXCLUDED_PACKAGE_COPY_DIRECTORY = 'node_modules';
|
|
21
21
|
|
|
22
22
|
const RETIRED_SKILL_DIRECTORIES = [
|
|
@@ -24,6 +24,7 @@ const RETIRED_SKILL_DIRECTORIES = [
|
|
|
24
24
|
'fixbugs',
|
|
25
25
|
'pr-scope-resolve',
|
|
26
26
|
'post-audit-findings',
|
|
27
|
+
'plugin-eval-standalone-skill',
|
|
27
28
|
'pr-consistency-audit',
|
|
28
29
|
'bdd-protocol',
|
|
29
30
|
'hitl',
|
|
@@ -607,6 +608,28 @@ test('a full reinstall over a pre-manifest dirty tree prunes retired skills and
|
|
|
607
608
|
}
|
|
608
609
|
});
|
|
609
610
|
|
|
611
|
+
test('a full install replaces the retired plugin eval name with build-eval', () => {
|
|
612
|
+
const sandbox = createSandbox();
|
|
613
|
+
const retiredSkillName = 'plugin-eval-standalone-skill';
|
|
614
|
+
try {
|
|
615
|
+
plantSkillDirectory(sandbox.skillsDirectory, retiredSkillName, true);
|
|
616
|
+
|
|
617
|
+
runInstaller(sandbox.homeDirectory, []);
|
|
618
|
+
|
|
619
|
+
assert.equal(existsSync(join(sandbox.skillsDirectory, retiredSkillName)), false);
|
|
620
|
+
assert.ok(prunedSkillBackupContains(sandbox.claudeDirectory, retiredSkillName));
|
|
621
|
+
assert.match(
|
|
622
|
+
readFileSync(join(sandbox.skillsDirectory, 'build-eval', 'SKILL.md'), 'utf8'),
|
|
623
|
+
/^---\r?\nname: build-eval\r?\n/,
|
|
624
|
+
);
|
|
625
|
+
const manifest = readManifest(sandbox.manifestPath);
|
|
626
|
+
assert.ok(manifest.skills.includes('build-eval'));
|
|
627
|
+
assert.equal(manifest.skills.includes(retiredSkillName), false);
|
|
628
|
+
} finally {
|
|
629
|
+
rmSync(sandbox.homeDirectory, { recursive: true, force: true });
|
|
630
|
+
}
|
|
631
|
+
});
|
|
632
|
+
|
|
610
633
|
test('a full reinstall over an old-format manifest without a skills key still prunes via the ever-shipped fallback', () => {
|
|
611
634
|
const sandbox = createSandbox();
|
|
612
635
|
try {
|
|
@@ -373,7 +373,13 @@ test('installer fault after_settings_write restores prior settings and files', (
|
|
|
373
373
|
const { claudeDirectory, rulesFile } = seedPriorInstall(homeDirectory);
|
|
374
374
|
const priorSettings = readFileSync(join(claudeDirectory, 'settings.json'), 'utf8');
|
|
375
375
|
const codexHooksPath = join(homeDirectory, '.codex', 'hooks.json');
|
|
376
|
-
writeFileWithParents(codexHooksPath,
|
|
376
|
+
writeFileWithParents(codexHooksPath, JSON.stringify({
|
|
377
|
+
enabled: false,
|
|
378
|
+
hooks: {
|
|
379
|
+
PreToolUse: [{ matcher: 'custom' }],
|
|
380
|
+
SessionEnd: [{ hooks: [{ type: 'command', command: `python "${join(homeDirectory, '.codex', 'hooks', 'lifecycle', 'session_end_cleanup.py')}"` }] }],
|
|
381
|
+
},
|
|
382
|
+
}) + '\n');
|
|
377
383
|
const priorCodexHooks = readFileSync(codexHooksPath, 'utf8');
|
|
378
384
|
|
|
379
385
|
const failedRun = runInstaller(homeDirectory, [], {
|
|
@@ -26,8 +26,17 @@ Run `codex-compat bridge --surface <name> --payload '<json-object>'`. The bridge
|
|
|
26
26
|
|
|
27
27
|
The package installer seeds `subagent-model-policy.json` in the agents-home
|
|
28
28
|
`rules/` directory when the file is absent. It registers one `PreToolUse` group
|
|
29
|
-
for `multi_agent_v1__spawn_agent` in `$CODEX_HOME/hooks.json`.
|
|
30
|
-
|
|
29
|
+
for `multi_agent_v1__spawn_agent` and one for `Agent|Task` in `$CODEX_HOME/hooks.json`.
|
|
30
|
+
Before adding these groups, the installer removes obsolete package-owned hook
|
|
31
|
+
registrations from every event. Custom hook entries and Codex's enabled or
|
|
32
|
+
disabled hook settings stay unchanged. Claude-only hook groups do not enter the
|
|
33
|
+
Codex file.
|
|
34
|
+
|
|
35
|
+
A detached Windows Codex daemon can open a console for each background command.
|
|
36
|
+
If this happens, run local interactive sessions with `codex --no-daemon` and keep
|
|
37
|
+
`codex exec` commands unchanged. [Codex issue #44768](https://github.com/openai/codex/issues/44768)
|
|
38
|
+
describes the symptom. [PR #49164](https://github.com/openai/codex/pull/49164)
|
|
39
|
+
adds the upstream console fix.
|
|
31
40
|
|
|
32
41
|
Edit the installed policy file to change the next routing decision. The resolver
|
|
33
42
|
loads that path for each invocation.
|
package/package.json
CHANGED
|
File without changes
|
/package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/baseline.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
/package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/evals/review/test_run.py
RENAMED
|
File without changes
|
/package/.agents/skills/{plugin-eval-standalone-skill → build-eval}/reference/build-evaluation.md
RENAMED
|
File without changes
|
|
File without changes
|