cyber-aced 0.3.0 → 0.4.0

package/license

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Unional
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -1,44 +1,44 @@
 {
-	"name": "cyber-aced",
-	"version": "0.3.0",
-	"description": "Agent Config Evaluation & Development. Spec, evaluate, and improve agent configurations: skills, AGENTS.md sections, subagent definitions, and commands.",
-	"repository": {
-		"type": "git",
-		"url": "git+https://github.com/cyberuni/cyber-sdd.git",
-		"directory": "plugins/aced"
-	},
-	"homepage": "https://github.com/cyberuni/cyber-sdd",
-	"license": "MIT",
-	"type": "module",
-	"bin": {
-		"aced-bench": "./dist/aced-bench.js"
-	},
-	"files": [
-		"dist",
-		"skills",
-		"agents",
-		"governances",
-		".plugin",
-		".codex-plugin",
-		".claude-plugin",
-		"!skills/**/*.test.mts"
-	],
-	"dependencies": {
-		"@cyberuni/agent-harness": "0.5.0"
-	},
-	"scripts": {
-		"build": "tsdown",
-		"check:spec": "node ../sdd/skills/check-project-specs/scripts/check-project-specs.mts",
-		"test": "node --test \"skills/*/scripts/*.test.mts\"",
-		"typecheck": "tsc -p tsconfig.json"
-	},
-	"publishConfig": {
-		"access": "public"
-	},
-	"devDependencies": {
-		"tsdown": "^0.23.0"
-	},
-	"engines": {
-		"node": ">=22.11"
-	}
-}
+  "name": "cyber-aced",
+  "version": "0.4.0",
+  "description": "Agent Config Evaluation & Development. Spec, evaluate, and improve agent configurations: skills, AGENTS.md sections, subagent definitions, and commands.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cyberuni/cyber-sdd.git",
+    "directory": "plugins/aced"
+  },
+  "homepage": "https://github.com/cyberuni/cyber-sdd",
+  "license": "MIT",
+  "type": "module",
+  "bin": {
+    "aced-bench": "./dist/aced-bench.js"
+  },
+  "files": [
+    "dist",
+    "skills",
+    "agents",
+    "governances",
+    ".plugin",
+    ".codex-plugin",
+    ".claude-plugin",
+    "!skills/**/*.test.mts"
+  ],
+  "dependencies": {
+    "@cyberuni/agent-harness": "0.5.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "tsdown": "^0.23.0"
+  },
+  "engines": {
+    "node": ">=22.11"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "check:spec": "node ../sdd/skills/check-project-specs/scripts/check-project-specs.mts",
+    "test": "node --test \"skills/*/scripts/*.test.mts\"",
+    "typecheck": "tsc -p tsconfig.json"
+  }
+}

package/skills/check-freshness/README.md

@@ -21,6 +21,14 @@ A directory the run expanded carries both its own entry and a `file` entry per f
 yielded — the directory hash covers names only, so without the per-file entries an edit inside that
 directory would read `current` forever.
 
+## Measured records
+
+The bench engine writes a measured record per arm to `.agents/aced/results/bench/<suite>/` with an
+`evaluated` set in the same entry shape but no `target`. `--suite <suite> --arm <label>` finds those
+records by the `suite` and `arm` they record, picks the greatest recorded `createdAt`, and compares.
+Every recorded input — `tasks.json`, a check, a `file` arm's source — changes what was measured, so
+any mismatch is `stale`; there is no `incomplete` here.
+
 ## Why the record and not the tree
 
 The first attempt at this capability inferred the subject's dependencies from outside: it guessed
@@ -50,7 +58,8 @@ oversight.
 ## Tests
 
 `scripts/check-freshness.test.mts` — one test per frozen scenario, with fixtures chosen so a
-guessing implementation cannot pass: the result is filed under a deliberately wrong directory name,
+guessing implementation cannot pass: the result is filed under a deliberately wrong directory name (a
+measured record under a file name ending in the wrong arm label),
 alphabetical filename order is the reverse of recorded timestamp order, and one fixture rewrites a
 file's modification time while leaving its bytes identical.
 

package/skills/check-freshness/SKILL.md

@@ -26,6 +26,18 @@ node "<skill>/scripts/check-freshness.mts" --node <node-dir>
 `.feature`. Exit status is **zero only for `current`**; every other verdict, and every fail-closed
 path, exits non-zero.
 
+For a **measured record** the bench engine wrote, name the suite and arm instead:
+
+```
+node "<skill>/scripts/check-freshness.mts" --suite <suite> --arm <label>
+```
+
+It reads `.agents/aced/results/bench/<suite>/`, keeps the records whose own `suite` and `arm` fields
+match (never the file name), takes the greatest recorded `createdAt`, and compares its evaluated set —
+the suite's `tasks.json`, its `checks/`, and a `file` arm's source. A measured record has no frozen
+suite, so any moved input reads `stale`; it never reads `incomplete`. Run it from inside the
+repository. `--suite` without `--arm`, or the reverse, fails closed with no verdict.
+
 It also exposes the hashing routine, so `run` records entries with the same implementation that
 reads them back — never a second one that merely agrees today:
 

package/skills/check-freshness/scripts/check-freshness.mts

@@ -282,6 +282,119 @@ export function findRepoRoot(from: string): string | null {
 
 export const RESULTS_DIR = join('.agents', 'aced', 'results')
 
+// ─── measured records ─────────────────────────────────────────────────────────
+//
+// The bench engine writes one measured record per arm to `.agents/aced/results/bench/<suite>/`.
+// It names no `target`, so it is found by the `suite` and `arm` it RECORDS — never by its file
+// name — and compared with the same hash rule. Every recorded input moves what was measured (the
+// prompts, what `pass` means, the arm itself), so there is no frozen suite to split off and no
+// `incomplete`: any moved input reads `stale`.
+
+export const MEASURED_RESULTS_DIR = join(RESULTS_DIR, 'bench')
+const MEASURED_BENCH_DIR = join('.agents', 'aced', 'bench')
+
+export interface MeasuredRecord {
+	layer?: string
+	suite?: string
+	arm?: string
+	createdAt?: string
+	subject?: { kind?: string; from?: string }
+	evaluated?: EvaluatedEntry[]
+}
+
+export interface MeasuredScan {
+	forArm: Array<{ file: string; record: MeasuredRecord }>
+	unreadable: string[]
+}
+
+/**
+ * Keeps the measured records whose OWN `suite` and `arm` fields match. A file that parses but is
+ * not a measured record — a compare record sitting beside them — is simply not a candidate.
+ */
+export function scanMeasured(files: string[], suite: string, arm: string, read: (f: string) => string): MeasuredScan {
+	const forArm: MeasuredScan['forArm'] = []
+	const unreadable: string[] = []
+	for (const file of files) {
+		let record: MeasuredRecord
+		try {
+			record = JSON.parse(read(file)) as MeasuredRecord
+		} catch {
+			unreadable.push(file)
+			continue
+		}
+		if (record && record.layer === 'measured' && record.suite === suite && record.arm === arm)
+			forArm.push({ file, record })
+	}
+	return { forArm, unreadable }
+}
+
+/** Newest by RECORDED `createdAt`, for the same reason `selectNewest` ignores file names. */
+export function selectNewestMeasured(records: MeasuredScan['forArm']): MeasuredScan['forArm'][number] | null {
+	let best: MeasuredScan['forArm'][number] | null = null
+	for (const r of records) {
+		const t = r.record.createdAt ?? ''
+		if (!best || t > (best.record.createdAt ?? '')) best = r
+	}
+	return best
+}
+
+/**
+ * The bench engine always records the suite's `tasks.json`, and records a `file` arm's source
+ * whenever the arm reads it from a path. A set omitting either contradicts the record carrying it.
+ */
+export function measuredIncoherence(record: MeasuredRecord, suite: string): string | null {
+	const paths = new Set((record.evaluated ?? []).map((e) => e.path))
+	const tasks = join(MEASURED_BENCH_DIR, suite, 'tasks.json')
+	if (!paths.has(tasks)) {
+		return `the recorded provenance contradicts the record it accompanies: it carries no entry for ${tasks}`
+	}
+	const from = record.subject?.kind === 'file' ? record.subject.from : undefined
+	if (from?.startsWith('path:')) {
+		const source = from.slice(5)
+		if (!paths.has(source)) {
+			return `the recorded provenance omits the file arm's source: no entry for ${source}`
+		}
+	}
+	return null
+}
+
+function measuredMain(repoRoot: string, suite: string, arm: string): number {
+	process.stdout.write(`suite: ${suite}\narm: ${arm}\n`)
+
+	const dir = join(repoRoot, MEASURED_RESULTS_DIR, suite)
+	if (!existsSync(dir)) {
+		report({ verdict: 'absent', reason: 'nothing is measured for this suite', mismatched: [], skipped: [] })
+		return 1
+	}
+
+	const scan = scanMeasured(collectResultFiles(dir), suite, arm, (f) => readFileSync(f, 'utf8'))
+	const skipped = scan.unreadable.map((f) => relative(repoRoot, f))
+	const newest = selectNewestMeasured(scan.forArm)
+	if (!newest) {
+		report({
+			verdict: 'absent',
+			reason: 'no measured record is recorded for this arm',
+			mismatched: [],
+			skipped,
+		})
+		return 1
+	}
+	if (!newest.record.evaluated) {
+		report({ verdict: 'absent', reason: 'the record carries no recorded provenance', mismatched: [], skipped })
+		return 1
+	}
+	const bad = measuredIncoherence(newest.record, suite)
+	if (bad) {
+		report({ verdict: 'absent', reason: bad, mismatched: [], skipped })
+		return 1
+	}
+
+	// No frozen suite: every entry is a subject input, so the verdict is `stale` or `current`.
+	const decision = decideVerdict(newest.record, repoRoot, '', skipped)
+	report(decision)
+	return decision.verdict === 'current' ? 0 : 1
+}
+
 // ─── CLI ──────────────────────────────────────────────────────────────────────
 
 function report(d: Decision): void {
@@ -314,10 +427,30 @@ export function main(argv: string[]): number {
 		return 0
 	}
 
+	const suiteIdx = argv.indexOf('--suite')
+	const armIdx = argv.indexOf('--arm')
+	if (suiteIdx !== -1 || armIdx !== -1) {
+		// Fail closed, as the --node path does: one option without the other emits no verdict.
+		const suite = suiteIdx === -1 ? undefined : argv[suiteIdx + 1]
+		const arm = armIdx === -1 ? undefined : argv[armIdx + 1]
+		if (!suite || !arm) {
+			console.error(
+				`✗ a measured check needs both --suite <suite> and --arm <label>; missing ${suite ? '--arm' : '--suite'}; no verdict`,
+			)
+			return 1
+		}
+		const repoRoot = findRepoRoot(process.cwd())
+		if (!repoRoot) {
+			console.error('✗ no repository root above the working directory; no verdict')
+			return 1
+		}
+		return measuredMain(repoRoot, suite, arm)
+	}
+
 	const nodeIdx = argv.indexOf('--node')
 	const nodeArg = nodeIdx === -1 ? undefined : argv[nodeIdx + 1]
 	if (!nodeArg) {
-		console.error('✗ usage: check-freshness --node <node-dir>')
+		console.error('✗ usage: check-freshness --node <node-dir> | --suite <suite> --arm <label>')
 		return 1
 	}
 	const nodeDir = resolve(nodeArg)

package/skills/improve-skill/scripts/validate.mts

@@ -172,6 +172,31 @@ export function recognizedScanRoots(cwd: string, dirGlobs: string[] = []): Set<s
 	return roots
 }
 
+// Every non-hidden file under a skill's scripts/, as a /-separated path relative to it. Recurses
+// into subfolders (e.g. scripts/vendor/) so the content checks see nested scripts too; does not
+// follow a symlinked folder, so a link cycle cannot loop the walk.
+export function listScriptFiles(scriptsDir: string, rel = ''): string[] {
+	const files: string[] = []
+	for (const entry of fs.readdirSync(path.join(scriptsDir, rel), { withFileTypes: true })) {
+		if (entry.name.startsWith('.')) continue
+		const relPath = rel ? `${rel}/${entry.name}` : entry.name
+		if (entry.isDirectory()) {
+			files.push(...listScriptFiles(scriptsDir, relPath))
+		} else if (entry.isFile() || (entry.isSymbolicLink() && isFileTarget(path.join(scriptsDir, relPath)))) {
+			files.push(relPath)
+		}
+	}
+	return files.sort()
+}
+
+function isFileTarget(filePath: string): boolean {
+	try {
+		return fs.statSync(filePath).isFile()
+	} catch {
+		return false
+	}
+}
+
 export function findSkillFiles(dirs: string[], cwd: string): string[] {
 	const seen = new Set<string>()
 	const results: string[] = []
@@ -509,6 +534,9 @@ export function runChecks(filePath: string, scanRoots?: Set<string>): CheckResul
 	const stripped = stripExamples(content)
 	const invisibleInSkill = findInvisibleUnicode(content)
 	const isPartialSkill = fmInternal
+	// ADR-0031: the exact description "By name only" is the by-name mechanism, independent of
+	// user-invocable (visibility only). A visible by-name skill must not be pushed to pad it (#81).
+	const isByNameDescription = fmDesc === 'By name only'
 	const isPublicShippedSkill = parent === 'skills' && skillBaseParent !== '.agents' && !fmInternal
 
 	// S1: the SKILL.md must sit in its own named subdirectory directly under a recognized scan root.
@@ -609,7 +637,7 @@ export function runChecks(filePath: string, scanRoots?: Set<string>): CheckResul
 	}
 
 	if (fmDesc) {
-		if (!isPartialSkill) {
+		if (!isPartialSkill && !isByNameDescription) {
 			const wordCount = fmDesc.split(/\s+/).filter(Boolean).length
 			if (wordCount < 12) {
 				warn(
@@ -712,7 +740,7 @@ export function runChecks(filePath: string, scanRoots?: Set<string>): CheckResul
 	}
 
 	if (hasScripts) {
-		const scriptFiles = fs.readdirSync(scriptsDir).filter((f) => !f.startsWith('.'))
+		const scriptFiles = listScriptFiles(scriptsDir)
 		const hasInteractive = scriptFiles.some((f) => {
 			const src = fs.readFileSync(path.join(scriptsDir, f), 'utf8')
 			return /readline|createInterface|\.question\(/.test(src)
@@ -784,7 +812,7 @@ export function runChecks(filePath: string, scanRoots?: Set<string>): CheckResul
 	}
 
 	if (hasScripts) {
-		for (const f of fs.readdirSync(scriptsDir).filter((name) => !name.startsWith('.'))) {
+		for (const f of listScriptFiles(scriptsDir)) {
 			const src = fs.readFileSync(path.join(scriptsDir, f), 'utf8')
 			const invisibleInScript = findInvisibleUnicode(src)
 			if (invisibleInScript) {