happyskills 0.50.0 → 0.52.0

package/src/validation/skill_md_rules.test.js

@@ -186,23 +186,26 @@ describe('validate_skill_md — optional fields', () => {
 		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', compatibility: 'x'.repeat(501) })
 		const [err, data] = await validate_skill_md(tmp, 'my-skill')
 		assert.ifError(err)
-		const check = data.results.find(r => r.field === 'compatibility')
+		const check = data.results.find(r => r.field === 'compatibility' && r.rule === 'max_length')
 		assert.strictEqual(check.severity, 'error')
 	})
 
-	it('warns for non-boolean disable-model-invocation', async () => {
+	it('errors for non-boolean disable-model-invocation (type mismatch)', async () => {
+		// Promoted from warning → error in the schema-driven validator: the
+		// loader treats this field as a YAML boolean strictly, so anything
+		// that doesn't parse as one breaks the contract.
 		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', 'disable-model-invocation': 'yes' })
 		const [err, data] = await validate_skill_md(tmp, 'my-skill')
 		assert.ifError(err)
 		const check = data.results.find(r => r.field === 'disable-model-invocation')
-		assert.strictEqual(check.severity, 'warning')
+		assert.strictEqual(check.severity, 'error')
 	})
 
 	it('warns when context is not fork', async () => {
 		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', context: 'other' })
 		const [err, data] = await validate_skill_md(tmp, 'my-skill')
 		assert.ifError(err)
-		const check = data.results.find(r => r.field === 'context')
+		const check = data.results.find(r => r.field === 'context' && r.rule === 'enum')
 		assert.strictEqual(check.severity, 'warning')
 	})
 
@@ -210,21 +213,21 @@ describe('validate_skill_md — optional fields', () => {
 		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', agent: 'my-agent' })
 		const [err, data] = await validate_skill_md(tmp, 'my-skill')
 		assert.ifError(err)
-		const check = data.results.find(r => r.field === 'agent')
+		const check = data.results.find(r => r.field === 'agent' && r.rule === 'requires_context')
 		assert.strictEqual(check.severity, 'warning')
 	})
 })
 
 describe('validate_skill_md — kit type', () => {
-	it('errors when SKILL.md is missing for a kit', async () => {
+	it('errors when README.md is missing for a kit', async () => {
 		const [err, data] = await validate_skill_md(tmp, 'my-kit', 'kit')
 		assert.ifError(err)
 		const check = data.results.find(r => r.rule === 'exists')
 		assert.strictEqual(check.severity, 'error')
 	})
 
-	it('passes when SKILL.md exists for a kit (no frontmatter required)', async () => {
-		fs.writeFileSync(path.join(tmp, 'SKILL.md'), '# My Kit\n\nThis is a kit.')
+	it('passes when README.md exists for a kit (no frontmatter required)', async () => {
+		fs.writeFileSync(path.join(tmp, 'README.md'), '# My Kit\n\nThis is a kit.')
 		const [err, data] = await validate_skill_md(tmp, 'my-kit', 'kit')
 		assert.ifError(err)
 		const exists_check = data.results.find(r => r.rule === 'exists')
@@ -234,10 +237,9 @@ describe('validate_skill_md — kit type', () => {
 	})
 
 	it('skips all frontmatter validation for kits', async () => {
-		fs.writeFileSync(path.join(tmp, 'SKILL.md'), '# My Kit\n\nNo frontmatter here.')
+		fs.writeFileSync(path.join(tmp, 'README.md'), '# My Kit\n\nNo frontmatter here.')
 		const [err, data] = await validate_skill_md(tmp, 'my-kit', 'kit')
 		assert.ifError(err)
-		// Should have no frontmatter, name, or description results
 		assert.ok(!data.results.some(r => r.rule === 'frontmatter'))
 		assert.ok(!data.results.some(r => r.field === 'name'))
 		assert.ok(!data.results.some(r => r.field === 'description'))
@@ -245,11 +247,22 @@ describe('validate_skill_md — kit type', () => {
 
 	it('skips line count validation for kits', async () => {
 		const long_content = '# My Kit\n' + 'line\n'.repeat(600)
-		fs.writeFileSync(path.join(tmp, 'SKILL.md'), long_content)
+		fs.writeFileSync(path.join(tmp, 'README.md'), long_content)
 		const [err, data] = await validate_skill_md(tmp, 'my-kit', 'kit')
 		assert.ifError(err)
 		assert.ok(!data.results.some(r => r.rule === 'line_count'))
 	})
+
+	it('errors when a kit also contains a SKILL.md', async () => {
+		fs.writeFileSync(path.join(tmp, 'README.md'), '# My Kit')
+		fs.writeFileSync(path.join(tmp, 'SKILL.md'), '# Should not be here')
+		const [err, data] = await validate_skill_md(tmp, 'my-kit', 'kit')
+		assert.ifError(err)
+		const check = data.results.find(r => r.rule === 'not_in_kit')
+		assert.ok(check)
+		assert.strictEqual(check.severity, 'error')
+		assert.strictEqual(check.file, 'SKILL.md')
+	})
 })
 
 describe('validate_skill_md — line count', () => {
@@ -270,3 +283,329 @@ describe('validate_skill_md — line count', () => {
 		assert.strictEqual(check.severity, 'pass')
 	})
 })
+
+describe('validate_skill_md — schema-driven type checks (regression: argument-hint bracket bug)', () => {
+	it('errors when argument-hint is an unquoted bracketed value (parses as YAML array)', async () => {
+		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', 'argument-hint': '[question or topic]' })
+		const [err, data] = await validate_skill_md(tmp, 'my-skill')
+		assert.ifError(err)
+		const check = data.results.find(r => r.field === 'argument-hint' && r.rule === 'type')
+		assert.strictEqual(check.severity, 'error')
+		assert.ok(/must be a string \(got array\)/.test(check.message), `got: ${check.message}`)
+		assert.ok(/Replace line 4 with: argument-hint: "\[question or topic\]"/.test(check.message), `got: ${check.message}`)
+	})
+
+	it('passes when argument-hint is properly quoted', async () => {
+		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', 'argument-hint': '"[question or topic]"' })
+		const [err, data] = await validate_skill_md(tmp, 'my-skill')
+		assert.ifError(err)
+		const errs = data.results.filter(r => r.field === 'argument-hint' && r.severity === 'error')
+		assert.deepEqual(errs, [])
+	})
+
+	it('errors when argument-hint is an unquoted flow mapping (parses as YAML object)', async () => {
+		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', 'argument-hint': '{a: 1}' })
+		const [err, data] = await validate_skill_md(tmp, 'my-skill')
+		assert.ifError(err)
+		const check = data.results.find(r => r.field === 'argument-hint' && r.rule === 'type')
+		assert.strictEqual(check.severity, 'error')
+		assert.ok(/got object/.test(check.message), `got: ${check.message}`)
+	})
+
+	it('errors (not warning) when disable-model-invocation is not actually a boolean', async () => {
+		// The schema makes type mismatches errors. The old per-field check used
+		// "warning" — we promote to error because Claude Code's loader treats
+		// the value strictly as a YAML boolean.
+		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', 'disable-model-invocation': 'yes' })
+		const [err, data] = await validate_skill_md(tmp, 'my-skill')
+		assert.ifError(err)
+		const check = data.results.find(r => r.field === 'disable-model-invocation' && r.rule === 'type')
+		assert.strictEqual(check.severity, 'error')
+	})
+
+	it('accepts disable-model-invocation: true', async () => {
+		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', 'disable-model-invocation': 'true' })
+		const [err, data] = await validate_skill_md(tmp, 'my-skill')
+		assert.ifError(err)
+		const errs = data.results.filter(r => r.field === 'disable-model-invocation' && r.severity === 'error')
+		assert.deepEqual(errs, [])
+	})
+
+	it('warns on a typo of a known frontmatter field (did-you-mean)', async () => {
+		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', argument_hint: '"[foo]"' })
+		const [err, data] = await validate_skill_md(tmp, 'my-skill')
+		assert.ifError(err)
+		const check = data.results.find(r => r.field === 'argument_hint' && r.rule === 'unknown_field')
+		assert.strictEqual(check.severity, 'warning')
+		assert.ok(/did you mean "argument-hint"\?/.test(check.message), `got: ${check.message}`)
+	})
+
+	it('does not warn on unknown fields that are not close to any known field (forward-compat)', async () => {
+		write_skill_md(tmp, { name: 'my-skill', description: 'Does things', 'some-future-anthropic-field': 'hello' })
+		const [err, data] = await validate_skill_md(tmp, 'my-skill')
+		assert.ifError(err)
+		const warnings = data.results.filter(r => r.field === 'some-future-anthropic-field')
+		assert.deepEqual(warnings, [])
+	})
+})
+
+// Error messages must be auto-correct-friendly: every frontmatter error
+// includes (1) the file, (2) the field, (3) the offending YAML line number,
+// and where applicable a structured `fix` field with the literal replacement
+// line. Without those, an LLM cannot mechanically apply the correction.
+describe('validate_skill_md — error message pinpointing (for LLM auto-correct)', () => {
+	const write_raw = (lines) => fs.writeFileSync(path.join(tmp, 'SKILL.md'), `---\n${lines.join('\n')}\n---\n\n# body\n`)
+
+	it('argument-hint bracket bug: error names the file, the field, the line, AND ships a literal fix', async () => {
+		write_raw([
+			'name: my-skill',                              // line 2
+			'description: Valid description for tests.',   // line 3
+			'argument-hint: [question or topic]'           // line 4
+		])
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const e = data.results.find(r => r.field === 'argument-hint' && r.rule === 'type' && r.severity === 'error')
+		assert.ok(e, 'expected a type error on argument-hint')
+		assert.strictEqual(e.file, 'SKILL.md')
+		assert.strictEqual(e.field, 'argument-hint')
+		assert.strictEqual(e.line, 4, 'line number must point at the actual offending YAML line')
+		assert.strictEqual(e.expected, 'string')
+		assert.strictEqual(e.actual, 'array')
+		assert.strictEqual(e.fix, 'argument-hint: "[question or topic]"',
+			'fix must be the literal corrected YAML line so an LLM can write it back verbatim')
+		assert.ok(/SKILL\.md line 4/.test(e.message), `message must lead with file:line — got: ${e.message}`)
+	})
+
+	it('multiple errors in the same file are each individually pinpointed', async () => {
+		// A pathologically broken frontmatter: bracket bug + forbidden char + wrong boolean type.
+		write_raw([
+			'name: my-skill',                                 // line 2 — ok
+			'description: Deploys to AWS: ECS and Lambda',    // line 3 — colon forbidden
+			'argument-hint: [foo]',                           // line 4 — bracket bug
+			'disable-model-invocation: maybe'                 // line 5 — boolean type mismatch
+		])
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const errors_by_field = new Map(
+			data.results.filter(r => r.severity === 'error').map(r => [r.field + '/' + r.rule, r])
+		)
+
+		const desc_err = errors_by_field.get('description/no_forbidden_characters')
+		assert.ok(desc_err, 'expected forbidden-char error on description')
+		assert.strictEqual(desc_err.line, 3)
+		assert.strictEqual(desc_err.character, ':')
+
+		const hint_err = errors_by_field.get('argument-hint/type')
+		assert.ok(hint_err, 'expected type error on argument-hint')
+		assert.strictEqual(hint_err.line, 4)
+		assert.strictEqual(hint_err.fix, 'argument-hint: "[foo]"')
+
+		const bool_err = errors_by_field.get('disable-model-invocation/type')
+		assert.ok(bool_err, 'expected type error on disable-model-invocation')
+		assert.strictEqual(bool_err.line, 5)
+		assert.strictEqual(bool_err.expected, 'boolean')
+		assert.strictEqual(bool_err.actual, 'string')
+	})
+
+	it('typo-of-known-field warning ships a structured `fix` with the corrected key + the same value', async () => {
+		write_raw([
+			'name: my-skill',
+			'description: Valid description here.',
+			'argument_hint: "[foo]"' // underscore typo
+		])
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const w = data.results.find(r => r.field === 'argument_hint' && r.rule === 'unknown_field')
+		assert.ok(w)
+		assert.strictEqual(w.line, 4)
+		assert.strictEqual(w.suggestion, 'argument-hint')
+		assert.strictEqual(w.fix, 'argument-hint: "[foo]"')
+	})
+
+	it('missing required field error includes a structured `fix` template', async () => {
+		write_raw(['name: my-skill']) // description missing
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const e = data.results.find(r => r.field === 'description' && r.rule === 'present')
+		assert.ok(e)
+		assert.strictEqual(e.fix, 'description: <value>')
+		assert.ok(/missing the required "description" field/.test(e.message))
+	})
+
+	it('forbidden-character error reports the character, its name, and its index in the value', async () => {
+		write_raw([
+			'name: my-skill',
+			'description: Hello; world'
+		])
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const e = data.results.find(r => r.field === 'description' && r.rule === 'no_forbidden_characters')
+		assert.ok(e)
+		assert.strictEqual(e.line, 3)
+		assert.strictEqual(e.character, ';')
+		assert.strictEqual(e.character_name, 'semicolon')
+		assert.strictEqual(e.char_index, 5) // "Hello" is 5 chars long; ';' is at index 5
+	})
+
+	it('agent-without-context-fork ships a structured `fix` for the missing context line', async () => {
+		write_raw([
+			'name: my-skill',
+			'description: Valid description for the agent test.',
+			'agent: my-agent'
+		])
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const w = data.results.find(r => r.field === 'agent' && r.rule === 'requires_context')
+		assert.ok(w)
+		assert.strictEqual(w.line, 4)
+		assert.strictEqual(w.fix, 'context: fork')
+	})
+
+	it('name-does-not-match-directory ships a structured `fix` line', async () => {
+		write_raw([
+			'name: wrong-name',
+			'description: Valid description here.'
+		])
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const w = data.results.find(r => r.field === 'name' && r.rule === 'matches_directory')
+		assert.ok(w)
+		assert.strictEqual(w.line, 2)
+		assert.strictEqual(w.fix, 'name: my-skill')
+	})
+
+	it('frontmatter parse error (malformed YAML line) carries the source line number', async () => {
+		// "this line has no colon" sits inside the frontmatter block and has no key:value structure.
+		fs.writeFileSync(path.join(tmp, 'SKILL.md'),
+			'---\nname: my-skill\nthis line has no colon\ndescription: ok desc here.\n---\n\nbody\n')
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const e = data.results.find(r => r.rule === 'frontmatter_parse')
+		assert.ok(e)
+		assert.strictEqual(e.severity, 'error')
+		assert.strictEqual(e.line, 3) // file line 3 — opening `---` is line 1, `name:` is line 2
+		assert.ok(/SKILL\.md line 3/.test(e.message), `got: ${e.message}`)
+	})
+})
+
+// Happy-path coverage: realistic, fully-valid skills must produce ZERO
+// errors (and, in the strict case, zero warnings). Without these, a future
+// schema tweak that accidentally rejects a legitimate field would slip
+// through every CI run.
+describe('validate_skill_md — valid skills (no false positives)', () => {
+	it('minimal valid skill: only name + description → no errors and no warnings', async () => {
+		write_skill_md(tmp, {
+			name: 'my-skill',
+			description: 'Deploy apps to AWS using ECS and Lambda'
+		})
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const errors = data.results.filter(r => r.severity === 'error')
+		const warnings = data.results.filter(r => r.severity === 'warning')
+		assert.deepEqual(errors, [], `unexpected errors: ${JSON.stringify(errors, null, 2)}`)
+		assert.deepEqual(warnings, [], `unexpected warnings: ${JSON.stringify(warnings, null, 2)}`)
+	})
+
+	it('full valid skill exercising every optional field → no errors', async () => {
+		write_skill_md(tmp, {
+			name: 'my-skill',
+			description: 'Deploy apps to AWS using ECS and Lambda — handles rollouts, rollbacks, and canary releases.',
+			'argument-hint': '"[your deploy request]"',
+			'allowed-tools': '"Bash, Read, Write, Edit"',
+			'disable-model-invocation': 'false',
+			'user-invocable': 'true',
+			compatibility: 'claude, cursor, windsurf',
+			keywords: 'deploy, aws, ecs'
+		})
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const errors = data.results.filter(r => r.severity === 'error')
+		assert.deepEqual(errors, [], `unexpected errors: ${JSON.stringify(errors, null, 2)}`)
+	})
+
+	it('valid agent skill (context: fork + agent) → no errors and no agent warning', async () => {
+		write_skill_md(tmp, {
+			name: 'my-skill',
+			description: 'A valid fork-context skill description.',
+			context: 'fork',
+			agent: 'my-agent'
+		})
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const errors = data.results.filter(r => r.severity === 'error')
+		const agent_warnings = data.results.filter(r => r.field === 'agent' && r.severity === 'warning')
+		assert.deepEqual(errors, [])
+		assert.deepEqual(agent_warnings, [])
+	})
+
+	it('valid allowed-tools as a YAML list (not a string) is accepted', async () => {
+		write_skill_md(tmp, {
+			name: 'my-skill',
+			description: 'A valid skill using allowed-tools in list form.',
+			'allowed-tools': '[Bash, Read, Write]'
+		})
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const errors = data.results.filter(r => r.severity === 'error')
+		assert.deepEqual(errors, [])
+	})
+
+	it('valid keywords as a YAML list is accepted', async () => {
+		write_skill_md(tmp, {
+			name: 'my-skill',
+			description: 'A valid skill using keywords in list form.',
+			keywords: '[deploy, aws, ecs]'
+		})
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const errors = data.results.filter(r => r.severity === 'error')
+		assert.deepEqual(errors, [])
+	})
+
+	it('valid skill emits the expected positive-confirmation pass results', async () => {
+		// Locks in the shape of the pass output an LLM/agent can rely on to
+		// confirm "yes, this skill is good to publish."
+		write_skill_md(tmp, {
+			name: 'my-skill',
+			description: 'A valid description that passes every rule cleanly.'
+		})
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const passes = data.results.filter(r => r.severity === 'pass').map(r => ({ field: r.field, rule: r.rule }))
+		// Required positive-confirmation rules. Order is not asserted — only presence.
+		const required_passes = [
+			{ field: null, rule: 'exists' },
+			{ field: null, rule: 'frontmatter' },
+			{ field: 'name', rule: 'type' },
+			{ field: 'name', rule: 'max_length' },
+			{ field: 'name', rule: 'matches_directory' },
+			{ field: 'description', rule: 'type' },
+			{ field: 'description', rule: 'max_length' },
+			{ field: 'description', rule: 'no_forbidden_characters' },
+			{ field: 'description', rule: 'soft_cap' },
+			{ field: null, rule: 'line_count' }
+		]
+		for (const expected of required_passes) {
+			assert.ok(
+				passes.some(p => p.field === expected.field && p.rule === expected.rule),
+				`missing pass for field=${expected.field} rule=${expected.rule}\nactual passes: ${JSON.stringify(passes)}`
+			)
+		}
+	})
+})
+
+// Sanity: pin the *shape* of an error so an LLM consumer can rely on the
+// contract (every error has file + field + line + message + rule + severity).
+// If the validator ever drops one of these for a fixable error, this fails.
+describe('validate_skill_md — error contract for LLM consumers', () => {
+	it('every error from a known field includes the full pinpointing contract', async () => {
+		fs.writeFileSync(path.join(tmp, 'SKILL.md'), [
+			'---',
+			'name: my-skill',
+			'description: Deploys to AWS; supports ECS', // forbidden semicolon
+			'argument-hint: [foo]',                      // bracket bug
+			'---',
+			'',
+			'# body'
+		].join('\n'))
+
+		const [, data] = await validate_skill_md(tmp, 'my-skill')
+		const field_errors = data.results.filter(r => r.severity === 'error' && r.field !== null)
+		assert.ok(field_errors.length >= 2, `expected at least 2 errors, got ${field_errors.length}`)
+
+		for (const e of field_errors) {
+			assert.strictEqual(e.file, 'SKILL.md', `error is missing file: ${JSON.stringify(e)}`)
+			assert.ok(typeof e.field === 'string' && e.field.length > 0, `error is missing field: ${JSON.stringify(e)}`)
+			assert.ok(typeof e.rule === 'string' && e.rule.length > 0, `error is missing rule: ${JSON.stringify(e)}`)
+			assert.ok(typeof e.message === 'string' && e.message.length > 0, `error is missing message: ${JSON.stringify(e)}`)
+			assert.ok(Number.isInteger(e.line) && e.line > 0, `error is missing/invalid line: ${JSON.stringify(e)}`)
+			assert.ok(e.message.startsWith(`SKILL.md line ${e.line}:`), `message must lead with "SKILL.md line N:" — got: ${e.message}`)
+		}
+	})
+})