happyskills 0.46.0 → 0.47.0

package/src/utils/slug_tokens.js

@@ -0,0 +1,68 @@
+// CLI mirror of api/app/utils/slug_tokens.js — used by `happyskills postlex`
+// to compute the same lexical-overlap signal the API uses for digest annotation
+// and slug-boost behavior.
+//
+// Spec 260521-01 v2 § 6 — invariant: the STOP_TOKENS list MUST stay
+// byte-identical to api/app/utils/slug_tokens.js. Drift causes the CLI's view
+// of which candidate is "exact" to disagree with the API's slug-boost, and the
+// post-lex stage promotes the wrong candidate (or none). The cross-import
+// test at slug_tokens.test.js asserts set-equality on both sides — it fails
+// CI on drift.
+//
+// If you change anything in this file, change the same thing in
+// api/app/utils/slug_tokens.js in the same commit.
+
+const STOP_TOKENS = new Set([
+	'dev', 'devs', 'labs', 'lab', 'ai', 'ml',
+	'skills', 'skill', 'agent', 'agents', 'agentic',
+	'io', 'co', 'inc', 'ltd', 'org', 'corp',
+	'team', 'group', 'project', 'projects',
+	'the', 'a', 'an', 'of', 'for', 'and', 'or',
+	'app', 'apps', 'tool', 'tools', 'kit', 'kits',
+])
+
+// Split slug-style identifier on `-_./` and whitespace, lowercase, drop
+// 1-char tokens and STOP_TOKENS. Returns a deduplicated array — order
+// preserved by first occurrence. Use slug_token_set() when you want a Set.
+const slug_tokens = (s) => {
+	if (!s || typeof s !== 'string') return []
+	const seen = new Set()
+	const out = []
+	for (const raw of s.toLowerCase().split(/[-_\s./]+/)) {
+		if (raw.length <= 1) continue
+		if (STOP_TOKENS.has(raw)) continue
+		if (seen.has(raw)) continue
+		seen.add(raw)
+		out.push(raw)
+	}
+	return out
+}
+
+const slug_token_set = (s) => new Set(slug_tokens(s))
+
+// Compute the lexical tier of a candidate name against a query
+// (spec 260521-01 v2 § 2.3):
+//   'exact'   — query token set equals name token set
+//   'strong'  — query tokens are a subset of name tokens (and not equal)
+//   'partial' — non-empty overlap but neither subset relation
+//   'none'    — no overlap or one side empty
+const compute_lex_tier = (q_tokens, name_tokens) => {
+	if (!q_tokens?.size || !name_tokens?.size) return 'none'
+	let overlap = 0
+	for (const t of q_tokens) if (name_tokens.has(t)) overlap++
+	if (overlap === 0) return 'none'
+	const q_subset = overlap === q_tokens.size
+	let n_in_q = 0
+	for (const t of name_tokens) if (q_tokens.has(t)) n_in_q++
+	const n_subset = n_in_q === name_tokens.size
+	if (q_subset && n_subset) return 'exact'
+	if (q_subset) return 'strong'
+	return 'partial'
+}
+
+module.exports = {
+	STOP_TOKENS,
+	slug_tokens,
+	slug_token_set,
+	compute_lex_tier,
+}

package/src/utils/slug_tokens.test.js

@@ -0,0 +1,126 @@
+// Unit tests + cross-import drift canary for cli/src/utils/slug_tokens.js.
+// Spec 260521-01 v2 § 6 — STOP_TOKENS MUST stay byte-identical between
+// CLI and API. The cross-import test at the bottom enforces that.
+
+const { describe, it } = require('node:test')
+const assert = require('node:assert/strict')
+const { STOP_TOKENS, slug_tokens, slug_token_set, compute_lex_tier } = require('./slug_tokens')
+
+describe('slug_tokens', () => {
+	it('splits on hyphens, underscores, dots, slashes, and whitespace', () => {
+		assert.deepEqual(slug_tokens('deploy-aws.lambda/handler edge_runtime'),
+			['deploy', 'aws', 'lambda', 'handler', 'edge', 'runtime'])
+	})
+
+	it('lowercases', () => {
+		assert.deepEqual(slug_tokens('Deploy-AWS'), ['deploy', 'aws'])
+	})
+
+	it('drops 1-character tokens', () => {
+		assert.deepEqual(slug_tokens('a-b-deploy-c'), ['deploy'])
+	})
+
+	it('drops STOP_TOKENS', () => {
+		assert.deepEqual(slug_tokens('the-deploy-tool-for-aws'), ['deploy', 'aws'])
+	})
+
+	it('deduplicates while preserving first-occurrence order', () => {
+		assert.deepEqual(slug_tokens('deploy-aws-deploy'), ['deploy', 'aws'])
+	})
+
+	it('returns [] for null / non-string / empty', () => {
+		assert.deepEqual(slug_tokens(null), [])
+		assert.deepEqual(slug_tokens(undefined), [])
+		assert.deepEqual(slug_tokens(''), [])
+		assert.deepEqual(slug_tokens(42), [])
+	})
+})
+
+describe('slug_token_set', () => {
+	it('returns a Set with the same members as slug_tokens', () => {
+		const set = slug_token_set('deploy-aws-lambda')
+		assert.ok(set instanceof Set)
+		assert.deepEqual([...set].sort(), ['aws', 'deploy', 'lambda'])
+	})
+})
+
+describe('compute_lex_tier', () => {
+	const set = s => slug_token_set(s)
+
+	it('returns "exact" when query and name token sets are equal', () => {
+		assert.equal(compute_lex_tier(set('deploy-aws'), set('deploy-aws')), 'exact')
+		assert.equal(compute_lex_tier(set('aws-deploy'), set('deploy-aws')), 'exact') // order-independent
+	})
+
+	it('returns "strong" when query tokens are a strict subset of name tokens', () => {
+		assert.equal(compute_lex_tier(set('deploy'), set('deploy-aws-lambda')), 'strong')
+	})
+
+	it('returns "partial" when there is overlap but neither is a subset', () => {
+		assert.equal(compute_lex_tier(set('deploy-aws'), set('deploy-gcp')), 'partial')
+	})
+
+	it('returns "none" when there is no overlap', () => {
+		assert.equal(compute_lex_tier(set('deploy-aws'), set('static-site')), 'none')
+	})
+
+	it('returns "none" when either side is empty', () => {
+		assert.equal(compute_lex_tier(set(''), set('deploy-aws')), 'none')
+		assert.equal(compute_lex_tier(set('deploy-aws'), set('')), 'none')
+		assert.equal(compute_lex_tier(new Set(), set('deploy-aws')), 'none')
+	})
+})
+
+describe('drift canary — cross-imports api/app/utils/slug_tokens.js', () => {
+	const api = require('../../../api/app/utils/slug_tokens')
+
+	it('STOP_TOKENS is byte-equal in both directions', () => {
+		// Forward: every CLI token exists in the API set
+		for (const t of STOP_TOKENS) {
+			assert.ok(api.STOP_TOKENS.has(t), `API STOP_TOKENS missing CLI token "${t}"`)
+		}
+		// Reverse: every API token exists in the CLI set
+		for (const t of api.STOP_TOKENS) {
+			assert.ok(STOP_TOKENS.has(t), `CLI STOP_TOKENS missing API token "${t}"`)
+		}
+		// Size also matches (defense-in-depth — a duplicate on one side would
+		// pass both forward+reverse but trip this).
+		assert.equal(STOP_TOKENS.size, api.STOP_TOKENS.size)
+	})
+
+	it('slug_tokens produces identical output across a battery of inputs', () => {
+		const inputs = [
+			'deploy-aws',
+			'Deploy-AWS.Lambda',
+			'the-deploy-tool-for-aws',
+			'a-b-c-deploy',
+			'acme/deploy-aws',
+			'underscores_and-hyphens.and/slashes',
+			'',
+			null,
+			undefined,
+			'日本語-skill',
+			'AGENTS-skill-kit',
+		]
+		for (const input of inputs) {
+			assert.deepEqual(slug_tokens(input), api.slug_tokens(input),
+				`slug_tokens drift on input ${JSON.stringify(input)}`)
+		}
+	})
+
+	it('compute_lex_tier produces identical labels across a battery of inputs', () => {
+		const pairs = [
+			['deploy-aws',            'deploy-aws'],            // exact
+			['deploy',                'deploy-aws-lambda'],     // strong
+			['deploy-aws',            'deploy-gcp'],            // partial
+			['deploy-aws',            'static-site'],           // none
+			['',                      'deploy-aws'],            // none
+			['agents-skill-the-deploy', 'deploy'],              // exact (stops drop)
+		]
+		for (const [q, n] of pairs) {
+			const cli_tier = compute_lex_tier(slug_token_set(q), slug_token_set(n))
+			const api_tier = api.compute_lex_tier(api.slug_token_set(q), api.slug_token_set(n))
+			assert.equal(cli_tier, api_tier, `compute_lex_tier drift on ("${q}", "${n}")`)
+		}
+	})
+})