@claude-flow/cli 3.51.0 → 3.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/helpers/.helpers-version +1 -1
- package/.claude/helpers/helpers.manifest.json +3 -3
- package/.claude/helpers/hook-handler.cjs +4 -3
- package/.claude/helpers/router.cjs +0 -0
- package/README.md +20 -1
- package/catalog-manifest.json +2 -2
- package/dist/src/commands/federation.js +2 -2
- package/dist/src/commands/guidance.js +43 -2
- package/dist/src/guidance/mod-projection.d.ts +120 -0
- package/dist/src/guidance/mod-projection.js +155 -0
- package/dist/src/guidance/mod-sources.d.ts +22 -0
- package/dist/src/guidance/mod-sources.js +123 -0
- package/dist/src/init/helpers-generator.js +79 -6
- package/dist/src/mcp-tools/seraphina-tools.js +15 -12
- package/dist/src/mcp-tools/service-url.d.ts +2 -0
- package/dist/src/mcp-tools/service-url.js +34 -0
- package/dist/src/mcp-tools/x-federation-tools.js +5 -4
- package/package.json +1 -1
- package/plugins/ruflo-metaharness/.claude-plugin/plugin.json +20 -4
- package/plugins/ruflo-metaharness/README.md +12 -0
- package/plugins/ruflo-metaharness/hooks/guard.ts +27 -0
- package/plugins/ruflo-metaharness/hooks/hooks.json +4 -0
- package/plugins/ruflo-metaharness/hooks/options.ts +14 -0
- package/plugins/ruflo-metaharness/hooks/register.ts +108 -0
- package/plugins/ruflo-metaharness/hooks/screen.ts +260 -0
- package/plugins/ruflo-metaharness/scripts/bench-parse-mcp-scan.mjs +13 -4
- package/plugins/ruflo-metaharness/scripts/smoke.sh +29 -7
- package/plugins/ruflo-metaharness/tests/register.test.ts +117 -0
- package/plugins/ruflo-metaharness/tests/textsof.test.ts +22 -0
- package/plugins/ruflo-metaharness/tsconfig.json +3 -0
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure text screening for the AgentDB mod (ADR-445). Two jobs: find secrets (so none is stored) and find prompt-injection phrasing (so
|
|
3
|
+
* retrieved memory cannot instruct the model). Findings are NAMES only: the matched text is never returned, logged or counted by value.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
// BEGIN SHARED SCREEN (generated from plugins/ruflo-agentdb/hooks/screen.ts by scripts/sync-mod-screen.mjs; do not edit in a copy)
|
|
7
|
+
export type Rules = readonly (readonly [string, RegExp])[]
|
|
8
|
+
|
|
9
|
+
/** The secret shapes every mod screens for. A plugin adds its own after these, outside the markers. */
|
|
10
|
+
export const COMMON_SECRETS: Rules = [
|
|
11
|
+
['private key', /-----BEGIN [A-Z ]*PRIVATE KEY-----/],
|
|
12
|
+
['aws access key', /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/],
|
|
13
|
+
['github token', /\b(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{40,})\b/],
|
|
14
|
+
['slack token', /\bxox[abprs]-[A-Za-z0-9-]{10,}/],
|
|
15
|
+
['slack webhook', /\bhooks\.slack\.com\/services\/T[A-Z0-9]{6,}\/B[A-Z0-9]{6,}\/[A-Za-z0-9]{16,}/],
|
|
16
|
+
['google api key', /\bAIza[0-9A-Za-z_-]{35}\b/],
|
|
17
|
+
['anthropic or openai key', /\bsk-(?:(?:ant|proj|svcacct|admin)-[A-Za-z0-9_-]{20,}|(?=[A-Za-z]{0,40}\d)[A-Za-z0-9]{32,})/],
|
|
18
|
+
['stripe key', /\b[rs]k_live_[A-Za-z0-9]{16,}/],
|
|
19
|
+
['npm token', /\bnpm_[A-Za-z0-9]{36}\b/],
|
|
20
|
+
['huggingface token', /\bhf_[A-Za-z0-9]{30,}\b/],
|
|
21
|
+
['sendgrid key', /\bSG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}/],
|
|
22
|
+
['twilio key', /\bSK[0-9a-f]{32}\b/],
|
|
23
|
+
['jwt', /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/],
|
|
24
|
+
['bearer token', /\bBearer\s+([A-Za-z0-9._~+/=-]{24,})/],
|
|
25
|
+
['database url with credentials', /\b[a-z][a-z0-9+.-]{1,20}:\/\/[^\s:@/]+:[^\s@/]{3,}@[^\s/]+/i],
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
export const INJECTION: Rules = [
|
|
29
|
+
['override instructions', /\b(?:ignore|disregard|forget|override)\b[^.\n]{0,40}\b(?:previous|prior|above|earlier|all|any|system)\b[^.\n]{0,30}\b(?:instructions?|rules?|prompts?|guidelines?)\b/i],
|
|
30
|
+
['role reassignment', /\byou are (?:now|no longer)\b|\bact as (?:an? )?(?:unrestricted|jailbroken)\b/i],
|
|
31
|
+
['new instructions', /\b(?:new|updated|real) (?:system )?instructions?\s*:/i],
|
|
32
|
+
['fake role tags', /<\/?\s*(?:system|assistant|developer|instructions?)\s*>|^\s*(?:system|assistant)\s*:/im],
|
|
33
|
+
['concealment', /\bdo not (?:tell|inform|mention|reveal)[^.\n]{0,30}\b(?:user|human|operator)\b/i],
|
|
34
|
+
['exfiltration', /\b(?:exfiltrate|send|post|upload)\b[^.\n]{0,50}\b(?:secrets?|credentials?|tokens?|api keys?|\.env)\b/i],
|
|
35
|
+
['shell pipe', /\b(?:curl|wget)\b[^|\n]{0,200}\|\s*(?:sudo\s+)?(?:ba|z)?sh\b/i],
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
// C0/C1 controls (keeping tab and newline), DEL, soft hyphen, combining grapheme joiner, Arabic letter mark, Hangul and Mongolian fillers/separators,
|
|
39
|
+
// zero-width, bidi (overrides and isolates) and invisible-format characters, variation selectors; built with escapes, never raw.
|
|
40
|
+
const INVISIBLE = new RegExp(
|
|
41
|
+
'[\\u0000-\\u0008\\u000b-\\u001f\\u007f-\\u009f\\u00ad\\u034f\\u061c\\u115f\\u1160\\u17b4\\u17b5\\u180b-\\u180e\\u200b-\\u200f\\u2028-\\u202e\\u2060-\\u206f\\u3164\\ufe00-\\ufe0f\\ufeff\\uffa0\\ufff9-\\ufffb]',
|
|
42
|
+
'g',
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
/** Longest input scanned in one pass; a longer one keeps its head and tail halves. One regex pass per rule, so cost stays linear. */
|
|
46
|
+
const MAX_SCAN = 200_000
|
|
47
|
+
|
|
48
|
+
/** Input bounded to MAX_SCAN characters with invisible characters removed, so none can hide a secret or a phrase. */
|
|
49
|
+
export const bare = (text: string) =>
|
|
50
|
+
(text.length > MAX_SCAN ? text.slice(0, MAX_SCAN / 2) + '\n' + text.slice(-MAX_SCAN / 2) : text).replace(INVISIBLE, '')
|
|
51
|
+
|
|
52
|
+
// A value is a secret CANDIDATE only when it is not a reference (env var, call, identifier path, placeholder, secret-manager path) and its
|
|
53
|
+
// shape is random enough: at least two character classes, one of them a digit or symbol, and Shannon entropy of at least 2.5 bits per character.
|
|
54
|
+
const PLACEHOLDER = /placeholder|your[-_ ]|example|changeme|change[-_]?me|redacted|dummy|replace[-_]?me|insert[-_]|\*{3,}|x{5,}|\.{3}|^(?:none|null|undefined|true|false)$/i
|
|
55
|
+
const REFERENCE =
|
|
56
|
+
/^(?:\$(?:\{[^}]*\}|\(|[A-Za-z_]\w*$)|%[^%]*%$|<[^>]*>$|\{\{|process\.env|os\.environ|env[.[]|import\.meta|System\.getenv|secrets?\.|vault:|op:\/\/|ref\+|arn:|projects\/[^/]+\/secrets\/|gcp:|kms:|aws:|file:)/i
|
|
57
|
+
const CALL = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\(|^[A-Za-z_$][\w$]*\[/
|
|
58
|
+
const IDENT_PATH = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+$/
|
|
59
|
+
const UUID = /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/i
|
|
60
|
+
const NAME_LIKE = /^[a-z][a-z0-9]*(?:[-_./][a-z0-9]+){2,}$/
|
|
61
|
+
|
|
62
|
+
function entropy(v: string): number {
|
|
63
|
+
const counts = new Map<string, number>()
|
|
64
|
+
for (const ch of v) counts.set(ch, (counts.get(ch) ?? 0) + 1)
|
|
65
|
+
let h = 0
|
|
66
|
+
for (const n of counts.values()) h -= (n / v.length) * Math.log2(n / v.length)
|
|
67
|
+
return h
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** True when `v` is a name, call, path or placeholder rather than a literal credential. */
|
|
71
|
+
function isReference(v: string): boolean {
|
|
72
|
+
if (PLACEHOLDER.test(v) || REFERENCE.test(v) || CALL.test(v) || IDENT_PATH.test(v) || URL_NO_CREDS.test(v)) return true
|
|
73
|
+
return NAME_LIKE.test(v) && v.replace(/\D/g, '').length / v.length < 0.15
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function plausibleSecret(v: string): boolean {
|
|
77
|
+
if (v.length < 8 || v.length > 256 || /\s/.test(v) || UUID.test(v) || isReference(v)) return false
|
|
78
|
+
const symbol = /[^A-Za-z0-9]/.test(v)
|
|
79
|
+
const digit = /\d/.test(v)
|
|
80
|
+
const classes = [/[a-z]/.test(v), /[A-Z]/.test(v), digit, symbol].filter(Boolean).length
|
|
81
|
+
return classes >= 2 && (digit || symbol) && entropy(v) >= 2.5
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Under a secret-named key a literal this long is a secret even with one character class or a UUID shape, unless it is a clear reference. */
|
|
85
|
+
const KEYED_MIN = 20
|
|
86
|
+
const PLACEHOLDER_WORD = /(?:^|[^a-z])(?:your|placeholder|changeme|change[-_]?me|example|redacted|dummy|replace[-_]?me|insert)(?:[^a-z]|$)|\*{3,}|x{5,}|\.{3}|^(?:none|null|undefined|true|false)$/i
|
|
87
|
+
const URL_NO_CREDS = /^[a-z][a-z0-9+.-]{1,20}:\/\/[^\s@]*$/i
|
|
88
|
+
|
|
89
|
+
/** Three or more lowercase hyphen-separated words (no hex or digit-only run of 8+, few digits), such as my-k8s-secret-name-for-database. */
|
|
90
|
+
function hyphenName(v: string): boolean {
|
|
91
|
+
const parts = v.split('-')
|
|
92
|
+
return parts.length >= 3 && parts.every(p => /^[a-z0-9]{2,}$/.test(p) && !/^[0-9a-f]{8,}$/.test(p)) && v.replace(/\D/g, '').length / v.length < 0.15
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function keyedSecret(v: string): boolean {
|
|
96
|
+
if (v.length < KEYED_MIN || v.length > 256 || /\s/.test(v)) return false
|
|
97
|
+
return !(PLACEHOLDER_WORD.test(v) || REFERENCE.test(v) || CALL.test(v) || IDENT_PATH.test(v) || URL_NO_CREDS.test(v) || (!UUID.test(v) && hyphenName(v)))
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const KEY_NAME = /(?:api[_-]?key|secret|token|passw(?:or)?d|passwd|pwd|credential|private[_-]?key|auth(?!or))s?[A-Za-z0-9_-]{0,40}["']?\s*[:=]\s*/gi
|
|
101
|
+
const QUOTED = /(["'\x60])((?:(?!\1)[^\n]){1,256})\1/y
|
|
102
|
+
const BARE_VALUE = /[^\s"'\x60,;]{1,256}/y
|
|
103
|
+
const QUERY_VALUE = /[^\s"'\x60,;&]{1,256}/y
|
|
104
|
+
|
|
105
|
+
/** A secret-named key assigned a literal value: env style, JSON, YAML, code. Values that are calls, references or placeholders do not count. */
|
|
106
|
+
function assignmentSecret(text: string): boolean {
|
|
107
|
+
let valueEnd = 0
|
|
108
|
+
for (const m of text.matchAll(KEY_NAME)) {
|
|
109
|
+
if (m.index < valueEnd) continue // a key-looking word inside the previous value, such as secretsmanager in an ARN
|
|
110
|
+
const at = m.index + m[0].length
|
|
111
|
+
let back = m.index
|
|
112
|
+
while (back > 0 && m.index - back < 64 && /[A-Za-z0-9_.-]/.test(text.charAt(back - 1))) back--
|
|
113
|
+
const re = /["'\x60]/.test(text.charAt(at)) ? QUOTED : /[?&]/.test(text.charAt(back - 1)) ? QUERY_VALUE : BARE_VALUE
|
|
114
|
+
re.lastIndex = at
|
|
115
|
+
const hit = re.exec(text)
|
|
116
|
+
const v = hit && (hit[2] ?? hit[0])
|
|
117
|
+
valueEnd = hit ? at + hit[0].length : at
|
|
118
|
+
if (v && (plausibleSecret(v) || keyedSecret(v))) return true
|
|
119
|
+
}
|
|
120
|
+
return false
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** A password in a URL's userinfo that is not a placeholder such as user:password or ${DB_PASSWORD}. */
|
|
124
|
+
function urlCredential(url: string): boolean {
|
|
125
|
+
const pass = /^[^:]+:\/\/[^\s:@/]+:([^\s@/]+)@/.exec(url)?.[1]
|
|
126
|
+
if (!pass || /\$\{|\{\{|%\(|%s/.test(pass)) return false
|
|
127
|
+
return !/^(?:password|passwd|pass|pwd|secret|changeme|dbpassword|db_password|\$\w*|<.*>|\{.*\}|\*+|x+)$/i.test(pass) && !PLACEHOLDER.test(pass)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const CHECKS: Readonly<Record<string, (m: RegExpMatchArray) => boolean>> = {
|
|
131
|
+
'bearer token': m => !isReference(m[1] ?? ''),
|
|
132
|
+
'database url with credentials': m => urlCredential(m[0]),
|
|
133
|
+
'database url with password': m => urlCredential(m[0]),
|
|
134
|
+
}
|
|
135
|
+
const globals = new WeakMap<RegExp, RegExp>()
|
|
136
|
+
|
|
137
|
+
function matches(name: string, re: RegExp, text: string): boolean {
|
|
138
|
+
if (name === 'key assignment') return assignmentSecret(text)
|
|
139
|
+
const check = CHECKS[name]
|
|
140
|
+
if (!check) return re.test(text)
|
|
141
|
+
let g = globals.get(re)
|
|
142
|
+
if (!g) globals.set(re, (g = new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g')))
|
|
143
|
+
for (const m of text.matchAll(g)) if (check(m)) return true
|
|
144
|
+
return false
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* The text textsOf appends when it had to drop input (a node, character or per-string budget ran out). It is never matched against a rule:
|
|
149
|
+
* `names` reports it as a finding of its own, so every guard that asks "is there a secret in these texts" refuses what it could not read in full.
|
|
150
|
+
*/
|
|
151
|
+
export const TRUNCATED = 'ruflo-screen: input exceeded the screening budget'
|
|
152
|
+
export const TRUNCATED_NAME = 'input too large to screen'
|
|
153
|
+
|
|
154
|
+
/** Names of the rules that match `text` (already bare'd). A rule named 'key assignment' is judged by assignmentSecret, whatever its regex. */
|
|
155
|
+
export const names = (rules: Rules, text: string) => text === TRUNCATED ? [TRUNCATED_NAME] : rules.filter(([name, re]) => matches(name, re, text)).map(([name]) => name)
|
|
156
|
+
|
|
157
|
+
export type Findings = { readonly secrets: readonly string[]; readonly injection: readonly string[] }
|
|
158
|
+
|
|
159
|
+
/** Names of every secret shape in `secrets` and every injection phrase found in `text`. Cost is linear in the capped input. */
|
|
160
|
+
export function screenWith(secrets: Rules, text: string): Findings {
|
|
161
|
+
const bounded = bare(text)
|
|
162
|
+
return { secrets: names(secrets, bounded), injection: names(INJECTION, bounded) }
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export const hasSecretIn = (secrets: Rules, text: string) => names(secrets, bare(text)).length > 0
|
|
166
|
+
|
|
167
|
+
/** Makes stored text safe to show: no control or bidi characters, whitespace collapsed, at most `max` characters. */
|
|
168
|
+
export function tidy(text: string, max: number): string {
|
|
169
|
+
const flat = text.replace(INVISIBLE, '').replace(/\s+/g, ' ').trim()
|
|
170
|
+
return flat.length > max ? `${flat.slice(0, Math.max(0, max - 1))}…` : flat
|
|
171
|
+
}
|
|
172
|
+
/** Bounds for textsOf: nodes visited, characters returned, the longest string read in full, the size of one returned chunk, chunk overlap. */
|
|
173
|
+
export type TextLimits = { readonly nodes?: number; readonly chars?: number; readonly perString?: number }
|
|
174
|
+
const NODES = 20_000
|
|
175
|
+
const CHARS = 2_000_000
|
|
176
|
+
const PER_STRING = 1_500_000
|
|
177
|
+
const OVERLAP = 2_048
|
|
178
|
+
const BARE_KEY_MIN = 8
|
|
179
|
+
|
|
180
|
+
/** A string as texts the screen can read whole: one text up to MAX_SCAN, else overlapping MAX_SCAN windows so a secret anywhere is inside one. */
|
|
181
|
+
function windows(text: string, out: string[]): void {
|
|
182
|
+
if (text.length <= MAX_SCAN) {
|
|
183
|
+
out.push(text)
|
|
184
|
+
return
|
|
185
|
+
}
|
|
186
|
+
for (let at = 0; ; at += MAX_SCAN - OVERLAP) {
|
|
187
|
+
out.push(text.slice(at, at + MAX_SCAN))
|
|
188
|
+
if (at + MAX_SCAN >= text.length) return
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Every string in a tool input, for the screen to read: iterative (no recursion, so nesting 5000 deep cannot overflow the stack) and
|
|
194
|
+
* breadth-first (siblings before depth, so a long list cannot hide a nested value). A string under an object key comes back as `key=value`,
|
|
195
|
+
* so a secret-named key is judged with its value; a key whose value is not a string is returned bare. Strings longer than the screen window
|
|
196
|
+
* come back as overlapping windows; one over `perString` keeps its head and tail. Work is bounded by `nodes` slots and `chars` characters.
|
|
197
|
+
* Anything dropped (slots or characters ran out, or a string lost its middle) is reported by a final TRUNCATED text, which `names` and
|
|
198
|
+
* `hasSecretIn` count as a finding, so the screen fails closed instead of passing what it did not read.
|
|
199
|
+
*/
|
|
200
|
+
export function textsOf(input: unknown, limits: TextLimits = {}): string[] {
|
|
201
|
+
const out: string[] = []
|
|
202
|
+
let slots = limits.nodes ?? NODES
|
|
203
|
+
let chars = limits.chars ?? CHARS
|
|
204
|
+
const perString = limits.perString ?? PER_STRING
|
|
205
|
+
let truncated = false
|
|
206
|
+
const take = (text: string): void => {
|
|
207
|
+
if (chars <= 0) {
|
|
208
|
+
truncated = true
|
|
209
|
+
return
|
|
210
|
+
}
|
|
211
|
+
if (text.length <= Math.min(perString, chars)) {
|
|
212
|
+
chars -= text.length
|
|
213
|
+
windows(text, out)
|
|
214
|
+
return
|
|
215
|
+
}
|
|
216
|
+
truncated = true
|
|
217
|
+
const half = Math.floor(Math.min(perString, chars) / 2)
|
|
218
|
+
chars -= 2 * half
|
|
219
|
+
windows(text.slice(0, half), out)
|
|
220
|
+
windows(text.slice(-half), out)
|
|
221
|
+
}
|
|
222
|
+
const queue: unknown[] = [input]
|
|
223
|
+
let head = 0
|
|
224
|
+
for (; head < queue.length && chars > 0; head++) {
|
|
225
|
+
const node = queue[head]
|
|
226
|
+
if (typeof node === 'string') take(node)
|
|
227
|
+
else if (Array.isArray(node)) {
|
|
228
|
+
let i = 0
|
|
229
|
+
for (; i < node.length && slots > 0; i++, slots--) if (i in node) queue.push(node[i])
|
|
230
|
+
if (i < node.length) truncated = true
|
|
231
|
+
} else if (typeof node === 'object' && node !== null) {
|
|
232
|
+
for (const k in node) {
|
|
233
|
+
if (slots-- <= 0) {
|
|
234
|
+
truncated = true
|
|
235
|
+
break
|
|
236
|
+
}
|
|
237
|
+
if (!Object.prototype.hasOwnProperty.call(node, k)) continue
|
|
238
|
+
const v = (node as Record<string, unknown>)[k]
|
|
239
|
+
if (typeof v === 'string') queue.push(k + '=' + v)
|
|
240
|
+
else {
|
|
241
|
+
if (k.length >= BARE_KEY_MIN) take(k)
|
|
242
|
+
queue.push(v)
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
if (queue.length > head) truncated = true
|
|
248
|
+
if (truncated) out.push(TRUNCATED)
|
|
249
|
+
return out
|
|
250
|
+
}
|
|
251
|
+
// END SHARED SCREEN
|
|
252
|
+
|
|
253
|
+
const SECRETS: Rules = [
|
|
254
|
+
...COMMON_SECRETS,
|
|
255
|
+
['key assignment', /\b(?:api[_-]?key|secret|token|passw(?:or)?d|credential)s?["']?\s*[:=]\s*["']?[A-Za-z0-9/+=_.-]{16,}/i],
|
|
256
|
+
]
|
|
257
|
+
|
|
258
|
+
export const scan = (text: string): Findings => screenWith(SECRETS, text)
|
|
259
|
+
|
|
260
|
+
export const hasSecret = (text: string) => hasSecretIn(SECRETS, text)
|
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
// node scripts/bench-parse-mcp-scan.mjs # default 100k iters
|
|
19
19
|
// node scripts/bench-parse-mcp-scan.mjs --iters 1000000
|
|
20
20
|
// node scripts/bench-parse-mcp-scan.mjs --format json
|
|
21
|
-
// node scripts/bench-parse-mcp-scan.mjs --max-mean-us 5 # CI gate
|
|
21
|
+
// node scripts/bench-parse-mcp-scan.mjs --max-mean-us 5 # CI gate (on the mean)
|
|
22
|
+
// node scripts/bench-parse-mcp-scan.mjs --max-mean-us 5 --gate-stat p50 # same ceiling on the median: robust to a loaded runner
|
|
22
23
|
//
|
|
23
24
|
// EXIT CODES
|
|
24
25
|
// 0 ok (or threshold satisfied)
|
|
@@ -33,15 +34,21 @@ guardCliArgs(import.meta.url, {
|
|
|
33
34
|
'--iters': 'value',
|
|
34
35
|
'--format': 'value',
|
|
35
36
|
'--max-mean-us': 'value',
|
|
37
|
+
'--gate-stat': 'value',
|
|
36
38
|
});
|
|
37
39
|
|
|
38
40
|
const ARGS = (() => {
|
|
39
|
-
const a = { iters: 100_000, format: 'table', maxMeanUs: null };
|
|
41
|
+
const a = { iters: 100_000, format: 'table', maxMeanUs: null, gateStat: 'mean' };
|
|
40
42
|
for (let i = 2; i < process.argv.length; i++) {
|
|
41
43
|
const v = process.argv[i];
|
|
42
44
|
if (v === '--iters') a.iters = parseInt(process.argv[++i], 10);
|
|
43
45
|
else if (v === '--format') a.format = process.argv[++i];
|
|
44
46
|
else if (v === '--max-mean-us') a.maxMeanUs = parseFloat(process.argv[++i]);
|
|
47
|
+
else if (v === '--gate-stat') a.gateStat = process.argv[++i];
|
|
48
|
+
}
|
|
49
|
+
if (a.gateStat !== 'mean' && a.gateStat !== 'p50') {
|
|
50
|
+
console.error(`bench-parse-mcp-scan: --gate-stat must be mean or p50; got ${a.gateStat}`);
|
|
51
|
+
process.exit(2);
|
|
45
52
|
}
|
|
46
53
|
return a;
|
|
47
54
|
})();
|
|
@@ -113,9 +120,11 @@ const results = [
|
|
|
113
120
|
let gate = { triggered: false, reasons: [] };
|
|
114
121
|
if (ARGS.maxMeanUs != null) {
|
|
115
122
|
for (const r of results) {
|
|
116
|
-
|
|
123
|
+
// p50 is the typical-call cost: one scheduler pause on a loaded CI runner moves the mean (iter 86 gate flaked on it) but not the median.
|
|
124
|
+
const value = ARGS.gateStat === 'p50' ? r.p50Ms * 1000 : r.meanUs;
|
|
125
|
+
if (value > ARGS.maxMeanUs) {
|
|
117
126
|
gate.triggered = true;
|
|
118
|
-
gate.reasons.push(`${r.label}:
|
|
127
|
+
gate.reasons.push(`${r.label}: ${ARGS.gateStat} ${value.toFixed(3)}μs > ceiling ${ARGS.maxMeanUs}μs`);
|
|
119
128
|
}
|
|
120
129
|
}
|
|
121
130
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
|
-
# Structural smoke test for ruflo-metaharness v0.
|
|
2
|
+
# Structural smoke test for ruflo-metaharness v0.2.3 (ADR-150 Phase 1).
|
|
3
3
|
set -u
|
|
4
4
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
5
5
|
PASS=0
|
|
@@ -32,10 +32,10 @@ else
|
|
|
32
32
|
bad "extraction-regex-rot: EXPECTED_TOOLS=$EXPECTED_TOOLS EXPECTED_SUBS=$EXPECTED_SUBS"
|
|
33
33
|
fi
|
|
34
34
|
|
|
35
|
-
step "1. plugin.json declares 0.
|
|
35
|
+
step "1. plugin.json declares 0.2.3 with adr-150 keywords"
|
|
36
36
|
v=$(grep -E '"version"' "$ROOT/.claude-plugin/plugin.json" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
|
37
|
-
if [[ "$v" != "0.
|
|
38
|
-
bad "expected 0.
|
|
37
|
+
if [[ "$v" != "0.2.3" ]]; then
|
|
38
|
+
bad "expected 0.2.3, got '$v'"
|
|
39
39
|
else
|
|
40
40
|
miss=""
|
|
41
41
|
for k in ruflo metaharness harness scorecard genome mcp-scan threat-model router adr-150 adr-148 adr-149 optional-dependency graceful-degradation subprocess phase-1-mvp; do
|
|
@@ -1066,10 +1066,13 @@ done
|
|
|
1066
1066
|
grep -q "from './_harness.mjs'" "$F" 2>/dev/null || miss="$miss not-using-production-parser"
|
|
1067
1067
|
# Gate flag exposed
|
|
1068
1068
|
grep -q -- "--max-mean-us" "$F" 2>/dev/null || miss="$miss no-gate-flag"
|
|
1069
|
-
# Runtime: bench
|
|
1070
|
-
|
|
1069
|
+
# Runtime: the parse bench stays an order of magnitude inside its regression ceiling. 30us on the MEDIAN: the CI runner measured p50 6.05us for the 5-finding payload (local 1us), so the old 5us ceiling failed there on speed alone; 30us is 5x the CI median and still trips on a 10x regression.
|
|
1070
|
+
# The median, not the mean: a scheduler pause on a loaded CI runner moves the mean and flaked this step in the fleet smoke.
|
|
1071
|
+
grep -q -- "--gate-stat" "$F" 2>/dev/null || miss="$miss no-gate-stat-flag"
|
|
1072
|
+
# On failure keep the bench's own output on stderr (the fleet JSON report captures stderrTail): the reason was invisible in CI before.
|
|
1073
|
+
bench_out=$(node "$F" --iters 10000 --max-mean-us 30 --gate-stat p50 2>&1) || { miss="$miss runtime-fails-or-perf-blew-30us"; printf '%s\n' "$bench_out" | tail -15 >&2; }
|
|
1071
1074
|
# Runtime: gate trips on absurd ceiling
|
|
1072
|
-
if node "$F" --iters 10000 --max-mean-us 0.0001 >/dev/null 2>&1; then
|
|
1075
|
+
if node "$F" --iters 10000 --max-mean-us 0.0001 --gate-stat p50 >/dev/null 2>&1; then
|
|
1073
1076
|
miss="$miss gate-failed-to-trip"
|
|
1074
1077
|
fi
|
|
1075
1078
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
@@ -2449,5 +2452,24 @@ LOADER="$ROOT/../../v3/@claude-flow/cli/src/commands/index.ts"
|
|
|
2449
2452
|
grep -q "eject: () => import" "$LOADER" 2>/dev/null || miss="$miss not-registered-in-loader"
|
|
2450
2453
|
[[ -z "$miss" ]] && ok || bad "$miss"
|
|
2451
2454
|
|
|
2455
|
+
# M1-M3. The mod (ADR-445): hooks module registered, files within the 500-line rule, no network or process access in the hooks
|
|
2456
|
+
step "M1. mod: hooks.json names register.ts, every hook file is present and under 500 lines"
|
|
2457
|
+
mod_ok=1
|
|
2458
|
+
grep -q '"./register.ts"' "$ROOT/hooks/hooks.json" || mod_ok=0
|
|
2459
|
+
for f in options screen guard register; do
|
|
2460
|
+
[[ -f "$ROOT/hooks/$f.ts" ]] || mod_ok=0
|
|
2461
|
+
[[ $(wc -l < "$ROOT/hooks/$f.ts" 2>/dev/null || echo 9999) -le 500 ]] || mod_ok=0
|
|
2462
|
+
done
|
|
2463
|
+
[[ $mod_ok -eq 1 ]] && ok || bad "mod hooks incomplete or too long"
|
|
2464
|
+
|
|
2465
|
+
step "M2. mod: guard defaults on (userConfig), the command name is metaharness-mod"
|
|
2466
|
+
node -e '
|
|
2467
|
+
const c = require(process.argv[1]).userConfig || {}
|
|
2468
|
+
process.exit(c.guard && c.guard.default === "on" ? 0 : 1)
|
|
2469
|
+
' "$ROOT/.claude-plugin/plugin.json" && grep -q "'metaharness-mod'" "$ROOT/hooks/register.ts" && ok || bad "userConfig defaults or command name wrong"
|
|
2470
|
+
|
|
2471
|
+
step "M3. mod: hooks never touch the network or spawn a process"
|
|
2472
|
+
if grep -nE '\$\.(http|process)\.|child_process|fetch\(' "$ROOT"/hooks/*.ts >/dev/null; then bad "network or process call in hooks"; else ok; fi
|
|
2473
|
+
|
|
2452
2474
|
printf "\n%s passed, %s failed\n" "$PASS" "$FAIL"
|
|
2453
2475
|
[[ $FAIL -eq 0 ]] || exit 1
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import type { On } from 'claude-code'
|
|
2
|
+
import { describe, expect, test, tier } from 'claude-code/testing'
|
|
3
|
+
|
|
4
|
+
tier('user')
|
|
5
|
+
|
|
6
|
+
const ROOT = '/work'
|
|
7
|
+
const P = 'mcp__plugin_ruflo-core_ruflo__'
|
|
8
|
+
const START = { surface: 'terminal', isInteractive: true, cwd: ROOT } as const
|
|
9
|
+
const slash = (args: string) => ({ command: 'metaharness-mod', args, origin: { kind: 'composer' }, presentation: { isFullscreen: false, columns: 100 } }) as const
|
|
10
|
+
const TOOLS = ["metaharness_score","metaharness_flywheel"].map(name => ({ name: P + name, description: '', mcp: true }))
|
|
11
|
+
|
|
12
|
+
/** The world beneath the mod: a project root, a file map, the connected tools, and a stub that answers every tool call. */
|
|
13
|
+
function world(on: On, tools = TOOLS) {
|
|
14
|
+
const files = new Map<string, string>()
|
|
15
|
+
const registered: string[] = []
|
|
16
|
+
on('session.start', ($, e) => ({ cwd: e.cwd }))
|
|
17
|
+
on('session.root', () => ({ value: ROOT }))
|
|
18
|
+
on('command.register', ($, e) => (registered.push(e.name), { value: { command: e.name } }))
|
|
19
|
+
on('fs.write', ($, e) => (files.set(e.path, e.text), { value: undefined }))
|
|
20
|
+
on('tool.list', () => ({ value: tools }))
|
|
21
|
+
on('clock.now', () => ({ value: Date.now() }))
|
|
22
|
+
on('tool.call', () => ({ result: 'ran' }))
|
|
23
|
+
return { files, registered, status: () => JSON.parse(files.get(`${ROOT}/.claude-flow/metaharness-mod/status.json`) ?? '{}') }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const attempt = async ($: Parameters<Parameters<typeof test>[2]>[0], call: object) =>
|
|
27
|
+
JSON.stringify(await $.tool.call(call as never).then(r => r, (e: unknown) => ({ text: String(e) })))
|
|
28
|
+
|
|
29
|
+
describe('guard', () => {
|
|
30
|
+
test('refuses: flywheel promote without confirm, and never echoes the secret', async ($, on) => {
|
|
31
|
+
world(on)
|
|
32
|
+
await $.session.start(START)
|
|
33
|
+
const out = await attempt($, { tool: P + 'metaharness_flywheel', operation: 'promote', receiptId: 'r1' })
|
|
34
|
+
expect(out).not.toContain('ran')
|
|
35
|
+
expect(out).toContain('ruflo-metaharness')
|
|
36
|
+
expect(out).not.toContain('ghp_')
|
|
37
|
+
expect(out).not.toContain('abcd1234')
|
|
38
|
+
expect(out).not.toContain('Zx9Qm2')
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
test('refuses: secret in an input, and never echoes the secret', async ($, on) => {
|
|
42
|
+
world(on)
|
|
43
|
+
await $.session.start(START)
|
|
44
|
+
const out = await attempt($, { tool: P + 'metaharness_score', path: `/repo token ghp_${'a1B2'.repeat(10)}` })
|
|
45
|
+
expect(out).not.toContain('ran')
|
|
46
|
+
expect(out).toContain('ruflo-metaharness')
|
|
47
|
+
expect(out).not.toContain('ghp_')
|
|
48
|
+
expect(out).not.toContain('abcd1234')
|
|
49
|
+
expect(out).not.toContain('Zx9Qm2')
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
test('passes good calls through', async ($, on) => {
|
|
53
|
+
world(on)
|
|
54
|
+
await $.session.start(START)
|
|
55
|
+
expect(await attempt($, { tool: P + 'metaharness_flywheel', operation: 'promote', receiptId: 'r1', confirm: true })).toContain('ran') // promote with confirm
|
|
56
|
+
expect(await attempt($, { tool: P + 'metaharness_flywheel', operation: 'status' })).toContain('ran') // flywheel status
|
|
57
|
+
expect(await attempt($, { tool: P + 'metaharness_score', path: '.' })).toContain('ran') // score
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
test('turning that option off lets the call through', { options: { guard: 'off' } }, async ($, on) => {
|
|
61
|
+
world(on)
|
|
62
|
+
await $.session.start(START)
|
|
63
|
+
expect(await attempt($, { tool: P + 'metaharness_flywheel', operation: 'promote', receiptId: 'r1' })).toContain('ran')
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
test('ignores tools that belong to someone else', async ($, on) => {
|
|
67
|
+
world(on)
|
|
68
|
+
await $.session.start(START)
|
|
69
|
+
expect(await attempt($, { tool: P + 'some_other_tool', value: `token ghp_${'a1B2'.repeat(10)}` })).toContain('ran')
|
|
70
|
+
})
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
describe('status file and /metaharness-mod', () => {
|
|
74
|
+
test('the status file is written at session start with the shape the console reads', async ($, on) => {
|
|
75
|
+
const w = world(on)
|
|
76
|
+
await $.session.start(START)
|
|
77
|
+
expect(w.registered).toEqual(['metaharness-mod'])
|
|
78
|
+
expect(w.status()).toMatchObject({ version: 1, guard: true, total: 0, blocked: 0, calls: {} })
|
|
79
|
+
expect(typeof w.status().updatedMs).toBe('number')
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
test('counters move when own tools run and when the guard refuses', async ($, on) => {
|
|
83
|
+
const w = world(on)
|
|
84
|
+
await $.session.start(START)
|
|
85
|
+
await attempt($, { tool: P + 'metaharness_score', path: '.' })
|
|
86
|
+
await attempt($, { tool: P + 'metaharness_flywheel', operation: 'promote', receiptId: 'r1' })
|
|
87
|
+
expect(w.status()).toMatchObject({ total: 1, blocked: 1 })
|
|
88
|
+
expect(Object.keys(w.status().calls)).toHaveLength(1)
|
|
89
|
+
expect(JSON.stringify(w.status())).not.toContain('ghp_')
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
test('status, tools, recent, scan and help answer locally', async ($, on) => {
|
|
93
|
+
world(on)
|
|
94
|
+
await $.session.start(START)
|
|
95
|
+
expect((await $.command.run(slash('status'))).text).toContain('guard on')
|
|
96
|
+
expect((await $.command.run(slash('tools'))).text).toContain('metaharness_score')
|
|
97
|
+
expect((await $.command.run(slash('recent'))).text).toContain('Nothing')
|
|
98
|
+
expect((await $.command.run(slash(`scan key ghp_${'a1B2'.repeat(10)}`))).text).toContain('github token')
|
|
99
|
+
expect((await $.command.run(slash('scan plain words'))).text).toContain('Nothing found')
|
|
100
|
+
expect((await $.command.run(slash(''))).text).toContain('/metaharness-mod status')
|
|
101
|
+
expect((await $.command.run(slash('what is this'))).text).toContain('Unknown')
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
test('tools says so plainly when none are connected', async ($, on) => {
|
|
105
|
+
world(on, [])
|
|
106
|
+
await $.session.start(START)
|
|
107
|
+
expect((await $.command.run(slash('tools'))).text).toContain('None of this plugin')
|
|
108
|
+
})
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
describe('defaults', () => {
|
|
112
|
+
test('with no options the guard is on', async ($, on) => {
|
|
113
|
+
const w = world(on)
|
|
114
|
+
await $.session.start(START)
|
|
115
|
+
expect(w.status().guard).toBe(true)
|
|
116
|
+
})
|
|
117
|
+
})
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { describe, expect, test, tier } from 'claude-code/testing'
|
|
2
|
+
|
|
3
|
+
import { textsOf } from '../hooks/guard'
|
|
4
|
+
import { hasSecret } from '../hooks/screen'
|
|
5
|
+
|
|
6
|
+
tier('user')
|
|
7
|
+
|
|
8
|
+
const AWS = `AKIA${'IOSFODNN7EXAMPLE'}`
|
|
9
|
+
const BIG = 'QUJD'.repeat(10_000)
|
|
10
|
+
const found = (input: unknown) => textsOf(input).some(hasSecret)
|
|
11
|
+
|
|
12
|
+
describe('what the guard reads', () => {
|
|
13
|
+
test('a key name leads its string value, so an object-shaped assignment is caught', () => {
|
|
14
|
+
expect(found({ meta: { api_key: 'abcdef1234567890abcdef' } })).toBe(true)
|
|
15
|
+
expect(found({ meta: { maxTokens: 4096, name: 'refresh policy' } })).toBe(false)
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
test('a long field does not starve the fields after it', () => {
|
|
19
|
+
expect(found({ value: BIG, note: AWS })).toBe(true)
|
|
20
|
+
expect(found({ a: BIG, b: BIG, c: BIG, d: 'plain text' })).toBe(false)
|
|
21
|
+
})
|
|
22
|
+
})
|