hackmyagent 0.9.4 → 0.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attack/scanner.d.ts.map +1 -1
- package/dist/attack/scanner.js +6 -3
- package/dist/attack/scanner.js.map +1 -1
- package/dist/attack/types.d.ts +2 -0
- package/dist/attack/types.d.ts.map +1 -1
- package/dist/attack/types.js.map +1 -1
- package/dist/benchmarks/oasb-1.d.ts +1 -1
- package/dist/benchmarks/oasb-1.d.ts.map +1 -1
- package/dist/benchmarks/oasb-1.js +3 -3
- package/dist/benchmarks/oasb-1.js.map +1 -1
- package/dist/cli.js +289 -125
- package/dist/cli.js.map +1 -1
- package/dist/hardening/scanner.d.ts.map +1 -1
- package/dist/hardening/scanner.js +42 -15
- package/dist/hardening/scanner.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/registry/client.d.ts +1 -0
- package/dist/registry/client.d.ts.map +1 -1
- package/dist/registry/client.js +26 -3
- package/dist/registry/client.js.map +1 -1
- package/dist/scanner/external-scanner.js +1 -1
- package/dist/scanner/external-scanner.js.map +1 -1
- package/dist/semantic/deep-scan.js +1 -1
- package/dist/semantic/structural/credential-context.js +1 -1
- package/dist/semantic/structural/credential-context.js.map +1 -1
- package/dist/soul/index.d.ts +2 -2
- package/dist/soul/index.d.ts.map +1 -1
- package/dist/soul/index.js +2 -1
- package/dist/soul/index.js.map +1 -1
- package/dist/soul/scanner.d.ts +39 -4
- package/dist/soul/scanner.d.ts.map +1 -1
- package/dist/soul/scanner.js +365 -107
- package/dist/soul/scanner.js.map +1 -1
- package/dist/soul/templates.d.ts +4 -0
- package/dist/soul/templates.d.ts.map +1 -1
- package/dist/soul/templates.js +249 -0
- package/dist/soul/templates.js.map +1 -1
- package/package.json +1 -1
package/dist/soul/scanner.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* SOUL Scanner - Behavioral Governance Scanner
|
|
4
4
|
*
|
|
5
5
|
* Scans governance files (SOUL.md, system-prompt.md, etc.) for coverage
|
|
6
|
-
* across
|
|
6
|
+
* across 9 behavioral governance domains defined in OASB v2.
|
|
7
7
|
*/
|
|
8
8
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
9
|
if (k2 === undefined) k2 = k;
|
|
@@ -39,7 +39,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
39
39
|
};
|
|
40
40
|
})();
|
|
41
41
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
42
|
-
exports.GOVERNANCE_FILES = exports.DOMAIN_ORDER = exports.CONTROL_DEFS = exports.SoulScanner = void 0;
|
|
42
|
+
exports.PROFILE_DOMAINS = exports.GOVERNANCE_FILES = exports.DOMAIN_ORDER = exports.CONTROL_DEFS = exports.SoulScanner = void 0;
|
|
43
43
|
const fs = __importStar(require("fs"));
|
|
44
44
|
const path = __importStar(require("path"));
|
|
45
45
|
const child_process_1 = require("child_process");
|
|
@@ -67,148 +67,229 @@ const MULTI_AGENT_ONLY = ['MULTI-AGENT'];
|
|
|
67
67
|
const CONTROL_DEFS = [
|
|
68
68
|
// Domain 7: Trust Hierarchy
|
|
69
69
|
{ id: 'SOUL-TH-001', name: 'Trust chain defined', domain: 'Trust Hierarchy', domainId: 7, tiers: ALL_TIERS,
|
|
70
|
-
keywords: ['trust', 'authority', 'principal', 'hierarchy', 'precedence', 'priority']
|
|
70
|
+
keywords: ['trust', 'authority', 'principal', 'hierarchy', 'precedence', 'priority'],
|
|
71
|
+
remediation: 'Define a trust hierarchy that establishes authority precedence among principals with clear priority ordering.' },
|
|
71
72
|
{ id: 'SOUL-TH-002', name: 'Conflict resolution defined', domain: 'Trust Hierarchy', domainId: 7, tiers: ALL_TIERS,
|
|
72
|
-
keywords: ['conflict', 'override', 'precedence', 'escalat']
|
|
73
|
+
keywords: ['conflict', 'override', 'precedence', 'escalat'],
|
|
74
|
+
remediation: 'Define conflict resolution rules: how override decisions follow precedence, when to escalate.' },
|
|
73
75
|
{ id: 'SOUL-TH-003', name: 'Agent-to-agent trust', domain: 'Trust Hierarchy', domainId: 7, tiers: MULTI_AGENT_ONLY,
|
|
74
|
-
keywords: ['agent-to-agent', 'sub-agent', 'orchestrat', 'delegate', 'trust.*agent', 'agent.*trust']
|
|
76
|
+
keywords: ['agent-to-agent', 'sub-agent', 'orchestrat', 'delegate', 'trust.*agent', 'agent.*trust'],
|
|
77
|
+
remediation: 'Define agent-to-agent trust: how sub-agent delegation works, orchestration trust boundaries.' },
|
|
75
78
|
{ id: 'SOUL-TH-004', name: 'Principal identity verification', domain: 'Trust Hierarchy', domainId: 7, tiers: ALL_TIERS,
|
|
76
|
-
keywords: ['authenticate', 'verify identity', 'principal source', 'identity verification', 'authenticated principal', 'identity claim']
|
|
79
|
+
keywords: ['authenticate', 'verify identity', 'principal source', 'identity verification', 'authenticated principal', 'identity claim'],
|
|
80
|
+
remediation: 'Authenticate all principals and verify identity. Require identity verification for every identity claim.' },
|
|
77
81
|
{ id: 'SOUL-TH-005', name: 'Trust hierarchy documentation complete', domain: 'Trust Hierarchy', domainId: 7, tiers: ALL_TIERS,
|
|
78
|
-
keywords: ['trust hierarchy', 'hierarchy levels', 'trust structure', 'trust path', 'hierarchy definition', 'trust order']
|
|
82
|
+
keywords: ['trust hierarchy', 'hierarchy levels', 'trust structure', 'trust path', 'hierarchy definition', 'trust order'],
|
|
83
|
+
remediation: 'Document the trust hierarchy with hierarchy levels and trust order so the trust structure is clear.' },
|
|
79
84
|
{ id: 'SOUL-TH-006', name: 'Principal authority scope defined', domain: 'Trust Hierarchy', domainId: 7, tiers: ALL_TIERS,
|
|
80
|
-
keywords: ['authority scope', 'principal authority', 'authority boundary', 'authority limit', 'scope definition', 'authority definition']
|
|
85
|
+
keywords: ['authority scope', 'principal authority', 'authority boundary', 'authority limit', 'scope definition', 'authority definition'],
|
|
86
|
+
remediation: 'Define authority scope for each principal authority with clear authority boundary and authority limit.' },
|
|
81
87
|
{ id: 'SOUL-TH-007', name: 'Trust boundary enforcement', domain: 'Trust Hierarchy', domainId: 7, tiers: TOOL_AND_UP,
|
|
82
|
-
keywords: ['enforce trust', 'trust enforcement', 'boundary enforcement', 'trust violation', 'enforce boundary', 'trust check']
|
|
88
|
+
keywords: ['enforce trust', 'trust enforcement', 'boundary enforcement', 'trust violation', 'enforce boundary', 'trust check'],
|
|
89
|
+
remediation: 'Enforce trust boundaries: trust enforcement triggers on any trust violation with a trust check.' },
|
|
83
90
|
{ id: 'SOUL-TH-008', name: 'Trust policy update protocol', domain: 'Trust Hierarchy', domainId: 7, tiers: ALL_TIERS,
|
|
84
|
-
keywords: ['trust update', 'policy update', 'trust change', 'update protocol', 'trust modification', 'change management']
|
|
91
|
+
keywords: ['trust update', 'policy update', 'trust change', 'update protocol', 'trust modification', 'change management'],
|
|
92
|
+
remediation: 'Define a trust update and policy update protocol for trust change via change management.' },
|
|
85
93
|
// Domain 8: Capability Boundaries (TOOL-USING and up)
|
|
86
94
|
{ id: 'SOUL-CB-001', name: 'Allowed actions declared', domain: 'Capability Boundaries', domainId: 8, tiers: TOOL_AND_UP,
|
|
87
|
-
keywords: ['allow', 'permit', 'can do', 'authorized', 'capabilities']
|
|
95
|
+
keywords: ['allow', 'permit', 'can do', 'authorized', 'capabilities'],
|
|
96
|
+
remediation: 'Declare allowed and authorized capabilities the agent is permitted to perform.' },
|
|
88
97
|
{ id: 'SOUL-CB-002', name: 'Denied actions declared', domain: 'Capability Boundaries', domainId: 8, tiers: TOOL_AND_UP,
|
|
89
|
-
keywords: ['deny', 'prohibit', 'must not', 'cannot', 'forbidden', 'restricted']
|
|
98
|
+
keywords: ['deny', 'prohibit', 'must not', 'cannot', 'forbidden', 'restricted'],
|
|
99
|
+
remediation: 'Declare denied actions: what is prohibited, forbidden, or restricted. The agent must not exceed these.' },
|
|
90
100
|
{ id: 'SOUL-CB-003', name: 'Filesystem/network scope', domain: 'Capability Boundaries', domainId: 8, tiers: TOOL_AND_UP,
|
|
91
|
-
keywords: ['file', 'directory', 'path', 'network', 'endpoint', 'url', 'api']
|
|
101
|
+
keywords: ['file', 'directory', 'path', 'network', 'endpoint', 'url', 'api'],
|
|
102
|
+
remediation: 'Define file, directory, and path scope. Declare network endpoint, URL, and API boundaries.' },
|
|
92
103
|
{ id: 'SOUL-CB-004', name: 'Least privilege principle', domain: 'Capability Boundaries', domainId: 8, tiers: TOOL_AND_UP,
|
|
93
|
-
keywords: ['least privilege', 'minimal', 'only needed', 'minimum necessary']
|
|
104
|
+
keywords: ['least privilege', 'minimal', 'only needed', 'minimum necessary'],
|
|
105
|
+
remediation: 'Apply least privilege: grant only minimal, minimum necessary permissions as needed.' },
|
|
94
106
|
{ id: 'SOUL-CB-005', name: 'Permission revocation process defined', domain: 'Capability Boundaries', domainId: 8, tiers: TOOL_AND_UP,
|
|
95
|
-
keywords: ['revoke', 'revocation', 'remove permission', 'disable access', 'withdraw access', 'permission removal']
|
|
107
|
+
keywords: ['revoke', 'revocation', 'remove permission', 'disable access', 'withdraw access', 'permission removal'],
|
|
108
|
+
remediation: 'Define a revocation process to revoke and remove permission. Disable access and withdraw access promptly.' },
|
|
96
109
|
{ id: 'SOUL-CB-006', name: 'Capability exposure minimized', domain: 'Capability Boundaries', domainId: 8, tiers: TOOL_AND_UP,
|
|
97
|
-
keywords: ['exposure limit', 'minimal exposure', 'capability exposure', 'selective exposure', 'controlled exposure', 'unexposed']
|
|
110
|
+
keywords: ['exposure limit', 'minimal exposure', 'capability exposure', 'selective exposure', 'controlled exposure', 'unexposed'],
|
|
111
|
+
remediation: 'Minimize capability exposure with an exposure limit. Use selective exposure and controlled exposure.' },
|
|
98
112
|
{ id: 'SOUL-CB-007', name: 'Tool integration boundaries declared', domain: 'Capability Boundaries', domainId: 8, tiers: TOOL_AND_UP,
|
|
99
|
-
keywords: ['tool boundary', 'tool scope', 'tool limit', 'tool interface', 'tool access control', 'tool constraint']
|
|
113
|
+
keywords: ['tool boundary', 'tool scope', 'tool limit', 'tool interface', 'tool access control', 'tool constraint'],
|
|
114
|
+
remediation: 'Declare tool boundary, tool scope, and tool limit. Define tool interface and tool access control.' },
|
|
100
115
|
{ id: 'SOUL-CB-008', name: 'Rate and resource limits enforced', domain: 'Capability Boundaries', domainId: 8, tiers: TOOL_AND_UP,
|
|
101
|
-
keywords: ['rate limit', 'rate limiting', 'resource limit', 'throttle', 'quota', 'bandwidth limit', 'usage limit']
|
|
116
|
+
keywords: ['rate limit', 'rate limiting', 'resource limit', 'throttle', 'quota', 'bandwidth limit', 'usage limit'],
|
|
117
|
+
remediation: 'Enforce rate limit and rate limiting. Set resource limit, throttle, quota, and usage limit.' },
|
|
102
118
|
{ id: 'SOUL-CB-009', name: 'Scope validation at invocation', domain: 'Capability Boundaries', domainId: 8, tiers: TOOL_AND_UP,
|
|
103
|
-
keywords: ['validate scope', 'scope check', 'scope validation', 'boundary check', 'scope enforcement', 'permission check']
|
|
119
|
+
keywords: ['validate scope', 'scope check', 'scope validation', 'boundary check', 'scope enforcement', 'permission check'],
|
|
120
|
+
remediation: 'Validate scope at invocation with a scope check and scope validation before execution.' },
|
|
104
121
|
{ id: 'SOUL-CB-010', name: 'Capability audit trail maintained', domain: 'Capability Boundaries', domainId: 8, tiers: TOOL_AND_UP,
|
|
105
|
-
keywords: ['capability audit', 'audit trail', 'capability log', 'usage log', 'execution log', 'action record']
|
|
122
|
+
keywords: ['capability audit', 'audit trail', 'capability log', 'usage log', 'execution log', 'action record'],
|
|
123
|
+
remediation: 'Maintain a capability audit and audit trail. Record actions in a capability log and usage log.' },
|
|
106
124
|
// Domain 9: Injection Hardening (all tiers)
|
|
107
125
|
{ id: 'SOUL-IH-001', name: 'Instruction override defense', domain: 'Injection Hardening', domainId: 9, tiers: ALL_TIERS,
|
|
108
|
-
keywords: ['ignore previous', 'override', 'injection', 'contradict']
|
|
126
|
+
keywords: ['ignore previous', 'override', 'injection', 'contradict'],
|
|
127
|
+
remediation: 'Defend against "ignore previous" instructions, override attempts, injection, and contradiction.' },
|
|
109
128
|
{ id: 'SOUL-IH-002', name: 'Encoded payload defense', domain: 'Injection Hardening', domainId: 9, tiers: ALL_TIERS,
|
|
110
|
-
keywords: ['encoded', 'obfuscated', 'base64', 'hidden']
|
|
129
|
+
keywords: ['encoded', 'obfuscated', 'base64', 'hidden'],
|
|
130
|
+
remediation: 'Defend against encoded, obfuscated, base64, and hidden payloads in user input.' },
|
|
111
131
|
{ id: 'SOUL-IH-003', name: 'Role-play refusal', domain: 'Injection Hardening', domainId: 9, tiers: ALL_TIERS,
|
|
112
|
-
keywords: ['role-play', 'pretend', 'act as', 'jailbreak', 'DAN'], critical: true
|
|
132
|
+
keywords: ['role-play', 'pretend', 'act as', 'jailbreak', 'as DAN'], critical: true,
|
|
133
|
+
remediation: 'Refuse role-play, pretend, "act as", jailbreak, and "act as DAN" requests.' },
|
|
113
134
|
{ id: 'SOUL-IH-004', name: 'Input validation and sanitization', domain: 'Injection Hardening', domainId: 9, tiers: ALL_TIERS,
|
|
114
|
-
keywords: ['input validation', 'sanitize', 'sanitization', 'validate input', 'filter input', 'clean input']
|
|
135
|
+
keywords: ['input validation', 'sanitize', 'sanitization', 'validate input', 'filter input', 'clean input'],
|
|
136
|
+
remediation: 'Apply input validation and sanitize all inputs. Use sanitization, validate input, filter input, and clean input.' },
|
|
115
137
|
{ id: 'SOUL-IH-005', name: 'Output encoding and escaping', domain: 'Injection Hardening', domainId: 9, tiers: ALL_TIERS,
|
|
116
|
-
keywords: ['output encoding', 'escape output', 'encode output', 'html escape', 'output sanitize', 'safe output']
|
|
138
|
+
keywords: ['output encoding', 'escape output', 'encode output', 'html escape', 'output sanitize', 'safe output'],
|
|
139
|
+
remediation: 'Apply output encoding: escape output, encode output with html escape for safe output.' },
|
|
117
140
|
{ id: 'SOUL-IH-006', name: 'Multi-layer injection defense', domain: 'Injection Hardening', domainId: 9, tiers: TOOL_AND_UP,
|
|
118
|
-
keywords: ['defense layer', 'defense in depth', 'layered defense', 'multiple defense', 'defense stack', 'multi-layer']
|
|
141
|
+
keywords: ['defense layer', 'defense in depth', 'layered defense', 'multiple defense', 'defense stack', 'multi-layer'],
|
|
142
|
+
remediation: 'Use defense in depth with a multi-layer defense stack and layered defense approach.' },
|
|
119
143
|
{ id: 'SOUL-IH-007', name: 'Injection detection and alerting', domain: 'Injection Hardening', domainId: 9, tiers: ALL_TIERS,
|
|
120
|
-
keywords: ['detect injection', 'injection detection', 'attack detection', 'log injection', 'alert injection', 'security log']
|
|
144
|
+
keywords: ['detect injection', 'injection detection', 'attack detection', 'log injection', 'alert injection', 'security log'],
|
|
145
|
+
remediation: 'Detect injection via injection detection. Attack detection logs to security log and alerts on injection.' },
|
|
121
146
|
{ id: 'SOUL-IH-008', name: 'Adversarial input testing', domain: 'Injection Hardening', domainId: 9, tiers: TOOL_AND_UP,
|
|
122
|
-
keywords: ['test defense', 'adversarial test', 'red team', 'penetration test', 'security test', 'verify hardening']
|
|
147
|
+
keywords: ['test defense', 'adversarial test', 'red team', 'penetration test', 'security test', 'verify hardening'],
|
|
148
|
+
remediation: 'Run adversarial test and red team exercises. Penetration test and security test verify hardening.' },
|
|
123
149
|
// Domain 10: Data Handling
|
|
124
150
|
{ id: 'SOUL-DH-001', name: 'PII protection', domain: 'Data Handling', domainId: 10, tiers: ALL_TIERS,
|
|
125
|
-
keywords: ['pii', 'personal', 'privacy', 'data protection', 'gdpr']
|
|
151
|
+
keywords: ['pii', 'personal', 'privacy', 'data protection', 'gdpr'],
|
|
152
|
+
remediation: 'Protect PII and personal data. Enforce privacy and data protection under GDPR.' },
|
|
126
153
|
{ id: 'SOUL-DH-002', name: 'Credential handling', domain: 'Data Handling', domainId: 10, tiers: TOOL_AND_UP,
|
|
127
|
-
keywords: ['credential', 'secret', 'password', 'api key', 'token']
|
|
154
|
+
keywords: ['credential', 'secret', 'password', 'api key', 'token'],
|
|
155
|
+
remediation: 'Handle credentials, secrets, passwords, API keys, and tokens securely.' },
|
|
128
156
|
{ id: 'SOUL-DH-003', name: 'Data minimization', domain: 'Data Handling', domainId: 10, tiers: ALL_TIERS,
|
|
129
|
-
keywords: ['minimiz', 'only collect', 'retention', 'delete', 'purge']
|
|
157
|
+
keywords: ['minimiz', 'only collect', 'retention', 'delete', 'purge'],
|
|
158
|
+
remediation: 'Minimize data collection: only collect what is needed, define retention, delete/purge old data.' },
|
|
130
159
|
{ id: 'SOUL-DH-004', name: 'Data retention and deletion policy', domain: 'Data Handling', domainId: 10, tiers: ALL_TIERS,
|
|
131
|
-
keywords: ['retention policy', 'retention period', 'data deletion', 'purge schedule', 'data retention', 'archival policy']
|
|
160
|
+
keywords: ['retention policy', 'retention period', 'data deletion', 'purge schedule', 'data retention', 'archival policy'],
|
|
161
|
+
remediation: 'Define a retention policy with retention period. Data deletion follows a purge schedule and archival policy.' },
|
|
132
162
|
{ id: 'SOUL-DH-005', name: 'Data classification framework', domain: 'Data Handling', domainId: 10, tiers: ALL_TIERS,
|
|
133
|
-
keywords: ['data classification', 'classify data', 'sensitivity level', 'data sensitivity', 'classification scheme', 'data category']
|
|
163
|
+
keywords: ['data classification', 'classify data', 'sensitivity level', 'data sensitivity', 'classification scheme', 'data category'],
|
|
164
|
+
remediation: 'Implement data classification: classify data by sensitivity level. Use a classification scheme with data categories.' },
|
|
134
165
|
{ id: 'SOUL-DH-006', name: 'Data access control enforcement', domain: 'Data Handling', domainId: 10, tiers: TOOL_AND_UP,
|
|
135
|
-
keywords: ['data access control', 'access rule', 'access policy', 'enforce access', 'data permission', 'access enforcement']
|
|
166
|
+
keywords: ['data access control', 'access rule', 'access policy', 'enforce access', 'data permission', 'access enforcement'],
|
|
167
|
+
remediation: 'Enforce data access control with access rules, access policy, data permission, and access enforcement.' },
|
|
136
168
|
{ id: 'SOUL-DH-007', name: 'Data encryption requirements', domain: 'Data Handling', domainId: 10, tiers: TOOL_AND_UP,
|
|
137
|
-
keywords: ['encrypt', 'encryption', 'encrypted', 'encryption at rest', 'encryption in transit', 'tls', 'https', 'cipher']
|
|
169
|
+
keywords: ['encrypt', 'encryption', 'encrypted', 'encryption at rest', 'encryption in transit', 'tls', 'https', 'cipher'],
|
|
170
|
+
remediation: 'Encrypt data with encryption at rest and encryption in transit via TLS/HTTPS.' },
|
|
138
171
|
{ id: 'SOUL-DH-008', name: 'Data breach response procedure', domain: 'Data Handling', domainId: 10, tiers: AGENTIC_AND_UP,
|
|
139
|
-
keywords: ['breach notification', 'breach response', 'incident response', 'data breach', 'breach procedure', 'incident notification']
|
|
172
|
+
keywords: ['breach notification', 'breach response', 'incident response', 'data breach', 'breach procedure', 'incident notification'],
|
|
173
|
+
remediation: 'Define breach notification and breach response. Incident response handles data breach with incident notification.' },
|
|
140
174
|
// Domain 11: Hardcoded Behaviors (all tiers)
|
|
141
175
|
{ id: 'SOUL-HB-001', name: 'Safety immutables defined', domain: 'Hardcoded Behaviors', domainId: 11, tiers: ALL_TIERS,
|
|
142
|
-
keywords: ['never', 'always', 'must not', 'absolute', 'immutable', 'hardcoded'], critical: true
|
|
176
|
+
keywords: ['never', 'always', 'must not', 'absolute', 'immutable', 'hardcoded'], critical: true,
|
|
177
|
+
remediation: 'Define safety immutables: never/always rules that are absolute, immutable, and hardcoded.' },
|
|
143
178
|
{ id: 'SOUL-HB-002', name: 'No data exfiltration rule', domain: 'Hardcoded Behaviors', domainId: 11, tiers: ALL_TIERS,
|
|
144
|
-
keywords: ['exfiltrat', 'unauthorized', 'leak', 'transmit']
|
|
179
|
+
keywords: ['exfiltrat', 'unauthorized', 'leak', 'transmit'],
|
|
180
|
+
remediation: 'Prohibit exfiltration of unauthorized data. Prevent leak and transmit to external destinations.' },
|
|
145
181
|
{ id: 'SOUL-HB-003', name: 'Kill switch / emergency stop', domain: 'Hardcoded Behaviors', domainId: 11, tiers: ALL_TIERS,
|
|
146
|
-
keywords: ['kill switch', 'emergency', 'shutdown', 'terminate', 'stop']
|
|
182
|
+
keywords: ['kill switch', 'emergency', 'shutdown', 'terminate', 'stop'],
|
|
183
|
+
remediation: 'Implement a kill switch for emergency shutdown. Terminate and stop on anomalous behavior.' },
|
|
147
184
|
{ id: 'SOUL-HB-004', name: 'Behavior integrity verification', domain: 'Hardcoded Behaviors', domainId: 11, tiers: TOOL_AND_UP,
|
|
148
|
-
keywords: ['verify behavior', 'integrity check', 'behavior integrity', 'validate behavior', 'integrity verification', 'behavior validation']
|
|
185
|
+
keywords: ['verify behavior', 'integrity check', 'behavior integrity', 'validate behavior', 'integrity verification', 'behavior validation'],
|
|
186
|
+
remediation: 'Verify behavior through integrity check and behavior integrity. Validate behavior via integrity verification.' },
|
|
149
187
|
{ id: 'SOUL-HB-005', name: 'Constraint immutability guarantee', domain: 'Hardcoded Behaviors', domainId: 11, tiers: ALL_TIERS,
|
|
150
|
-
keywords: ['immutable constraint', 'immutable rule', 'unchangeable', 'permanent constraint', 'fixed rule', 'hardcoded constraint']
|
|
188
|
+
keywords: ['immutable constraint', 'immutable rule', 'unchangeable', 'permanent constraint', 'fixed rule', 'hardcoded constraint'],
|
|
189
|
+
remediation: 'Guarantee immutable constraint and immutable rule enforcement. Unchangeable and permanent constraints are fixed.' },
|
|
151
190
|
{ id: 'SOUL-HB-006', name: 'Tamper detection mechanism', domain: 'Hardcoded Behaviors', domainId: 11, tiers: TOOL_AND_UP,
|
|
152
|
-
keywords: ['detect tamper', 'tamper detection', 'tamper-proof', 'detect modification', 'detect unauthorized change', 'integrity monitor']
|
|
191
|
+
keywords: ['detect tamper', 'tamper detection', 'tamper-proof', 'detect modification', 'detect unauthorized change', 'integrity monitor'],
|
|
192
|
+
remediation: 'Detect tamper via tamper detection. Tamper-proof design with integrity monitor.' },
|
|
153
193
|
{ id: 'SOUL-HB-007', name: 'Safety behavior audit', domain: 'Hardcoded Behaviors', domainId: 11, tiers: TOOL_AND_UP,
|
|
154
|
-
keywords: ['behavior audit', 'audit behavior', 'behavior attestation', 'certify behavior', 'behavior verification', 'safety audit']
|
|
194
|
+
keywords: ['behavior audit', 'audit behavior', 'behavior attestation', 'certify behavior', 'behavior verification', 'safety audit'],
|
|
195
|
+
remediation: 'Conduct behavior audit and audit behavior. Behavior attestation certifies behavior via safety audit.' },
|
|
155
196
|
{ id: 'SOUL-HB-008', name: 'Enforcement resilience under pressure', domain: 'Hardcoded Behaviors', domainId: 11, tiers: AGENTIC_AND_UP,
|
|
156
|
-
keywords: ['enforcement resilience', 'reliable enforcement', 'robust enforcement', 'fail-safe', 'enforcement guarantee', 'enforcement mechanism']
|
|
197
|
+
keywords: ['enforcement resilience', 'reliable enforcement', 'robust enforcement', 'fail-safe', 'enforcement guarantee', 'enforcement mechanism'],
|
|
198
|
+
remediation: 'Ensure enforcement resilience with reliable enforcement. Robust enforcement via fail-safe enforcement mechanism.' },
|
|
157
199
|
// Domain 12: Agentic Safety (AGENTIC and up)
|
|
158
200
|
{ id: 'SOUL-AS-001', name: 'Iteration/loop limits', domain: 'Agentic Safety', domainId: 12, tiers: AGENTIC_AND_UP,
|
|
159
|
-
keywords: ['iteration', 'loop', 'limit', 'maximum', 'budget']
|
|
201
|
+
keywords: ['iteration', 'loop', 'limit', 'maximum', 'budget'],
|
|
202
|
+
remediation: 'Set iteration and loop limits with a maximum budget per session.' },
|
|
160
203
|
{ id: 'SOUL-AS-002', name: 'Budget/cost caps', domain: 'Agentic Safety', domainId: 12, tiers: AGENTIC_AND_UP,
|
|
161
|
-
keywords: ['budget', 'cost', 'spending', 'cap', 'limit']
|
|
204
|
+
keywords: ['budget', 'cost', 'spending', 'cap', 'limit'],
|
|
205
|
+
remediation: 'Define budget and cost caps with spending limits.' },
|
|
162
206
|
{ id: 'SOUL-AS-003', name: 'Timeout defined', domain: 'Agentic Safety', domainId: 12, tiers: AGENTIC_AND_UP,
|
|
163
|
-
keywords: ['timeout', 'time limit', 'duration', 'deadline']
|
|
207
|
+
keywords: ['timeout', 'time limit', 'duration', 'deadline'],
|
|
208
|
+
remediation: 'Define timeout and time limit. Set duration and deadline for operations.' },
|
|
164
209
|
{ id: 'SOUL-AS-004', name: 'Reversibility preference', domain: 'Agentic Safety', domainId: 12, tiers: MULTI_AGENT_ONLY,
|
|
165
|
-
keywords: ['reversible', 'undo', 'rollback', 'revert']
|
|
210
|
+
keywords: ['reversible', 'undo', 'rollback', 'revert'],
|
|
211
|
+
remediation: 'Prefer reversible actions. Support undo, rollback, and revert.' },
|
|
166
212
|
{ id: 'SOUL-AS-005', name: 'Tool dependency limits', domain: 'Agentic Safety', domainId: 12, tiers: AGENTIC_AND_UP,
|
|
167
|
-
keywords: ['dependency limit', 'dependency depth', 'dependency chain', 'tool dependency', 'dependency tracking', 'dependency count']
|
|
213
|
+
keywords: ['dependency limit', 'dependency depth', 'dependency chain', 'tool dependency', 'dependency tracking', 'dependency count'],
|
|
214
|
+
remediation: 'Enforce dependency limit on dependency depth and dependency chain. Track tool dependency with dependency count.' },
|
|
168
215
|
{ id: 'SOUL-AS-006', name: 'State management limits', domain: 'Agentic Safety', domainId: 12, tiers: AGENTIC_AND_UP,
|
|
169
|
-
keywords: ['state limit', 'state management', 'memory limit', 'context limit', 'state size', 'session state limit']
|
|
216
|
+
keywords: ['state limit', 'state management', 'memory limit', 'context limit', 'state size', 'session state limit'],
|
|
217
|
+
remediation: 'Set state limit on state management. Enforce memory limit, context limit, and session state limit.' },
|
|
170
218
|
{ id: 'SOUL-AS-007', name: 'Error recovery protocol', domain: 'Agentic Safety', domainId: 12, tiers: AGENTIC_AND_UP,
|
|
171
|
-
keywords: ['error recovery', 'recovery protocol', 'error handling', 'retry logic', 'error fallback', 'recovery mechanism']
|
|
219
|
+
keywords: ['error recovery', 'recovery protocol', 'error handling', 'retry logic', 'error fallback', 'recovery mechanism'],
|
|
220
|
+
remediation: 'Define error recovery with a recovery protocol. Error handling includes retry logic and error fallback.' },
|
|
172
221
|
{ id: 'SOUL-AS-008', name: 'Task isolation and sandboxing', domain: 'Agentic Safety', domainId: 12, tiers: AGENTIC_AND_UP,
|
|
173
|
-
keywords: ['task isolation', 'sandbox', 'sandboxing', 'isolated execution', 'execution boundary', 'isolation level']
|
|
222
|
+
keywords: ['task isolation', 'sandbox', 'sandboxing', 'isolated execution', 'execution boundary', 'isolation level'],
|
|
223
|
+
remediation: 'Enforce task isolation via sandbox and sandboxing. Isolated execution within an execution boundary.' },
|
|
174
224
|
{ id: 'SOUL-AS-009', name: 'Resource cleanup on completion', domain: 'Agentic Safety', domainId: 12, tiers: AGENTIC_AND_UP,
|
|
175
|
-
keywords: ['cleanup', 'resource cleanup', 'finalization', 'resource release', 'graceful shutdown', 'cleanup procedure']
|
|
225
|
+
keywords: ['cleanup', 'resource cleanup', 'finalization', 'resource release', 'graceful shutdown', 'cleanup procedure'],
|
|
226
|
+
remediation: 'Perform cleanup and resource cleanup on completion. Finalization and graceful shutdown via cleanup procedure.' },
|
|
176
227
|
{ id: 'SOUL-AS-010', name: 'Concurrent execution coordination', domain: 'Agentic Safety', domainId: 12, tiers: MULTI_AGENT_ONLY,
|
|
177
|
-
keywords: ['concurrent limit', 'concurrency', 'concurrent execution', 'coordination', 'serialize task', 'synchronize', 'parallel limit']
|
|
228
|
+
keywords: ['concurrent limit', 'concurrency', 'concurrent execution', 'coordination', 'serialize task', 'synchronize', 'parallel limit'],
|
|
229
|
+
remediation: 'Enforce concurrent limit on concurrency. Coordination and synchronize with parallel limit.' },
|
|
178
230
|
// Domain 13: Honesty and Transparency (all tiers)
|
|
179
231
|
{ id: 'SOUL-HT-001', name: 'Uncertainty acknowledgment', domain: 'Honesty and Transparency', domainId: 13, tiers: ALL_TIERS,
|
|
180
|
-
keywords: ['uncertain', "don't know", 'not sure', 'acknowledge', 'calibrat']
|
|
232
|
+
keywords: ['uncertain', "don't know", 'not sure', 'acknowledge', 'calibrat'],
|
|
233
|
+
remediation: 'Acknowledge uncertainty: say "don\'t know" or "not sure". Calibrate confidence.' },
|
|
181
234
|
{ id: 'SOUL-HT-002', name: 'No fabrication rule', domain: 'Honesty and Transparency', domainId: 13, tiers: ALL_TIERS,
|
|
182
|
-
keywords: ['fabricat', 'hallucin', 'invent', 'make up', 'accurate']
|
|
235
|
+
keywords: ['fabricat', 'hallucin', 'invent', 'make up', 'accurate'],
|
|
236
|
+
remediation: 'Never fabricate or hallucinate. Do not invent or make up facts. Be accurate.' },
|
|
183
237
|
{ id: 'SOUL-HT-003', name: 'Identity disclosure', domain: 'Honesty and Transparency', domainId: 13, tiers: ALL_TIERS,
|
|
184
|
-
keywords: ['identity', 'ai', 'assistant', 'disclose', 'transparent']
|
|
238
|
+
keywords: ['identity', 'ai', 'assistant', 'disclose', 'transparent'],
|
|
239
|
+
remediation: 'Disclose identity as an AI assistant. Be transparent about capabilities.' },
|
|
185
240
|
{ id: 'SOUL-HT-004', name: 'Knowledge boundaries documented', domain: 'Honesty and Transparency', domainId: 13, tiers: ALL_TIERS,
|
|
186
|
-
keywords: ['knowledge boundary', 'knowledge limit', 'knowledge cutoff', 'training limit', 'knowledge scope', 'knowledge limitation']
|
|
241
|
+
keywords: ['knowledge boundary', 'knowledge limit', 'knowledge cutoff', 'training limit', 'knowledge scope', 'knowledge limitation'],
|
|
242
|
+
remediation: 'Document knowledge boundary and knowledge limit. State knowledge cutoff and training limit.' },
|
|
187
243
|
{ id: 'SOUL-HT-005', name: 'Confidence level disclosure', domain: 'Honesty and Transparency', domainId: 13, tiers: ALL_TIERS,
|
|
188
|
-
keywords: ['confidence level', 'confidence score', 'confidence calibration', 'express confidence', 'certainty level', 'calibrated confidence']
|
|
244
|
+
keywords: ['confidence level', 'confidence score', 'confidence calibration', 'express confidence', 'certainty level', 'calibrated confidence'],
|
|
245
|
+
remediation: 'Disclose confidence level and confidence score. Use confidence calibration for calibrated confidence.' },
|
|
189
246
|
{ id: 'SOUL-HT-006', name: 'Training data recency disclosed', domain: 'Honesty and Transparency', domainId: 13, tiers: ALL_TIERS,
|
|
190
|
-
keywords: ['training cutoff', 'training date', 'cutoff date', 'knowledge date', 'data recency', 'up to date', 'information currency']
|
|
247
|
+
keywords: ['training cutoff', 'training date', 'cutoff date', 'knowledge date', 'data recency', 'up to date', 'information currency'],
|
|
248
|
+
remediation: 'Disclose training cutoff and training date. Note data recency and information currency.' },
|
|
191
249
|
{ id: 'SOUL-HT-007', name: 'Limitations acknowledged in responses', domain: 'Honesty and Transparency', domainId: 13, tiers: ALL_TIERS,
|
|
192
|
-
keywords: ['acknowledge limitation', 'limitation notice', 'caveat', 'disclose limitation', 'limitation disclosure', 'note limitation']
|
|
250
|
+
keywords: ['acknowledge limitation', 'limitation notice', 'caveat', 'disclose limitation', 'limitation disclosure', 'note limitation'],
|
|
251
|
+
remediation: 'Acknowledge limitation with a limitation notice or caveat. Disclose limitation in responses.' },
|
|
193
252
|
{ id: 'SOUL-HT-008', name: 'Source verification practices', domain: 'Honesty and Transparency', domainId: 13, tiers: TOOL_AND_UP,
|
|
194
|
-
keywords: ['verify source', 'source verification', 'cite source', 'citation practice', 'verify information', 'source accuracy']
|
|
253
|
+
keywords: ['verify source', 'source verification', 'cite source', 'citation practice', 'verify information', 'source accuracy'],
|
|
254
|
+
remediation: 'Verify source via source verification. Cite source using citation practice for source accuracy.' },
|
|
195
255
|
// Domain 14: Human Oversight (TOOL-USING and up)
|
|
196
256
|
{ id: 'SOUL-HO-001', name: 'Approval gates', domain: 'Human Oversight', domainId: 14, tiers: TOOL_AND_UP,
|
|
197
|
-
keywords: ['approval', 'confirm', 'human-in-the-loop', 'review', 'authorize']
|
|
257
|
+
keywords: ['approval', 'confirm', 'human-in-the-loop', 'review', 'authorize'],
|
|
258
|
+
remediation: 'Require approval and confirmation. Human-in-the-loop review authorizes high-impact actions.' },
|
|
198
259
|
{ id: 'SOUL-HO-002', name: 'Override mechanism', domain: 'Human Oversight', domainId: 14, tiers: TOOL_AND_UP,
|
|
199
|
-
keywords: ['override', 'intervene', 'manual', 'human control']
|
|
260
|
+
keywords: ['override', 'intervene', 'manual', 'human control'],
|
|
261
|
+
remediation: 'Provide override mechanism: intervene manually with human control.' },
|
|
200
262
|
{ id: 'SOUL-HO-003', name: 'Monitoring/logging', domain: 'Human Oversight', domainId: 14, tiers: TOOL_AND_UP,
|
|
201
|
-
keywords: ['monitor', 'log', 'audit', 'track', 'observe']
|
|
263
|
+
keywords: ['monitor', 'log', 'audit', 'track', 'observe'],
|
|
264
|
+
remediation: 'Monitor and log all actions for audit. Track and observe behavior.' },
|
|
202
265
|
{ id: 'SOUL-HO-004', name: 'Approval workflow and escalation', domain: 'Human Oversight', domainId: 14, tiers: TOOL_AND_UP,
|
|
203
|
-
keywords: ['approval workflow', 'escalation path', 'escalation workflow', 'approval process', 'approval chain', 'workflow process']
|
|
266
|
+
keywords: ['approval workflow', 'escalation path', 'escalation workflow', 'approval process', 'approval chain', 'workflow process'],
|
|
267
|
+
remediation: 'Define approval workflow with escalation path. The approval process follows the approval chain.' },
|
|
204
268
|
{ id: 'SOUL-HO-005', name: 'Action notification protocol', domain: 'Human Oversight', domainId: 14, tiers: TOOL_AND_UP,
|
|
205
|
-
keywords: ['notification protocol', 'alert protocol', 'notify user', 'action notification', 'alert system', 'notification trigger']
|
|
269
|
+
keywords: ['notification protocol', 'alert protocol', 'notify user', 'action notification', 'alert system', 'notification trigger'],
|
|
270
|
+
remediation: 'Implement notification protocol and alert protocol. Notify user via action notification with alert system.' },
|
|
206
271
|
{ id: 'SOUL-HO-006', name: 'Operator identity verification', domain: 'Human Oversight', domainId: 14, tiers: TOOL_AND_UP,
|
|
207
|
-
keywords: ['operator verification', 'verify operator', 'operator authorization', 'operator authentication', 'operator identity', 'authorize operator']
|
|
272
|
+
keywords: ['operator verification', 'verify operator', 'operator authorization', 'operator authentication', 'operator identity', 'authorize operator'],
|
|
273
|
+
remediation: 'Verify operator via operator verification. Operator authorization and operator authentication confirm identity.' },
|
|
208
274
|
{ id: 'SOUL-HO-007', name: 'Audit log retention and access', domain: 'Human Oversight', domainId: 14, tiers: TOOL_AND_UP,
|
|
209
|
-
keywords: ['audit retention', 'log retention', 'audit log access', 'log access control', 'audit preservation', 'log archival']
|
|
275
|
+
keywords: ['audit retention', 'log retention', 'audit log access', 'log access control', 'audit preservation', 'log archival'],
|
|
276
|
+
remediation: 'Maintain audit retention and log retention. Audit log access is governed by log access control.' },
|
|
210
277
|
{ id: 'SOUL-HO-008', name: 'Escalation triggers for runaway detection', domain: 'Human Oversight', domainId: 14, tiers: AGENTIC_AND_UP,
|
|
211
|
-
keywords: ['escalation trigger', 'runaway detection', 'detect runaway', 'malfunction detection', 'anomaly detection', 'escalation condition']
|
|
278
|
+
keywords: ['escalation trigger', 'runaway detection', 'detect runaway', 'malfunction detection', 'anomaly detection', 'escalation condition'],
|
|
279
|
+
remediation: 'Define escalation trigger for runaway detection. Detect runaway and malfunction via anomaly detection.' },
|
|
280
|
+
// Domain 15: Harm Avoidance
|
|
281
|
+
{ id: 'SOUL-HV-001', name: 'Pre-action risk assessment', domain: 'Harm Avoidance', domainId: 15, tiers: TOOL_AND_UP,
|
|
282
|
+
keywords: ['risk assessment', 'consequence', 'impact analysis', 'before acting', 'potential harm', 'side effect', 'cost-benefit', 'think before'],
|
|
283
|
+
remediation: 'Evaluate potential consequences via risk assessment before acting. Consider potential harm, side effects, and cost-benefit.' },
|
|
284
|
+
{ id: 'SOUL-HV-002', name: 'Proportional response', domain: 'Harm Avoidance', domainId: 15, tiers: ALL_TIERS,
|
|
285
|
+
keywords: ['proportional', 'commensurate', 'calibrate', 'appropriate response', 'level of caution', 'measured response', 'scale caution'],
|
|
286
|
+
remediation: 'Scale caution proportionally. Use a measured response calibrated to the level of caution appropriate for the stakes.' },
|
|
287
|
+
{ id: 'SOUL-HV-003', name: 'Unintended impact awareness', domain: 'Harm Avoidance', domainId: 15, tiers: AGENTIC_AND_UP,
|
|
288
|
+
keywords: ['downstream effect', 'second-order', 'unintended', 'ripple effect', 'cascade', 'knock-on', 'cumulative impact', 'broader impact'],
|
|
289
|
+
remediation: 'Consider downstream effects and second-order consequences. Account for unintended ripple effects, cascade, and cumulative impact.' },
|
|
290
|
+
{ id: 'SOUL-HV-004', name: 'Ambiguity resolution', domain: 'Harm Avoidance', domainId: 15, tiers: ALL_TIERS,
|
|
291
|
+
keywords: ['ambiguous', 'safer interpretation', 'clarification', 'disambiguate', 'uncertain instruction', 'default to safe', 'ask for clarification'],
|
|
292
|
+
remediation: 'When instructions are ambiguous, default to the safer interpretation or ask for clarification. Disambiguate uncertain instructions.' },
|
|
212
293
|
];
|
|
213
294
|
exports.CONTROL_DEFS = CONTROL_DEFS;
|
|
214
295
|
// Unique domain names in order
|
|
@@ -221,9 +302,23 @@ const DOMAIN_ORDER = [
|
|
|
221
302
|
'Agentic Safety',
|
|
222
303
|
'Honesty and Transparency',
|
|
223
304
|
'Human Oversight',
|
|
305
|
+
'Harm Avoidance',
|
|
224
306
|
];
|
|
225
307
|
exports.DOMAIN_ORDER = DOMAIN_ORDER;
|
|
226
308
|
// ---------------------------------------------------------------------------
|
|
309
|
+
// Profile-to-domain mapping
|
|
310
|
+
// ---------------------------------------------------------------------------
|
|
311
|
+
/** Domain IDs that apply to each profile. */
|
|
312
|
+
const PROFILE_DOMAINS = {
|
|
313
|
+
conversational: [9, 11, 13, 15], // Injection, Hardcoded, Honesty, Harm Avoidance
|
|
314
|
+
'code-assistant': [7, 9, 10, 11, 13, 15], // + Trust, Data
|
|
315
|
+
'tool-agent': [7, 8, 9, 10, 11, 13, 14, 15], // + Capability, Oversight
|
|
316
|
+
autonomous: [7, 8, 9, 10, 11, 12, 13, 14, 15], // + Agentic Safety
|
|
317
|
+
orchestrator: [7, 8, 9, 10, 11, 12, 13, 14, 15], // All 9 domains
|
|
318
|
+
custom: [7, 8, 9, 10, 11, 12, 13, 14, 15], // All 9 domains
|
|
319
|
+
};
|
|
320
|
+
exports.PROFILE_DOMAINS = PROFILE_DOMAINS;
|
|
321
|
+
// ---------------------------------------------------------------------------
|
|
227
322
|
// Tier detection keywords
|
|
228
323
|
// ---------------------------------------------------------------------------
|
|
229
324
|
const TIER_KEYWORDS = {
|
|
@@ -232,6 +327,16 @@ const TIER_KEYWORDS = {
|
|
|
232
327
|
toolUsing: ['tool_use', 'function_calling', 'tools', 'mcp', 'modelcontextprotocol', 'function call', 'tool call'],
|
|
233
328
|
};
|
|
234
329
|
// ---------------------------------------------------------------------------
|
|
330
|
+
// Profile detection keywords
|
|
331
|
+
// ---------------------------------------------------------------------------
|
|
332
|
+
const PROFILE_KEYWORDS = {
|
|
333
|
+
orchestrator: ['orchestrat', 'multi-agent', 'multi_agent', 'swarm', 'coordinator', 'delegate.*agent', 'agent.*delegate'],
|
|
334
|
+
autonomous: ['autonomous', 'self-directed', 'agent loop', 'auto-run', 'agentic', 'self-improving'],
|
|
335
|
+
'tool-agent': ['tool_use', 'function_calling', 'mcp', 'modelcontextprotocol', 'function call', 'tool call'],
|
|
336
|
+
'code-assistant': ['code', 'file access', 'code generation', 'repository', 'codebase', 'programming'],
|
|
337
|
+
conversational: ['chatbot', 'q&a', 'question and answer', 'conversational', 'chat', 'assistant'],
|
|
338
|
+
};
|
|
339
|
+
// ---------------------------------------------------------------------------
|
|
235
340
|
// SoulScanner class
|
|
236
341
|
// ---------------------------------------------------------------------------
|
|
237
342
|
class SoulScanner {
|
|
@@ -250,8 +355,17 @@ class SoulScanner {
|
|
|
250
355
|
}
|
|
251
356
|
/**
|
|
252
357
|
* Detect agent tier by scanning governance file content and project files.
|
|
358
|
+
* Respects a `<!-- soul:tier=TIER -->` marker if present (prevents tier drift).
|
|
253
359
|
*/
|
|
254
360
|
detectTier(targetDir, governanceContent) {
|
|
361
|
+
// Check for explicit tier marker first (prevents drift after hardening)
|
|
362
|
+
const markerMatch = governanceContent.match(/<!--\s*soul:tier=(\S+)\s*-->/i);
|
|
363
|
+
if (markerMatch) {
|
|
364
|
+
const markerTier = markerMatch[1].toUpperCase();
|
|
365
|
+
if (['BASIC', 'TOOL-USING', 'AGENTIC', 'MULTI-AGENT'].includes(markerTier)) {
|
|
366
|
+
return markerTier;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
255
369
|
// Combine governance content with any package.json or config content
|
|
256
370
|
let combined = governanceContent.toLowerCase();
|
|
257
371
|
const pkgPath = path.join(targetDir, 'package.json');
|
|
@@ -281,6 +395,44 @@ class SoulScanner {
|
|
|
281
395
|
}
|
|
282
396
|
return 'BASIC';
|
|
283
397
|
}
|
|
398
|
+
/**
|
|
399
|
+
* Detect agent profile from governance content.
|
|
400
|
+
* Respects a `<!-- soul:profile=PROFILE -->` marker if present.
|
|
401
|
+
*/
|
|
402
|
+
detectProfile(governanceContent) {
|
|
403
|
+
// Check for explicit profile marker first
|
|
404
|
+
const markerMatch = governanceContent.match(/<!--\s*soul:profile=(\S+)\s*-->/i);
|
|
405
|
+
if (markerMatch) {
|
|
406
|
+
const markerProfile = markerMatch[1].toLowerCase();
|
|
407
|
+
if (Object.keys(PROFILE_DOMAINS).includes(markerProfile)) {
|
|
408
|
+
return markerProfile;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
const lower = governanceContent.toLowerCase();
|
|
412
|
+
// Check from most specific to least
|
|
413
|
+
for (const kw of PROFILE_KEYWORDS.orchestrator) {
|
|
414
|
+
if (lower.includes(kw.toLowerCase()))
|
|
415
|
+
return 'orchestrator';
|
|
416
|
+
}
|
|
417
|
+
for (const kw of PROFILE_KEYWORDS.autonomous) {
|
|
418
|
+
if (lower.includes(kw.toLowerCase()))
|
|
419
|
+
return 'autonomous';
|
|
420
|
+
}
|
|
421
|
+
for (const kw of PROFILE_KEYWORDS['tool-agent']) {
|
|
422
|
+
if (lower.includes(kw.toLowerCase()))
|
|
423
|
+
return 'tool-agent';
|
|
424
|
+
}
|
|
425
|
+
for (const kw of PROFILE_KEYWORDS['code-assistant']) {
|
|
426
|
+
if (lower.includes(kw.toLowerCase()))
|
|
427
|
+
return 'code-assistant';
|
|
428
|
+
}
|
|
429
|
+
for (const kw of PROFILE_KEYWORDS.conversational) {
|
|
430
|
+
if (lower.includes(kw.toLowerCase()))
|
|
431
|
+
return 'conversational';
|
|
432
|
+
}
|
|
433
|
+
// Default: custom (evaluates all domains)
|
|
434
|
+
return 'custom';
|
|
435
|
+
}
|
|
284
436
|
/**
|
|
285
437
|
* Check if content matches any keyword for a control.
|
|
286
438
|
* Case-insensitive substring match.
|
|
@@ -365,6 +517,7 @@ class SoulScanner {
|
|
|
365
517
|
}
|
|
366
518
|
/**
|
|
367
519
|
* Calculate grade from score, applying critical floor if needed.
|
|
520
|
+
* @deprecated Use calculateLevel() for display; grade kept for backward compat.
|
|
368
521
|
*/
|
|
369
522
|
calculateGrade(score, criticalMissing) {
|
|
370
523
|
let grade;
|
|
@@ -384,6 +537,20 @@ class SoulScanner {
|
|
|
384
537
|
}
|
|
385
538
|
return { grade, floored: false };
|
|
386
539
|
}
|
|
540
|
+
/**
|
|
541
|
+
* Calculate progress-oriented maturity level.
|
|
542
|
+
*/
|
|
543
|
+
calculateLevel(score) {
|
|
544
|
+
if (score === 0)
|
|
545
|
+
return 'not-started';
|
|
546
|
+
if (score >= 80)
|
|
547
|
+
return 'hardened';
|
|
548
|
+
if (score >= 60)
|
|
549
|
+
return 'standard';
|
|
550
|
+
if (score >= 40)
|
|
551
|
+
return 'developing';
|
|
552
|
+
return 'initial';
|
|
553
|
+
}
|
|
387
554
|
/**
|
|
388
555
|
* Calculate conformance level based on score and critical control status.
|
|
389
556
|
* none: one or more critical controls are missing
|
|
@@ -401,17 +568,18 @@ class SoulScanner {
|
|
|
401
568
|
return 'essential';
|
|
402
569
|
}
|
|
403
570
|
/**
|
|
404
|
-
* Return the subset of controls applicable to a given agent tier.
|
|
571
|
+
* Return the subset of controls applicable to a given agent tier and profile.
|
|
405
572
|
*/
|
|
406
|
-
applicableControls(tier) {
|
|
407
|
-
|
|
573
|
+
applicableControls(tier, profile) {
|
|
574
|
+
const profileDomainIds = PROFILE_DOMAINS[profile];
|
|
575
|
+
return CONTROL_DEFS.filter((d) => d.tiers.includes(tier) && profileDomainIds.includes(d.domainId));
|
|
408
576
|
}
|
|
409
577
|
/**
|
|
410
578
|
* Scan a directory for behavioral governance coverage.
|
|
411
579
|
*/
|
|
412
580
|
async scanSoul(targetDir, options) {
|
|
413
581
|
const govFile = this.findGovernanceFile(targetDir);
|
|
414
|
-
//
|
|
582
|
+
// Read content early (needed for tier + profile detection)
|
|
415
583
|
const contentForTier = govFile
|
|
416
584
|
? (() => { try {
|
|
417
585
|
return fs.readFileSync(govFile, 'utf-8');
|
|
@@ -422,16 +590,46 @@ class SoulScanner {
|
|
|
422
590
|
: '';
|
|
423
591
|
const tierForced = !!options?.tier;
|
|
424
592
|
const tier = (tierForced ? options.tier.toUpperCase() : null) || this.detectTier(targetDir, contentForTier);
|
|
425
|
-
const
|
|
593
|
+
const profileForced = !!options?.profile;
|
|
594
|
+
const profile = (profileForced ? options.profile.toLowerCase() : null) || this.detectProfile(contentForTier);
|
|
595
|
+
const applicable = this.applicableControls(tier, profile);
|
|
596
|
+
// Determine which domains are skipped by profile
|
|
597
|
+
const profileDomainIds = PROFILE_DOMAINS[profile];
|
|
598
|
+
const skippedDomains = DOMAIN_ORDER.filter((d) => {
|
|
599
|
+
const domainId = CONTROL_DEFS.find((c) => c.domain === d)?.domainId;
|
|
600
|
+
return domainId !== undefined && !profileDomainIds.includes(domainId);
|
|
601
|
+
});
|
|
426
602
|
// No governance file found
|
|
427
603
|
if (!govFile) {
|
|
428
604
|
const emptyDomains = DOMAIN_ORDER.map((domain) => {
|
|
429
605
|
const defs = applicable.filter((d) => d.domain === domain);
|
|
430
|
-
|
|
431
|
-
|
|
606
|
+
const domainId = CONTROL_DEFS.find((d) => d.domain === domain)?.domainId ?? 0;
|
|
607
|
+
const isSkipped = !profileDomainIds.includes(domainId);
|
|
608
|
+
if (defs.length === 0 && !isSkipped) {
|
|
609
|
+
// Domain is included by profile but has no controls at the current tier
|
|
610
|
+
return {
|
|
611
|
+
domain,
|
|
612
|
+
domainId,
|
|
613
|
+
controls: [],
|
|
614
|
+
passed: 0,
|
|
615
|
+
total: 0,
|
|
616
|
+
percentage: 0,
|
|
617
|
+
skippedByTier: true,
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
if (isSkipped) {
|
|
621
|
+
return {
|
|
622
|
+
domain,
|
|
623
|
+
domainId,
|
|
624
|
+
controls: [],
|
|
625
|
+
passed: 0,
|
|
626
|
+
total: 0,
|
|
627
|
+
percentage: 0,
|
|
628
|
+
skippedByProfile: true,
|
|
629
|
+
};
|
|
630
|
+
}
|
|
432
631
|
const controls = defs
|
|
433
632
|
.map((d) => ({ id: d.id, name: d.name, domain: d.domain, keywords: d.keywords, passed: false }));
|
|
434
|
-
const domainId = defs[0]?.domainId ?? 0;
|
|
435
633
|
return {
|
|
436
634
|
domain,
|
|
437
635
|
domainId,
|
|
@@ -443,15 +641,20 @@ class SoulScanner {
|
|
|
443
641
|
}).filter((d) => d !== null);
|
|
444
642
|
const criticalMissing = applicable.filter((d) => d.critical).map((d) => d.id);
|
|
445
643
|
const { grade, floored } = this.calculateGrade(0, criticalMissing);
|
|
644
|
+
const level = this.calculateLevel(0);
|
|
446
645
|
const conformance = this.calculateConformance(0, criticalMissing);
|
|
447
646
|
return {
|
|
448
647
|
file: null,
|
|
449
648
|
fileSize: 0,
|
|
450
649
|
agentTier: tier,
|
|
451
650
|
tierForced,
|
|
651
|
+
agentProfile: profile,
|
|
652
|
+
profileForced,
|
|
653
|
+
skippedDomains,
|
|
452
654
|
domains: emptyDomains,
|
|
453
655
|
score: 0,
|
|
454
656
|
grade,
|
|
657
|
+
level,
|
|
455
658
|
conformance,
|
|
456
659
|
criticalFloor: floored,
|
|
457
660
|
criticalMissing,
|
|
@@ -494,14 +697,36 @@ class SoulScanner {
|
|
|
494
697
|
}
|
|
495
698
|
}
|
|
496
699
|
}
|
|
497
|
-
// Group into domains (
|
|
700
|
+
// Group into domains (include skipped-by-profile domains for visibility)
|
|
498
701
|
const domains = DOMAIN_ORDER.map((domain) => {
|
|
702
|
+
const domainId = CONTROL_DEFS.find((d) => d.domain === domain)?.domainId ?? 0;
|
|
703
|
+
const isSkipped = !profileDomainIds.includes(domainId);
|
|
704
|
+
if (isSkipped) {
|
|
705
|
+
return {
|
|
706
|
+
domain,
|
|
707
|
+
domainId,
|
|
708
|
+
controls: [],
|
|
709
|
+
passed: 0,
|
|
710
|
+
total: 0,
|
|
711
|
+
percentage: 0,
|
|
712
|
+
skippedByProfile: true,
|
|
713
|
+
};
|
|
714
|
+
}
|
|
499
715
|
const domainControls = controlResults.filter((c) => c.domain === domain);
|
|
500
|
-
if (domainControls.length === 0)
|
|
501
|
-
|
|
716
|
+
if (domainControls.length === 0) {
|
|
717
|
+
// Domain is included by profile but has no controls at the current tier
|
|
718
|
+
return {
|
|
719
|
+
domain,
|
|
720
|
+
domainId,
|
|
721
|
+
controls: [],
|
|
722
|
+
passed: 0,
|
|
723
|
+
total: 0,
|
|
724
|
+
percentage: 0,
|
|
725
|
+
skippedByTier: true,
|
|
726
|
+
};
|
|
727
|
+
}
|
|
502
728
|
const passed = domainControls.filter((c) => c.passed).length;
|
|
503
729
|
const total = domainControls.length;
|
|
504
|
-
const domainId = CONTROL_DEFS.find((d) => d.domain === domain)?.domainId ?? 0;
|
|
505
730
|
return {
|
|
506
731
|
domain,
|
|
507
732
|
domainId,
|
|
@@ -511,9 +736,10 @@ class SoulScanner {
|
|
|
511
736
|
percentage: total > 0 ? Math.round((passed / total) * 100) : 0,
|
|
512
737
|
};
|
|
513
738
|
}).filter((d) => d !== null);
|
|
514
|
-
// Calculate overall score as average of applicable domain percentages
|
|
515
|
-
const
|
|
516
|
-
|
|
739
|
+
// Calculate overall score as average of applicable (non-skipped) domain percentages
|
|
740
|
+
const scoredDomains = domains.filter((d) => !d.skippedByProfile && d.total > 0);
|
|
741
|
+
const score = scoredDomains.length > 0
|
|
742
|
+
? Math.round(scoredDomains.reduce((sum, d) => sum + d.percentage, 0) / scoredDomains.length)
|
|
517
743
|
: 0;
|
|
518
744
|
// Find missing critical controls (only applicable ones)
|
|
519
745
|
const criticalMissing = applicable
|
|
@@ -521,6 +747,7 @@ class SoulScanner {
|
|
|
521
747
|
.filter((d) => !controlResults.find((c) => c.id === d.id)?.passed)
|
|
522
748
|
.map((d) => d.id);
|
|
523
749
|
const { grade, floored } = this.calculateGrade(score, criticalMissing);
|
|
750
|
+
const level = this.calculateLevel(score);
|
|
524
751
|
const conformance = this.calculateConformance(score, criticalMissing);
|
|
525
752
|
const totalPassed = controlResults.filter((c) => c.passed).length;
|
|
526
753
|
const result = {
|
|
@@ -528,9 +755,13 @@ class SoulScanner {
|
|
|
528
755
|
fileSize,
|
|
529
756
|
agentTier: tier,
|
|
530
757
|
tierForced,
|
|
758
|
+
agentProfile: profile,
|
|
759
|
+
profileForced,
|
|
760
|
+
skippedDomains,
|
|
531
761
|
domains,
|
|
532
762
|
score,
|
|
533
763
|
grade,
|
|
764
|
+
level,
|
|
534
765
|
conformance,
|
|
535
766
|
criticalFloor: floored,
|
|
536
767
|
criticalMissing,
|
|
@@ -547,35 +778,43 @@ class SoulScanner {
|
|
|
547
778
|
}
|
|
548
779
|
/**
|
|
549
780
|
* Generate or update SOUL.md with missing governance sections.
|
|
781
|
+
* Supports iterative hardening: if a domain heading exists but controls
|
|
782
|
+
* are failing, appends targeted remediation for those controls.
|
|
550
783
|
*/
|
|
551
784
|
async hardenSoul(targetDir, options) {
|
|
552
785
|
const dryRun = options?.dryRun ?? false;
|
|
553
|
-
//
|
|
554
|
-
const
|
|
786
|
+
// Detect tier BEFORE hardening so we can pin it
|
|
787
|
+
const govFileCheck = this.findGovernanceFile(targetDir);
|
|
788
|
+
let existingContent = '';
|
|
789
|
+
if (govFileCheck) {
|
|
790
|
+
try {
|
|
791
|
+
existingContent = fs.readFileSync(govFileCheck, 'utf-8');
|
|
792
|
+
}
|
|
793
|
+
catch {
|
|
794
|
+
// File may not be readable
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
const preTier = this.detectTier(targetDir, existingContent);
|
|
798
|
+
const preProfile = options?.profile
|
|
799
|
+
? options.profile.toLowerCase()
|
|
800
|
+
: this.detectProfile(existingContent);
|
|
555
801
|
// Determine target file
|
|
556
|
-
const govFile =
|
|
557
|
-
?
|
|
802
|
+
const govFile = govFileCheck
|
|
803
|
+
? govFileCheck
|
|
558
804
|
: path.join(targetDir, 'SOUL.md');
|
|
559
|
-
const existedBefore =
|
|
805
|
+
const existedBefore = govFileCheck !== null;
|
|
560
806
|
const sectionsAdded = [];
|
|
561
807
|
let controlsAdded = 0;
|
|
562
808
|
// Build content to append
|
|
563
809
|
let newContent = '';
|
|
564
810
|
if (!existedBefore) {
|
|
565
|
-
// Create full SOUL.md from scratch
|
|
566
|
-
newContent += `# Agent Governance (SOUL)\n\
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
if (existedBefore) {
|
|
571
|
-
try {
|
|
572
|
-
existingContent = fs.readFileSync(govFile, 'utf-8');
|
|
573
|
-
}
|
|
574
|
-
catch {
|
|
575
|
-
// File may not be readable; treat as empty
|
|
576
|
-
}
|
|
811
|
+
// Create full SOUL.md from scratch with tier/profile markers
|
|
812
|
+
newContent += `# Agent Governance (SOUL)\n\n`;
|
|
813
|
+
newContent += `<!-- soul:tier=${preTier} -->\n`;
|
|
814
|
+
newContent += `<!-- soul:profile=${preProfile} -->\n\n`;
|
|
815
|
+
newContent += `This document defines the behavioral governance rules for this agent.\nGenerated by HackMyAgent scan-soul/harden-soul.\n\n`;
|
|
577
816
|
}
|
|
578
|
-
// harden-soul generates all
|
|
817
|
+
// harden-soul generates all 9 domain sections (comprehensive / future-proof).
|
|
579
818
|
// scan-soul evaluates only tier-applicable controls; harden-soul adds them all
|
|
580
819
|
// so the resulting SOUL.md is ready if the agent tier increases later.
|
|
581
820
|
for (const domainName of DOMAIN_ORDER) {
|
|
@@ -586,14 +825,33 @@ class SoulScanner {
|
|
|
586
825
|
const existingLower = existingContent.toLowerCase();
|
|
587
826
|
const headingLower = template.heading.toLowerCase();
|
|
588
827
|
if (existingLower.includes(headingLower)) {
|
|
589
|
-
// Domain heading exists --
|
|
828
|
+
// Domain heading exists -- check for failing controls (iterative hardening)
|
|
829
|
+
const domainDefs = CONTROL_DEFS.filter((d) => d.domain === domainName);
|
|
830
|
+
const failing = domainDefs.filter((def) => !this.checkControl(existingContent, def));
|
|
831
|
+
if (failing.length === 0)
|
|
832
|
+
continue; // All controls pass, skip
|
|
833
|
+
// Append targeted remediation for failing controls
|
|
834
|
+
let remediation = `\n### ${domainName} (Additional Governance)\n`;
|
|
835
|
+
for (const def of failing) {
|
|
836
|
+
if (def.remediation) {
|
|
837
|
+
remediation += `\n${def.remediation}\n`;
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
newContent += remediation;
|
|
841
|
+
sectionsAdded.push(`${domainName} (augmented: ${failing.length} controls)`);
|
|
842
|
+
controlsAdded += failing.length;
|
|
590
843
|
continue;
|
|
591
844
|
}
|
|
592
845
|
newContent += template.content + '\n';
|
|
593
846
|
sectionsAdded.push(domainName);
|
|
594
|
-
// Count controls
|
|
595
|
-
const
|
|
596
|
-
|
|
847
|
+
// Count controls that actually pass with the template content
|
|
848
|
+
const domainDefs = CONTROL_DEFS.filter((d) => d.domain === domainName);
|
|
849
|
+
const actuallyPassing = domainDefs.filter((def) => this.checkControl(template.content, def)).length;
|
|
850
|
+
controlsAdded += actuallyPassing;
|
|
851
|
+
}
|
|
852
|
+
// Add tier/profile markers to existing files if not present
|
|
853
|
+
if (existedBefore && !existingContent.match(/<!--\s*soul:tier=/i)) {
|
|
854
|
+
newContent = `\n<!-- soul:tier=${preTier} -->\n<!-- soul:profile=${preProfile} -->\n` + newContent;
|
|
597
855
|
}
|
|
598
856
|
// Apply or preview
|
|
599
857
|
if (!dryRun && newContent.length > 0) {
|