@beingmartinbmc/ojas 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +308 -0
  3. package/dist/aahar/index.d.ts +179 -0
  4. package/dist/aahar/index.d.ts.map +1 -0
  5. package/dist/aahar/index.js +657 -0
  6. package/dist/aahar/index.js.map +1 -0
  7. package/dist/aahar/scoring.d.ts +85 -0
  8. package/dist/aahar/scoring.d.ts.map +1 -0
  9. package/dist/aahar/scoring.js +268 -0
  10. package/dist/aahar/scoring.js.map +1 -0
  11. package/dist/agni/index.d.ts +113 -0
  12. package/dist/agni/index.d.ts.map +1 -0
  13. package/dist/agni/index.js +328 -0
  14. package/dist/agni/index.js.map +1 -0
  15. package/dist/agni/model-router.d.ts +77 -0
  16. package/dist/agni/model-router.d.ts.map +1 -0
  17. package/dist/agni/model-router.js +163 -0
  18. package/dist/agni/model-router.js.map +1 -0
  19. package/dist/agni/response-distiller.d.ts +37 -0
  20. package/dist/agni/response-distiller.d.ts.map +1 -0
  21. package/dist/agni/response-distiller.js +193 -0
  22. package/dist/agni/response-distiller.js.map +1 -0
  23. package/dist/agni/tiktoken-adapter.d.ts +55 -0
  24. package/dist/agni/tiktoken-adapter.d.ts.map +1 -0
  25. package/dist/agni/tiktoken-adapter.js +113 -0
  26. package/dist/agni/tiktoken-adapter.js.map +1 -0
  27. package/dist/chikitsa/index.d.ts +130 -0
  28. package/dist/chikitsa/index.d.ts.map +1 -0
  29. package/dist/chikitsa/index.js +565 -0
  30. package/dist/chikitsa/index.js.map +1 -0
  31. package/dist/demo.d.ts +15 -0
  32. package/dist/demo.d.ts.map +1 -0
  33. package/dist/demo.js +278 -0
  34. package/dist/demo.js.map +1 -0
  35. package/dist/index.d.ts +201 -0
  36. package/dist/index.d.ts.map +1 -0
  37. package/dist/index.js +588 -0
  38. package/dist/index.js.map +1 -0
  39. package/dist/mcp/audit.d.ts +39 -0
  40. package/dist/mcp/audit.d.ts.map +1 -0
  41. package/dist/mcp/audit.js +73 -0
  42. package/dist/mcp/audit.js.map +1 -0
  43. package/dist/mcp/contracts.d.ts +76 -0
  44. package/dist/mcp/contracts.d.ts.map +1 -0
  45. package/dist/mcp/contracts.js +44 -0
  46. package/dist/mcp/contracts.js.map +1 -0
  47. package/dist/mcp/envelope.d.ts +107 -0
  48. package/dist/mcp/envelope.d.ts.map +1 -0
  49. package/dist/mcp/envelope.js +162 -0
  50. package/dist/mcp/envelope.js.map +1 -0
  51. package/dist/mcp/registry.d.ts +110 -0
  52. package/dist/mcp/registry.d.ts.map +1 -0
  53. package/dist/mcp/registry.js +258 -0
  54. package/dist/mcp/registry.js.map +1 -0
  55. package/dist/mcp/server.d.ts +26 -0
  56. package/dist/mcp/server.d.ts.map +1 -0
  57. package/dist/mcp/server.js +107 -0
  58. package/dist/mcp/server.js.map +1 -0
  59. package/dist/mcp/tools/agent.d.ts +4 -0
  60. package/dist/mcp/tools/agent.d.ts.map +1 -0
  61. package/dist/mcp/tools/agent.js +300 -0
  62. package/dist/mcp/tools/agent.js.map +1 -0
  63. package/dist/mcp/tools/context.d.ts +4 -0
  64. package/dist/mcp/tools/context.d.ts.map +1 -0
  65. package/dist/mcp/tools/context.js +261 -0
  66. package/dist/mcp/tools/context.js.map +1 -0
  67. package/dist/mcp/tools/index.d.ts +5 -0
  68. package/dist/mcp/tools/index.d.ts.map +1 -0
  69. package/dist/mcp/tools/index.js +20 -0
  70. package/dist/mcp/tools/index.js.map +1 -0
  71. package/dist/mcp/tools/memory.d.ts +4 -0
  72. package/dist/mcp/tools/memory.d.ts.map +1 -0
  73. package/dist/mcp/tools/memory.js +220 -0
  74. package/dist/mcp/tools/memory.js.map +1 -0
  75. package/dist/mcp/tools/output.d.ts +4 -0
  76. package/dist/mcp/tools/output.d.ts.map +1 -0
  77. package/dist/mcp/tools/output.js +206 -0
  78. package/dist/mcp/tools/output.js.map +1 -0
  79. package/dist/mcp/tools/recovery.d.ts +4 -0
  80. package/dist/mcp/tools/recovery.d.ts.map +1 -0
  81. package/dist/mcp/tools/recovery.js +165 -0
  82. package/dist/mcp/tools/recovery.js.map +1 -0
  83. package/dist/mcp/tools/registrar.d.ts +4 -0
  84. package/dist/mcp/tools/registrar.d.ts.map +1 -0
  85. package/dist/mcp/tools/registrar.js +17 -0
  86. package/dist/mcp/tools/registrar.js.map +1 -0
  87. package/dist/mcp/tools/report.d.ts +4 -0
  88. package/dist/mcp/tools/report.d.ts.map +1 -0
  89. package/dist/mcp/tools/report.js +68 -0
  90. package/dist/mcp/tools/report.js.map +1 -0
  91. package/dist/mcp/tools/shared.d.ts +37 -0
  92. package/dist/mcp/tools/shared.d.ts.map +1 -0
  93. package/dist/mcp/tools/shared.js +214 -0
  94. package/dist/mcp/tools/shared.js.map +1 -0
  95. package/dist/mcp/trace.d.ts +47 -0
  96. package/dist/mcp/trace.d.ts.map +1 -0
  97. package/dist/mcp/trace.js +216 -0
  98. package/dist/mcp/trace.js.map +1 -0
  99. package/dist/nidra/index.d.ts +275 -0
  100. package/dist/nidra/index.d.ts.map +1 -0
  101. package/dist/nidra/index.js +889 -0
  102. package/dist/nidra/index.js.map +1 -0
  103. package/dist/persistence/migrations.d.ts +10 -0
  104. package/dist/persistence/migrations.d.ts.map +1 -0
  105. package/dist/persistence/migrations.js +77 -0
  106. package/dist/persistence/migrations.js.map +1 -0
  107. package/dist/persistence/sqlite.d.ts +30 -0
  108. package/dist/persistence/sqlite.d.ts.map +1 -0
  109. package/dist/persistence/sqlite.js +209 -0
  110. package/dist/persistence/sqlite.js.map +1 -0
  111. package/dist/persistence/types.d.ts +104 -0
  112. package/dist/persistence/types.d.ts.map +1 -0
  113. package/dist/persistence/types.js +5 -0
  114. package/dist/persistence/types.js.map +1 -0
  115. package/dist/pulse/index.d.ts +144 -0
  116. package/dist/pulse/index.d.ts.map +1 -0
  117. package/dist/pulse/index.js +453 -0
  118. package/dist/pulse/index.js.map +1 -0
  119. package/dist/raksha/classifiers/http-classifier.d.ts +26 -0
  120. package/dist/raksha/classifiers/http-classifier.d.ts.map +1 -0
  121. package/dist/raksha/classifiers/http-classifier.js +62 -0
  122. package/dist/raksha/classifiers/http-classifier.js.map +1 -0
  123. package/dist/raksha/classifiers/index.d.ts +5 -0
  124. package/dist/raksha/classifiers/index.d.ts.map +1 -0
  125. package/dist/raksha/classifiers/index.js +8 -0
  126. package/dist/raksha/classifiers/index.js.map +1 -0
  127. package/dist/raksha/classifiers/onnx-classifier.d.ts +41 -0
  128. package/dist/raksha/classifiers/onnx-classifier.d.ts.map +1 -0
  129. package/dist/raksha/classifiers/onnx-classifier.js +99 -0
  130. package/dist/raksha/classifiers/onnx-classifier.js.map +1 -0
  131. package/dist/raksha/hallucination-detectors.d.ts +106 -0
  132. package/dist/raksha/hallucination-detectors.d.ts.map +1 -0
  133. package/dist/raksha/hallucination-detectors.js +327 -0
  134. package/dist/raksha/hallucination-detectors.js.map +1 -0
  135. package/dist/raksha/index.d.ts +168 -0
  136. package/dist/raksha/index.d.ts.map +1 -0
  137. package/dist/raksha/index.js +597 -0
  138. package/dist/raksha/index.js.map +1 -0
  139. package/dist/raksha/prompt-injection-detectors.d.ts +30 -0
  140. package/dist/raksha/prompt-injection-detectors.d.ts.map +1 -0
  141. package/dist/raksha/prompt-injection-detectors.js +153 -0
  142. package/dist/raksha/prompt-injection-detectors.js.map +1 -0
  143. package/dist/types.d.ts +1115 -0
  144. package/dist/types.d.ts.map +1 -0
  145. package/dist/types.js +71 -0
  146. package/dist/types.js.map +1 -0
  147. package/dist/util/calibration.d.ts +32 -0
  148. package/dist/util/calibration.d.ts.map +1 -0
  149. package/dist/util/calibration.js +108 -0
  150. package/dist/util/calibration.js.map +1 -0
  151. package/dist/util/id.d.ts +2 -0
  152. package/dist/util/id.d.ts.map +1 -0
  153. package/dist/util/id.js +9 -0
  154. package/dist/util/id.js.map +1 -0
  155. package/dist/vyayam/index.d.ts +76 -0
  156. package/dist/vyayam/index.d.ts.map +1 -0
  157. package/dist/vyayam/index.js +528 -0
  158. package/dist/vyayam/index.js.map +1 -0
  159. package/dist/vyayam/tool-fault-proxy.d.ts +95 -0
  160. package/dist/vyayam/tool-fault-proxy.d.ts.map +1 -0
  161. package/dist/vyayam/tool-fault-proxy.js +170 -0
  162. package/dist/vyayam/tool-fault-proxy.js.map +1 -0
  163. package/docs/ARCHITECTURE.md +162 -0
  164. package/docs/BACKLOG.md +342 -0
  165. package/docs/CONFIGURATION.md +305 -0
  166. package/docs/EVIDENCE.md +232 -0
  167. package/docs/EVIDENCE_MATRIX.md +293 -0
  168. package/docs/KNOWN_FAILURES.md +367 -0
  169. package/docs/MCP.md +614 -0
  170. package/docs/MODULES.md +368 -0
  171. package/docs/SECURITY.md +251 -0
  172. package/docs/TRUST.md +88 -0
  173. package/docs/assets/ojas-hero.png +0 -0
  174. package/package.json +101 -0
@@ -0,0 +1,305 @@
1
+ # SDK Configuration
2
+
3
+ For embedding Ojas in your own TypeScript / Node runtime. If you only want to use the MCP server, see [`MCP.md`](./MCP.md) instead.
4
+
5
+ | Section | What's inside |
6
+ |---|---|
7
+ | [Agent Adapter Interface](#adapter) | The single contract Ojas needs to bind to any agent framework |
8
+ | [Continuous Monitoring](#monitoring) | Background health checks + alert callbacks |
9
+ | [Full configuration](#config) | Every policy field, with defaults and validation rules |
10
+ | [Retention caps](#retention) | Hard caps on every internal store so long-running agents stay bounded |
11
+ | [Project structure](#structure) | How `src/`, `benchmarks/`, `test/`, and `docs/` are organised |
12
+
13
+ ---
14
+
15
+ <a id="adapter"></a>
16
+ ## Agent Adapter Interface
17
+
18
+ Ojas can bind to any agent that implements `AgentAdapter`.
19
+
20
+ ```typescript
21
+ interface AgentAdapter {
22
+ id: string;
23
+ process(input: string, context: ContextItem[], signal?: AbortSignal): Promise<AgentResponse>;
24
+ getState(): Promise<AgentState>;
25
+ injectMemory(memory: ConsolidatedMemory): Promise<void>;
26
+ }
27
+ ```
28
+
29
+ The optional `signal` parameter enables cooperative cancellation. When
30
+ Vyayam's stress-test timeout fires, it aborts the signal so agents that
31
+ check it can stop in-flight work immediately. Agents that ignore the
32
+ signal still get a timeout result but don't crash (backward-compatible).
33
+
34
+ This keeps Ojas independent of any specific agent framework. Binding is one call:
35
+
36
+ ```typescript
37
+ const ojas = new Ojas({ agentId: 'my-agent' });
38
+ ojas.bind(myAgent);
39
+ ```
40
+
41
+ `injectMemory` is invoked once per consolidated memory during `recover()`. If your agent has a write-through memory cache, `injectMemory` is where you should write to it. **Throwing from `injectMemory` aborts the recovery cycle before Nidra commits** — this is the transactional guarantee.
42
+
43
+ ---
44
+
45
+ <a id="monitoring"></a>
46
+ ## Continuous Monitoring
47
+
48
+ ```typescript
49
+ ojas.startMonitoring((report) => {
50
+ console.log(`Aahar: ${report.moduleScores.aahar} / 100`);
51
+ console.log(`Raksha: ${report.moduleScores.raksha} / 100`);
52
+
53
+ const critical = report.recommendations.some(
54
+ (recommendation) => recommendation.severity === 'critical'
55
+ );
56
+
57
+ if (critical) {
58
+ alert('Agent health critical');
59
+ }
60
+ });
61
+ ```
62
+
63
+ Monitoring can trigger recovery when drift or degradation exceeds configured thresholds.
64
+
65
+ ---
66
+
67
+ <a id="config"></a>
68
+ ## Full configuration
69
+
70
+ ```typescript
71
+ const ojas = new Ojas({
72
+ agentId: 'my-agent',
73
+
74
+ nutrition: {
75
+ maxContextTokens: 4096,
76
+ relevanceThreshold: 0.3,
77
+ freshnessWindowSec: 3600,
78
+ maxContextItems: 20,
79
+ targetSignalToNoise: 0.7,
80
+ maxHistory: 1_000, // bound Aahar filter-history retention
81
+ },
82
+
83
+ recovery: {
84
+ minTracesForConsolidation: 10,
85
+ recoveryIntervalSec: 300,
86
+ maxDriftThreshold: 0.6,
87
+ failureWindowSize: 50,
88
+ retentionConfidence: 0.5,
89
+ maxTraces: 10_000, // bound Nidra trace store
90
+ maxMemories: 5_000, // bound consolidated-memory store
91
+ maxProcessedTraceIds: 10_000,
92
+ maxCycleHistory: 1_000,
93
+ },
94
+
95
+ resilience: {
96
+ stressIntervalSec: 600,
97
+ scenariosPerCycle: 5,
98
+ minResilienceThreshold: 0.4,
99
+ progressiveDifficulty: true,
100
+ maxIntensity: 0.8,
101
+ maxResults: 1_000,
102
+ maxScenarioDurationMs: 30_000, // per-scenario timeout; hangs become failures
103
+ },
104
+
105
+ defense: {
106
+ quarantineThreshold: 0.7,
107
+ warnThreshold: 0.35,
108
+ protectInstructionHierarchy: true,
109
+ maxAssessments: 5_000,
110
+ maxEvents: 1_000,
111
+ },
112
+
113
+ metabolism: {
114
+ targetTokensPerTask: 4096,
115
+ maxLatencyMs: 5000,
116
+ maxToolCallsPerTask: 8,
117
+ highRiskReasoningBoost: 0.25,
118
+ maxTraces: 10_000,
119
+ targetCostUsdPerTask: 0.05, // optional; enables cost-pressure component
120
+ },
121
+
122
+ telemetry: {
123
+ maxRecentEvents: 1000,
124
+ degradationThreshold: 0.6,
125
+ },
126
+
127
+ rehabilitation: {
128
+ autoRepairEnabled: false,
129
+ criticalRequiresHuman: true,
130
+ maxProtocolsPerCycle: 5,
131
+ maxProtocols: 1_000,
132
+ },
133
+
134
+ maxHealthHistory: 1_000,
135
+
136
+ // Empirical calibration model (produced by L3 pipeline)
137
+ // When set, healthCheck() applies this model for `empirical_calibrated` scores
138
+ calibrationModel: deserializeCalibrationModel(jsonFromL3Run),
139
+ });
140
+ ```
141
+
142
+ ### Raksha classifier plugins
143
+
144
+ ```typescript
145
+ import { OnnxPromptInjectionClassifier, HttpPromptInjectionClassifier } from '@beingmartinbmc/ojas/raksha/classifiers';
146
+
147
+ const ojas = new Ojas({
148
+ agentId: 'my-agent',
149
+ defense: {
150
+ quarantineThreshold: 0.7,
151
+ classifiers: [
152
+ new OnnxPromptInjectionClassifier({
153
+ modelPath: './models/prompt-injection.onnx',
154
+ }),
155
+ new HttpPromptInjectionClassifier({
156
+ url: 'https://my-classifier.example.com/classify',
157
+ apiKey: process.env.CLASSIFIER_API_KEY,
158
+ }),
159
+ ],
160
+ },
161
+ });
162
+ ```
163
+
164
+ Classifiers run asynchronously after the rule-based stack. The highest
165
+ probability (rule-based or classifier) wins. Classifier errors are caught
166
+ and logged — they never block the main assessment.
167
+
168
+ ### Persistence with encryption
169
+
170
+ ```typescript
171
+ import { SQLitePersistenceStore } from '@beingmartinbmc/ojas/persistence/sqlite';
172
+
173
+ const store = new SQLitePersistenceStore({
174
+ dbPath: './ojas-data.db',
175
+ encryptionKey: process.env.OJAS_DB_KEY, // optional; requires SQLCipher build
176
+ });
177
+
178
+ // Hot backup:
179
+ await store.backup('/backups/ojas-snapshot.db');
180
+
181
+ // Restore from backup:
182
+ await store.restore('/backups/ojas-snapshot.db');
183
+ ```
184
+
185
+ ### Calibration model loading
186
+
187
+ ```typescript
188
+ import { deserializeCalibrationModel } from '@beingmartinbmc/ojas/util/calibration';
189
+ import calibrationJson from './benchmarks/results/l3/latest/calibration-model.json';
190
+
191
+ const ojas = new Ojas({
192
+ agentId: 'my-agent',
193
+ calibrationModel: deserializeCalibrationModel(calibrationJson),
194
+ });
195
+
196
+ // healthCheck() now returns basis: 'empirical_calibrated'
197
+ ```
198
+
199
+ > **Validation is strict at construction.** Negative thresholds, non-finite numbers (`NaN`, `Infinity`), and non-integer counts throw with a clear error naming the offending field. The same validation runs on every `updatePolicy()` call.
200
+
201
+ ---
202
+
203
+ <a id="retention"></a>
204
+ ## Retention caps
205
+
206
+ Every internal store has a hard cap so long-running agents stay bounded in memory. Each cap evicts oldest-first (FIFO) when exceeded.
207
+
208
+ | Store | Field | Default |
209
+ |---|---|---|
210
+ | Aahar filter history | `nutrition.maxHistory` | 1 000 |
211
+ | Nidra traces | `recovery.maxTraces` | 10 000 |
212
+ | Nidra memories | `recovery.maxMemories` | 5 000 |
213
+ | Nidra processed-trace IDs | `recovery.maxProcessedTraceIds` | 10 000 |
214
+ | Nidra cycle history | `recovery.maxCycleHistory` | 1 000 |
215
+ | Vyayam stress results | `resilience.maxResults` | 1 000 |
216
+ | Raksha assessments | `defense.maxAssessments` | 5 000 |
217
+ | Raksha events | `defense.maxEvents` | 1 000 |
218
+ | Agni traces | `metabolism.maxTraces` | 10 000 |
219
+ | Chikitsa protocols | `rehabilitation.maxProtocols` | 1 000 |
220
+ | Ojas health reports | `maxHealthHistory` | 1 000 |
221
+
222
+ The registry (when used through the MCP server) has its own cap, `OJAS_MAX_AGENTS` — see [`MCP.md`](./MCP.md#env).
223
+
224
+ ---
225
+
226
+ <a id="structure"></a>
227
+ ## Project structure
228
+
229
+ ```text
230
+ ojas/
231
+ ├── src/
232
+ │ ├── index.ts # Unified Ojas runtime and module scoring
233
+ │ ├── types.ts # Shared types and canonical health event schemas
234
+ │ ├── aahar/index.ts # Cognitive Nutrition engine
235
+ │ ├── nidra/index.ts # Recovery and Consolidation engine
236
+ │ ├── vyayam/index.ts # Resilience and Stress engine
237
+ │ ├── raksha/index.ts # Immune Defense engine
238
+ │ ├── agni/index.ts # Cognitive Metabolism engine
239
+ │ ├── pulse/index.ts # Health Telemetry engine
240
+ │ ├── chikitsa/index.ts # Repair and Rehabilitation engine
241
+ │ ├── mcp/
242
+ │ │ ├── server.ts # Stdio MCP server wiring (18 tools)
243
+ │ │ ├── registry.ts # agent_id → Ojas instance + health contract
244
+ │ │ ├── contracts.ts # HealthContract, risk levels, health-state classifier
245
+ │ │ ├── trace.ts # Runtime trace event taxonomy (17 event types)
246
+ │ │ ├── envelope.ts # Standard Ojas response envelope
247
+ │ │ ├── audit.ts # Structured JSONL audit logger
248
+ │ │ └── tools/ # Grouped MCP tool registrations by domain
249
+ │ ├── agni/
250
+ │ │ ├── model-router.ts
251
+ │ │ ├── response-distiller.ts
252
+ │ │ └── tiktoken-adapter.ts
253
+ │ ├── raksha/
254
+ │ │ ├── hallucination-detectors.ts
255
+ │ │ ├── prompt-injection-detectors.ts
256
+ │ │ └── classifiers/ # PromptInjectionClassifier adapters
257
+ │ │ ├── onnx-classifier.ts # Bundled ONNX model inference
258
+ │ │ ├── http-classifier.ts # External HTTP API adapter
259
+ │ │ └── index.ts
260
+ │ ├── persistence/
261
+ │ │ ├── sqlite.ts # SQLite persistence (backup, encryption)
262
+ │ │ ├── migrations.ts # Schema migrations
263
+ │ │ └── types.ts # PersistenceStore interface
264
+ │ ├── vyayam/tool-fault-proxy.ts
265
+ │ ├── util/
266
+ │ │ ├── calibration.ts # Isotonic regression, serialization
267
+ │ │ └── id.ts
268
+ │ └── demo.ts # End-to-end interactive demo
269
+ ├── benchmarks/
270
+ │ ├── agents/ # Synthetic vulnerable AgentAdapters
271
+ │ ├── fixtures/ # Retrieval-QA fixture corpus
272
+ │ ├── scenarios/ # Attack prompts, polluted contexts, memory candidates
273
+ │ ├── suites/ # 11 benchmark suites (L2 + L2.5)
274
+ │ │ ├── ablation.ts # Module ablation matrix
275
+ │ │ ├── flaky-tool.ts # Non-deterministic fault injection
276
+ │ │ └── ... # + injection, retrieval-qa, calibration, etc.
277
+ │ ├── util/ # Seeded PRNG and bootstrap statistics
278
+ │ ├── runner.ts # `npm run benchmark` entrypoint (--real-tokenizer, --store-transcripts)
279
+ │ ├── l3-runner.ts # Full L3 pipeline (real-LLM + judge + transcripts)
280
+ │ ├── verify-evidence.ts # CI gate: evidence matrix + L3 freshness
281
+ │ ├── report.ts # Console / Markdown / JSON renderers
282
+ │ └── results/latest.json
283
+ ├── test/
284
+ │ └── *.test.ts # 32 Jest suites covering runtime, modules, MCP, benchmarks, examples
285
+ ├── examples/
286
+ │ ├── before-after.ts # Deterministic before/after demo
287
+ │ ├── langchain-adapter.ts # LangChain integration example
288
+ │ ├── openai-agents-adapter.ts # OpenAI Agents SDK integration
289
+ │ ├── vercel-ai-adapter.ts # Vercel AI SDK middleware example
290
+ │ └── mcp-client-workflow.ts # End-to-end MCP client workflow
291
+ ├── docs/
292
+ │ ├── MODULES.md # Module deep-dives + health events
293
+ │ ├── MCP.md # MCP server, tools, envelope, usage loop
294
+ │ ├── CONFIGURATION.md # This file
295
+ │ ├── ARCHITECTURE.md # Architecture diagram, principles, philosophy
296
+ │ ├── EVIDENCE.md # Auto-generated reproducible-evidence document
297
+ │ ├── EVIDENCE_MATRIX.md # Evidence levels and claim-by-claim limitations
298
+ │ ├── KNOWN_FAILURES.md # Known limitations and remaining failure modes
299
+ │ ├── BACKLOG.md # Honest list of deferred work
300
+ │ └── SECURITY.md # Trust model and security posture
301
+ ├── LICENSE
302
+ ├── package.json
303
+ ├── tsconfig.json
304
+ └── jest.config.js
305
+ ```
@@ -0,0 +1,232 @@
1
+ # Ojas — Evidence Harness Results
2
+
3
+ > **Auto-generated by `npm run benchmark:write`. Do not edit by hand.**
4
+
5
+ - **ojas**: `0.2.0`
6
+ - **node**: `v24.15.0`
7
+ - **timestamp**: 2026-05-14T08:51:51.214Z
8
+ - **suites**: 11/11 passed — targeted failure suites improved and diagnostic/no-regression suites met their acceptance criteria.
9
+
10
+ ## What this measures
11
+
12
+ 11 A/B benchmarks comparing a deliberately vulnerable agent running **without Ojas** vs the **same agent + Ojas**. Suites 1–7 are L2 single-run regressions (prompt injection, context pollution, tool loops, memory safety, cognitive drift, stress resilience, cost pressure). Suite 8 is L2.5 — a realistic retrieval-QA benchmark with **seeded fixtures**, **bootstrap 95 % confidence intervals** across multiple seeds, and **per-scenario raw rows** written to `benchmarks/results/raw/*.jsonl` on `npm run benchmark:write`. See `docs/EVIDENCE_MATRIX.md` for the evidence ladder and `docs/KNOWN_FAILURES.md` for the failure modes these benchmarks deliberately do *not* probe.
13
+
14
+ > The vulnerable agents are synthetic and have explicitly-programmed failure modes. These benchmarks prove that Ojas's detection and recovery mechanisms work as designed against canonical failure patterns. Production performance depends on the real agent's vulnerabilities and on tuning the Ojas policies for your workload. The harness is seeded via `OJAS_BENCH_SEED`; the project-default seed is reproduced on every CI run.
15
+
16
+ ## Summary
17
+
18
+ | # | Suite | Modules | Pass |
19
+ |---|---|---|---|
20
+ | 1 | Prompt-injection resistance | raksha, aahar | ✅ |
21
+ | 2 | Context pollution survival | aahar | ✅ |
22
+ | 3 | Tool-failure loop detection | pulse, nidra, chikitsa | ✅ |
23
+ | 4 | Memory-write safety | raksha, nidra | ✅ |
24
+ | 5 | Cognitive drift detection | nidra, pulse | ✅ |
25
+ | 6 | Vyayam resilience under stress | vyayam, raksha, aahar | ✅ |
26
+ | 7 | Cost pressure on bloated contexts | aahar, agni | ✅ |
27
+ | 8 | Retrieval-QA realistic synthetic benchmark (L2.5) | aahar, raksha | ✅ |
28
+ | 9 | Health-score calibration (L2.5) | aahar, nidra, vyayam, raksha, agni, pulse, chikitsa | ✅ |
29
+ | 10 | Ablation matrix — per-module contribution | raksha, aahar, nidra, vyayam, agni, pulse | ✅ |
30
+ | 11 | Flaky-tool resilience | vyayam, pulse | ✅ |
31
+
32
+ ## Per-suite results
33
+
34
+ ### 1. Prompt-injection resistance ✅
35
+
36
+ *Modules: raksha, aahar*
37
+
38
+ 33 adversarial inputs across direct override, markup boundary, role confusion, memory poisoning, authority claim, embedded, obfuscated, and policy-laundering categories — evaluated against 30 benign controls across plain technical docs, security-topic discussions, Cyrillic/Greek prose, JWT-like base64 tokens, and marketing / customer-support copy to surface false_positive_rate honestly.
39
+
40
+ | Metric | Baseline | With Ojas | Δ | Better |
41
+ |---|---:|---:|---:|:---:|
42
+ | `attacks_succeeded` | 19/33 | 0/33 | | ↓ |
43
+ | `compliance_rate` | 57.6 % | 0 % | −100.0% | ↓ |
44
+ | `attacks_quarantined_by_raksha` | 0/33 | 33/33 | 100.0% | ↑ |
45
+ | `benign_controls_preserved` | 30/30 | 30/30 | | ✓ |
46
+ | `false_positive_rate` | 0 | 0 | 0.0% | ↓ |
47
+
48
+ > Every adversarial input was caught by Raksha.
49
+ > false_positive_rate = 0.0% on 30 benign controls (tolerance ≤ 5%).
50
+
51
+ ### 2. Context pollution survival ✅
52
+
53
+ *Modules: aahar*
54
+
55
+ 2 fixtures of mixed-quality retrieval (signal + noise + duplicates + stale items) fed to a NoisyAgent.
56
+
57
+ | Metric | Baseline | With Ojas | Δ | Better |
58
+ |---|---:|---:|---:|:---:|
59
+ | `signal_to_noise_ratio` | 0.53 | 1 | 1.9× | ↑ |
60
+ | `tokens_spent` | 3540 tokens | 1340 tokens | −62.1% | ↓ |
61
+ | `agent_confidence` | 0.54 | 0.76 | −47.9% | ↑ |
62
+ | `duplicates_reaching_agent` | 1 | 0 | | ↓ |
63
+
64
+ > Ojas dropped 2200 wasted context tokens across 2 fixtures.
65
+
66
+ ### 3. Tool-failure loop detection ✅
67
+
68
+ *Modules: pulse, nidra, chikitsa*
69
+
70
+ 3 simulated tools whose calls always fail. Measures how quickly Ojas detects the loop and whether Chikitsa emits a usable repair plan.
71
+
72
+ | Metric | Baseline | With Ojas | Δ | Better |
73
+ |---|---:|---:|---:|:---:|
74
+ | `avg_failures_before_intervention` | 20 failures | 2 failures | | ↓ |
75
+ | `repair_protocols_emitted` | 0/3 | 3/3 | | ↑ |
76
+ | `repair_includes_fallback_action` | 0/3 | 3/3 | | ↑ |
77
+
78
+ ### 4. Memory-write safety ✅
79
+
80
+ *Modules: raksha, nidra*
81
+
82
+ 16 candidate memory writes (mix of safe / low-confidence / injection-tainted) evaluated by the policy used by ojas_validate_memory_write.
83
+
84
+ | Metric | Baseline | With Ojas | Δ | Better |
85
+ |---|---:|---:|---:|:---:|
86
+ | `malicious_writes_committed` | 6/6 | 1/6 | | ↓ |
87
+ | `low_confidence_downgraded_to_session_note` | 0/5 | 5/5 | | ↑ |
88
+ | `safe_writes_preserved` | 5/5 | 5/5 | | ✓ |
89
+ | `total_writes_evaluated` | 16 | 16 | | − |
90
+
91
+ ### 5. Cognitive drift detection ✅
92
+
93
+ *Modules: nidra, pulse*
94
+
95
+ 5 simulated long-horizon sessions of 40 traces each, with monotonically increasing failure probability. Measures how many traces Nidra needs before flagging drift.
96
+
97
+ | Metric | Baseline | With Ojas | Δ | Better |
98
+ |---|---:|---:|---:|:---:|
99
+ | `drift_detection_runs` | 0/5 | 5/5 | | ↑ |
100
+ | `avg_traces_until_detection` | ∞ (no detector) traces | 19.6 traces | | ↓ |
101
+ | `recovery_recommended_after_detection` | never | always | | ✓ |
102
+
103
+ ### 6. Vyayam resilience under stress ✅
104
+
105
+ *Modules: vyayam, raksha, aahar*
106
+
107
+ All 8 Vyayam stress types (intensity 0.7) executed against the raw NaiveComplianceAgent vs the same agent wrapped in an Ojas shield (Raksha + Aahar pre-filter).
108
+
109
+ | Metric | Baseline | With Ojas | Δ | Better |
110
+ |---|---:|---:|---:|:---:|
111
+ | `stress_scenarios_passed` | 7/8 | 7/8 | | ↑ |
112
+ | `hallucinations_detected` | 0 | 0 | | ↓ |
113
+ | `secret_leaks_under_stress` | 0 | 0 | | ↓ |
114
+
115
+ > Pass kind: `no_regression`.
116
+
117
+ > Baseline and Ojas pass the same number of Vyayam stressors at this intensity, so this suite demonstrates **no regression** from inserting Ojas into the pipeline rather than a targeted improvement. See suites 1 and 4 for the contrastive proof against more targeted attacks.
118
+
119
+ ### 7. Cost pressure on bloated contexts ✅
120
+
121
+ *Modules: aahar, agni*
122
+
123
+ 3 heavy-retrieval tasks (5 signal items + 60 noise items each) compared with and without Aahar pre-filtering. Cost pressure is measured by Agni.
124
+
125
+ | Metric | Baseline | With Ojas | Δ | Better |
126
+ |---|---:|---:|---:|:---:|
127
+ | `avg_tokens_per_complex_task` | 12680 tokens | 680 tokens | −94.6% | ↓ |
128
+ | `avg_latency_ms` | 320 ms | 80 ms | −75.0% | ↓ |
129
+ | `agni_cost_pressure` | 0.02 | 0 | | ↓ |
130
+
131
+ ### 8. Retrieval-QA realistic synthetic benchmark (L2.5) ✅
132
+
133
+ *Modules: aahar, raksha*
134
+
135
+ 20 questions × 5 seeded shuffles = 100 trials per config. Each scenario mixes 1 relevant + 8 benign-noisy + 1 adversarial doc(s) and feeds the same set to QAAgent baseline vs Ojas-filtered. Bootstrap 95% CIs across 1000 resamples.
136
+
137
+ | Metric | Baseline | With Ojas | Δ | Better |
138
+ |---|---:|---:|---:|:---:|
139
+ | `task_success_rate` | 0.35 [0.260, 0.440] | 0.95 [0.910, 0.990] | +60.0pp | ↑ |
140
+ | `relevant_context_recall` | 1 | 1 | 0.0pp | ↑ |
141
+ | `irrelevant_context_rejection` | 0 | 1 | +100.0pp | ↑ |
142
+ | `adversarial_inclusion_rate` | 1 | 0.11 [0.050, 0.180] | −89.0pp | ↓ |
143
+ | `adversarial_leak_rate` | 0.65 [0.550, 0.740] | 0 | −65.0pp | ↓ |
144
+ | `relevant_doc_drop_rate` | 0 | 0 | 0.0pp | ↓ |
145
+
146
+ > Seeds: `[101, 202, 303, 404, 505]` · evidence level `L2.5` · pass kind `improvement` · CI bounds in brackets are bootstrap 95% intervals across these seeds.
147
+
148
+ > relevant_doc_drop_rate = 0.0% (Aahar dropped the answer-bearing doc this often; tolerance ≤ 5%). This is *not* a generic false-positive rate over the benign corpus — it's specifically the answer-doc-recall failure rate.
149
+ > Raw per-scenario rows: benchmarks/results/raw/retrieval-qa-<timestamp>.jsonl (one row per scenario × config).
150
+ > Limitations: deterministic QAAgent (not a real LLM); keyword answer matching; fixed corpus. Set OJAS_BENCH_LLM=1 with an OpenAI-compatible provider to run the opt-in real-LLM path.
151
+
152
+ ### 9. Health-score calibration (L2.5) ✅
153
+
154
+ *Modules: aahar, nidra, vyayam, raksha, agni, pulse, chikitsa*
155
+
156
+ 5 seeds × 100 synthetic agent instances per seed = 500 (latent-quality, Ojas-score, failure-outcome) triples. Each instance is fed synthetic traces / threat assessments scaled by its latent quality q ∈ [0,1]; the ground-truth failure outcome uses a different weight formula than Ojas, so a positive result is evidence that the score is meaningful, not just self-consistent.
157
+
158
+ | Metric | Baseline | With Ojas | Δ | Better |
159
+ |---|---:|---:|---:|:---:|
160
+ | `spearman_rho_score_vs_failure` | 0 | -0.313 | -0.313 | ↓ |
161
+ | `score_range_observed_min` | n/a | 0.306 | | − |
162
+ | `score_range_observed_max` | n/a | 0.869 | | − |
163
+ | `failure_rate_bucket_[0.0,0.2)` | n/a | empty | | − |
164
+ | `failure_rate_bucket_[0.2,0.4)` | n/a | 0.667 | | − |
165
+ | `failure_rate_bucket_[0.4,0.6)` | n/a | 0.57 | | − |
166
+ | `failure_rate_bucket_[0.6,0.8)` | n/a | 0.392 | | − |
167
+ | `failure_rate_bucket_[0.8,1.0]` | n/a | 0.242 | | − |
168
+ | `monotonicity_holds` | n/a | yes | | ✓ |
169
+ | `isotonic_bins` | n/a | 16 | | − |
170
+ | `brier_score_raw_vs_synthetic_success` | n/a | 0.23 | | ↓ |
171
+ | `brier_score_isotonic_calibrated` | 0.23 | 0.219 | 0.011 | ↓ |
172
+
173
+ > Seeds: `[101, 202, 303, 404, 505]` · evidence level `L2.5` · pass kind `diagnostic` · CI bounds in brackets are bootstrap 95% intervals across these seeds.
174
+
175
+ > Spearman ρ = -0.313 (target ≤ -0.2 — clearly negative). Score has real but modest predictive power.
176
+ > Observed score range: [0.306, 0.869]. **Calibration finding:** Ojas's overall score is *squashed* into this band even for synthetic agents driven to extreme quality / unhealthiness. Operator implication: treat <0.4 as "very unhealthy" and >0.8 as "very healthy"; interpret deltas inside the band, not the absolute numbers.
177
+ > Bucket failure rates: [0.0, 0.2)=empty (n=0), [0.2, 0.4)=67% (n=93), [0.4, 0.6)=57% (n=165), [0.6, 0.8)=39% (n=176), [0.8, 1.0]=24% (n=66).
178
+ > Monotonicity: holds within 5pp slack.
179
+ > Synthetic isotonic calibration: 16 bins, Brier 0.230 raw → 0.219 calibrated. This validates an advisory diagnostic mapping on the synthetic suite only, not a production probability model.
180
+ > Limitations: synthetic q→telemetry mapping; not validated against real LLM degradation. See docs/EVIDENCE_MATRIX.md and docs/KNOWN_FAILURES.md.
181
+
182
+ ### 10. Ablation matrix — per-module contribution ✅
183
+
184
+ *Modules: raksha, aahar, nidra, vyayam, agni, pulse*
185
+
186
+ Disables each Ojas module individually and measures the impact on injection catch rate and token retention.
187
+
188
+ | Metric | Baseline | With Ojas | Δ | Better |
189
+ |---|---:|---:|---:|:---:|
190
+ | `injections_caught_full` | 0/10 attacks | 10/10 attacks | | ↑ |
191
+ | `raksha_contribution` | 0 | 10 | +10 catches | ↑ |
192
+ | `tokens_retained_full` | 7600 tokens | 4000 tokens | | ↓ |
193
+ | `aahar_contribution` | 7600 | 4000 | -3600 tokens | ↓ |
194
+
195
+ > Pass kind: `diagnostic`.
196
+
197
+ > Ablation matrix: each row disables one module and measures the delta.
198
+ > Raksha contributes +10 injection catches.
199
+ > Aahar reduces retained tokens by 3600.
200
+
201
+ ### 11. Flaky-tool resilience ✅
202
+
203
+ *Modules: vyayam, pulse*
204
+
205
+ Non-deterministic fault profiles (intermittent 500s, high latency, connection resets, mixed chaos) applied via ToolFaultProxy. Measures graceful degradation and health score sensitivity.
206
+
207
+ | Metric | Baseline | With Ojas | Δ | Better |
208
+ |---|---:|---:|---:|:---:|
209
+ | `total_runs` | 32 | 32 | | − |
210
+ | `faults_injected` | 0 | 9 | | − |
211
+ | `crashes_handled` | 0 crashes | 2 crashes | | − |
212
+ | `fault_injection_rate` | 0 | 0.281 | | − |
213
+ | `degradation_detection_rate` | 0 | 0 | | ↑ |
214
+
215
+ > Pass kind: `diagnostic`.
216
+
217
+ > Tested 4 flaky-tool profiles × 8 runs each.
218
+ > 9 faults injected across 32 runs (28.1% fault rate).
219
+ > 2 throw-mode crashes handled gracefully.
220
+
221
+ ## Reproduce
222
+
223
+ ```bash
224
+ npm install
225
+ npm run build # produces dist/
226
+ npm run benchmark # prints the console table (project-default seed)
227
+ npm run benchmark:write # regenerates docs/EVIDENCE.md, benchmarks/results/latest.json,
228
+ # and benchmarks/results/raw/*.jsonl per-scenario rows
229
+ OJAS_BENCH_SEED=4242 npm run benchmark # override the seed to test seed sensitivity
230
+ ```
231
+
232
+ Source: `@benchmarks/runner.ts`, `@benchmarks/suites/`, `@benchmarks/fixtures/`. Seeded PRNG: `@benchmarks/util/prng.ts`. Bootstrap CIs: `@benchmarks/util/stats.ts`.