@rune-kit/rune 2.14.0 → 2.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -11
- package/compiler/__tests__/adapter-model-mapping.test.js +152 -0
- package/compiler/__tests__/context-preview-signal.test.js +166 -0
- package/compiler/__tests__/oracle-bundle-format.test.js +210 -0
- package/compiler/__tests__/oracle-pending-schema.test.js +209 -0
- package/compiler/__tests__/skill-description-quality.test.js +147 -0
- package/compiler/adapters/antigravity.js +17 -1
- package/compiler/adapters/claude.js +4 -0
- package/compiler/adapters/codex.js +16 -1
- package/compiler/adapters/cursor.js +4 -0
- package/compiler/adapters/generic.js +14 -1
- package/compiler/adapters/openclaw.js +15 -1
- package/compiler/adapters/opencode.js +17 -1
- package/compiler/adapters/windsurf.js +4 -0
- package/package.json +1 -1
- package/skills/adversary/SKILL.md +61 -1
- package/skills/adversary/evals.md +79 -0
- package/skills/adversary/references/context-bundle-format.md +122 -0
- package/skills/adversary/references/oracle-mode.md +116 -0
- package/skills/asset-creator/SKILL.md +1 -1
- package/skills/audit/SKILL.md +2 -1
- package/skills/autopsy/SKILL.md +1 -1
- package/skills/ba/SKILL.md +1 -1
- package/skills/brainstorm/SKILL.md +1 -1
- package/skills/browser-pilot/SKILL.md +1 -1
- package/skills/completion-gate/SKILL.md +1 -1
- package/skills/constraint-check/SKILL.md +1 -1
- package/skills/context-engine/SKILL.md +85 -1
- package/skills/context-engine/references/preview-gate.md +139 -0
- package/skills/cook/SKILL.md +1 -0
- package/skills/db/SKILL.md +1 -1
- package/skills/debug/SKILL.md +6 -4
- package/skills/dependency-doctor/SKILL.md +1 -1
- package/skills/design/SKILL.md +1 -1
- package/skills/doc-processor/SKILL.md +1 -1
- package/skills/docs/SKILL.md +1 -1
- package/skills/docs-seeker/SKILL.md +1 -1
- package/skills/fix/SKILL.md +4 -3
- package/skills/git/SKILL.md +1 -1
- package/skills/hallucination-guard/SKILL.md +1 -1
- package/skills/integrity-check/SKILL.md +1 -1
- package/skills/journal/SKILL.md +1 -1
- package/skills/logic-guardian/SKILL.md +1 -1
- package/skills/marketing/SKILL.md +1 -1
- package/skills/mcp-builder/SKILL.md +1 -1
- package/skills/onboard/SKILL.md +1 -1
- package/skills/perf/SKILL.md +1 -1
- package/skills/plan/SKILL.md +1 -1
- package/skills/preflight/SKILL.md +1 -1
- package/skills/research/SKILL.md +1 -1
- package/skills/retro/SKILL.md +1 -1
- package/skills/review/SKILL.md +2 -2
- package/skills/review-intake/SKILL.md +1 -1
- package/skills/safeguard/SKILL.md +1 -1
- package/skills/scaffold/SKILL.md +1 -1
- package/skills/sentinel/SKILL.md +1 -1
- package/skills/sentinel-env/SKILL.md +1 -1
- package/skills/session-bridge/SKILL.md +91 -4
- package/skills/session-bridge/references/detach-protocol.md +111 -0
- package/skills/skill-forge/SKILL.md +1 -1
- package/skills/slides/SKILL.md +1 -1
- package/skills/surgeon/SKILL.md +1 -1
- package/skills/team/SKILL.md +3 -0
- package/skills/trend-scout/SKILL.md +1 -1
- package/skills/video-creator/SKILL.md +1 -1
- package/skills/watchdog/SKILL.md +1 -1
- package/skills/worktree/SKILL.md +1 -1
package/README.md
CHANGED
|
@@ -83,17 +83,25 @@ _Methodology: Claude Code CLI headless mode (`claude -p --output-format json`),
|
|
|
83
83
|
|
|
84
84
|
---
|
|
85
85
|
|
|
86
|
-
## What's New (v2.
|
|
87
|
-
|
|
88
|
-
- **`
|
|
89
|
-
-
|
|
90
|
-
-
|
|
91
|
-
- **
|
|
92
|
-
- **
|
|
93
|
-
- **
|
|
94
|
-
- **
|
|
95
|
-
|
|
96
|
-
|
|
86
|
+
## What's New (v2.15.0 — Second Opinion + Cross-Provider + Routing Clarity)
|
|
87
|
+
|
|
88
|
+
- **`adversary` v0.2.0 — Mode: oracle** — when `agent.stuck` fires from `debug` (3 disproved hypotheses) or `fix` (2+ failed attempts), oracle-mode dispatches a stateless second-model pass with explicit "no prior context" framing. Bundle format is regex-validated (`[SYSTEM]` invariant role-priming + `[USER]` template + `### File N`), token-capped (100k bundle, 4k per file, 12 files max), citation-required reply contract. Secrets auto-redacted. Breaks the confirmation-bias loop that scout's zoom-out (structural pivot) cannot.
|
|
89
|
+
- **`session-bridge` v0.8.0 — Detach Mode** — async escalation primitive. Heavy-model second-opinion calls (1-10 min wall time) no longer block the primary agent. `.rune/oracle-pending/<sessionId>.json` is the rendezvous file; idempotent dispatch (bundleHash-keyed); 10min default timeout; 24h orphan cleanup on session start. `cook` Phase 4 and `team` Phase 3 reattach via filesystem poll between adjacent tasks.
|
|
90
|
+
- **`context-engine` v1.1.0 — Mode: preview** — pre-flight cost gate. Caller emits `context.preview` BEFORE bundle build with file list + estimated tokens (chars × 0.25). Per-caller thresholds: adversary 50k/100k, team 30k/80k (per worker), review 40k/100k, audit 60k/120k. Action enum `proceed | warn | block`. Override via `RUNE_CONTEXT_THRESHOLDS_<CALLER>`. Stops `team` parallel workstreams from silently blowing $20 of Opus tokens.
|
|
91
|
+
- **Cross-provider model mapping** — 5 non-Anthropic adapters now translate `model: opus|sonnet|haiku` to provider-correct names. **codex** → gpt-5-pro / gpt-5 / gpt-5-mini. **antigravity** → gemini-3-pro / gemini-3-flash / gemini-3-flash-lite. **opencode / openclaw / generic** → tier:heavy / tier:mid / tier:light (provider-agnostic). claude / cursor / windsurf remain no-op (Anthropic backend understands native names).
|
|
92
|
+
- **Routing clarity sweep** — all 63 SKILL.md descriptions now double-quoted (YAML safety). 13 ambiguous-name skills got explicit "Use when…" routing hints so skill-router doesn't have to guess: ba, completion-gate, constraint-check, doc-processor, integrity-check, logic-guardian, onboard, preflight, sentinel-env, watchdog, worktree, hallucination-guard, mcp-builder.
|
|
93
|
+
- **4 new mesh signals** — `oracle.dispatched`, `oracle.response`, `oracle.failed`, `context.preview`. All registered in Signal Catalog with full emit/listen mapping. `agent.stuck` listeners updated to include adversary in addition to scout.
|
|
94
|
+
- **1,331 tests** — +71 from v2.14.0 across 5 new test files: adapter-model-mapping (18), oracle-bundle-format (19), oracle-pending-schema (16), context-preview-signal (13), skill-description-quality (5).
|
|
95
|
+
|
|
96
|
+
### Previous (v2.14.0 — Deep Modules)
|
|
97
|
+
|
|
98
|
+
- **`improve-architecture` skill (NEW L2, opus)** — controlled vocabulary (Module / Interface / Implementation / Depth / Seam / Adapter / Leverage / Locality), numeric depth-leverage-locality scoring (1–5 each), 4 dependency categories, structured proposal payloads.
|
|
99
|
+
- **TDD vertical-slicing HARD-GATE** — `test` v1.3.0 catches "horizontal slicing" (5 tests before any GREEN), commit-pair audit trail, shape-test smell detector.
|
|
100
|
+
- **`.out-of-scope/` knowledge base** — `ba` v0.11.0 reads, `review-intake` v1.2.0 writes. Stops re-litigation of rejected features.
|
|
101
|
+
- **CONTEXT.md inline-sharpen + ADR 3-criteria gate** — `journal` v0.4.0 only opens an ADR when sum ≥ 11 + each axis ≥ 3.
|
|
102
|
+
- **Agent Brief durability** — `context-pack` v0.2.0 regex smell tests block stale paths/line numbers.
|
|
103
|
+
- **Design-It-Twice mode** — `brainstorm` v0.6.0 with constraint-pinned parallel subagents + diversity score gate.
|
|
104
|
+
- **Zoom-out + explore-first micro-utilities** — `scout` v0.4.0 listens for `agent.stuck`; `ba` Step 2.0 HARD-GATE requires tool-call evidence.
|
|
97
105
|
|
|
98
106
|
### Previous (v2.13.0 — Script Contract + Media Pack)
|
|
99
107
|
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adapter model tier mapping (v2.15+) — verify non-Anthropic adapters
|
|
3
|
+
* translate `model: opus|sonnet|haiku` to provider-correct names while
|
|
4
|
+
* Anthropic-backed adapters (claude/cursor/windsurf) remain no-op.
|
|
5
|
+
*
|
|
6
|
+
* Validates:
|
|
7
|
+
* - codex/antigravity emit concrete provider model names
|
|
8
|
+
* - opencode/openclaw/generic emit semantic tier hints (provider-agnostic)
|
|
9
|
+
* - claude/cursor/windsurf do NOT translate (no-op)
|
|
10
|
+
* - Skills without `model:` produce no model line
|
|
11
|
+
* - Unknown tier values pass through unchanged
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import assert from 'node:assert';
|
|
15
|
+
import { describe, test } from 'node:test';
|
|
16
|
+
import { getAdapter } from '../adapters/index.js';
|
|
17
|
+
|
|
18
|
+
const baseSkill = {
|
|
19
|
+
name: 'cook',
|
|
20
|
+
description: 'Feature implementation orchestrator',
|
|
21
|
+
layer: 'L1',
|
|
22
|
+
group: 'orchestration',
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
describe('codex adapter model mapping', () => {
|
|
26
|
+
const codex = getAdapter('codex');
|
|
27
|
+
|
|
28
|
+
test('opus translates to gpt-5-pro', () => {
|
|
29
|
+
const header = codex.generateHeader({ ...baseSkill, model: 'opus' });
|
|
30
|
+
assert.match(header, /model: gpt-5-pro/);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test('sonnet translates to gpt-5', () => {
|
|
34
|
+
const header = codex.generateHeader({ ...baseSkill, model: 'sonnet' });
|
|
35
|
+
assert.match(header, /model: gpt-5(?!-)/);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test('haiku translates to gpt-5-mini', () => {
|
|
39
|
+
const header = codex.generateHeader({ ...baseSkill, model: 'haiku' });
|
|
40
|
+
assert.match(header, /model: gpt-5-mini/);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test('omitted model produces no model line', () => {
|
|
44
|
+
const header = codex.generateHeader({ ...baseSkill });
|
|
45
|
+
assert.doesNotMatch(header, /model:/);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test('unknown tier passes through unchanged', () => {
|
|
49
|
+
const header = codex.generateHeader({ ...baseSkill, model: 'custom-fine-tune' });
|
|
50
|
+
assert.match(header, /model: custom-fine-tune/);
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe('antigravity adapter model mapping', () => {
|
|
55
|
+
const antigravity = getAdapter('antigravity');
|
|
56
|
+
|
|
57
|
+
test('opus translates to gemini-3-pro', () => {
|
|
58
|
+
const header = antigravity.generateHeader({ ...baseSkill, model: 'opus' });
|
|
59
|
+
assert.match(header, /model: gemini-3-pro/);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('sonnet translates to gemini-3-flash', () => {
|
|
63
|
+
const header = antigravity.generateHeader({ ...baseSkill, model: 'sonnet' });
|
|
64
|
+
assert.match(header, /model: gemini-3-flash(?!-)/);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test('haiku translates to gemini-3-flash-lite', () => {
|
|
68
|
+
const header = antigravity.generateHeader({ ...baseSkill, model: 'haiku' });
|
|
69
|
+
assert.match(header, /model: gemini-3-flash-lite/);
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
describe('opencode adapter model mapping (provider-agnostic)', () => {
|
|
74
|
+
const opencode = getAdapter('opencode');
|
|
75
|
+
|
|
76
|
+
test('opus translates to tier:heavy', () => {
|
|
77
|
+
const header = opencode.generateHeader({ ...baseSkill, model: 'opus' });
|
|
78
|
+
assert.match(header, /model: tier:heavy/);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test('sonnet translates to tier:mid', () => {
|
|
82
|
+
const header = opencode.generateHeader({ ...baseSkill, model: 'sonnet' });
|
|
83
|
+
assert.match(header, /model: tier:mid/);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test('haiku translates to tier:light', () => {
|
|
87
|
+
const header = opencode.generateHeader({ ...baseSkill, model: 'haiku' });
|
|
88
|
+
assert.match(header, /model: tier:light/);
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
describe('openclaw adapter model mapping (markdown header)', () => {
|
|
93
|
+
const openclaw = getAdapter('openclaw');
|
|
94
|
+
|
|
95
|
+
test('opus appears as tier:heavy in metadata line', () => {
|
|
96
|
+
const header = openclaw.generateHeader({ ...baseSkill, model: 'opus' });
|
|
97
|
+
assert.match(header, /model: tier:heavy/);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test('omitted model produces no model line', () => {
|
|
101
|
+
const header = openclaw.generateHeader({ ...baseSkill });
|
|
102
|
+
assert.doesNotMatch(header, /model:/);
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
describe('generic adapter model mapping (markdown header)', () => {
|
|
107
|
+
const generic = getAdapter('generic');
|
|
108
|
+
|
|
109
|
+
test('opus appears as tier:heavy in metadata line', () => {
|
|
110
|
+
const header = generic.generateHeader({ ...baseSkill, model: 'opus' });
|
|
111
|
+
assert.match(header, /model: tier:heavy/);
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
describe('Anthropic-native adapters are no-op for model field', () => {
|
|
116
|
+
test('claude generateHeader returns empty (passthrough)', () => {
|
|
117
|
+
const claude = getAdapter('claude');
|
|
118
|
+
const header = claude.generateHeader({ ...baseSkill, model: 'opus' });
|
|
119
|
+
assert.strictEqual(header, '');
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test('cursor + windsurf do NOT rewrite anthropic tier names', () => {
|
|
123
|
+
const cursor = getAdapter('cursor');
|
|
124
|
+
const windsurf = getAdapter('windsurf');
|
|
125
|
+
const cursorHeader = cursor.generateHeader({ ...baseSkill, model: 'opus' });
|
|
126
|
+
const windsurfHeader = windsurf.generateHeader({ ...baseSkill, model: 'opus' });
|
|
127
|
+
assert.doesNotMatch(cursorHeader, /gpt-5/);
|
|
128
|
+
assert.doesNotMatch(cursorHeader, /gemini/);
|
|
129
|
+
assert.doesNotMatch(windsurfHeader, /gpt-5/);
|
|
130
|
+
assert.doesNotMatch(windsurfHeader, /gemini/);
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
describe('cross-adapter consistency', () => {
|
|
135
|
+
test('all 5 non-Anthropic adapters emit a model line for opus skills', () => {
|
|
136
|
+
const adapterNames = ['codex', 'antigravity', 'opencode', 'openclaw', 'generic'];
|
|
137
|
+
for (const name of adapterNames) {
|
|
138
|
+
const adapter = getAdapter(name);
|
|
139
|
+
const header = adapter.generateHeader({ ...baseSkill, model: 'opus' });
|
|
140
|
+
assert.match(header, /model: /, `${name} should emit a model line for opus skill`);
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test('all 5 non-Anthropic adapters omit model line when skill has no model', () => {
|
|
145
|
+
const adapterNames = ['codex', 'antigravity', 'opencode', 'openclaw', 'generic'];
|
|
146
|
+
for (const name of adapterNames) {
|
|
147
|
+
const adapter = getAdapter(name);
|
|
148
|
+
const header = adapter.generateHeader({ ...baseSkill });
|
|
149
|
+
assert.doesNotMatch(header, /model:/, `${name} should NOT emit a model line for skill without model`);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
});
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context preview signal (v2.15+) — verify the schema and threshold logic
|
|
3
|
+
* for context-engine's preview-gate mode. Caller emits context.preview before
|
|
4
|
+
* bundling expensive context, gating dispatch on token budget.
|
|
5
|
+
*
|
|
6
|
+
* Validates:
|
|
7
|
+
* - Signal name conforms to dot-notation pattern
|
|
8
|
+
* - Payload schema (caller, estimated_tokens, file_count, top_5, threshold, action)
|
|
9
|
+
* - Threshold action enum: proceed | warn | block
|
|
10
|
+
* - Caller-specific threshold table (adversary 50k/100k, team 30k/80k, etc.)
|
|
11
|
+
* - Token estimate function: chars × 0.25
|
|
12
|
+
* - Decision function: tokens >= block_at → block, >= warn_at → warn, else proceed
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import assert from 'node:assert';
|
|
16
|
+
import { describe, test } from 'node:test';
|
|
17
|
+
|
|
18
|
+
const SIGNAL_PATTERN = /^[a-z][a-z0-9]*(\.[a-z][a-z0-9]*)+$/;
|
|
19
|
+
|
|
20
|
+
const VALID_CALLERS = new Set(['adversary', 'team', 'review', 'audit']);
|
|
21
|
+
const VALID_ACTIONS = new Set(['proceed', 'warn', 'block']);
|
|
22
|
+
|
|
23
|
+
const DEFAULT_THRESHOLDS = {
|
|
24
|
+
adversary: { warn_at: 50_000, block_at: 100_000 },
|
|
25
|
+
team: { warn_at: 30_000, block_at: 80_000 },
|
|
26
|
+
review: { warn_at: 40_000, block_at: 100_000 },
|
|
27
|
+
audit: { warn_at: 60_000, block_at: 120_000 },
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
function estimateTokens(chars) {
|
|
31
|
+
return Math.floor(chars * 0.25);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function decideAction(estimatedTokens, threshold) {
|
|
35
|
+
if (estimatedTokens >= threshold.block_at) return 'block';
|
|
36
|
+
if (estimatedTokens >= threshold.warn_at) return 'warn';
|
|
37
|
+
return 'proceed';
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function buildPreviewPayload({ caller, files, promptChars = 200 }) {
|
|
41
|
+
if (!VALID_CALLERS.has(caller)) throw new Error(`unknown caller: ${caller}`);
|
|
42
|
+
const threshold = DEFAULT_THRESHOLDS[caller];
|
|
43
|
+
const totalChars = promptChars + files.reduce((sum, f) => sum + f.chars, 0);
|
|
44
|
+
const estimated_tokens = estimateTokens(totalChars);
|
|
45
|
+
const top_5_files_by_size = [...files].sort((a, b) => b.chars - a.chars).slice(0, 5);
|
|
46
|
+
return {
|
|
47
|
+
caller,
|
|
48
|
+
estimated_tokens,
|
|
49
|
+
file_count: files.length,
|
|
50
|
+
top_5_files_by_size,
|
|
51
|
+
threshold,
|
|
52
|
+
action: decideAction(estimated_tokens, threshold),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function validatePayload(payload) {
|
|
57
|
+
const errors = [];
|
|
58
|
+
if (!VALID_CALLERS.has(payload.caller)) errors.push('invalid_caller');
|
|
59
|
+
if (typeof payload.estimated_tokens !== 'number' || payload.estimated_tokens < 0) errors.push('invalid_tokens');
|
|
60
|
+
if (typeof payload.file_count !== 'number' || payload.file_count < 0) errors.push('invalid_file_count');
|
|
61
|
+
if (!Array.isArray(payload.top_5_files_by_size) || payload.top_5_files_by_size.length > 5)
|
|
62
|
+
errors.push('invalid_top_5');
|
|
63
|
+
if (typeof payload.threshold?.warn_at !== 'number' || typeof payload.threshold?.block_at !== 'number')
|
|
64
|
+
errors.push('invalid_threshold');
|
|
65
|
+
if (!VALID_ACTIONS.has(payload.action)) errors.push('invalid_action');
|
|
66
|
+
return { valid: errors.length === 0, errors };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
describe('Signal naming compliance', () => {
|
|
70
|
+
test('context.preview conforms to signal naming pattern', () => {
|
|
71
|
+
assert.match('context.preview', SIGNAL_PATTERN);
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
describe('Payload schema', () => {
|
|
76
|
+
test('valid payload passes', () => {
|
|
77
|
+
const payload = buildPreviewPayload({
|
|
78
|
+
caller: 'adversary',
|
|
79
|
+
files: [{ path: 'a.ts', chars: 1000 }],
|
|
80
|
+
});
|
|
81
|
+
assert.strictEqual(validatePayload(payload).valid, true);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test('rejects unknown caller', () => {
|
|
85
|
+
const payload = buildPreviewPayload({
|
|
86
|
+
caller: 'adversary',
|
|
87
|
+
files: [{ path: 'a.ts', chars: 1000 }],
|
|
88
|
+
});
|
|
89
|
+
payload.caller = 'unknown_skill';
|
|
90
|
+
assert.ok(validatePayload(payload).errors.includes('invalid_caller'));
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test('rejects unknown action', () => {
|
|
94
|
+
const payload = buildPreviewPayload({
|
|
95
|
+
caller: 'adversary',
|
|
96
|
+
files: [{ path: 'a.ts', chars: 1000 }],
|
|
97
|
+
});
|
|
98
|
+
payload.action = 'maybe';
|
|
99
|
+
assert.ok(validatePayload(payload).errors.includes('invalid_action'));
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test('top_5 capped at 5 entries even with many files', () => {
|
|
103
|
+
const files = Array.from({ length: 12 }, (_, i) => ({ path: `f${i}.ts`, chars: 100 + i * 10 }));
|
|
104
|
+
const payload = buildPreviewPayload({ caller: 'team', files });
|
|
105
|
+
assert.ok(payload.top_5_files_by_size.length <= 5);
|
|
106
|
+
assert.strictEqual(payload.file_count, 12);
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
describe('Token estimation', () => {
|
|
111
|
+
test('chars × 0.25 = tokens', () => {
|
|
112
|
+
assert.strictEqual(estimateTokens(4000), 1000);
|
|
113
|
+
assert.strictEqual(estimateTokens(0), 0);
|
|
114
|
+
assert.strictEqual(estimateTokens(100_000), 25_000);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
describe('Action decision logic', () => {
|
|
119
|
+
test('proceed when under warn threshold', () => {
|
|
120
|
+
const payload = buildPreviewPayload({
|
|
121
|
+
caller: 'adversary',
|
|
122
|
+
files: [{ path: 'a.ts', chars: 10_000 }],
|
|
123
|
+
});
|
|
124
|
+
assert.strictEqual(payload.action, 'proceed');
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test('warn when between warn and block', () => {
|
|
128
|
+
const payload = buildPreviewPayload({
|
|
129
|
+
caller: 'adversary',
|
|
130
|
+
files: [{ path: 'a.ts', chars: 280_000 }],
|
|
131
|
+
});
|
|
132
|
+
assert.strictEqual(payload.action, 'warn');
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
test('block when at or over block threshold', () => {
|
|
136
|
+
const payload = buildPreviewPayload({
|
|
137
|
+
caller: 'adversary',
|
|
138
|
+
files: [{ path: 'a.ts', chars: 500_000 }],
|
|
139
|
+
});
|
|
140
|
+
assert.strictEqual(payload.action, 'block');
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
describe('Caller-specific thresholds', () => {
|
|
145
|
+
test('adversary uses 50k/100k', () => {
|
|
146
|
+
assert.deepStrictEqual(DEFAULT_THRESHOLDS.adversary, { warn_at: 50_000, block_at: 100_000 });
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test('team uses lower per-worker thresholds (30k/80k)', () => {
|
|
150
|
+
assert.deepStrictEqual(DEFAULT_THRESHOLDS.team, { warn_at: 30_000, block_at: 80_000 });
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
test('audit uses higher cross-pack thresholds (60k/120k)', () => {
|
|
154
|
+
assert.deepStrictEqual(DEFAULT_THRESHOLDS.audit, { warn_at: 60_000, block_at: 120_000 });
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test('same file scope produces different actions per caller', () => {
|
|
158
|
+
const files = [{ path: 'big.ts', chars: 200_000 }]; // ~50k tokens
|
|
159
|
+
const adv = buildPreviewPayload({ caller: 'adversary', files });
|
|
160
|
+
const team = buildPreviewPayload({ caller: 'team', files });
|
|
161
|
+
// Adversary: 50k → exactly at warn_at → warn
|
|
162
|
+
assert.strictEqual(adv.action, 'warn');
|
|
163
|
+
// Team: 50k > block_at (80k worker cap not hit, but...) actually 50k < 80k → warn
|
|
164
|
+
assert.strictEqual(team.action, 'warn');
|
|
165
|
+
});
|
|
166
|
+
});
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adversary oracle-mode context bundle format — verify the regex-validated
|
|
3
|
+
* structure that adversary emits when dispatching to a second model.
|
|
4
|
+
*
|
|
5
|
+
* The bundle has 3 mandatory sections:
|
|
6
|
+
* 1. [SYSTEM] line with exact role-priming text
|
|
7
|
+
* 2. [USER] line with template-driven problem statement
|
|
8
|
+
* 3. ### File N: <path> blocks with normalized content
|
|
9
|
+
*
|
|
10
|
+
* Validates:
|
|
11
|
+
* - System prompt is invariant (must match exact text)
|
|
12
|
+
* - User section non-empty, within length bounds
|
|
13
|
+
* - File headers numbered sequentially
|
|
14
|
+
* - Hard caps enforced (12 files, 4k chars per file, 100k bundle)
|
|
15
|
+
* - Forbidden content stripped (secrets redacted)
|
|
16
|
+
* - Citation validation (reply must cite files in bundle)
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import assert from 'node:assert';
|
|
20
|
+
import { describe, test } from 'node:test';
|
|
21
|
+
|
|
22
|
+
const SYSTEM_LINE_REGEX =
|
|
23
|
+
/^\[SYSTEM\] You are Oracle, a focused one-shot problem solver\. You have NO prior context — assume zero project knowledge\. Cite file:line for every claim\. Reject any claim you cannot ground in the provided files\.$/;
|
|
24
|
+
|
|
25
|
+
const USER_LINE_REGEX = /^\[USER\] .{20,2000}$/;
|
|
26
|
+
|
|
27
|
+
const FILE_HEADER_REGEX = /^### File \d+: [^\n]+$/;
|
|
28
|
+
|
|
29
|
+
const HARD_CAPS = {
|
|
30
|
+
bundleChars: 400_000,
|
|
31
|
+
perFileChars: 4_000,
|
|
32
|
+
fileCount: 12,
|
|
33
|
+
userMaxChars: 2_000,
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
function buildSystemLine() {
|
|
37
|
+
return '[SYSTEM] You are Oracle, a focused one-shot problem solver. You have NO prior context — assume zero project knowledge. Cite file:line for every claim. Reject any claim you cannot ground in the provided files.';
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function buildUserLine(source, payload) {
|
|
41
|
+
if (source === 'debug') {
|
|
42
|
+
return `[USER] Agent stuck after ${payload.cycles} hypothesis cycles. Disproved: ${payload.disproved}. Error: ${payload.error}. What is the most likely root cause not yet considered?`;
|
|
43
|
+
}
|
|
44
|
+
if (source === 'fix') {
|
|
45
|
+
return `[USER] Agent stuck after ${payload.attempts} failed fix attempts on ${payload.file}. Each attempt's tests failed with: ${payload.testFailure}. What is the most likely cause of the test failure?`;
|
|
46
|
+
}
|
|
47
|
+
return `[USER] ${payload}`;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function buildBundle({ files = [], userLine, systemLine = buildSystemLine() }) {
|
|
51
|
+
const sections = [systemLine, '', userLine, ''];
|
|
52
|
+
files.forEach((f, i) => {
|
|
53
|
+
sections.push(`### File ${i + 1}: ${f.path}`);
|
|
54
|
+
sections.push(f.content);
|
|
55
|
+
sections.push('');
|
|
56
|
+
});
|
|
57
|
+
return sections.join('\n');
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function validateBundle(bundle) {
|
|
61
|
+
const errors = [];
|
|
62
|
+
if (bundle.length > HARD_CAPS.bundleChars) errors.push('bundle_too_large');
|
|
63
|
+
const lines = bundle.split('\n');
|
|
64
|
+
const sysLine = lines.find((l) => l.startsWith('[SYSTEM]'));
|
|
65
|
+
if (!sysLine || !SYSTEM_LINE_REGEX.test(sysLine)) errors.push('invalid_system_line');
|
|
66
|
+
const userLine = lines.find((l) => l.startsWith('[USER]'));
|
|
67
|
+
if (!userLine || !USER_LINE_REGEX.test(userLine)) errors.push('invalid_user_line');
|
|
68
|
+
const fileHeaders = lines.filter((l) => l.startsWith('### File '));
|
|
69
|
+
if (fileHeaders.length > HARD_CAPS.fileCount) errors.push('too_many_files');
|
|
70
|
+
for (let i = 0; i < fileHeaders.length; i++) {
|
|
71
|
+
if (!FILE_HEADER_REGEX.test(fileHeaders[i])) errors.push(`invalid_file_header_${i}`);
|
|
72
|
+
const expectedNum = i + 1;
|
|
73
|
+
if (!fileHeaders[i].startsWith(`### File ${expectedNum}:`)) errors.push(`out_of_order_file_${i}`);
|
|
74
|
+
}
|
|
75
|
+
return { valid: errors.length === 0, errors };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function redactSecrets(content) {
|
|
79
|
+
return content
|
|
80
|
+
.replace(/(api[_-]?key|secret|password|token)\s*[:=]\s*["'][^"']{8,}["']/gi, '$1=<REDACTED>')
|
|
81
|
+
.replace(/Authorization:\s*Bearer\s+\S{20,}/g, 'Authorization: Bearer <REDACTED>')
|
|
82
|
+
.replace(/process\.env\.[A-Z_]+\s*=\s*["'][^"']+["']/g, (match) => match.replace(/["'][^"']+["']/, '"<REDACTED>"'));
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
describe('System prompt invariance', () => {
|
|
86
|
+
test('exact system line matches regex', () => {
|
|
87
|
+
assert.match(buildSystemLine(), SYSTEM_LINE_REGEX);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test('rejects modified system prompt (no improvising)', () => {
|
|
91
|
+
const tampered = '[SYSTEM] You are Oracle, an improved problem solver with extra context.';
|
|
92
|
+
assert.doesNotMatch(tampered, SYSTEM_LINE_REGEX);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test('rejects empty system line', () => {
|
|
96
|
+
assert.doesNotMatch('[SYSTEM]', SYSTEM_LINE_REGEX);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
describe('User section template', () => {
|
|
101
|
+
test('debug-source user line matches regex', () => {
|
|
102
|
+
const line = buildUserLine('debug', { cycles: 3, disproved: 'H1; H2; H3', error: '401 intermittent' });
|
|
103
|
+
assert.match(line, USER_LINE_REGEX);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test('fix-source user line matches regex', () => {
|
|
107
|
+
const line = buildUserLine('fix', {
|
|
108
|
+
attempts: 2,
|
|
109
|
+
file: 'src/auth.ts',
|
|
110
|
+
testFailure: 'expected 200, got 401',
|
|
111
|
+
});
|
|
112
|
+
assert.match(line, USER_LINE_REGEX);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test('rejects user line under 20 chars', () => {
|
|
116
|
+
assert.doesNotMatch('[USER] short', USER_LINE_REGEX);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test('manual mode accepts free-form within bounds', () => {
|
|
120
|
+
const line = `[USER] ${'x'.repeat(100)}`;
|
|
121
|
+
assert.match(line, USER_LINE_REGEX);
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
describe('File header structure', () => {
|
|
126
|
+
test('valid file header passes regex', () => {
|
|
127
|
+
assert.match('### File 1: src/auth/login.ts', FILE_HEADER_REGEX);
|
|
128
|
+
assert.match('### File 12: tests/e2e/checkout.spec.ts', FILE_HEADER_REGEX);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test('rejects malformed file headers', () => {
|
|
132
|
+
assert.doesNotMatch('### File: src/auth.ts', FILE_HEADER_REGEX);
|
|
133
|
+
assert.doesNotMatch('### file 1: src/auth.ts', FILE_HEADER_REGEX);
|
|
134
|
+
assert.doesNotMatch('## File 1: src/auth.ts', FILE_HEADER_REGEX);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
test('files numbered sequentially from 1', () => {
|
|
138
|
+
const bundle = buildBundle({
|
|
139
|
+
systemLine: buildSystemLine(),
|
|
140
|
+
userLine: buildUserLine('manual', 'Why does the test fail intermittently?'),
|
|
141
|
+
files: [
|
|
142
|
+
{ path: 'a.ts', content: 'a' },
|
|
143
|
+
{ path: 'b.ts', content: 'b' },
|
|
144
|
+
{ path: 'c.ts', content: 'c' },
|
|
145
|
+
],
|
|
146
|
+
});
|
|
147
|
+
const result = validateBundle(bundle);
|
|
148
|
+
assert.strictEqual(result.valid, true, `unexpected errors: ${result.errors.join(', ')}`);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
describe('Hard caps', () => {
|
|
153
|
+
test('rejects bundle with 13+ files', () => {
|
|
154
|
+
const files = Array.from({ length: 13 }, (_, i) => ({ path: `f${i}.ts`, content: 'x' }));
|
|
155
|
+
const bundle = buildBundle({ userLine: buildUserLine('manual', 'Why does this break under load?'), files });
|
|
156
|
+
const result = validateBundle(bundle);
|
|
157
|
+
assert.ok(result.errors.includes('too_many_files'));
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test('accepts bundle with exactly 12 files', () => {
|
|
161
|
+
const files = Array.from({ length: 12 }, (_, i) => ({ path: `f${i}.ts`, content: 'x' }));
|
|
162
|
+
const bundle = buildBundle({ userLine: buildUserLine('manual', 'Why does this break under load?'), files });
|
|
163
|
+
const result = validateBundle(bundle);
|
|
164
|
+
assert.strictEqual(result.valid, true);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test('rejects bundle exceeding 400k chars total', () => {
|
|
168
|
+
const oversize = 'x'.repeat(50_000);
|
|
169
|
+
const files = Array.from({ length: 10 }, () => ({ path: 'big.ts', content: oversize }));
|
|
170
|
+
const bundle = buildBundle({ userLine: buildUserLine('manual', 'Why does this break under load?'), files });
|
|
171
|
+
const result = validateBundle(bundle);
|
|
172
|
+
assert.ok(result.errors.includes('bundle_too_large'));
|
|
173
|
+
});
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
describe('Secret redaction', () => {
|
|
177
|
+
test('redacts hardcoded api keys', () => {
|
|
178
|
+
const input = 'const apiKey = "sk-1234567890abcdef"';
|
|
179
|
+
const out = redactSecrets(input);
|
|
180
|
+
assert.match(out, /<REDACTED>/);
|
|
181
|
+
assert.doesNotMatch(out, /sk-1234567890abcdef/);
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
test('redacts Bearer tokens', () => {
|
|
185
|
+
const input = 'Authorization: Bearer abc123def456ghi789jkl000mno111';
|
|
186
|
+
const out = redactSecrets(input);
|
|
187
|
+
assert.match(out, /Bearer <REDACTED>/);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test('preserves non-secret content', () => {
|
|
191
|
+
const input = 'function login(user) { return user.id; }';
|
|
192
|
+
assert.strictEqual(redactSecrets(input), input);
|
|
193
|
+
});
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
describe('Signal naming compliance', () => {
|
|
197
|
+
const SIGNAL_PATTERN = /^[a-z][a-z0-9]*(\.[a-z][a-z0-9]*)+$/;
|
|
198
|
+
|
|
199
|
+
test('oracle.dispatched conforms to signal naming', () => {
|
|
200
|
+
assert.match('oracle.dispatched', SIGNAL_PATTERN);
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
test('oracle.response conforms to signal naming', () => {
|
|
204
|
+
assert.match('oracle.response', SIGNAL_PATTERN);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
test('oracle.failed conforms to signal naming', () => {
|
|
208
|
+
assert.match('oracle.failed', SIGNAL_PATTERN);
|
|
209
|
+
});
|
|
210
|
+
});
|