nubos-pilot 1.2.4 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -1
- package/README.md +2 -1
- package/SECURITY.md +3 -4
- package/bin/np-tools/_commands.cjs +3 -0
- package/bin/np-tools/_elision-proxy-entry.cjs +13 -0
- package/bin/np-tools/elision-bench.cjs +67 -0
- package/bin/np-tools/elision-get.cjs +48 -0
- package/bin/np-tools/elision-get.test.cjs +66 -0
- package/bin/np-tools/learnings.cjs +1 -1
- package/bin/np-tools/loop-run-round.cjs +25 -11
- package/bin/np-tools/plan-milestone.cjs +1 -0
- package/bin/np-tools/research-phase.cjs +1 -1
- package/bin/np-tools/resolve-model.cjs +55 -1
- package/bin/np-tools/resolve-model.test.cjs +139 -0
- package/bin/np-tools/security.cjs +1 -1
- package/bin/np-tools/spawn-headless.cjs +155 -3
- package/bin/np-tools/spawn-headless.test.cjs +108 -58
- package/bin/np-tools/spawn-offhost.cjs +93 -0
- package/bin/np-tools/spawn-offhost.test.cjs +38 -0
- package/lib/agents.cjs +16 -2
- package/lib/cache-align.cjs +78 -0
- package/lib/cache-align.test.cjs +69 -0
- package/lib/compress.cjs +495 -0
- package/lib/compress.test.cjs +267 -0
- package/lib/config-defaults.cjs +39 -0
- package/lib/config-schema.cjs +45 -5
- package/lib/elision-bench.cjs +409 -0
- package/lib/elision-bench.test.cjs +89 -0
- package/lib/elision-proxy.cjs +158 -0
- package/lib/elision-proxy.test.cjs +243 -0
- package/lib/elision.cjs +163 -0
- package/lib/elision.test.cjs +143 -0
- package/lib/learnings/extract.cjs +4 -4
- package/lib/learnings/extract.test.cjs +8 -8
- package/lib/model-providers.cjs +118 -0
- package/lib/model-providers.test.cjs +85 -0
- package/lib/nubosloop.cjs +1 -1
- package/lib/output-steering.cjs +68 -0
- package/lib/output-steering.test.cjs +74 -0
- package/lib/researcher-swarm.cjs +14 -3
- package/lib/runtime/agent-loop.cjs +94 -0
- package/lib/runtime/agent-loop.test.cjs +240 -0
- package/lib/runtime/dispatch.cjs +174 -0
- package/lib/runtime/dispatch.test.cjs +207 -0
- package/lib/runtime/preflight.cjs +68 -0
- package/lib/runtime/preflight.test.cjs +62 -0
- package/lib/runtime/providers/openai-compat.cjs +103 -0
- package/lib/runtime/providers/openai-compat.test.cjs +112 -0
- package/lib/runtime/tools/index.cjs +447 -0
- package/lib/runtime/tools/index.test.cjs +254 -0
- package/lib/schemas/data/elision-entry.v1.json +16 -0
- package/lib/security/review.cjs +4 -4
- package/lib/security/review.test.cjs +6 -6
- package/lib/token-cost.cjs +46 -0
- package/lib/token-cost.test.cjs +42 -0
- package/np-tools.cjs +3 -0
- package/package.json +1 -1
- package/workflows/add-tests.md +41 -0
- package/workflows/architect-phase.md +19 -0
- package/workflows/discuss-phase.md +29 -10
- package/workflows/execute-phase.md +93 -4
- package/workflows/plan-phase.md +57 -16
- package/workflows/research-phase.md +45 -0
- package/workflows/scan-codebase.md +21 -3
- package/workflows/validate-phase.md +30 -13
- package/workflows/verify-work.md +17 -0
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
const os = require('node:os');
|
|
5
|
+
const path = require('node:path');
|
|
6
|
+
const http = require('node:http');
|
|
7
|
+
const child_process = require('node:child_process');
|
|
8
|
+
const { test, afterEach } = require('node:test');
|
|
9
|
+
const assert = require('node:assert/strict');
|
|
10
|
+
|
|
11
|
+
const proxy = require('./elision-proxy.cjs');
|
|
12
|
+
|
|
13
|
+
function bigLog() {
|
|
14
|
+
const lines = [];
|
|
15
|
+
for (let i = 0; i < 300; i++) {
|
|
16
|
+
lines.push(i % 71 === 0 ? ('ERROR: boom at svc_' + i) : ('[info] step ' + i + ' ok ' + 'x'.repeat(40)));
|
|
17
|
+
}
|
|
18
|
+
return lines.join('\n');
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function fakeCx(store) {
|
|
22
|
+
return { enabled: true, minBlockBytes: 100, verifyMaxBytes: 2000, store: store || (() => 'abcdef012345') };
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const _servers = [];
|
|
26
|
+
const _dirs = [];
|
|
27
|
+
const _procs = [];
|
|
28
|
+
afterEach(() => {
|
|
29
|
+
while (_procs.length) { try { _procs.pop().kill(); } catch {} }
|
|
30
|
+
while (_servers.length) { try { _servers.pop().close(); } catch {} }
|
|
31
|
+
while (_dirs.length) { try { fs.rmSync(_dirs.pop(), { recursive: true, force: true }); } catch {} }
|
|
32
|
+
});
|
|
33
|
+
function ws(files) {
|
|
34
|
+
const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'np-proxy-')));
|
|
35
|
+
for (const [rel, content] of Object.entries(files || {})) {
|
|
36
|
+
const abs = path.join(root, rel);
|
|
37
|
+
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
|
38
|
+
fs.writeFileSync(abs, content, 'utf-8');
|
|
39
|
+
}
|
|
40
|
+
_dirs.push(root);
|
|
41
|
+
return root;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
test('PXY-1: compresses a large tool_result and stores the original; cache_control + ids survive', () => {
|
|
45
|
+
const seen = [];
|
|
46
|
+
const body = {
|
|
47
|
+
model: 'claude',
|
|
48
|
+
system: [{ type: 'text', text: 'SYSTEM PROMPT '.repeat(50), cache_control: { type: 'ephemeral' } }],
|
|
49
|
+
messages: [
|
|
50
|
+
{ role: 'assistant', content: [{ type: 'text', text: 'thinking' }] },
|
|
51
|
+
{ role: 'user', content: [
|
|
52
|
+
{ type: 'tool_result', tool_use_id: 'tu_1', is_error: false, cache_control: { type: 'ephemeral' }, content: bigLog() },
|
|
53
|
+
] },
|
|
54
|
+
],
|
|
55
|
+
};
|
|
56
|
+
const out = proxy.compressAnthropicBody(body, fakeCx((orig) => { seen.push(orig); return 'abcdef012345'; }));
|
|
57
|
+
assert.equal(out.stats.blocks_compressed, 1);
|
|
58
|
+
assert.ok(out.stats.bytes_after < out.stats.bytes_before);
|
|
59
|
+
const tr = out.body.messages[1].content[0];
|
|
60
|
+
assert.ok(tr.content.includes('⟦elided:abcdef012345'), 'marker injected into tool_result');
|
|
61
|
+
assert.deepEqual(tr.cache_control, { type: 'ephemeral' }, 'cache_control preserved');
|
|
62
|
+
assert.equal(tr.tool_use_id, 'tu_1');
|
|
63
|
+
assert.equal(tr.is_error, false);
|
|
64
|
+
assert.equal(seen.length, 1, 'original stored once');
|
|
65
|
+
assert.ok(seen[0].includes('ERROR: boom at svc_0'), 'raw original captured');
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test('PXY-2: system blocks and ordinary text are never touched (cached prefix stays byte-identical)', () => {
|
|
69
|
+
const sys = 'SYSTEM '.repeat(500);
|
|
70
|
+
const body = {
|
|
71
|
+
system: [{ type: 'text', text: sys, cache_control: { type: 'ephemeral' } }],
|
|
72
|
+
messages: [{ role: 'assistant', content: [{ type: 'text', text: 'A'.repeat(5000) }] }],
|
|
73
|
+
};
|
|
74
|
+
const out = proxy.compressAnthropicBody(body, fakeCx());
|
|
75
|
+
assert.equal(out.stats.blocks_compressed, 0);
|
|
76
|
+
assert.equal(out.body.system[0].text, sys);
|
|
77
|
+
assert.equal(out.body.messages[0].content[0].text, 'A'.repeat(5000));
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test('PXY-3: small tool_result is left verbatim', () => {
|
|
81
|
+
const body = { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 't', content: 'tiny' }] }] };
|
|
82
|
+
const out = proxy.compressAnthropicBody(body, fakeCx());
|
|
83
|
+
assert.equal(out.stats.blocks_compressed, 0);
|
|
84
|
+
assert.equal(out.body.messages[0].content[0].content, 'tiny');
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test('PXY-4: deterministic — same body compresses to the same bytes (cache-stable across turns)', () => {
|
|
88
|
+
const mk = () => ({ messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 't', content: bigLog() }] }] });
|
|
89
|
+
const a = proxy.compressAnthropicBody(mk(), fakeCx());
|
|
90
|
+
const b = proxy.compressAnthropicBody(mk(), fakeCx());
|
|
91
|
+
assert.equal(JSON.stringify(a.body), JSON.stringify(b.body));
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test('PXY-5: array-form tool_result content compresses its text blocks', () => {
|
|
95
|
+
const body = { messages: [{ role: 'user', content: [
|
|
96
|
+
{ type: 'tool_result', tool_use_id: 't', content: [{ type: 'text', text: bigLog() }] },
|
|
97
|
+
] }] };
|
|
98
|
+
const out = proxy.compressAnthropicBody(body, fakeCx());
|
|
99
|
+
assert.equal(out.stats.blocks_compressed, 1);
|
|
100
|
+
assert.ok(out.body.messages[0].content[0].content[0].text.includes('⟦elided:'));
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test('PXY-6: disabled compression context is a no-op', () => {
|
|
104
|
+
const body = { messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 't', content: bigLog() }] }] };
|
|
105
|
+
const out = proxy.compressAnthropicBody(body, { enabled: false, store: () => 'x' });
|
|
106
|
+
assert.equal(out.stats.blocks_compressed, 0);
|
|
107
|
+
assert.equal(out.body, body);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test('PXY-7: end-to-end — child request is crushed in flight, forwarded, response piped back', async () => {
|
|
111
|
+
const cwd = ws({ '.nubos-pilot/config.json': JSON.stringify({ compression: { enabled: true } }) });
|
|
112
|
+
|
|
113
|
+
let received = null;
|
|
114
|
+
const upstream = http.createServer((req, res) => {
|
|
115
|
+
const cs = [];
|
|
116
|
+
req.on('data', (c) => cs.push(c));
|
|
117
|
+
req.on('end', () => {
|
|
118
|
+
received = { path: req.url, auth: req.headers['x-api-key'], body: Buffer.concat(cs).toString('utf-8') };
|
|
119
|
+
res.writeHead(200, { 'content-type': 'application/json' });
|
|
120
|
+
res.end(JSON.stringify({ ok: true, id: 'msg_123' }));
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
_servers.push(upstream);
|
|
124
|
+
await new Promise((r) => upstream.listen(0, '127.0.0.1', r));
|
|
125
|
+
const upstreamUrl = 'http://127.0.0.1:' + upstream.address().port;
|
|
126
|
+
|
|
127
|
+
const { server, baseUrl } = await proxy.start({ cwd, upstream: upstreamUrl });
|
|
128
|
+
_servers.push(server);
|
|
129
|
+
|
|
130
|
+
const reqBody = JSON.stringify({
|
|
131
|
+
model: 'claude',
|
|
132
|
+
messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tu', content: bigLog() }] }],
|
|
133
|
+
});
|
|
134
|
+
const resp = await new Promise((resolve, reject) => {
|
|
135
|
+
const u = new URL(baseUrl + '/v1/messages');
|
|
136
|
+
const r = http.request({ hostname: u.hostname, port: u.port, path: u.pathname, method: 'POST',
|
|
137
|
+
headers: { 'content-type': 'application/json', 'x-api-key': 'sk-test', 'content-length': Buffer.byteLength(reqBody) } },
|
|
138
|
+
(res) => { const cs = []; res.on('data', (c) => cs.push(c)); res.on('end', () => resolve({ status: res.statusCode, body: Buffer.concat(cs).toString('utf-8') })); });
|
|
139
|
+
r.on('error', reject);
|
|
140
|
+
r.end(reqBody);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
assert.equal(resp.status, 200);
|
|
144
|
+
assert.deepEqual(JSON.parse(resp.body), { ok: true, id: 'msg_123' }, 'upstream response piped through unchanged');
|
|
145
|
+
assert.equal(received.path, '/v1/messages', 'path forwarded');
|
|
146
|
+
assert.equal(received.auth, 'sk-test', 'auth header forwarded');
|
|
147
|
+
assert.ok(received.body.length < reqBody.length, 'upstream got a smaller body');
|
|
148
|
+
assert.ok(received.body.includes('⟦elided:'), 'tool_result was crushed in flight');
|
|
149
|
+
const hash = JSON.parse(received.body).messages[0].content[0].content.match(/⟦elided:([a-f0-9]{12})/)[1];
|
|
150
|
+
const back = require('./elision.cjs').retrieve(hash, cwd);
|
|
151
|
+
assert.equal(back.status, 'ok');
|
|
152
|
+
assert.ok(back.original.includes('ERROR: boom at svc_0'), 'original recoverable from the ledger');
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test('PXY-10: cache_align normalizes tools and adds a breakpoint in flight when opted in', async () => {
|
|
156
|
+
const cwd = ws({ '.nubos-pilot/config.json': JSON.stringify({ compression: { enabled: true, cache_align: { enabled: true } } }) });
|
|
157
|
+
let received = null;
|
|
158
|
+
const upstream = http.createServer((req, res) => {
|
|
159
|
+
const cs = []; req.on('data', (c) => cs.push(c));
|
|
160
|
+
req.on('end', () => { received = Buffer.concat(cs).toString('utf-8'); res.writeHead(200, { 'content-type': 'application/json' }); res.end('{"ok":true}'); });
|
|
161
|
+
});
|
|
162
|
+
_servers.push(upstream);
|
|
163
|
+
await new Promise((r) => upstream.listen(0, '127.0.0.1', r));
|
|
164
|
+
const { server, baseUrl } = await proxy.start({ cwd, upstream: 'http://127.0.0.1:' + upstream.address().port });
|
|
165
|
+
_servers.push(server);
|
|
166
|
+
|
|
167
|
+
const reqBody = JSON.stringify({
|
|
168
|
+
model: 'claude',
|
|
169
|
+
system: 'plain stable system',
|
|
170
|
+
tools: [{ name: 'zeta' }, { name: 'alpha' }],
|
|
171
|
+
messages: [{ role: 'user', content: 'hi' }],
|
|
172
|
+
});
|
|
173
|
+
await new Promise((resolve, reject) => {
|
|
174
|
+
const u = new URL(baseUrl + '/v1/messages');
|
|
175
|
+
const r = http.request({ hostname: u.hostname, port: u.port, path: u.pathname, method: 'POST',
|
|
176
|
+
headers: { 'content-type': 'application/json', 'content-length': Buffer.byteLength(reqBody) } },
|
|
177
|
+
(res) => { const cs = []; res.on('data', (c) => cs.push(c)); res.on('end', resolve); });
|
|
178
|
+
r.on('error', reject);
|
|
179
|
+
r.end(reqBody);
|
|
180
|
+
});
|
|
181
|
+
const got = JSON.parse(received);
|
|
182
|
+
assert.deepEqual(got.tools.map((t) => t.name), ['alpha', 'zeta'], 'tools sorted for a stable prefix');
|
|
183
|
+
assert.deepEqual(got.tools[1].cache_control, { type: 'ephemeral' }, 'breakpoint added on the last tool');
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
test('PXY-9: forked entry reports its baseUrl over IPC and crushes traffic (spawn-headless mechanism)', async () => {
|
|
187
|
+
const cwd = ws({ '.nubos-pilot/config.json': JSON.stringify({ compression: { enabled: true, proxy: { enabled: true } } }) });
|
|
188
|
+
let received = null;
|
|
189
|
+
const upstream = http.createServer((req, res) => {
|
|
190
|
+
const cs = []; req.on('data', (c) => cs.push(c));
|
|
191
|
+
req.on('end', () => { received = Buffer.concat(cs).toString('utf-8'); res.writeHead(200, { 'content-type': 'application/json' }); res.end('{"ok":true}'); });
|
|
192
|
+
});
|
|
193
|
+
_servers.push(upstream);
|
|
194
|
+
await new Promise((r) => upstream.listen(0, '127.0.0.1', r));
|
|
195
|
+
|
|
196
|
+
const entry = path.join(__dirname, '..', 'bin', 'np-tools', '_elision-proxy-entry.cjs');
|
|
197
|
+
const proc = child_process.fork(entry, [], {
|
|
198
|
+
env: Object.assign({}, process.env, { ELISION_PROXY_CWD: cwd, ELISION_PROXY_UPSTREAM: 'http://127.0.0.1:' + upstream.address().port }),
|
|
199
|
+
stdio: ['ignore', 'ignore', 'inherit', 'ipc'],
|
|
200
|
+
});
|
|
201
|
+
_procs.push(proc);
|
|
202
|
+
const baseUrl = await new Promise((resolve, reject) => {
|
|
203
|
+
proc.once('message', (m) => (m && m.ready ? resolve(m.baseUrl) : reject(new Error(m && m.error))));
|
|
204
|
+
proc.once('error', reject);
|
|
205
|
+
});
|
|
206
|
+
assert.match(baseUrl, /^http:\/\/127\.0\.0\.1:\d+$/);
|
|
207
|
+
|
|
208
|
+
const reqBody = JSON.stringify({ model: 'claude', messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tu', content: bigLog() }] }] });
|
|
209
|
+
const status = await new Promise((resolve, reject) => {
|
|
210
|
+
const u = new URL(baseUrl + '/v1/messages');
|
|
211
|
+
const r = http.request({ hostname: u.hostname, port: u.port, path: u.pathname, method: 'POST',
|
|
212
|
+
headers: { 'content-type': 'application/json', 'content-length': Buffer.byteLength(reqBody) } },
|
|
213
|
+
(res) => { res.resume(); res.on('end', () => resolve(res.statusCode)); });
|
|
214
|
+
r.on('error', reject); r.end(reqBody);
|
|
215
|
+
});
|
|
216
|
+
assert.equal(status, 200);
|
|
217
|
+
assert.ok(received && received.includes('⟦elided:'), 'forked proxy crushed the tool_result before forwarding');
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
test('PXY-8: non-JSON body is forwarded untouched (no crash)', async () => {
|
|
221
|
+
const cwd = ws({ '.nubos-pilot/config.json': JSON.stringify({ compression: { enabled: true } }) });
|
|
222
|
+
let received = null;
|
|
223
|
+
const upstream = http.createServer((req, res) => {
|
|
224
|
+
const cs = []; req.on('data', (c) => cs.push(c));
|
|
225
|
+
req.on('end', () => { received = Buffer.concat(cs).toString('utf-8'); res.writeHead(200); res.end('pong'); });
|
|
226
|
+
});
|
|
227
|
+
_servers.push(upstream);
|
|
228
|
+
await new Promise((r) => upstream.listen(0, '127.0.0.1', r));
|
|
229
|
+
const { server, baseUrl } = await proxy.start({ cwd, upstream: 'http://127.0.0.1:' + upstream.address().port });
|
|
230
|
+
_servers.push(server);
|
|
231
|
+
|
|
232
|
+
const resp = await new Promise((resolve, reject) => {
|
|
233
|
+
const u = new URL(baseUrl + '/v1/messages');
|
|
234
|
+
const r = http.request({ hostname: u.hostname, port: u.port, path: u.pathname, method: 'POST',
|
|
235
|
+
headers: { 'content-type': 'text/plain', 'content-length': 11 } }, (res) => {
|
|
236
|
+
const cs = []; res.on('data', (c) => cs.push(c)); res.on('end', () => resolve({ status: res.statusCode, body: Buffer.concat(cs).toString('utf-8') }));
|
|
237
|
+
});
|
|
238
|
+
r.on('error', reject); r.end('not a json!');
|
|
239
|
+
});
|
|
240
|
+
assert.equal(resp.status, 200);
|
|
241
|
+
assert.equal(resp.body, 'pong');
|
|
242
|
+
assert.equal(received, 'not a json!', 'non-JSON forwarded verbatim');
|
|
243
|
+
});
|
package/lib/elision.cjs
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
const path = require('node:path');
|
|
5
|
+
const crypto = require('node:crypto');
|
|
6
|
+
|
|
7
|
+
const { NubosPilotError, atomicWriteFileSync, withFileLock, findProjectRoot } = require('./core.cjs');
|
|
8
|
+
const { validate } = require('./validate.cjs');
|
|
9
|
+
const config = require('./config.cjs');
|
|
10
|
+
const { DEFAULT_COMPRESSION } = require('./config-defaults.cjs');
|
|
11
|
+
const logger = require('./logger.cjs').child('elision');
|
|
12
|
+
|
|
13
|
+
const STORE_VERSION = 1;
|
|
14
|
+
const STORE_SCHEMA = 'elision-entry.v1';
|
|
15
|
+
const HASH_LEN = 12;
|
|
16
|
+
const HASH_RE = /^[a-f0-9]{12}$/;
|
|
17
|
+
const DEFAULT_TTL_MS = 30 * 60 * 1000;
|
|
18
|
+
const DEFAULT_VERIFY_MAX_BYTES = 2000;
|
|
19
|
+
|
|
20
|
+
function hashOf(text) {
|
|
21
|
+
return crypto.createHash('sha256').update(text == null ? '' : String(text)).digest('hex').slice(0, HASH_LEN);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function _elisionDir(cwd) {
|
|
25
|
+
let root;
|
|
26
|
+
try { root = findProjectRoot(cwd); }
|
|
27
|
+
catch { root = cwd || process.cwd(); }
|
|
28
|
+
return path.join(root, '.nubos-pilot', 'elision');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function _entryPath(cwd, hash) {
|
|
32
|
+
return path.join(_elisionDir(cwd), hash + '.json');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function _isExpired(entry, now) {
|
|
36
|
+
if (!entry || typeof entry.created_at !== 'string') return true;
|
|
37
|
+
const created = Date.parse(entry.created_at);
|
|
38
|
+
if (!Number.isFinite(created)) return true;
|
|
39
|
+
const ttl = Number.isFinite(entry.ttl_ms) ? entry.ttl_ms : DEFAULT_TTL_MS;
|
|
40
|
+
if (ttl <= 0) return false;
|
|
41
|
+
return now - created > ttl;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function store(original, meta, cwd) {
|
|
45
|
+
const text = original == null ? '' : String(original);
|
|
46
|
+
const m = meta || {};
|
|
47
|
+
const hash = hashOf(text);
|
|
48
|
+
if (!HASH_RE.test(hash)) {
|
|
49
|
+
throw new NubosPilotError('elision-hash-invalid', 'computed Elision hash is malformed', { hash });
|
|
50
|
+
}
|
|
51
|
+
const target = _entryPath(cwd, hash);
|
|
52
|
+
return withFileLock(target, () => {
|
|
53
|
+
if (fs.existsSync(target)) return hash;
|
|
54
|
+
const entry = {
|
|
55
|
+
version: STORE_VERSION,
|
|
56
|
+
hash,
|
|
57
|
+
original: text,
|
|
58
|
+
type: typeof m.type === 'string' ? m.type.slice(0, 64) : 'plain',
|
|
59
|
+
created_at: new Date().toISOString(),
|
|
60
|
+
ttl_ms: Number.isFinite(m.ttlMs) ? Math.max(0, Math.round(m.ttlMs)) : DEFAULT_TTL_MS,
|
|
61
|
+
original_bytes: Buffer.byteLength(text, 'utf-8'),
|
|
62
|
+
compressed_bytes: Number.isFinite(m.compressedBytes) ? Math.max(0, Math.round(m.compressedBytes)) : 0,
|
|
63
|
+
};
|
|
64
|
+
const errors = validate(entry, STORE_SCHEMA);
|
|
65
|
+
if (errors.length) {
|
|
66
|
+
throw new NubosPilotError('elision-entry-invalid', 'refusing to persist malformed Elision entry', { errors });
|
|
67
|
+
}
|
|
68
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
69
|
+
atomicWriteFileSync(target, JSON.stringify(entry), 'utf-8', 0o600);
|
|
70
|
+
logger.debug('stored', { hash, type: entry.type, original_bytes: entry.original_bytes });
|
|
71
|
+
return hash;
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function retrieve(hash, cwd) {
|
|
76
|
+
if (typeof hash !== 'string' || !HASH_RE.test(hash)) {
|
|
77
|
+
return { status: 'not_found', hash: String(hash) };
|
|
78
|
+
}
|
|
79
|
+
const target = _entryPath(cwd, hash);
|
|
80
|
+
let raw;
|
|
81
|
+
try { raw = fs.readFileSync(target, 'utf-8'); }
|
|
82
|
+
catch { return { status: 'not_found', hash }; }
|
|
83
|
+
let entry;
|
|
84
|
+
try { entry = JSON.parse(raw); }
|
|
85
|
+
catch { return { status: 'not_found', hash }; }
|
|
86
|
+
if (_isExpired(entry, Date.now())) {
|
|
87
|
+
return { status: 'expired', hash, ttl_ms: entry.ttl_ms, created_at: entry.created_at };
|
|
88
|
+
}
|
|
89
|
+
return {
|
|
90
|
+
status: 'ok',
|
|
91
|
+
hash,
|
|
92
|
+
original: typeof entry.original === 'string' ? entry.original : '',
|
|
93
|
+
type: entry.type || 'plain',
|
|
94
|
+
original_bytes: entry.original_bytes || 0,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function prune(cwd) {
|
|
99
|
+
const dir = _elisionDir(cwd);
|
|
100
|
+
let names;
|
|
101
|
+
try { names = fs.readdirSync(dir); }
|
|
102
|
+
catch { return { removed: 0 }; }
|
|
103
|
+
const now = Date.now();
|
|
104
|
+
let removed = 0;
|
|
105
|
+
for (const name of names) {
|
|
106
|
+
if (!name.endsWith('.json')) continue;
|
|
107
|
+
const p = path.join(dir, name);
|
|
108
|
+
let entry;
|
|
109
|
+
try { entry = JSON.parse(fs.readFileSync(p, 'utf-8')); }
|
|
110
|
+
catch { continue; }
|
|
111
|
+
if (_isExpired(entry, now)) {
|
|
112
|
+
try { fs.unlinkSync(p); removed += 1; }
|
|
113
|
+
catch { /* leave lapsed entry for the next prune */ }
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (removed) logger.info('pruned', { removed });
|
|
117
|
+
return { removed };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function compressionContext(cwd) {
|
|
121
|
+
let cfg;
|
|
122
|
+
try { cfg = config.tryReadConfigPath(cwd, 'compression', DEFAULT_COMPRESSION) || DEFAULT_COMPRESSION; }
|
|
123
|
+
catch { cfg = DEFAULT_COMPRESSION; }
|
|
124
|
+
const enabled = cfg.enabled === true;
|
|
125
|
+
const elisionEnabled = enabled && (!cfg.elision || cfg.elision.enabled !== false);
|
|
126
|
+
const ttlMs = cfg.elision && Number.isFinite(cfg.elision.ttl_ms) ? cfg.elision.ttl_ms : undefined;
|
|
127
|
+
const storeFn = elisionEnabled
|
|
128
|
+
? (original, type) => {
|
|
129
|
+
try { return store(original, { type, ttlMs }, cwd); }
|
|
130
|
+
catch { return null; }
|
|
131
|
+
}
|
|
132
|
+
: null;
|
|
133
|
+
const os = cfg.output_steering || {};
|
|
134
|
+
const er = os.effort_routing || {};
|
|
135
|
+
const baseEffort = typeof er.base_effort === 'string' && er.base_effort ? er.base_effort : null;
|
|
136
|
+
const outputSteering = {
|
|
137
|
+
enabled: enabled && os.enabled === true,
|
|
138
|
+
profile: typeof os.verbosity_profile === 'string' ? os.verbosity_profile : 'balanced',
|
|
139
|
+
effortRouting: enabled && os.enabled === true && er.enabled === true && baseEffort !== null,
|
|
140
|
+
baseEffort,
|
|
141
|
+
mechanicalEffort: typeof er.mechanical_effort === 'string' ? er.mechanical_effort : 'low',
|
|
142
|
+
};
|
|
143
|
+
return {
|
|
144
|
+
enabled,
|
|
145
|
+
store: storeFn,
|
|
146
|
+
minBlockBytes: Number.isFinite(cfg.min_block_bytes) ? cfg.min_block_bytes : undefined,
|
|
147
|
+
verifyMaxBytes: Number.isFinite(cfg.verify_max_bytes) ? cfg.verify_max_bytes : DEFAULT_VERIFY_MAX_BYTES,
|
|
148
|
+
outputSteering,
|
|
149
|
+
cacheAlign: enabled && !!(cfg.cache_align && cfg.cache_align.enabled === true),
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
module.exports = {
|
|
154
|
+
STORE_VERSION,
|
|
155
|
+
HASH_LEN,
|
|
156
|
+
HASH_RE,
|
|
157
|
+
DEFAULT_TTL_MS,
|
|
158
|
+
hashOf,
|
|
159
|
+
store,
|
|
160
|
+
retrieve,
|
|
161
|
+
prune,
|
|
162
|
+
compressionContext,
|
|
163
|
+
};
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { test } = require('node:test');
|
|
4
|
+
const assert = require('node:assert/strict');
|
|
5
|
+
const fs = require('node:fs');
|
|
6
|
+
const os = require('node:os');
|
|
7
|
+
const path = require('node:path');
|
|
8
|
+
|
|
9
|
+
const elision = require('./elision.cjs');
|
|
10
|
+
|
|
11
|
+
function sandbox() {
|
|
12
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'np-elision-'));
|
|
13
|
+
fs.mkdirSync(path.join(root, '.nubos-pilot'), { recursive: true });
|
|
14
|
+
return root;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function writeConfig(root, compression) {
|
|
18
|
+
fs.writeFileSync(
|
|
19
|
+
path.join(root, '.nubos-pilot', 'config.json'),
|
|
20
|
+
JSON.stringify({ compression }),
|
|
21
|
+
);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
test('ELISION-1: store then retrieve is lossless', () => {
|
|
25
|
+
const root = sandbox();
|
|
26
|
+
try {
|
|
27
|
+
const original = 'line one\nERROR boom\nline three';
|
|
28
|
+
const hash = elision.store(original, { type: 'log' }, root);
|
|
29
|
+
assert.match(hash, /^[a-f0-9]{12}$/);
|
|
30
|
+
const got = elision.retrieve(hash, root);
|
|
31
|
+
assert.equal(got.status, 'ok');
|
|
32
|
+
assert.equal(got.original, original);
|
|
33
|
+
assert.equal(got.type, 'log');
|
|
34
|
+
} finally {
|
|
35
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test('ELISION-2: store is idempotent for identical content', () => {
|
|
40
|
+
const root = sandbox();
|
|
41
|
+
try {
|
|
42
|
+
const h1 = elision.store('same', { type: 'plain' }, root);
|
|
43
|
+
const p = path.join(root, '.nubos-pilot', 'elision', h1 + '.json');
|
|
44
|
+
const mtime1 = fs.statSync(p).mtimeMs;
|
|
45
|
+
const h2 = elision.store('same', { type: 'plain' }, root);
|
|
46
|
+
assert.equal(h2, h1);
|
|
47
|
+
assert.equal(fs.statSync(p).mtimeMs, mtime1);
|
|
48
|
+
} finally {
|
|
49
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test('ELISION-3: unknown and malformed hashes return not_found', () => {
|
|
54
|
+
const root = sandbox();
|
|
55
|
+
try {
|
|
56
|
+
assert.equal(elision.retrieve('aaaaaaaaaaaa', root).status, 'not_found');
|
|
57
|
+
assert.equal(elision.retrieve('NOT-A-HASH', root).status, 'not_found');
|
|
58
|
+
assert.equal(elision.retrieve(null, root).status, 'not_found');
|
|
59
|
+
} finally {
|
|
60
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test('ELISION-4: an entry past its ttl reports expired', () => {
|
|
65
|
+
const root = sandbox();
|
|
66
|
+
try {
|
|
67
|
+
const hash = elision.hashOf('old payload');
|
|
68
|
+
const dir = path.join(root, '.nubos-pilot', 'elision');
|
|
69
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
70
|
+
const entry = {
|
|
71
|
+
version: 1,
|
|
72
|
+
hash,
|
|
73
|
+
original: 'old payload',
|
|
74
|
+
type: 'plain',
|
|
75
|
+
created_at: new Date(Date.now() - 60 * 60 * 1000).toISOString(),
|
|
76
|
+
ttl_ms: 1000,
|
|
77
|
+
original_bytes: 11,
|
|
78
|
+
compressed_bytes: 0,
|
|
79
|
+
};
|
|
80
|
+
fs.writeFileSync(path.join(dir, hash + '.json'), JSON.stringify(entry));
|
|
81
|
+
assert.equal(elision.retrieve(hash, root).status, 'expired');
|
|
82
|
+
} finally {
|
|
83
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test('ELISION-5: prune removes only expired entries', () => {
|
|
88
|
+
const root = sandbox();
|
|
89
|
+
try {
|
|
90
|
+
const fresh = elision.store('fresh', { type: 'plain' }, root);
|
|
91
|
+
const staleHash = elision.hashOf('stale');
|
|
92
|
+
const dir = path.join(root, '.nubos-pilot', 'elision');
|
|
93
|
+
fs.writeFileSync(path.join(dir, staleHash + '.json'), JSON.stringify({
|
|
94
|
+
version: 1, hash: staleHash, original: 'stale', type: 'plain',
|
|
95
|
+
created_at: new Date(Date.now() - 60 * 60 * 1000).toISOString(),
|
|
96
|
+
ttl_ms: 1000, original_bytes: 5, compressed_bytes: 0,
|
|
97
|
+
}));
|
|
98
|
+
const res = elision.prune(root);
|
|
99
|
+
assert.equal(res.removed, 1);
|
|
100
|
+
assert.equal(elision.retrieve(fresh, root).status, 'ok');
|
|
101
|
+
assert.equal(elision.retrieve(staleHash, root).status, 'not_found');
|
|
102
|
+
} finally {
|
|
103
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test('ELISION-6: compressionContext is disabled by default', () => {
|
|
108
|
+
const root = sandbox();
|
|
109
|
+
try {
|
|
110
|
+
const cx = elision.compressionContext(root);
|
|
111
|
+
assert.equal(cx.enabled, false);
|
|
112
|
+
assert.equal(cx.store, null);
|
|
113
|
+
assert.equal(cx.verifyMaxBytes, 2000);
|
|
114
|
+
} finally {
|
|
115
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test('ELISION-7: compressionContext yields a working store when enabled', () => {
|
|
120
|
+
const root = sandbox();
|
|
121
|
+
try {
|
|
122
|
+
writeConfig(root, { enabled: true });
|
|
123
|
+
const cx = elision.compressionContext(root);
|
|
124
|
+
assert.equal(cx.enabled, true);
|
|
125
|
+
assert.equal(typeof cx.store, 'function');
|
|
126
|
+
const hash = cx.store('cached body', 'plain');
|
|
127
|
+
assert.equal(elision.retrieve(hash, root).original, 'cached body');
|
|
128
|
+
} finally {
|
|
129
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test('ELISION-8: compressionContext store is null when elision is disabled', () => {
|
|
134
|
+
const root = sandbox();
|
|
135
|
+
try {
|
|
136
|
+
writeConfig(root, { enabled: true, elision: { enabled: false } });
|
|
137
|
+
const cx = elision.compressionContext(root);
|
|
138
|
+
assert.equal(cx.enabled, true);
|
|
139
|
+
assert.equal(cx.store, null);
|
|
140
|
+
} finally {
|
|
141
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
142
|
+
}
|
|
143
|
+
});
|
|
@@ -124,7 +124,7 @@ function parseExtractorOutput(raw) {
|
|
|
124
124
|
return { candidates, parse_ok: true };
|
|
125
125
|
}
|
|
126
126
|
|
|
127
|
-
function _defaultSpawn(promptText, opts) {
|
|
127
|
+
async function _defaultSpawn(promptText, opts) {
|
|
128
128
|
const spawnHeadless = require('../../bin/np-tools/spawn-headless.cjs');
|
|
129
129
|
const tmp = os.tmpdir();
|
|
130
130
|
const tag = process.pid + '-' + crypto.randomBytes(4).toString('hex');
|
|
@@ -132,7 +132,7 @@ function _defaultSpawn(promptText, opts) {
|
|
|
132
132
|
const outputPath = path.join(tmp, 'np-learn-out-' + tag + '.json');
|
|
133
133
|
fs.writeFileSync(promptPath, promptText, 'utf-8');
|
|
134
134
|
try {
|
|
135
|
-
spawnHeadless.run(
|
|
135
|
+
await spawnHeadless.run(
|
|
136
136
|
['--agent', EXTRACTOR_AGENT, '--prompt-path', promptPath, '--output-path', outputPath,
|
|
137
137
|
'--timeout-ms', String(opts.timeoutMs)],
|
|
138
138
|
{ cwd: opts.cwd, stdout: { write: () => {} } },
|
|
@@ -144,7 +144,7 @@ function _defaultSpawn(promptText, opts) {
|
|
|
144
144
|
}
|
|
145
145
|
}
|
|
146
146
|
|
|
147
|
-
function runExtract(opts) {
|
|
147
|
+
async function runExtract(opts) {
|
|
148
148
|
const o = opts || {};
|
|
149
149
|
const cwd = o.cwd || process.cwd();
|
|
150
150
|
const config = o.config || {};
|
|
@@ -164,7 +164,7 @@ function runExtract(opts) {
|
|
|
164
164
|
const promptText = buildExtractorPrompt(diff);
|
|
165
165
|
let raw = '';
|
|
166
166
|
try {
|
|
167
|
-
raw = spawn(promptText, { cwd, timeoutMs: config.timeout_ms || 120000 });
|
|
167
|
+
raw = await spawn(promptText, { cwd, timeoutMs: config.timeout_ms || 120000 });
|
|
168
168
|
} catch {
|
|
169
169
|
return { ran: true, logged: 0, reason: 'spawn-failed' };
|
|
170
170
|
}
|
|
@@ -67,31 +67,31 @@ test('EX-6: non-JSON output → parse_ok false', () => {
|
|
|
67
67
|
assert.strictEqual(extract.parseExtractorOutput('').parse_ok, false);
|
|
68
68
|
});
|
|
69
69
|
|
|
70
|
-
test('EX-7: runExtract on a non-repo returns not-a-repo, logs nothing', () => {
|
|
70
|
+
test('EX-7: runExtract on a non-repo returns not-a-repo, logs nothing', async () => {
|
|
71
71
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'np-norepo-'));
|
|
72
72
|
try {
|
|
73
73
|
const logged = [];
|
|
74
|
-
const r = extract.runExtract({ cwd: dir, spawnImpl: () => '{}', logImpl: (c) => logged.push(c) });
|
|
74
|
+
const r = await extract.runExtract({ cwd: dir, spawnImpl: () => '{}', logImpl: (c) => logged.push(c) });
|
|
75
75
|
assert.strictEqual(r.ran, false);
|
|
76
76
|
assert.strictEqual(r.reason, 'not-a-repo');
|
|
77
77
|
assert.strictEqual(logged.length, 0);
|
|
78
78
|
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
|
79
79
|
});
|
|
80
80
|
|
|
81
|
-
test('EX-8: runExtract on empty repo (no commit, no changes) → empty-diff', () => {
|
|
81
|
+
test('EX-8: runExtract on empty repo (no commit, no changes) → empty-diff', async () => {
|
|
82
82
|
const dir = _gitRepo(false);
|
|
83
83
|
try {
|
|
84
|
-
const r = extract.runExtract({ cwd: dir, spawnImpl: () => '{}', logImpl: () => {} });
|
|
84
|
+
const r = await extract.runExtract({ cwd: dir, spawnImpl: () => '{}', logImpl: () => {} });
|
|
85
85
|
assert.strictEqual(r.ran, true);
|
|
86
86
|
assert.strictEqual(r.reason, 'empty-diff');
|
|
87
87
|
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
|
88
88
|
});
|
|
89
89
|
|
|
90
|
-
test('EX-9: runExtract over a commit logs parsed candidates', () => {
|
|
90
|
+
test('EX-9: runExtract over a commit logs parsed candidates', async () => {
|
|
91
91
|
const dir = _gitRepo(true);
|
|
92
92
|
try {
|
|
93
93
|
const logged = [];
|
|
94
|
-
const r = extract.runExtract({
|
|
94
|
+
const r = await extract.runExtract({
|
|
95
95
|
cwd: dir,
|
|
96
96
|
spawnImpl: () => JSON.stringify({ result: JSON.stringify({ learnings: [
|
|
97
97
|
{ pattern: 'keep add() pure and total', outcome: 'verified' },
|
|
@@ -104,11 +104,11 @@ test('EX-9: runExtract over a commit logs parsed candidates', () => {
|
|
|
104
104
|
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
|
105
105
|
});
|
|
106
106
|
|
|
107
|
-
test('EX-10: runExtract with unparseable spawn output → parse-failed, no log', () => {
|
|
107
|
+
test('EX-10: runExtract with unparseable spawn output → parse-failed, no log', async () => {
|
|
108
108
|
const dir = _gitRepo(true);
|
|
109
109
|
try {
|
|
110
110
|
const logged = [];
|
|
111
|
-
const r = extract.runExtract({ cwd: dir, spawnImpl: () => 'garbage', logImpl: (c) => logged.push(c) });
|
|
111
|
+
const r = await extract.runExtract({ cwd: dir, spawnImpl: () => 'garbage', logImpl: (c) => logged.push(c) });
|
|
112
112
|
assert.strictEqual(r.reason, 'parse-failed');
|
|
113
113
|
assert.strictEqual(logged.length, 0);
|
|
114
114
|
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|