@polderlabs/bizar 5.2.0 → 5.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bizar-dash/dist/assets/{MobileChat-TCputYzr.js → MobileChat-OidrxXR9.js} +1 -1
- package/bizar-dash/dist/assets/{MobileChat-TCputYzr.js.map → MobileChat-OidrxXR9.js.map} +1 -1
- package/bizar-dash/dist/assets/{MobileSettings-BxcbL5XT.js → MobileSettings-B91gVFii.js} +1 -1
- package/bizar-dash/dist/assets/{MobileSettings-BxcbL5XT.js.map → MobileSettings-B91gVFii.js.map} +1 -1
- package/bizar-dash/dist/assets/{icons-DRDXfbBP.js → icons-Bd8piSb2.js} +18 -13
- package/bizar-dash/dist/assets/icons-Bd8piSb2.js.map +1 -0
- package/bizar-dash/dist/assets/{main-xFpWMd32.css → main-3G6mOhiK.css} +1 -1
- package/bizar-dash/dist/assets/main-6YUZY968.js +19 -0
- package/bizar-dash/dist/assets/main-6YUZY968.js.map +1 -0
- package/bizar-dash/dist/assets/{mobile-j3rOZK6v.js → mobile-CSsp30lr.js} +2 -2
- package/bizar-dash/dist/assets/{mobile-j3rOZK6v.js.map → mobile-CSsp30lr.js.map} +1 -1
- package/bizar-dash/dist/assets/{mobile--17fkfrl.js → mobile-DtwK7DOV.js} +1 -1
- package/bizar-dash/dist/assets/{mobile--17fkfrl.js.map → mobile-DtwK7DOV.js.map} +1 -1
- package/bizar-dash/dist/assets/{useSlashCommands-BG-DhEck.js → useSlashCommands-DEkxEvOO.js} +2 -2
- package/bizar-dash/dist/assets/{useSlashCommands-BG-DhEck.js.map → useSlashCommands-DEkxEvOO.js.map} +1 -1
- package/bizar-dash/dist/index.html +5 -5
- package/bizar-dash/dist/mobile.html +3 -3
- package/bizar-dash/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json +1 -1
- package/bizar-dash/src/server/auth.mjs +14 -1
- package/bizar-dash/src/server/eval-store.mjs +55 -0
- package/bizar-dash/src/server/eval.mjs +63 -0
- package/bizar-dash/src/server/plugins/permission-audit.mjs +109 -0
- package/bizar-dash/src/server/plugins/sandbox.mjs +76 -2
- package/bizar-dash/src/server/plugins/store.mjs +4 -0
- package/bizar-dash/src/server/routes/eval.mjs +42 -2
- package/bizar-dash/src/server/routes/plugins.mjs +44 -4
- package/bizar-dash/src/server/routes/users.mjs +18 -0
- package/bizar-dash/src/server/schedules-runner.mjs +61 -0
- package/bizar-dash/src/web/components/EvalExportButton.tsx +27 -0
- package/bizar-dash/src/web/components/InstallConfirmDialog.tsx +62 -0
- package/bizar-dash/src/web/components/MarketplacePluginCard.tsx +51 -0
- package/bizar-dash/src/web/components/PluginCard.tsx +2 -2
- package/bizar-dash/src/web/components/PluginPermissions.tsx +1 -1
- package/bizar-dash/src/web/components/ScheduleTemplateCard.tsx +1 -0
- package/bizar-dash/src/web/components/Topbar.tsx +3 -0
- package/bizar-dash/src/web/components/VoiceNotesPanel.tsx +28 -0
- package/bizar-dash/src/web/styles/main.css +139 -0
- package/bizar-dash/src/web/views/Doctor.tsx +3 -3
- package/bizar-dash/src/web/views/Eval.tsx +225 -25
- package/bizar-dash/src/web/views/EvalReport.tsx +7 -3
- package/bizar-dash/src/web/views/Marketplace.tsx +135 -0
- package/bizar-dash/src/web/views/Plugins.tsx +9 -3
- package/bizar-dash/src/web/views/Schedules.tsx +1 -1
- package/bizar-dash/tests/a11y/components.test.tsx +149 -0
- package/bizar-dash/tests/a11y/forms.test.tsx +191 -0
- package/bizar-dash/tests/a11y/navigation.test.tsx +112 -0
- package/bizar-dash/tests/components/marketplace-plugin-card.test.tsx +144 -0
- package/bizar-dash/tests/eval-csv.test.mjs +67 -0
- package/bizar-dash/tests/eval-scheduled.test.mjs +84 -0
- package/bizar-dash/tests/plugins-permissions.test.mjs +525 -0
- package/cli/commands/dash.mjs +5 -0
- package/cli/commands/tailscale.mjs +8 -1
- package/package.json +1 -1
- package/bizar-dash/dist/assets/icons-DRDXfbBP.js.map +0 -1
- package/bizar-dash/dist/assets/main-D5Ditnrd.js +0 -19
- package/bizar-dash/dist/assets/main-D5Ditnrd.js.map +0 -1
package/bizar-dash/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":"4.1.9","results":[[":tests/lib/utils.test.ts",{"duration":
|
|
1
|
+
{"version":"4.1.9","results":[[":tests/lib/utils.test.ts",{"duration":15.788036999999974,"failed":false}],[":tests/components/Button.test.tsx",{"duration":84.90294800000004,"failed":false}],[":tests/components/Card.test.tsx",{"duration":18.837508999999955,"failed":false}],[":tests/hooks/useModal.test.tsx",{"duration":120.20212400000003,"failed":false}],[":tests/components/Toast.test.tsx",{"duration":161.860364,"failed":false}],[":tests/components/Modal.test.tsx",{"duration":169.36771700000008,"failed":false}],[":tests/lib/i18n.test.ts",{"duration":2.9599100000000362,"failed":false}],[":tests/components/StatusBadge.test.tsx",{"duration":22.451562999999965,"failed":false}],[":tests/hooks/useToast.test.tsx",{"duration":113.03238799999997,"failed":false}],[":tests/components/Spinner.test.tsx",{"duration":56.80167199999994,"failed":false}],[":tests/backup-restore.test.tsx",{"duration":209.892064,"failed":false}],[":tests/a11y.test.tsx",{"duration":679.8717469999999,"failed":false}],[":tests/lib/search-fuzzy.test.ts",{"duration":5.03047799999996,"failed":false}],[":tests/components/settings-search.test.tsx",{"duration":718.4375819999999,"failed":false}],[":tests/memory-graph-view.test.tsx",{"duration":29.26699000000002,"failed":false}],[":tests/components/workspace-selector.test.tsx",{"duration":120.47785299999998,"failed":false}],[":tests/components/screenshot-ocr.test.tsx",{"duration":27.81256400000001,"failed":false}],[":tests/voice-recorder.test.tsx",{"duration":114.61372900000003,"failed":false}],[":tests/autosave.test.tsx",{"duration":254.917053,"failed":false}],[":tests/settings-layout.test.tsx",{"duration":217.68918099999996,"failed":false}],[":tests/settings-nav.test.tsx",{"duration":290.68972699999995,"failed":false}],[":tests/components/doctor-panel.test.tsx",{"duration":70.02781100000004,"failed":false}],[":tests/chat-composer.test.tsx",{"duration":232.71696299999996,"failed":false}],[":tests/settings-mode-wiring.test.tsx",{"duration":276.79545799999994,"failed":false}],[":tests/minimax-bar.test.ts",{"duration":0,"failed":true}],[":tests/minimax-bar.test.tsx",{"duration":22.24368499999997,"failed":false}],[":tests/components/plugin-permissions.test.tsx",{"duration":92.75674500000002,"failed":false}],[":tests/eval-web-ui.test.tsx",{"duration":117.119639,"failed":false}],[":tests/components/marketplace-plugin-card.test.tsx",{"duration":188.3587500000001,"failed":false}],[":tests/a11y/components.test.tsx",{"duration":127.70726300000001,"failed":false}],[":tests/a11y/forms.test.tsx",{"duration":103.14186000000007,"failed":true}],[":tests/a11y/navigation.test.tsx",{"duration":120.83297300000004,"failed":true}]]}
|
|
@@ -484,7 +484,20 @@ function extractToken(req) {
|
|
|
484
484
|
*/
|
|
485
485
|
export function getCurrentUserId(req) {
|
|
486
486
|
const token = extractToken(req);
|
|
487
|
-
if (!token)
|
|
487
|
+
if (!token) {
|
|
488
|
+
// v5.3.1 — Tailscale trust: when the dashboard is reached via
|
|
489
|
+
// tailscale serve and the operator has opted in to trust tailnet
|
|
490
|
+
// clients, derive a deterministic userId from the tailnet hostname
|
|
491
|
+
// so the rest of the v5.0 multi-user code paths (workspaces, etc.)
|
|
492
|
+
// work without requiring an explicit token.
|
|
493
|
+
if (process.env.BIZAR_DASHBOARD_TRUST_TAILSCALE === '1' && isLoopback(req)) {
|
|
494
|
+
const hostname = String(req?.headers?.host || '').split(':')[0];
|
|
495
|
+
if (hostname && hostname.endsWith('.ts.net')) {
|
|
496
|
+
return 'usr_ts_' + createHash('sha256').update(hostname).digest('hex').slice(0, 12);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
return null;
|
|
500
|
+
}
|
|
488
501
|
|
|
489
502
|
const secret = getOrCreateSecret();
|
|
490
503
|
|
|
@@ -22,6 +22,10 @@ const STORE_HOME = process.env.BIZAR_STORE_HOME
|
|
|
22
22
|
: join(homedir(), '.local', 'share', 'bizar');
|
|
23
23
|
const EVAL_DIR = join(STORE_HOME, 'eval');
|
|
24
24
|
const INDEX_FILE = join(EVAL_DIR, '.index.json');
|
|
25
|
+
const CONFIG_DIR = process.env.BIZAR_EVAL_CONFIG_DIR
|
|
26
|
+
? process.env.BIZAR_EVAL_CONFIG_DIR
|
|
27
|
+
: join(homedir(), '.config', 'bizar');
|
|
28
|
+
const SCHEDULES_FILE = join(CONFIG_DIR, 'eval-schedules.json');
|
|
25
29
|
|
|
26
30
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
|
27
31
|
|
|
@@ -210,6 +214,57 @@ export function buildRunId() {
|
|
|
210
214
|
return `run_${ts}_${rand}`;
|
|
211
215
|
}
|
|
212
216
|
|
|
217
|
+
// ── Eval schedules ───────────────────────────────────────────────────────────
|
|
218
|
+
|
|
219
|
+
function loadSchedules() {
|
|
220
|
+
return safeReadJSON(SCHEDULES_FILE, []);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function saveSchedulesIndex(schedules) {
|
|
224
|
+
ensureDir(join(homedir(), '.config', 'bizar'));
|
|
225
|
+
atomicWriteJson(SCHEDULES_FILE, schedules);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Register a new eval schedule.
|
|
230
|
+
*
|
|
231
|
+
* @param {{ name: string, suitePath: string, cron: string, agent?: string }} opts
|
|
232
|
+
* @returns {{ id: string, name: string, suitePath: string, cron: string, agent: string }}
|
|
233
|
+
*/
|
|
234
|
+
export function registerEvalSchedule({ name, suitePath, cron, agent = 'thor' }) {
|
|
235
|
+
const schedules = loadSchedules();
|
|
236
|
+
const id = `eval_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
|
|
237
|
+
const schedule = { id, name, suitePath, cron, agent };
|
|
238
|
+
schedules.push(schedule);
|
|
239
|
+
saveSchedulesIndex(schedules);
|
|
240
|
+
return schedule;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* List all eval schedules.
|
|
245
|
+
*
|
|
246
|
+
* @returns {Array<{ id: string, name: string, suitePath: string, cron: string, agent: string }>}
|
|
247
|
+
*/
|
|
248
|
+
export function listEvalSchedules() {
|
|
249
|
+
return loadSchedules();
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Delete an eval schedule by id.
|
|
254
|
+
*
|
|
255
|
+
* @param {string} id
|
|
256
|
+
* @returns {{ ok: boolean }}
|
|
257
|
+
*/
|
|
258
|
+
export function deleteEvalSchedule(id) {
|
|
259
|
+
if (!id) return { ok: false };
|
|
260
|
+
const schedules = loadSchedules();
|
|
261
|
+
const idx = schedules.findIndex((s) => s.id === id);
|
|
262
|
+
if (idx < 0) return { ok: false };
|
|
263
|
+
schedules.splice(idx, 1);
|
|
264
|
+
saveSchedulesIndex(schedules);
|
|
265
|
+
return { ok: true };
|
|
266
|
+
}
|
|
267
|
+
|
|
213
268
|
// ── Test reset ────────────────────────────────────────────────────────────────
|
|
214
269
|
|
|
215
270
|
/**
|
|
@@ -358,6 +358,69 @@ export async function runFixtureById(
|
|
|
358
358
|
return runFixture(fixture, { llmCall, timeoutMs, toolCalls });
|
|
359
359
|
}
|
|
360
360
|
|
|
361
|
+
// ── CSV export ────────────────────────────────────────────────────────────────
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Convert an array of EvalResult to CSV format.
|
|
365
|
+
*
|
|
366
|
+
* @param {object[]} results
|
|
367
|
+
* @returns {string}
|
|
368
|
+
*/
|
|
369
|
+
export function toCSV(results) {
|
|
370
|
+
const headers = [
|
|
371
|
+
'fixture_id',
|
|
372
|
+
'name',
|
|
373
|
+
'agent',
|
|
374
|
+
'ok',
|
|
375
|
+
'pass_count',
|
|
376
|
+
'fail_count',
|
|
377
|
+
'latency_ms',
|
|
378
|
+
'tokens',
|
|
379
|
+
'timestamp',
|
|
380
|
+
];
|
|
381
|
+
const rows = results.map((r) => [
|
|
382
|
+
r.fixtureId,
|
|
383
|
+
r.name || '',
|
|
384
|
+
r.agent || '',
|
|
385
|
+
r.ok ? '1' : '0',
|
|
386
|
+
r.checks?.filter((c) => c.pass).length || 0,
|
|
387
|
+
r.checks?.filter((c) => !c.pass).length || 0,
|
|
388
|
+
r.latencyMs || 0,
|
|
389
|
+
r.usage?.total_tokens || 0,
|
|
390
|
+
r.timestamp || '',
|
|
391
|
+
]);
|
|
392
|
+
|
|
393
|
+
return [headers, ...rows]
|
|
394
|
+
.map((row) =>
|
|
395
|
+
row
|
|
396
|
+
.map((cell) => {
|
|
397
|
+
const s = String(cell);
|
|
398
|
+
return s.includes(',') || s.includes('"') || s.includes('\n')
|
|
399
|
+
? `"${s.replace(/"/g, '""')}"`
|
|
400
|
+
: s;
|
|
401
|
+
})
|
|
402
|
+
.join(',')
|
|
403
|
+
)
|
|
404
|
+
.join('\n');
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Run a suite and return both the run result and CSV export.
|
|
409
|
+
*
|
|
410
|
+
* @param {string} suitePath
|
|
411
|
+
* @param {{ llmCall?: Function, concurrency?: number, timeoutMs?: number }} opts
|
|
412
|
+
* @returns {Promise<{ run: object, csv: string }>}
|
|
413
|
+
*/
|
|
414
|
+
export async function runSuiteAndExport(suitePath, opts) {
|
|
415
|
+
const run = await runSuite(suitePath, opts);
|
|
416
|
+
return {
|
|
417
|
+
run,
|
|
418
|
+
csv: toCSV(run.results),
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// ── Fixture loading ─────────────────────────────────────────────────────────
|
|
423
|
+
|
|
361
424
|
/**
|
|
362
425
|
* Load all JSON fixtures from a directory.
|
|
363
426
|
*
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* src/server/plugins/permission-audit.mjs
|
|
3
|
+
*
|
|
4
|
+
* v5.3.0 — Audit log for plugin permission uses.
|
|
5
|
+
*
|
|
6
|
+
* Every call to `safeInvoke` (and every successful run after the
|
|
7
|
+
* permission check) emits one entry here. The audit log is in-memory
|
|
8
|
+
* (bounded to 1000 entries — older entries are evicted FIFO) because
|
|
9
|
+
* it is meant for live UI ("who invoked what on which plugin in the
|
|
10
|
+
* last hour") and for incident triage, not for permanent record
|
|
11
|
+
* keeping. For long-term storage, extend this module with a sink that
|
|
12
|
+
* writes to disk or to a structured-log aggregator.
|
|
13
|
+
*
|
|
14
|
+
* Why a separate module?
|
|
15
|
+
* - `sandbox.mjs` stays focused on the execution boundary.
|
|
16
|
+
* - `routes/plugins.mjs` and any future admin UI can read the log
|
|
17
|
+
* without importing the entire sandbox.
|
|
18
|
+
* - Tests can `clearPermissionAuditLog()` between cases.
|
|
19
|
+
*/
|
|
20
|
+
import * as logger from '../logger.mjs';
|
|
21
|
+
|
|
22
|
+
/** Bounded ring buffer of audit entries; trimmed at 1000. */
|
|
23
|
+
const auditLog = [];
|
|
24
|
+
|
|
25
|
+
/** Hard cap to bound memory; oldest entries fall off. */
|
|
26
|
+
const MAX_AUDIT_ENTRIES = 1000;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @typedef {{
|
|
30
|
+
* timestamp: string,
|
|
31
|
+
* pluginId: string,
|
|
32
|
+
* method: string,
|
|
33
|
+
* permission: string,
|
|
34
|
+
* allowed: boolean
|
|
35
|
+
* }} PermissionAuditEntry
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Record a permission use (allowed or denied) for one method call.
|
|
40
|
+
*
|
|
41
|
+
* @param {string} pluginId plugin identifier (matches `loaded.id` or
|
|
42
|
+
* `installed.id`)
|
|
43
|
+
* @param {string} method method name invoked
|
|
44
|
+
* @param {string} permission comma-joined permission names that the
|
|
45
|
+
* method was checked against; pass `'none'`
|
|
46
|
+
* if no permissions were required
|
|
47
|
+
* @param {boolean} allowed true if the call was allowed, false if it
|
|
48
|
+
* was rejected with permission_denied
|
|
49
|
+
*/
|
|
50
|
+
export function logPermissionUse(pluginId, method, permission, allowed) {
|
|
51
|
+
const entry = {
|
|
52
|
+
timestamp: new Date().toISOString(),
|
|
53
|
+
pluginId: pluginId || 'unknown',
|
|
54
|
+
method: method || '',
|
|
55
|
+
permission: permission || 'none',
|
|
56
|
+
allowed: !!allowed,
|
|
57
|
+
};
|
|
58
|
+
auditLog.push(entry);
|
|
59
|
+
// FIFO eviction — keep memory bounded without losing the most recent
|
|
60
|
+
// activity, which is what the UI typically wants.
|
|
61
|
+
while (auditLog.length > MAX_AUDIT_ENTRIES) {
|
|
62
|
+
auditLog.shift();
|
|
63
|
+
}
|
|
64
|
+
logger.info('plugin.permission.use', {
|
|
65
|
+
module: 'plugin-audit',
|
|
66
|
+
pluginId: entry.pluginId,
|
|
67
|
+
method: entry.method,
|
|
68
|
+
permission: entry.permission,
|
|
69
|
+
allowed: entry.allowed,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Read recent audit entries, optionally filtered by plugin.
|
|
75
|
+
* Returns newest-first so the UI can render a live list directly.
|
|
76
|
+
*
|
|
77
|
+
* @param {object} [opts]
|
|
78
|
+
* @param {string} [opts.pluginId] if set, only entries for this plugin
|
|
79
|
+
* @param {number} [opts.limit=50] max entries to return
|
|
80
|
+
* @returns {PermissionAuditEntry[]}
|
|
81
|
+
*/
|
|
82
|
+
export function getPermissionAuditLog({ pluginId, limit = 50 } = {}) {
|
|
83
|
+
let entries = auditLog;
|
|
84
|
+
if (pluginId) {
|
|
85
|
+
entries = entries.filter((e) => e.pluginId === pluginId);
|
|
86
|
+
}
|
|
87
|
+
// `slice(-limit)` gives the last `limit` entries in chronological order;
|
|
88
|
+
// reversing once gives newest-first for the API consumer.
|
|
89
|
+
return entries.slice(-limit).reverse();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Wipe all audit entries. Used by tests in `beforeEach` so cases are
|
|
94
|
+
* isolated. Not exposed via any HTTP route — admin tools that want to
|
|
95
|
+
* reset the buffer should not exist (the audit log is read-only).
|
|
96
|
+
*/
|
|
97
|
+
export function clearPermissionAuditLog() {
|
|
98
|
+
auditLog.length = 0;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Test/internal: expose the current entry count so routes can detect
|
|
103
|
+
* when the buffer was emptied by an outside test runner. Not exported
|
|
104
|
+
* in the public surface; consumers should treat the audit log as
|
|
105
|
+
* opaque.
|
|
106
|
+
*/
|
|
107
|
+
export function _permissionAuditLength() {
|
|
108
|
+
return auditLog.length;
|
|
109
|
+
}
|
|
@@ -33,6 +33,7 @@ import { readFileSync, statSync } from 'node:fs';
|
|
|
33
33
|
import { resolve as pathResolve } from 'node:path';
|
|
34
34
|
import { randomUUID } from 'node:crypto';
|
|
35
35
|
import * as logger from '../logger.mjs';
|
|
36
|
+
import { logPermissionUse } from './permission-audit.mjs';
|
|
36
37
|
|
|
37
38
|
/** Default timeout for both script compile + async method execution. */
|
|
38
39
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
@@ -380,7 +381,20 @@ function safeStringify(value) {
|
|
|
380
381
|
* @param {number} [opts.timeoutMs]
|
|
381
382
|
* @param {number} [opts.memoryLimitMb]
|
|
382
383
|
* @param {typeof globalThis.fetch} [opts.fetchImpl]
|
|
383
|
-
* @
|
|
384
|
+
* @param {Array<{ name: string, permissions?: string[] }>} [opts.methodSpecs]
|
|
385
|
+
* Per-method permission declarations from `plugin.json`'s
|
|
386
|
+
* `exports` array. Each entry is `{ name, permissions }`; used by
|
|
387
|
+
* `safeInvoke` to enforce method-level permission checks before
|
|
388
|
+
* invoking. Optional — a plugin loaded without this field gets no
|
|
389
|
+
* method-level gating (legacy behaviour).
|
|
390
|
+
* @returns {Promise<{
|
|
391
|
+
* exports: Record<string, Function>,
|
|
392
|
+
* permissions: Set<string>,
|
|
393
|
+
* invalidPermissions: string[],
|
|
394
|
+
* memoryLimitMb: number,
|
|
395
|
+
* api: object,
|
|
396
|
+
* methodSpecs: Array<{ name: string, permissions?: string[] }>
|
|
397
|
+
* }>}
|
|
384
398
|
*/
|
|
385
399
|
export async function loadPlugin(opts) {
|
|
386
400
|
const {
|
|
@@ -393,6 +407,7 @@ export async function loadPlugin(opts) {
|
|
|
393
407
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
394
408
|
memoryLimitMb = DEFAULT_MEMORY_LIMIT_MB,
|
|
395
409
|
fetchImpl,
|
|
410
|
+
methodSpecs = [],
|
|
396
411
|
} = opts;
|
|
397
412
|
|
|
398
413
|
if (!mainFile || typeof mainFile !== 'string') {
|
|
@@ -494,24 +509,79 @@ export async function loadPlugin(opts) {
|
|
|
494
509
|
invalidPermissions: invalid,
|
|
495
510
|
memoryLimitMb,
|
|
496
511
|
api,
|
|
512
|
+
methodSpecs: Array.isArray(methodSpecs) ? methodSpecs : [],
|
|
513
|
+
// Stash the plugin id on the loaded object so safeInvoke can
|
|
514
|
+
// write it to the audit log without callers having to thread it
|
|
515
|
+
// through as a separate argument (compatible with the v5.0
|
|
516
|
+
// invocation shape).
|
|
517
|
+
pluginId,
|
|
497
518
|
};
|
|
498
519
|
}
|
|
499
520
|
|
|
521
|
+
/**
|
|
522
|
+
* v5.3.0 helper — normalise whatever shape `loaded.permissions` was
|
|
523
|
+
* stored as into a plain array. `loadPlugin` uses a `Set` internally,
|
|
524
|
+
* but tests may pass an `Array`, and `loaded.permissions` may also be
|
|
525
|
+
* missing entirely (older callers). Always returns an array of strings.
|
|
526
|
+
*
|
|
527
|
+
* @param {unknown} perms
|
|
528
|
+
* @returns {string[]}
|
|
529
|
+
*/
|
|
530
|
+
function permissionsAsArray(perms) {
|
|
531
|
+
if (!perms) return [];
|
|
532
|
+
if (perms instanceof Set) return [...perms];
|
|
533
|
+
if (Array.isArray(perms)) return perms.filter((p) => typeof p === 'string');
|
|
534
|
+
return [];
|
|
535
|
+
}
|
|
536
|
+
|
|
500
537
|
/**
|
|
501
538
|
* Invoke a method on a loaded plugin inside a sandbox timeout race.
|
|
502
539
|
* Returns `{ ok: true, result }` on success, `{ ok: false, error }`
|
|
503
540
|
* on any throw (including permission errors, timeouts, plugin bugs).
|
|
504
541
|
*
|
|
542
|
+
* v5.3.0 — method-level permission enforcement: when `loaded.methodSpecs`
|
|
543
|
+
* is populated (from plugin.json's `exports` array), `safeInvoke` checks
|
|
544
|
+
* the called method's declared permissions against the plugin's granted
|
|
545
|
+
* permissions BEFORE running. A denial is recorded in the audit log
|
|
546
|
+
* (`permission-audit.mjs`) and returned with `code: 'permission_denied'`
|
|
547
|
+
* and a `missing` array listing the ungranted permissions.
|
|
548
|
+
*
|
|
505
549
|
* @param {object} loaded return value of loadPlugin()
|
|
506
550
|
* @param {string} method method name on the plugin's exports
|
|
507
551
|
* @param {unknown[]} args
|
|
508
552
|
* @param {number} [timeoutMs]
|
|
509
|
-
* @returns {Promise<{ ok: true, result: unknown } | { ok: false, error: string, code?: string }>}
|
|
553
|
+
* @returns {Promise<{ ok: true, result: unknown } | { ok: false, error: string, code?: string, missing?: string[] }>}
|
|
510
554
|
*/
|
|
511
555
|
export async function safeInvoke(loaded, method, args = [], timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
512
556
|
if (!loaded || typeof loaded !== 'object') {
|
|
513
557
|
return { ok: false, error: 'plugin not loaded', code: 'not_loaded' };
|
|
514
558
|
}
|
|
559
|
+
|
|
560
|
+
// v5.3.0 — method-level permission enforcement. Uses
|
|
561
|
+
// `loaded.methodSpecs` (a copy of plugin.json's `exports` array, each
|
|
562
|
+
// entry shaped `{ name, permissions?: string[] }`). When the plugin
|
|
563
|
+
// declares no method-level permissions, we skip the check entirely
|
|
564
|
+
// (backward compatibility for plugins loaded without a manifest).
|
|
565
|
+
const pluginId = loaded.pluginId || loaded.id || 'unknown';
|
|
566
|
+
const methodSpecs = Array.isArray(loaded.methodSpecs) ? loaded.methodSpecs : [];
|
|
567
|
+
const methodSpec = methodSpecs.find((s) => s && s.name === method);
|
|
568
|
+
if (methodSpec) {
|
|
569
|
+
const required = Array.isArray(methodSpec.permissions) ? methodSpec.permissions : [];
|
|
570
|
+
const grantedArr = permissionsAsArray(loaded.permissions);
|
|
571
|
+
const missing = required.filter((p) => !grantedArr.includes(p));
|
|
572
|
+
if (missing.length > 0) {
|
|
573
|
+
logPermissionUse(pluginId, method, missing.join(','), false);
|
|
574
|
+
return {
|
|
575
|
+
ok: false,
|
|
576
|
+
error: `Method "${method}" requires permissions: ${missing.join(', ')}`,
|
|
577
|
+
code: 'permission_denied',
|
|
578
|
+
missing,
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
// Allowed — log the granted set (or 'none' if no perms required).
|
|
582
|
+
logPermissionUse(pluginId, method, required.join(',') || 'none', true);
|
|
583
|
+
}
|
|
584
|
+
|
|
515
585
|
const fn = loaded.exports && loaded.exports[method];
|
|
516
586
|
if (typeof fn !== 'function') {
|
|
517
587
|
return {
|
|
@@ -586,6 +656,9 @@ export async function loadAndInvoke(opts) {
|
|
|
586
656
|
pluginRoot: opts.pluginRoot,
|
|
587
657
|
timeoutMs: opts.timeoutMs,
|
|
588
658
|
fetchImpl: opts.fetchImpl,
|
|
659
|
+
// v5.3.0 — wire method-level permissions through from the manifest
|
|
660
|
+
// so safeInvoke can enforce them.
|
|
661
|
+
methodSpecs: Array.isArray(manifest.exports) ? manifest.exports : [],
|
|
589
662
|
});
|
|
590
663
|
} catch (err) {
|
|
591
664
|
return {
|
|
@@ -652,4 +725,5 @@ export const __testing = {
|
|
|
652
725
|
DEFAULT_MEMORY_LIMIT_MB,
|
|
653
726
|
ALLOWED_GLOBALS,
|
|
654
727
|
KNOWN_PERMS,
|
|
728
|
+
permissionsAsArray,
|
|
655
729
|
};
|
|
@@ -628,6 +628,10 @@ export async function invokePlugin(pluginId, method, args = [], opts = {}) {
|
|
|
628
628
|
pluginRoot: pluginDir,
|
|
629
629
|
timeoutMs: opts.timeoutMs,
|
|
630
630
|
fetchImpl: opts.fetchImpl,
|
|
631
|
+
// v5.3.0 — pass method-level permission declarations through to
|
|
632
|
+
// the sandbox so `safeInvoke` can enforce them. Manifest defaults
|
|
633
|
+
// to an empty array (readManifest normalises missing exports).
|
|
634
|
+
methodSpecs: Array.isArray(manifest.exports) ? manifest.exports : [],
|
|
631
635
|
});
|
|
632
636
|
} catch (err) {
|
|
633
637
|
return { ok: false, error: err.message, code: err.code || 'load_error' };
|
|
@@ -14,8 +14,8 @@
|
|
|
14
14
|
* via the state for custom providers.
|
|
15
15
|
*/
|
|
16
16
|
import { Router } from 'express';
|
|
17
|
-
import { loadFixtures, runFixture, runSuite } from '../eval.mjs';
|
|
18
|
-
import { saveRun, listRuns, getRun, compareRuns, buildRunId } from '../eval-store.mjs';
|
|
17
|
+
import { loadFixtures, runFixture, runSuite, toCSV } from '../eval.mjs';
|
|
18
|
+
import { saveRun, listRuns, getRun, compareRuns, buildRunId, registerEvalSchedule, listEvalSchedules, deleteEvalSchedule } from '../eval-store.mjs';
|
|
19
19
|
import { chatCompletion } from '../minimax.mjs';
|
|
20
20
|
import { wrap } from './_shared.mjs';
|
|
21
21
|
|
|
@@ -45,6 +45,19 @@ export function createEvalRouter({ state, broadcast }) {
|
|
|
45
45
|
res.json(run);
|
|
46
46
|
}));
|
|
47
47
|
|
|
48
|
+
// GET /api/eval/runs/:id/export.csv — download run results as CSV
|
|
49
|
+
router.get('/eval/runs/:id/export.csv', wrap(async (req, res) => {
|
|
50
|
+
const run = await getRun(req.params.id);
|
|
51
|
+
if (!run) {
|
|
52
|
+
res.status(404).json({ error: 'not_found', message: `run ${req.params.id} not found` });
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
const csv = toCSV(run.results || []);
|
|
56
|
+
res.set('Content-Type', 'text/csv; charset=utf-8');
|
|
57
|
+
res.set('Content-Disposition', `attachment; filename="eval-${req.params.id}.csv"`);
|
|
58
|
+
res.send(csv);
|
|
59
|
+
}));
|
|
60
|
+
|
|
48
61
|
// GET /api/eval/fixtures — list fixtures in a suite path
|
|
49
62
|
router.get('/eval/fixtures', wrap(async (req, res) => {
|
|
50
63
|
const { path: suitePath } = req.query;
|
|
@@ -143,5 +156,32 @@ export function createEvalRouter({ state, broadcast }) {
|
|
|
143
156
|
res.json(diff);
|
|
144
157
|
}));
|
|
145
158
|
|
|
159
|
+
// GET /api/eval/schedules — list all eval schedules
|
|
160
|
+
router.get('/eval/schedules', wrap(async (req, res) => {
|
|
161
|
+
const schedules = await listEvalSchedules();
|
|
162
|
+
res.json({ schedules });
|
|
163
|
+
}));
|
|
164
|
+
|
|
165
|
+
// POST /api/eval/schedules — create a new eval schedule
|
|
166
|
+
router.post('/eval/schedules', wrap(async (req, res) => {
|
|
167
|
+
const { name, suitePath, cron, agent } = req.body || {};
|
|
168
|
+
if (!name || !suitePath || !cron) {
|
|
169
|
+
res.status(400).json({ error: 'bad_request', message: 'name, suitePath, and cron are required' });
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
const schedule = await registerEvalSchedule({ name, suitePath, cron, agent });
|
|
173
|
+
res.status(201).json(schedule);
|
|
174
|
+
}));
|
|
175
|
+
|
|
176
|
+
// DELETE /api/eval/schedules/:id — delete an eval schedule
|
|
177
|
+
router.delete('/eval/schedules/:id', wrap(async (req, res) => {
|
|
178
|
+
const result = await deleteEvalSchedule(req.params.id);
|
|
179
|
+
if (!result.ok) {
|
|
180
|
+
res.status(404).json({ error: 'not_found', message: `schedule ${req.params.id} not found` });
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
res.json({ ok: true });
|
|
184
|
+
}));
|
|
185
|
+
|
|
146
186
|
return router;
|
|
147
187
|
}
|
|
@@ -29,6 +29,7 @@ import { withSpan, setCommonAttributes } from '../otel.mjs';
|
|
|
29
29
|
import { recordTrace } from '../metrics.mjs';
|
|
30
30
|
import * as registry from '../plugins/registry.mjs';
|
|
31
31
|
import * as store from '../plugins/store.mjs';
|
|
32
|
+
import { getPermissionAuditLog } from '../plugins/permission-audit.mjs';
|
|
32
33
|
|
|
33
34
|
/**
|
|
34
35
|
* @returns {import('express').Router}
|
|
@@ -315,21 +316,60 @@ export function createPluginsRouter() {
|
|
|
315
316
|
}
|
|
316
317
|
// Distinguish "this is a plugin bug" (500) from "this is a usage
|
|
317
318
|
// error" (400/404) so the client can decide whether to retry.
|
|
319
|
+
// v5.3.0 — `permission_denied` returns HTTP 403 (Forbidden) since
|
|
320
|
+
// the request is well-formed but the caller is not authorised to
|
|
321
|
+
// invoke the method. All other usage errors stay at 400.
|
|
318
322
|
const usageCodes = new Set([
|
|
319
323
|
'not_installed',
|
|
320
324
|
'no_such_method',
|
|
321
325
|
'bad_manifest',
|
|
322
|
-
'permission_denied',
|
|
323
326
|
'corrupt_install',
|
|
324
327
|
]);
|
|
325
|
-
|
|
328
|
+
let status;
|
|
329
|
+
if (result.code === 'permission_denied') {
|
|
330
|
+
status = 403;
|
|
331
|
+
} else if (usageCodes.has(result.code)) {
|
|
332
|
+
status = 400;
|
|
333
|
+
} else {
|
|
334
|
+
status = 500;
|
|
335
|
+
}
|
|
326
336
|
span.setAttribute('plugin.invoke.error_code', result.code || 'unknown');
|
|
327
|
-
|
|
337
|
+
if (result.code === 'permission_denied') {
|
|
338
|
+
span.setAttribute('plugin.invoke.permission_missing',
|
|
339
|
+
(result.missing || []).join(','));
|
|
340
|
+
}
|
|
341
|
+
res.status(status).json({
|
|
342
|
+
error: result.code || 'plugin_error',
|
|
343
|
+
message: result.error,
|
|
344
|
+
...(result.code === 'permission_denied'
|
|
345
|
+
? { missing: result.missing }
|
|
346
|
+
: {}),
|
|
347
|
+
...(result.permission ? { permission: result.permission } : {}),
|
|
348
|
+
});
|
|
328
349
|
recordTrace('plugin.invoke', {
|
|
329
|
-
outcome: 'error',
|
|
350
|
+
outcome: result.code === 'permission_denied' ? 'permission_denied' : 'error',
|
|
330
351
|
code: result.code || 'unknown',
|
|
331
352
|
});
|
|
332
353
|
})));
|
|
333
354
|
|
|
355
|
+
// ── GET /api/plugins/:id/audit ─────────────────────────────────────────
|
|
356
|
+
// v5.3.0 — return the in-memory permission audit log for one plugin
|
|
357
|
+
// (newest-first). Bounded to 50 entries by default; pass `?limit=N`
|
|
358
|
+
// to fetch up to the buffer's full capacity. The route is read-only
|
|
359
|
+
// — admin tools should not clear the log from here.
|
|
360
|
+
router.get('/plugins/:id/audit', wrap(withSpan('plugin.audit.get', async (span, req, res) => {
|
|
361
|
+
setCommonAttributes(span, {
|
|
362
|
+
ip: req.ip || req.socket?.remoteAddress,
|
|
363
|
+
userAgent: req.headers?.['user-agent'],
|
|
364
|
+
});
|
|
365
|
+
span.setAttribute('plugin.id', req.params.id || '');
|
|
366
|
+
const limit = Number.isFinite(Number(req.query?.limit))
|
|
367
|
+
? Math.max(1, Math.min(1000, Number(req.query.limit)))
|
|
368
|
+
: 50;
|
|
369
|
+
const audit = getPermissionAuditLog({ pluginId: req.params.id, limit });
|
|
370
|
+
res.json({ audit });
|
|
371
|
+
recordTrace('plugin.audit.get', { count: audit.length });
|
|
372
|
+
})));
|
|
373
|
+
|
|
334
374
|
return router;
|
|
335
375
|
}
|
|
@@ -26,7 +26,25 @@ export function createUsersRouter() {
|
|
|
26
26
|
return res.status(401).json({ error: 'unauthorized', message: 'Not authenticated' });
|
|
27
27
|
}
|
|
28
28
|
const user = await getUser(userId);
|
|
29
|
+
// v5.3.1 — Tailscale-trust: when getCurrentUserId returns a derived
|
|
30
|
+
// userId (usr_ts_*) that doesn't exist in the store, synthesize a
|
|
31
|
+
// minimal user record from the request Host header so the dashboard
|
|
32
|
+
// can render a profile without needing a full signup.
|
|
29
33
|
if (!user) {
|
|
34
|
+
if (userId.startsWith('usr_ts_')) {
|
|
35
|
+
const host = String(req.headers?.host || '').split(':')[0] || 'tailnet';
|
|
36
|
+
const tailnetName = host.split('.')[0] || 'tailnet';
|
|
37
|
+
const workspaces = await listWorkspacesWithRoles(userId);
|
|
38
|
+
return res.json({
|
|
39
|
+
user: {
|
|
40
|
+
id: userId,
|
|
41
|
+
email: `${tailnetName}@tailscale.local`,
|
|
42
|
+
name: `Tailnet user (${tailnetName})`,
|
|
43
|
+
tailnet: true
|
|
44
|
+
},
|
|
45
|
+
workspaces
|
|
46
|
+
});
|
|
47
|
+
}
|
|
30
48
|
return res.status(404).json({ error: 'not_found', message: 'User not found' });
|
|
31
49
|
}
|
|
32
50
|
const workspaces = await listWorkspacesWithRoles(userId);
|
|
@@ -199,6 +199,9 @@ async function runAction(action, ctx = {}) {
|
|
|
199
199
|
if (action.type === 'agent') {
|
|
200
200
|
return runAgentAction(action, ctx);
|
|
201
201
|
}
|
|
202
|
+
if (action.type === 'eval-run') {
|
|
203
|
+
return runEvalAction(action, ctx);
|
|
204
|
+
}
|
|
202
205
|
throw new Error(`unknown action type: ${action.type}`);
|
|
203
206
|
}
|
|
204
207
|
|
|
@@ -370,6 +373,64 @@ async function runWebhook(action) {
|
|
|
370
373
|
}
|
|
371
374
|
}
|
|
372
375
|
|
|
376
|
+
async function runEvalAction(action, ctx = {}) {
|
|
377
|
+
const logger = ctx.logger || {
|
|
378
|
+
info: (msg, meta) => console.log(`[eval-run] ${msg}`, meta || ''),
|
|
379
|
+
error: (msg, meta) => console.error(`[eval-run] ${msg}`, meta || ''),
|
|
380
|
+
};
|
|
381
|
+
try {
|
|
382
|
+
const { runSuite } = await import('./eval.mjs');
|
|
383
|
+
const { saveRun, buildRunId } = await import('./eval-store.mjs');
|
|
384
|
+
const { chatCompletion } = await import('./minimax.mjs');
|
|
385
|
+
|
|
386
|
+
const runId = buildRunId();
|
|
387
|
+
const startedAt = new Date().toISOString();
|
|
388
|
+
|
|
389
|
+
const llmCall = async (prompt, opts = {}) => {
|
|
390
|
+
const result = await chatCompletion({ prompt, model: 'MiniMax-M3' });
|
|
391
|
+
if (!result.ok) throw new Error(result.message || 'llm call failed');
|
|
392
|
+
return {
|
|
393
|
+
content: result.content,
|
|
394
|
+
usage: {
|
|
395
|
+
inputTokens: result.usage?.prompt_tokens ?? 0,
|
|
396
|
+
outputTokens: result.usage?.completion_tokens ?? 0,
|
|
397
|
+
totalTokens: result.usage?.total_tokens ?? 0,
|
|
398
|
+
},
|
|
399
|
+
};
|
|
400
|
+
};
|
|
401
|
+
|
|
402
|
+
const result = await runSuite(action.suitePath, {
|
|
403
|
+
llmCall,
|
|
404
|
+
concurrency: 5,
|
|
405
|
+
timeoutMs: 120_000,
|
|
406
|
+
agent: action.agent,
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
const finishedAt = new Date().toISOString();
|
|
410
|
+
const run = {
|
|
411
|
+
id: runId,
|
|
412
|
+
startedAt,
|
|
413
|
+
finishedAt,
|
|
414
|
+
suitePath: action.suitePath,
|
|
415
|
+
total: result.total,
|
|
416
|
+
passed: result.passed,
|
|
417
|
+
failed: result.failed,
|
|
418
|
+
results: result.results,
|
|
419
|
+
};
|
|
420
|
+
|
|
421
|
+
await saveRun(run);
|
|
422
|
+
logger.info('scheduled eval completed', {
|
|
423
|
+
suite: action.suitePath,
|
|
424
|
+
passed: result.passed,
|
|
425
|
+
total: result.total,
|
|
426
|
+
});
|
|
427
|
+
return { ok: result.failed === 0, passed: result.passed, failed: result.failed, total: result.total };
|
|
428
|
+
} catch (err) {
|
|
429
|
+
logger.error('scheduled eval failed', { error: err.message });
|
|
430
|
+
return { ok: false, error: err.message };
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
373
434
|
export const schedulesRunner = {
|
|
374
435
|
LOG_FILE,
|
|
375
436
|
|