@polderlabs/bizar 4.8.0 → 5.0.0

package/templates/eval-fixtures/regression-suite.json

@@ -0,0 +1,79 @@
+{
+  "id": "regression-suite",
+  "name": "Full regression suite",
+  "description": "Runs all smoke tests to catch regressions",
+  "fixtures": [
+    {
+      "id": "code-search-basic",
+      "name": "Basic code search",
+      "description": "Verifies the agent can find a function definition",
+      "agent": "thor",
+      "prompt": "Find the function named 'add' in this repository. Show its signature and parameter names.",
+      "expected": {
+        "contains": ["function add", "a", "b"],
+        "notContains": ["TODO", "FIXME"],
+        "regex": ["function\\s+add\\s*\\("],
+        "jsonSchema": null,
+        "maxTokens": 500,
+        "maxLatencyMs": 30000
+      },
+      "tags": ["code-search", "smoke"]
+    },
+    {
+      "id": "tool-call-correctness",
+      "name": "Tool call correctness",
+      "description": "Verifies correct tool usage",
+      "agent": "thor",
+      "prompt": "Create a new file called /tmp/eval-test.txt with the content 'eval test output'.",
+      "expected": {
+        "contains": ["write", "/tmp/eval-test.txt"],
+        "notContains": ["error", "failed"],
+        "regex": [],
+        "jsonSchema": null,
+        "maxTokens": 500,
+        "maxLatencyMs": 30000
+      },
+      "tags": ["tools", "smoke"]
+    },
+    {
+      "id": "response-format",
+      "name": "Response format validation",
+      "description": "Verifies JSON output with schema",
+      "agent": "thor",
+      "prompt": "Return JSON: { \"status\": \"ok\", \"version\": \"1.0.0\", \"data\": { \"count\": 42 } }. No markdown.",
+      "expected": {
+        "contains": [],
+        "notContains": [],
+        "regex": [],
+        "jsonSchema": {
+          "type": "object",
+          "required": ["status", "version", "data"],
+          "properties": {
+            "status": { "type": "string" },
+            "version": { "type": "string" },
+            "data": { "type": "object", "required": ["count"], "properties": { "count": { "type": "number" } } }
+          }
+        },
+        "maxTokens": 300,
+        "maxLatencyMs": 20000
+      },
+      "tags": ["format", "json", "smoke"]
+    },
+    {
+      "id": "latency-bounds",
+      "name": "Latency bounds",
+      "description": "Verifies response within latency budget",
+      "agent": "thor",
+      "prompt": "Respond with exactly the word 'ok' and nothing else.",
+      "expected": {
+        "contains": ["ok"],
+        "notContains": [],
+        "regex": [],
+        "jsonSchema": null,
+        "maxTokens": 50,
+        "maxLatencyMs": 10000
+      },
+      "tags": ["latency", "performance"]
+    }
+  ]
+}

package/templates/eval-fixtures/response-format.json

@@ -0,0 +1,30 @@
+{
+  "id": "response-format",
+  "name": "Response format validation",
+  "description": "Verifies the agent returns valid JSON with a known schema",
+  "agent": "thor",
+  "prompt": "Return a JSON object with exactly these fields: { \"status\": \"ok\", \"version\": \"1.0.0\", \"data\": { \"count\": 42 } }. No extra fields. Valid JSON only, no markdown fences.",
+  "expected": {
+    "contains": [],
+    "notContains": [],
+    "regex": [],
+    "jsonSchema": {
+      "type": "object",
+      "required": ["status", "version", "data"],
+      "properties": {
+        "status": { "type": "string" },
+        "version": { "type": "string" },
+        "data": {
+          "type": "object",
+          "required": ["count"],
+          "properties": {
+            "count": { "type": "number" }
+          }
+        }
+      }
+    },
+    "maxTokens": 300,
+    "maxLatencyMs": 20000
+  },
+  "tags": ["format", "json", "smoke"]
+}

package/templates/eval-fixtures/tool-call-correctness.json

@@ -0,0 +1,24 @@
+{
+  "id": "tool-call-correctness",
+  "name": "Tool call correctness",
+  "description": "Verifies the agent uses the correct tool for a file system operation",
+  "agent": "thor",
+  "prompt": "Create a new file called /tmp/eval-test.txt with the content 'eval test output' using the write tool.",
+  "expected": {
+    "contains": [
+      "write",
+      "/tmp/eval-test.txt",
+      "eval test output"
+    ],
+    "notContains": [
+      "error",
+      "failed",
+      "could not"
+    ],
+    "regex": [],
+    "jsonSchema": null,
+    "maxTokens": 500,
+    "maxLatencyMs": 30000
+  },
+  "tags": ["tools", "smoke", "file-ops"]
+}

package/templates/plugin-template/README.md

@@ -0,0 +1,121 @@
+# Echo Plugin Template
+
+The smallest possible Bizar marketplace plugin. Use it as a starting
+point for your own — copy this folder, rename the fields in
+`plugin.json`, replace the body of `index.js`, and ship it as a
+`.tar.gz`.
+
+## What you get
+
+```
+plugin-template/
+├── plugin.json     # manifest (id, name, version, main, exports, permissions)
+├── index.js        # the plugin's main entry — exports { init, echo, shout }
+├── README.md       # you are here
+└── tests/
+    └── plugin.test.js   # node:test smoke tests
+```
+
+## Trying it locally
+
+You can sanity-check the template without publishing it: install it
+from a local path (or just point the host at this folder directly):
+
+```bash
+# From inside your dashboard's project:
+bizar plugin install <path-to-plugin-template>
+bizar plugin info echo
+bizar plugin invoke echo echo "hello world"
+# → echo: hello world
+
+bizar plugin config echo greeting "yo"
+bizar plugin invoke echo shout "hello world"
+# → SHOUT: HELLO WORLD
+
+bizar plugin uninstall echo
+```
+
+## Publishing
+
+1. **Pick an id.** It must match `/^[a-z0-9][a-z0-9-]*$/`. Lowercase
+   kebab-case is the convention (`vercel-deploy`, `github-pr-watcher`).
+
+2. **Edit `plugin.json`** — at minimum:
+   - `id`, `name`, `version`, `description`
+   - `main` (relative to the tarball root)
+   - `exports` (array of method descriptors; drives the dashboard UI)
+   - `permissions` (every `api.*` method you'll use, except `console`,
+     `api.config.*`, and `api.log` which are always available)
+
+3. **Write the plugin code** in `index.js`. Use `module.exports = { ... }`
+   and export plain async functions. Every async method is awaited by
+   the host; a thrown error becomes `{ ok: false, error, code }` on the
+   wire.
+
+4. **Package the tarball.** Standard convention:
+
+   ```bash
+   tar czf echo-0.1.0.tar.gz plugin.json index.js tests
+   ```
+
+   The tarball root must contain `plugin.json` (either at the very
+   top of the archive or inside a single top-level directory — both
+   work).
+
+5. **Compute the SHA-256** of the tarball:
+
+   ```bash
+   sha256sum echo-0.1.0.tar.gz
+   # → 8a3f... (64 hex chars)
+   ```
+
+6. **Submit to the registry** by opening a PR against
+   [DrB0rk/bizar-plugins](https://github.com/DrB0rk/bizar-plugins).
+   Add your entry under `plugins:` in `registry.json`:
+
+   ```json
+   {
+     "id": "echo",
+     "name": "Echo",
+     "version": "0.1.0",
+     "description": "Echo plugin",
+     "author": "you",
+     "category": "utility",
+     "tags": ["example", "template"],
+     "tarball": "https://github.com/.../echo-0.1.0.tar.gz",
+     "checksum": "sha256:8a3f...",
+     "permissions": [],
+     "minBizarVersion": "4.9.0"
+   }
+   ```
+
+   The host verifies the checksum before extracting — a tampered
+   tarball is rejected outright.
+
+7. **Install.** Once merged:
+
+   ```bash
+   bizar plugin install echo
+   ```
+
+## Sandbox permissions
+
+| Permission   | What it unlocks                                      |
+|--------------|------------------------------------------------------|
+| `net`        | `api.http.get(url)`, `api.http.post(url, body)`      |
+| `fs:read`    | `api.fs.read(path)` — sandboxed to the plugin root   |
+| *(none)*     | `api.config.*`, `api.log`, `console.*` are free      |
+
+Anything else (`process`, `child_process`, raw `fs`, `net`) is
+stripped from the plugin's global scope by `node:vm`. A plugin that
+tries `require('node:fs')` will throw `require is not defined`.
+
+## Versioning
+
+Bizar plugins use [semver](https://semver.org/). Bump the major
+version for any breaking change to `plugin.json`'s `exports[]`; minor
+for new methods; patch for bugfixes.
+
+## License
+
+MIT, same as the rest of BizarHarness.

package/templates/plugin-template/index.js

@@ -0,0 +1,66 @@
+/**
+ * Echo plugin — the minimal marketplace plugin.
+ *
+ * Plugins export their public methods via `module.exports`. Each
+ * method receives `(args..., { config, api, plugin })`. The host
+ * (bizar-dash's plugin store) handles sandboxing, timeouts, and
+ * permission enforcement — your code just needs to focus on the
+ * domain logic.
+ *
+ * What you CAN do from inside a plugin:
+ *   - console.log / console.error    → routed through the dashboard's logger
+ *   - api.http.get(url)              → only if you declare "net" permission
+ *   - api.fs.read(path)              → only if you declare "fs:read" permission
+ *   - api.config.get(key) / set(key, value)   → in-memory + persisted to disk
+ *   - api.log(level, msg)            → structured logger with your plugin id
+ *
+ * What you CANNOT do (the sandbox strips these globals):
+ *   - require('node:fs') directly    → use api.fs.read instead
+ *   - require('node:child_process')  → not allowed in v1
+ *   - process.exit / process.kill    → use api.log instead
+ *   - require('node:net')            → use api.http instead
+ */
+'use strict';
+
+/**
+ * The plugin's `init` hook. Called once after install and on every
+ * dashboard start (the host caches the loaded plugin). Use it to
+ * warm caches, validate config, or subscribe to events.
+ *
+ * Returning a Promise is fine — the host awaits it.
+ */
+async function init() {
+  // Pull a config value the user may have set.
+  const greeting = api.config.get('greeting') || 'echo';
+  api.log('info', `${plugin.id} v${plugin.version} initialized (greeting="${greeting}")`);
+}
+
+/**
+ * The simplest possible plugin method: return the input verbatim,
+ * prefixed with the configured greeting.
+ *
+ * @param {string} message
+ */
+async function echo(message) {
+  if (typeof message !== 'string') {
+    throw new Error(`echo: expected string, got ${typeof message}`);
+  }
+  const greeting = api.config.get('greeting') || 'echo';
+  return `${greeting}: ${message}`;
+}
+
+/**
+ * Upper-cased variant — demonstrates that async / await / Promises
+ * all work transparently inside the sandbox.
+ *
+ * @param {string} message
+ */
+async function shout(message) {
+  return `SHOUT: ${String(message).toUpperCase()}`;
+}
+
+module.exports = {
+  init,
+  echo,
+  shout,
+};

package/templates/plugin-template/plugin.json

@@ -0,0 +1,42 @@
+{
+  "id": "echo",
+  "name": "Echo",
+  "version": "0.1.0",
+  "description": "A minimal example plugin — echoes whatever you send it. Use this as a starting template for your own marketplace plugin.",
+  "author": "DrB0rk",
+  "main": "index.js",
+  "exports": [
+    {
+      "name": "echo",
+      "args": [
+        { "name": "message", "type": "string" }
+      ],
+      "returns": "string",
+      "description": "Return the input message prefixed with 'echo:'."
+    },
+    {
+      "name": "shout",
+      "args": [
+        { "name": "message", "type": "string" }
+      ],
+      "returns": "string",
+      "description": "Return the input message upper-cased and prefixed with 'SHOUT:'."
+    },
+    {
+      "name": "init",
+      "args": [],
+      "returns": "void",
+      "description": "Optional setup hook — called once after install / dashboard start."
+    }
+  ],
+  "permissions": [],
+  "configSchema": {
+    "greeting": {
+      "type": "string",
+      "required": false,
+      "default": "echo"
+    }
+  },
+  "homepage": "https://github.com/DrB0rk/bizar-plugins",
+  "minBizarVersion": "4.9.0"
+}

package/templates/plugin-template/tests/plugin.test.js

@@ -0,0 +1,83 @@
+/**
+ * tests/plugin.test.js
+ *
+ * Smoke tests for the Echo plugin template.
+ *
+ * These run with `node --test tests/plugin.test.js` from the
+ * template folder. They exercise the plugin by loading it directly
+ * through the host's `sandbox.mjs` (so the same code path that runs
+ * in production is exercised here), not by re-implementing the
+ * loading logic.
+ *
+ * If you copy this template for your own plugin, replace the
+ * expectations with real ones for your plugin's behaviour.
+ */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { fileURLToPath } from 'node:url';
+import { dirname, resolve } from 'node:path';
+import { readFileSync } from 'node:fs';
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const TEMPLATE_ROOT = resolve(HERE, '..');
+
+test('plugin.json is valid JSON with required fields', () => {
+  const raw = readFileSync(resolve(TEMPLATE_ROOT, 'plugin.json'), 'utf8');
+  const m = JSON.parse(raw);
+  assert.equal(typeof m.id, 'string');
+  assert.ok(m.id, 'id must be non-empty');
+  assert.equal(typeof m.version, 'string');
+  assert.ok(m.version, 'version must be non-empty');
+  assert.equal(typeof m.main, 'string');
+  assert.ok(m.main, 'main must be non-empty');
+  assert.ok(Array.isArray(m.exports), 'exports must be an array');
+  assert.ok(Array.isArray(m.permissions), 'permissions must be an array');
+});
+
+test('index.js exists and is non-empty', () => {
+  const src = readFileSync(resolve(TEMPLATE_ROOT, 'index.js'), 'utf8');
+  assert.ok(src.length > 100, 'index.js should be a real file');
+  assert.match(src, /module\.exports\s*=/, 'must use module.exports');
+});
+
+test('plugin exports echo, shout, and init', async () => {
+  // We exercise the actual sandbox so this test fails if the plugin
+  // ever drifts from the host's API surface.
+  const sandboxUrl = new URL('../../../bizar-dash/src/server/plugins/sandbox.mjs',
+    import.meta.url);
+  const { loadPlugin, safeInvoke } = await import(sandboxUrl);
+  const loaded = await loadPlugin({
+    mainFile: resolve(TEMPLATE_ROOT, 'index.js'),
+    config: { greeting: 'test' },
+    permissions: [],
+    pluginId: 'echo',
+    pluginRoot: TEMPLATE_ROOT,
+  });
+  assert.equal(typeof loaded.exports.echo, 'function');
+  assert.equal(typeof loaded.exports.shout, 'function');
+  assert.equal(typeof loaded.exports.init, 'function');
+
+  const r1 = await safeInvoke(loaded, 'echo', ['hello']);
+  assert.equal(r1.ok, true);
+  assert.equal(r1.result, 'test: hello');
+
+  const r2 = await safeInvoke(loaded, 'shout', ['hi']);
+  assert.equal(r2.ok, true);
+  assert.equal(r2.result, 'SHOUT: HI');
+});
+
+test('echo validates its argument', async () => {
+  const sandboxUrl = new URL('../../../bizar-dash/src/server/plugins/sandbox.mjs',
+    import.meta.url);
+  const { loadPlugin, safeInvoke } = await import(sandboxUrl);
+  const loaded = await loadPlugin({
+    mainFile: resolve(TEMPLATE_ROOT, 'index.js'),
+    config: {},
+    permissions: [],
+    pluginId: 'echo',
+    pluginRoot: TEMPLATE_ROOT,
+  });
+  const r = await safeInvoke(loaded, 'echo', [42]);
+  assert.equal(r.ok, false);
+  assert.match(r.error, /expected string/);
+});