jishushell 0.4.24 → 0.5.15

package/scripts/check-new-file-tests.mjs

@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+/**
+ * CI check: warn when newly added source files have no corresponding test.
+ *
+ * Usage:
+ *   node scripts/check-new-file-tests.mjs [--base <ref>]
+ *
+ * Defaults:
+ *   --base: CI_MERGE_REQUEST_TARGET_BRANCH_NAME (MR) or origin/main (fallback)
+ *
+ * Exit codes:
+ *   0 — all new files have tests (or no new files)
+ *   1 — some new files are missing tests (warning)
+ */
+
+import { execSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+
+// ── Configuration ──────────────────────────────────────────────────────
+
+/** File basenames exempt from the test requirement. */
+const EXEMPT_BASENAMES = new Set([
+  "index.ts",
+  "index.tsx",
+  "types.ts",
+  "types.tsx",
+  "constants.ts",
+  "constants.tsx",
+  "main.tsx",
+  "App.tsx",
+  "vite-env.d.ts",
+]);
+
+/** Glob-style suffix patterns to exempt. */
+const EXEMPT_SUFFIXES = [
+  ".d.ts",
+  ".test.ts",
+  ".test.tsx",
+  ".spec.ts",
+  ".spec.tsx",
+  ".visual.spec.ts",
+  ".browser.test.ts",
+  ".browser.test.tsx",
+];
+
+/** Path prefixes to exempt entirely. */
+const EXEMPT_PATH_PREFIXES = [
+  "src/test-setup",
+  "frontend/src/test-setup",
+  "frontend/src/visual-tests/",
+];
+
+/** Directories where backend tests live (searched for basename match). */
+const BACKEND_TEST_DIRS = ["tests/unit", "tests/integration"];
+
+// ── Helpers ────────────────────────────────────────────────────────────
+
+function git(cmd) {
+  return execSync(`git ${cmd}`, { encoding: "utf-8" }).trim();
+}
+
+function resolveBase() {
+  // CLI --base override
+  const baseIdx = process.argv.indexOf("--base");
+  if (baseIdx !== -1 && process.argv[baseIdx + 1]) {
+    return process.argv[baseIdx + 1];
+  }
+
+  // GitLab MR pipeline
+  const mrTarget = process.env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME;
+  if (mrTarget) {
+    try {
+      git(`fetch origin ${mrTarget} --depth=1 2>/dev/null`);
+    } catch {
+      // already fetched or shallow — proceed
+    }
+    return `origin/${mrTarget}`;
+  }
+
+  // GitLab push pipeline
+  const beforeSha = process.env.CI_COMMIT_BEFORE_SHA;
+  if (beforeSha && beforeSha !== "0000000000000000000000000000000000000000") {
+    return beforeSha;
+  }
+
+  // Local fallback
+  return "origin/main";
+}
+
+function isExempt(filePath) {
+  const base = basename(filePath);
+
+  if (EXEMPT_BASENAMES.has(base)) return true;
+  if (EXEMPT_SUFFIXES.some((s) => filePath.endsWith(s))) return true;
+  if (EXEMPT_PATH_PREFIXES.some((p) => filePath.startsWith(p))) return true;
+
+  return false;
+}
+
+/**
+ * Search for a test file matching the given source file basename.
+ * Accepts exact basename match or partial token match
+ * (e.g., `app.ts` matches `app-cli.test.ts`).
+ */
+function findBackendTest(sourceFile) {
+  const base = basename(sourceFile, ".ts");
+
+  for (const dir of BACKEND_TEST_DIRS) {
+    try {
+      const files = execSync(
+        `find ${dir} -name "*.test.ts" -type f 2>/dev/null`,
+        { encoding: "utf-8" },
+      ).trim();
+
+      if (!files) continue;
+
+      for (const testFile of files.split("\n")) {
+        const testBase = basename(testFile, ".test.ts");
+        // Exact match or token-contains match
+        if (testBase === base || testBase.includes(base) || base.includes(testBase)) {
+          return testFile;
+        }
+      }
+    } catch {
+      continue;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Check if a co-located frontend test file exists.
+ */
+function findFrontendTest(sourceFile) {
+  const dir = dirname(sourceFile);
+  const base = basename(sourceFile).replace(/\.(tsx?)$/, "");
+
+  const candidates = [
+    join(dir, `${base}.test.ts`),
+    join(dir, `${base}.test.tsx`),
+  ];
+
+  for (const candidate of candidates) {
+    if (existsSync(candidate)) return candidate;
+  }
+
+  return null;
+}
+
+// ── Main ───────────────────────────────────────────────────────────────
+
+const base = resolveBase();
+let mergeBase;
+try {
+  mergeBase = git(`merge-base HEAD ${base}`);
+} catch {
+  // merge-base may fail on shallow clones — fall back to direct base
+  mergeBase = base;
+}
+
+let newFiles;
+try {
+  const output = git(`diff --name-only --diff-filter=A ${mergeBase}...HEAD`);
+  newFiles = output ? output.split("\n") : [];
+} catch {
+  console.log("⚠ Could not compute diff — skipping new-file test check.");
+  process.exit(0);
+}
+
+// Filter to source files only
+const backendSources = newFiles.filter(
+  (f) => f.startsWith("src/") && f.endsWith(".ts") && !isExempt(f),
+);
+const frontendSources = newFiles.filter(
+  (f) =>
+    f.startsWith("frontend/src/") &&
+    (f.endsWith(".ts") || f.endsWith(".tsx")) &&
+    !isExempt(f),
+);
+
+const uncovered = [];
+
+for (const file of backendSources) {
+  const testFile = findBackendTest(file);
+  if (!testFile) {
+    uncovered.push({ source: file, kind: "backend" });
+  }
+}
+
+for (const file of frontendSources) {
+  const testFile = findFrontendTest(file);
+  if (!testFile) {
+    uncovered.push({ source: file, kind: "frontend" });
+  }
+}
+
+// ── Report ─────────────────────────────────────────────────────────────
+
+const totalChecked = backendSources.length + frontendSources.length;
+
+if (totalChecked === 0) {
+  console.log("✔ No new source files to check.");
+  process.exit(0);
+}
+
+console.log(`Checked ${totalChecked} new source file(s) against ${mergeBase.slice(0, 8)}...\n`);
+
+if (uncovered.length === 0) {
+  console.log("✔ All new source files have matching test files.");
+  process.exit(0);
+}
+
+console.log("⚠ The following new source files have no obvious matching test:\n");
+for (const { source, kind } of uncovered) {
+  const hint =
+    kind === "backend"
+      ? "→ Expected: tests/unit/**/<name>.test.ts or tests/integration/**/<name>.test.ts"
+      : "→ Expected: co-located <name>.test.{ts,tsx}";
+  console.log(`  ${source}`);
+  console.log(`    ${hint}\n`);
+}
+console.log(
+  "If these files are covered by existing integration/e2e tests, " +
+  "note the covering test in the MR description.\n" +
+  "See docs/testing.md for testing conventions.\n",
+);
+
+process.exit(1);

package/scripts/check-quarantine-expiry.mjs

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+/**
+ * Check quarantine expiry — fails CI if any quarantined test has expired.
+ *
+ * Usage: node scripts/check-quarantine-expiry.mjs
+ * Exit codes:
+ *   0 — all quarantined tests are within their expiry window (or none exist)
+ *   1 — one or more quarantined tests have expired
+ */
+
+import { readFileSync } from "fs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const quarantinePath = resolve(__dirname, "../tests/quarantine.json");
+
+const MAX_QUARANTINE_DAYS = 14;
+
+try {
+  const raw = readFileSync(quarantinePath, "utf-8");
+  const data = JSON.parse(raw);
+  const tests = data.tests || [];
+
+  if (tests.length === 0) {
+    console.log("✅ No quarantined tests.");
+    process.exit(0);
+  }
+
+  const today = new Date();
+  today.setHours(0, 0, 0, 0);
+
+  const expired = [];
+  const active = [];
+  const tooLong = [];
+
+  for (const entry of tests) {
+    const expiryDate = new Date(entry.expiry);
+    expiryDate.setHours(0, 0, 0, 0);
+    const createdDate = new Date(entry.created);
+    createdDate.setHours(0, 0, 0, 0);
+
+    const durationDays = Math.ceil(
+      (expiryDate.getTime() - createdDate.getTime()) / (1000 * 60 * 60 * 24),
+    );
+
+    if (durationDays > MAX_QUARANTINE_DAYS) {
+      tooLong.push({
+        ...entry,
+        durationDays,
+      });
+    }
+
+    if (expiryDate < today) {
+      expired.push(entry);
+    } else {
+      active.push(entry);
+    }
+  }
+
+  if (active.length > 0) {
+    console.log(`⏳ ${active.length} quarantined test(s) (active):`);
+    for (const t of active) {
+      console.log(`   - ${t.file} > "${t.testName}" (expires: ${t.expiry}, owner: ${t.owner})`);
+    }
+  }
+
+  if (tooLong.length > 0) {
+    console.error(`\n❌ ${tooLong.length} quarantine(s) exceed ${MAX_QUARANTINE_DAYS}-day limit:`);
+    for (const t of tooLong) {
+      console.error(
+        `   - ${t.file} > "${t.testName}" (${t.durationDays} days, max ${MAX_QUARANTINE_DAYS})`,
+      );
+    }
+    console.error("\nShorten expiry to at most 14 days from creation date.");
+  }
+
+  if (expired.length > 0) {
+    console.error(`\n❌ ${expired.length} quarantined test(s) have EXPIRED:`);
+    for (const t of expired) {
+      console.error(
+        `   - ${t.file} > "${t.testName}" (expired: ${t.expiry}, owner: ${t.owner})`,
+      );
+      console.error(`     Reason: ${t.reason}`);
+      if (t.issueLink) {
+        console.error(`     Issue: ${t.issueLink}`);
+      }
+    }
+    console.error("\nFix or remove expired quarantine entries before merging.");
+  }
+
+  if (tooLong.length > 0 || expired.length > 0) {
+    process.exit(1);
+  }
+
+  console.log("\n✅ All quarantine entries are within their expiry window.");
+  process.exit(0);
+} catch (err) {
+  if (err.code === "ENOENT") {
+    console.log("✅ No quarantine file found (tests/quarantine.json).");
+    process.exit(0);
+  }
+  console.error("Failed to check quarantine:", err.message);
+  process.exit(1);
+}

package/scripts/perf/README.md

@@ -0,0 +1,49 @@
+# Performance Baseline Tests (k6)
+
+Latency baseline scripts for core API endpoints. Run nightly in CI to detect
+performance regressions.
+
+## Prerequisites
+
+- [k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) installed on the runner
+- JishuShell server running (the CI job starts one automatically)
+
+## Scripts
+
+| Script | Endpoints covered |
+|--------|-------------------|
+| `auth.js` | `/api/auth/status`, `/api/auth/login`, `/api/auth/logout` |
+| `proxy.js` | `/proxy/v1/models`, `/api/llm/providers` |
+| `instances.js` | `/api/instances` CRUD lifecycle |
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `BASE_URL` | `http://127.0.0.1:8090` | Server base URL |
+| `PASSWORD` | `perf-test-password-2026` | Admin password |
+| `SMOKE` | `false` | Set to `true` for quick 10s validation |
+
+## Usage
+
+```bash
+# Smoke test (quick validation)
+k6 run -e SMOKE=true scripts/perf/auth.js
+
+# Full baseline run
+k6 run scripts/perf/auth.js
+
+# All scripts with custom base URL
+for script in scripts/perf/auth.js scripts/perf/proxy.js scripts/perf/instances.js; do
+  k6 run -e BASE_URL=http://localhost:9090 "$script"
+done
+```
+
+## Thresholds
+
+Default thresholds (see `config.js`):
+- **p(95) < 500ms** for general endpoints
+- **p(99) < 1000ms** tail latency
+- **Error rate < 1%**
+
+Per-endpoint overrides are defined in each script's `options.thresholds`.

package/scripts/perf/auth.js

@@ -0,0 +1,99 @@
+import http from "k6/http";
+import { check, group, sleep } from "k6";
+import {
+  BASE_URL,
+  PASSWORD,
+  DEFAULT_THRESHOLDS,
+  DEFAULT_STAGES,
+  SMOKE_STAGES,
+  authHeaders,
+} from "./config.js";
+
+const isSmoke = __ENV.SMOKE === "true";
+
+export const options = {
+  stages: isSmoke ? SMOKE_STAGES : DEFAULT_STAGES,
+  thresholds: {
+    ...DEFAULT_THRESHOLDS,
+    "http_req_duration{name:login}": ["p(95)<300"],
+    "http_req_duration{name:auth_status}": ["p(95)<100"],
+  },
+  tags: { suite: "auth" },
+};
+
+/**
+ * Setup: initialize password and obtain a baseline token.
+ * Runs once before VUs start.
+ */
+export function setup() {
+  // Check if already initialized
+  const statusRes = http.get(`${BASE_URL}/api/auth/status`);
+  const initialized = statusRes.json("initialized");
+
+  let token;
+  if (!initialized) {
+    const initRes = http.post(
+      `${BASE_URL}/api/auth/init`,
+      JSON.stringify({ password: PASSWORD }),
+      { headers: { "Content-Type": "application/json" } },
+    );
+    check(initRes, { "init 200": (r) => r.status === 200 });
+    token = initRes.json("token");
+  } else {
+    const loginRes = http.post(
+      `${BASE_URL}/api/auth/login`,
+      JSON.stringify({ password: PASSWORD }),
+      { headers: { "Content-Type": "application/json" } },
+    );
+    check(loginRes, { "login 200": (r) => r.status === 200 });
+    token = loginRes.json("token");
+  }
+
+  if (!token) {
+    throw new Error("Failed to obtain auth token during setup");
+  }
+  return { token };
+}
+
+export default function (data) {
+  group("auth-status", () => {
+    const res = http.get(`${BASE_URL}/api/auth/status`, {
+      tags: { name: "auth_status" },
+    });
+    check(res, {
+      "status 200": (r) => r.status === 200,
+      "has initialized field": (r) => r.json("initialized") !== undefined,
+    });
+  });
+
+  sleep(0.5);
+
+  group("login", () => {
+    const res = http.post(
+      `${BASE_URL}/api/auth/login`,
+      JSON.stringify({ password: PASSWORD }),
+      {
+        headers: { "Content-Type": "application/json" },
+        tags: { name: "login" },
+      },
+    );
+    check(res, {
+      "login 200": (r) => r.status === 200,
+      "returns token": (r) => r.json("token") !== undefined,
+    });
+  });
+
+  sleep(0.5);
+
+  group("logout", () => {
+    const res = http.post(`${BASE_URL}/api/auth/logout`, null, {
+      headers: authHeaders(data.token),
+      tags: { name: "logout" },
+    });
+    check(res, {
+      "logout 200": (r) => r.status === 200,
+    });
+  });
+
+  sleep(1);
+}

package/scripts/perf/config.js

@@ -0,0 +1,63 @@
+/**
+ * Shared configuration and helpers for k6 performance tests.
+ *
+ * Environment variables:
+ *   BASE_URL  — server base URL (default: http://127.0.0.1:8090)
+ *   PASSWORD  — admin password for auth (default: perf-test-password-2026)
+ */
+
+export const BASE_URL = __ENV.BASE_URL || "http://127.0.0.1:8090";
+export const PASSWORD = __ENV.PASSWORD || "perf-test-password-2026";
+
+/**
+ * Standard k6 thresholds for API endpoints.
+ * - p(95) < 500ms: acceptable for most CRUD ops
+ * - p(99) < 1000ms: tail latency budget
+ * - error rate < 1%
+ */
+export const DEFAULT_THRESHOLDS = {
+  http_req_duration: ["p(95)<500", "p(99)<1000"],
+  http_req_failed: ["rate<0.01"],
+};
+
+/**
+ * Standard load profile for baseline tests.
+ * Ramp up to 10 VUs over 30s, hold for 2m, ramp down.
+ */
+export const DEFAULT_STAGES = [
+  { duration: "30s", target: 10 },
+  { duration: "2m", target: 10 },
+  { duration: "10s", target: 0 },
+];
+
+/**
+ * Smoke profile for quick validation (CI gate).
+ * 1 VU for 10s — just verify the script works.
+ */
+export const SMOKE_STAGES = [
+  { duration: "10s", target: 1 },
+];
+
+/**
+ * Build standard headers with auth cookie.
+ * @param {string} token - JWT token from login/init
+ * @returns {object} headers object for k6 http calls
+ */
+export function authHeaders(token) {
+  return {
+    "Content-Type": "application/json",
+    Cookie: `jishushell_session=${token}`,
+  };
+}
+
+/**
+ * Build standard headers with Bearer token (for proxy routes).
+ * @param {string} token - JWT token
+ * @returns {object} headers object for k6 http calls
+ */
+export function bearerHeaders(token) {
+  return {
+    "Content-Type": "application/json",
+    Authorization: `Bearer ${token}`,
+  };
+}

package/scripts/perf/instances.js

@@ -0,0 +1,143 @@
+import http from "k6/http";
+import { check, group, sleep } from "k6";
+import {
+  BASE_URL,
+  PASSWORD,
+  DEFAULT_THRESHOLDS,
+  DEFAULT_STAGES,
+  SMOKE_STAGES,
+  authHeaders,
+} from "./config.js";
+
+const isSmoke = __ENV.SMOKE === "true";
+
+export const options = {
+  stages: isSmoke ? SMOKE_STAGES : DEFAULT_STAGES,
+  thresholds: {
+    ...DEFAULT_THRESHOLDS,
+    "http_req_duration{name:list_instances}": ["p(95)<300"],
+    "http_req_duration{name:create_instance}": ["p(95)<1000"],
+    "http_req_duration{name:get_instance}": ["p(95)<300"],
+    "http_req_duration{name:delete_instance}": ["p(95)<1000"],
+  },
+  tags: { suite: "instances" },
+};
+
+/**
+ * Setup: obtain auth token.
+ */
+export function setup() {
+  const statusRes = http.get(`${BASE_URL}/api/auth/status`);
+  const initialized = statusRes.json("initialized");
+
+  let token;
+  if (!initialized) {
+    const initRes = http.post(
+      `${BASE_URL}/api/auth/init`,
+      JSON.stringify({ password: PASSWORD }),
+      { headers: { "Content-Type": "application/json" } },
+    );
+    token = initRes.json("token");
+  } else {
+    const loginRes = http.post(
+      `${BASE_URL}/api/auth/login`,
+      JSON.stringify({ password: PASSWORD }),
+      { headers: { "Content-Type": "application/json" } },
+    );
+    token = loginRes.json("token");
+  }
+
+  if (!token) {
+    throw new Error("Failed to obtain auth token during setup");
+  }
+  return { token };
+}
+
+export default function (data) {
+  const headers = authHeaders(data.token);
+  const instanceId = `perf-${__VU}-${__ITER}`;
+
+  group("list-instances", () => {
+    const res = http.get(`${BASE_URL}/api/instances`, {
+      headers,
+      tags: { name: "list_instances" },
+    });
+    check(res, {
+      "list 200": (r) => r.status === 200,
+      "returns array": (r) => {
+        try {
+          return Array.isArray(r.json());
+        } catch {
+          return false;
+        }
+      },
+    });
+  });
+
+  sleep(0.3);
+
+  group("create-instance", () => {
+    const payload = JSON.stringify({
+      id: instanceId,
+      name: `Perf Test ${instanceId}`,
+      description: "Created by k6 performance test",
+      agentType: "openclaw",
+    });
+    const res = http.post(`${BASE_URL}/api/instances`, payload, {
+      headers,
+      tags: { name: "create_instance" },
+    });
+    check(res, {
+      "create 200/201": (r) => r.status === 200 || r.status === 201,
+    });
+  });
+
+  sleep(0.3);
+
+  group("get-instance", () => {
+    const res = http.get(`${BASE_URL}/api/instances/${instanceId}`, {
+      headers,
+      tags: { name: "get_instance" },
+    });
+    check(res, {
+      "get 200": (r) => r.status === 200,
+      "has id": (r) => {
+        try {
+          return r.json("id") === instanceId;
+        } catch {
+          return false;
+        }
+      },
+    });
+  });
+
+  sleep(0.3);
+
+  group("update-instance", () => {
+    const payload = JSON.stringify({
+      name: `Updated ${instanceId}`,
+      description: "Updated by k6 performance test",
+    });
+    const res = http.put(`${BASE_URL}/api/instances/${instanceId}`, payload, {
+      headers,
+      tags: { name: "update_instance" },
+    });
+    check(res, {
+      "update 200": (r) => r.status === 200,
+    });
+  });
+
+  sleep(0.3);
+
+  group("delete-instance", () => {
+    const res = http.del(`${BASE_URL}/api/instances/${instanceId}`, null, {
+      headers,
+      tags: { name: "delete_instance" },
+    });
+    check(res, {
+      "delete 200": (r) => r.status === 200,
+    });
+  });
+
+  sleep(1);
+}