@sleep2agi/commhub-server 0.8.5 → 0.8.6-preview.1

package/src/uploads.test.ts

@@ -0,0 +1,294 @@
+// #221 — unit tests for the upload primitives. The HTTP wire-up + Bun
+// multipart parsing lives in server/src/index.ts; this file pins the
+// path / regex / rate-limit / attachment-schema invariants that the
+// 6-item security checklist depends on.
+//
+// Coverage matrix:
+//   1. file_id regex accepts server-generated ids + rejects traversal attempts
+//   2. sanitizeExt strips client-side filename paths, lowercases ext
+//   3. buildStoragePath rejects malformed id / ext, places file under date bucket
+//   4. isPathInsideUploadsRoot defends against symlink / traversal escape
+//   5. UploadRateLimiter — first call allowed, exhausting bucket blocks, window expiry resets
+//   6. validateAttachments accepts MCP-compatible shape, rejects too-many / bad type / bad file_id
+//   7. validateIndexEntry refuses tampered shapes
+//   8. indexEntryPath rejects malformed file_id
+import { describe, expect, test } from "bun:test";
+import {
+  FILE_ID_REGEX,
+  MAX_UPLOAD_BYTES,
+  UploadRateLimiter,
+  UPLOAD_RATE_MAX_PER_WINDOW,
+  UPLOAD_RATE_WINDOW_MS,
+  buildStoragePath,
+  generateFileId,
+  getDateBucket,
+  indexEntryPath,
+  isPathInsideUploadsRoot,
+  sanitizeExt,
+  validateAttachments,
+  validateIndexEntry,
+} from "./uploads";
+
+describe("FILE_ID_REGEX", () => {
+  test("accepts the server-generated uuidv4-without-hyphens shape (32 hex chars)", () => {
+    for (let i = 0; i < 20; i++) {
+      const id = generateFileId();
+      expect(FILE_ID_REGEX.test(id)).toBe(true);
+      expect(id.length).toBe(32);
+    }
+  });
+
+  test("rejects every directory-traversal attempt a client might try", () => {
+    const traversals = [
+      "../foo",
+      "..%2Ffoo",
+      "foo/bar",
+      "foo\\bar",
+      "foo.bar",
+      ".bashrc",
+      "/etc/passwd",
+      "foo bar",
+      "",
+      "abc", // too short
+      "x".repeat(65), // too long
+    ];
+    for (const t of traversals) {
+      expect(FILE_ID_REGEX.test(t)).toBe(false);
+    }
+  });
+});
+
+describe("sanitizeExt", () => {
+  test("returns a leading-dot lowercase extension for a normal filename", () => {
+    expect(sanitizeExt("photo.PNG")).toBe(".png");
+    expect(sanitizeExt("report.pdf")).toBe(".pdf");
+    expect(sanitizeExt("archive.tar.gz")).toBe(".gz");
+  });
+
+  test("strips any client-supplied directory prefix before reading the ext", () => {
+    // The handler must NEVER use the client filename for storage paths,
+    // but the ext is allowed through for nicer URLs. We still strip any
+    // path separator the client tries to sneak in.
+    expect(sanitizeExt("../../etc/passwd.jpg")).toBe(".jpg");
+    expect(sanitizeExt("..\\..\\windows\\boot.ini")).toBe(".ini");
+    expect(sanitizeExt("foo/bar/baz.html")).toBe(".html");
+  });
+
+  test("returns empty string for missing / weird inputs", () => {
+    expect(sanitizeExt(undefined)).toBe("");
+    expect(sanitizeExt(null)).toBe("");
+    expect(sanitizeExt("")).toBe("");
+    expect(sanitizeExt("noext")).toBe("");
+    // ".bashrc" → ".bashrc" is acceptable: storage uses the server-generated
+    // file_id, so the file lands as "<file_id>.bashrc". The download handler
+    // forces Content-Disposition: attachment + X-Content-Type-Options: nosniff,
+    // so the ext can't be leveraged to execute / mis-render the file.
+    expect(sanitizeExt(".bashrc")).toBe(".bashrc");
+    // ext > 16 chars is rejected; "extensiontoolong1" is 17 chars.
+    expect(sanitizeExt("foo.extensiontoolong1")).toBe("");
+  });
+
+  test("rejects exts with non-alphanumeric characters", () => {
+    expect(sanitizeExt("payload.exe;.png")).toBe(".png");
+    expect(sanitizeExt("foo.<script>")).toBe("");
+  });
+});
+
+describe("buildStoragePath", () => {
+  test("places the file under a YYYY-MM-DD bucket with file_id+ext as the name", () => {
+    const now = new Date("2026-06-11T03:00:00Z");
+    const r = buildStoragePath("abcdef1234567890abcdef1234567890", ".png", {
+      uploadsRoot: "/srv/uploads",
+      now,
+    });
+    expect(r.dateBucket).toBe("2026-06-11");
+    expect(r.relativePath).toBe("2026-06-11/abcdef1234567890abcdef1234567890.png");
+    expect(r.absolutePath).toBe("/srv/uploads/2026-06-11/abcdef1234567890abcdef1234567890.png");
+  });
+
+  test("rejects malformed file_id", () => {
+    expect(() => buildStoragePath("../etc/passwd", ".png", { uploadsRoot: "/srv" })).toThrow(/invalid/);
+    expect(() => buildStoragePath("ok", ".png", { uploadsRoot: "/srv" })).toThrow(/invalid/);
+  });
+
+  test("rejects malformed ext", () => {
+    expect(() => buildStoragePath(generateFileId(), "../passwd", { uploadsRoot: "/srv" })).toThrow(/invalid/);
+    expect(() => buildStoragePath(generateFileId(), ".x".repeat(40), { uploadsRoot: "/srv" })).toThrow(/invalid/);
+  });
+
+  test("allows empty ext", () => {
+    const r = buildStoragePath(generateFileId(), "", { uploadsRoot: "/srv" });
+    expect(r.relativePath.endsWith("/")).toBe(false);
+    expect(r.relativePath.includes(".")).toBe(false);
+  });
+});
+
+describe("getDateBucket", () => {
+  test("uses UTC so the bucket is consistent across hub timezones", () => {
+    expect(getDateBucket(new Date("2026-06-11T23:59:59Z"))).toBe("2026-06-11");
+    expect(getDateBucket(new Date("2026-06-12T00:00:01Z"))).toBe("2026-06-12");
+  });
+});
+
+describe("isPathInsideUploadsRoot", () => {
+  test("true for a normal nested path", () => {
+    expect(isPathInsideUploadsRoot("/srv/uploads/2026-06-11/abc.png", "/srv/uploads")).toBe(true);
+  });
+
+  test("false for an upward traversal that lands outside the root", () => {
+    expect(isPathInsideUploadsRoot("/srv/uploads/../etc/passwd", "/srv/uploads")).toBe(false);
+    expect(isPathInsideUploadsRoot("/etc/passwd", "/srv/uploads")).toBe(false);
+  });
+
+  test("false for the root itself (must be a file underneath, not the root dir)", () => {
+    expect(isPathInsideUploadsRoot("/srv/uploads", "/srv/uploads")).toBe(false);
+  });
+});
+
+describe("UploadRateLimiter", () => {
+  test("first N calls within window are allowed; the (N+1)th is blocked", () => {
+    const lim = new UploadRateLimiter(60_000, 3); // 3 per minute
+    const t0 = 1_700_000_000_000;
+    const a1 = lim.check("token-x", t0);
+    const a2 = lim.check("token-x", t0 + 100);
+    const a3 = lim.check("token-x", t0 + 200);
+    const a4 = lim.check("token-x", t0 + 300);
+    expect(a1.allowed).toBe(true);
+    expect(a2.allowed).toBe(true);
+    expect(a3.allowed).toBe(true);
+    expect(a4.allowed).toBe(false);
+    if (!a4.allowed) {
+      expect(a4.retryAfterMs).toBeGreaterThan(0);
+      expect(a4.retryAfterMs).toBeLessThanOrEqual(60_000);
+    }
+  });
+
+  test("different keys are isolated", () => {
+    const lim = new UploadRateLimiter(60_000, 1);
+    expect(lim.check("alice", 1).allowed).toBe(true);
+    expect(lim.check("alice", 2).allowed).toBe(false);
+    expect(lim.check("bob", 3).allowed).toBe(true);
+  });
+
+  test("window expiry resets the bucket", () => {
+    const lim = new UploadRateLimiter(60_000, 1);
+    const t0 = 1_700_000_000_000;
+    expect(lim.check("token-x", t0).allowed).toBe(true);
+    expect(lim.check("token-x", t0 + 30_000).allowed).toBe(false);
+    expect(lim.check("token-x", t0 + 60_001).allowed).toBe(true);
+  });
+
+  test("default constructor matches documented public defaults", () => {
+    const lim = new UploadRateLimiter();
+    expect(UPLOAD_RATE_WINDOW_MS).toBe(60 * 60 * 1000);
+    expect(UPLOAD_RATE_MAX_PER_WINDOW).toBe(60);
+    for (let i = 0; i < UPLOAD_RATE_MAX_PER_WINDOW; i++) {
+      expect(lim.check("token-y", 1 + i).allowed).toBe(true);
+    }
+    expect(lim.check("token-y", 1000).allowed).toBe(false);
+  });
+});
+
+describe("validateAttachments", () => {
+  test("undefined / null input → empty array", () => {
+    const a = validateAttachments(undefined);
+    expect(a.ok).toBe(true);
+    if (a.ok) expect(a.attachments).toEqual([]);
+    const b = validateAttachments(null);
+    expect(b.ok).toBe(true);
+    if (b.ok) expect(b.attachments).toEqual([]);
+  });
+
+  test("rejects non-array input", () => {
+    expect(validateAttachments("nope").ok).toBe(false);
+    expect(validateAttachments({}).ok).toBe(false);
+  });
+
+  test("accepts a well-formed attachment matching MCP send_task shape", () => {
+    const fileId = generateFileId();
+    const res = validateAttachments([
+      { type: "file", file_id: fileId, name: "photo.png", mime: "image/png", size: 12345 },
+    ]);
+    expect(res.ok).toBe(true);
+    if (res.ok) {
+      expect(res.attachments[0].file_id).toBe(fileId);
+      expect(res.attachments[0].mime).toBe("image/png");
+    }
+  });
+
+  test("rejects too many attachments (>20)", () => {
+    const fileId = generateFileId();
+    const many = Array.from({ length: 21 }, () => ({ type: "file" as const, file_id: fileId }));
+    const res = validateAttachments(many);
+    expect(res.ok).toBe(false);
+  });
+
+  test("rejects invalid file_id (path traversal)", () => {
+    const res = validateAttachments([{ type: "file", file_id: "../etc/passwd" }]);
+    expect(res.ok).toBe(false);
+  });
+
+  test("rejects size larger than MAX_UPLOAD_BYTES", () => {
+    const fileId = generateFileId();
+    const res = validateAttachments([{ type: "file", file_id: fileId, size: MAX_UPLOAD_BYTES + 1 }]);
+    expect(res.ok).toBe(false);
+  });
+
+  test("rejects unexpected type", () => {
+    const res = validateAttachments([{ type: "image", file_id: generateFileId() }]);
+    expect(res.ok).toBe(false);
+  });
+});
+
+describe("validateIndexEntry", () => {
+  test("accepts a well-formed entry", () => {
+    const id = generateFileId();
+    expect(
+      validateIndexEntry({
+        file_id: id,
+        date_bucket: "2026-06-11",
+        ext: ".png",
+        name: "anything",
+        mime: "image/png",
+        size: 12345,
+        uploaded_at: "2026-06-11T03:00:00Z",
+      }),
+    ).toBe(true);
+  });
+
+  test("rejects bad date_bucket (anti-tamper)", () => {
+    expect(
+      validateIndexEntry({ file_id: generateFileId(), date_bucket: "../etc", ext: ".png", size: 1 }),
+    ).toBe(false);
+  });
+
+  test("rejects bad ext", () => {
+    expect(
+      validateIndexEntry({ file_id: generateFileId(), date_bucket: "2026-06-11", ext: "/etc/passwd", size: 1 }),
+    ).toBe(false);
+  });
+
+  test("rejects size beyond MAX_UPLOAD_BYTES (anti-tamper)", () => {
+    expect(
+      validateIndexEntry({
+        file_id: generateFileId(),
+        date_bucket: "2026-06-11",
+        ext: "",
+        size: MAX_UPLOAD_BYTES + 1,
+      }),
+    ).toBe(false);
+  });
+});
+
+describe("indexEntryPath", () => {
+  test("returns null on malformed file_id", () => {
+    expect(indexEntryPath("../etc/passwd", { uploadsRoot: "/srv" })).toBeNull();
+    expect(indexEntryPath("", { uploadsRoot: "/srv" })).toBeNull();
+  });
+
+  test("places the index file under <uploadsRoot>/.index/", () => {
+    const id = generateFileId();
+    const p = indexEntryPath(id, { uploadsRoot: "/srv/up" });
+    expect(p).toBe(`/srv/up/.index/${id}.json`);
+  });
+});

package/src/uploads.ts

@@ -0,0 +1,240 @@
+// #221 — commhub-server file upload primitives.
+//
+// Hub upload surface for the mobile APP (#220). The hub started reverse-
+// proxying over HTTPS to the public internet last night, so the upload
+// endpoint is a brand-new public attack surface. The six security items
+// in the issue are HARD verification items, encoded here:
+//
+//   1. file_id is server-generated (uuidv4, alphanumeric only), client-
+//      supplied filenames never enter the storage path.
+//   2. Storage root sits in $HOME/.anet/server/uploads (outside any
+//      web-served directory); downloads go through an explicit handler.
+//   3. Download handler forces Content-Disposition: attachment +
+//      X-Content-Type-Options: nosniff — uploaded files are never
+//      executed or served as HTML.
+//   4. 12 MB cap enforced two-stage per 通信龙: Content-Length header
+//      pre-check (reject before reading body) + post-parse size verify.
+//   5. Auth is the existing requireAuth — anonymous + bad tokens 401.
+//   6. Naive per-token rate limit: 60 uploads/hour per token (or IP for
+//      legacy/dev mode).
+//
+// This module stays pure-ish: file I/O lives in the HTTP handlers, the
+// path / id / rate-limit primitives are unit-testable.
+
+import { homedir } from "os";
+import { join, relative, isAbsolute } from "path";
+
+export const MAX_UPLOAD_BYTES = 12 * 1024 * 1024;
+
+// Allow ~1 MiB of multipart envelope overhead on top of the 12 MiB
+// payload limit when checking Content-Length. The post-parse check is
+// still done against MAX_UPLOAD_BYTES so the actual binary payload
+// never exceeds the documented cap.
+export const MAX_REQUEST_CONTENT_LENGTH = MAX_UPLOAD_BYTES + 1024 * 1024;
+
+// Per-token rate-limit constants. 60 uploads/hour matches the issue
+// "如 60 req/h" suggestion and is enforced in-memory (one process holds
+// the source of truth; resets on restart).
+export const UPLOAD_RATE_WINDOW_MS = 60 * 60 * 1000;
+export const UPLOAD_RATE_MAX_PER_WINDOW = 60;
+
+// Strict file_id regex. Generated server-side via crypto.randomUUID()
+// with hyphens stripped, so the natural output is 32 hex chars. The
+// regex is intentionally permissive on the upper bound (64 chars) to
+// allow future scheme changes (e.g. prefix), but it rejects ANY of `.`
+// `/` `\` `..` so the same regex used for path generation can also be
+// used to validate a /api/files/:file_id lookup against directory
+// traversal.
+export const FILE_ID_REGEX = /^[A-Za-z0-9_-]{8,64}$/;
+
+export type GeneratedFile = {
+  fileId: string;
+  ext: string;
+  dateBucket: string;
+  relativePath: string;
+  absolutePath: string;
+};
+
+export function getUploadsRoot(env: NodeJS.ProcessEnv = process.env): string {
+  return env.COMMHUB_UPLOADS_DIR || join(homedir(), ".anet/server/uploads");
+}
+
+export function getDateBucket(now: Date = new Date()): string {
+  // YYYY-MM-DD partitioning per issue spec — keeps a single directory
+  // from accumulating millions of entries while leaving the lookup
+  // path predictable from the disk index (file_id → date bucket).
+  return now.toISOString().slice(0, 10);
+}
+
+export function generateFileId(): string {
+  return crypto.randomUUID().replace(/-/g, "");
+}
+
+/**
+ * Extract a safe file extension from a client-supplied filename.
+ * Server NEVER uses the rest of the filename. Returns "" if no
+ * acceptable extension is present.
+ *
+ * Accepts: alphanumeric extensions of 1–16 chars (`.png`, `.tar`,
+ * `.docx`, `.tar.gz` returns only `.gz`). Rejects: anything containing
+ * path separators, dots in unusual positions, or non-ASCII characters.
+ */
+export function sanitizeExt(filename: string | undefined | null): string {
+  if (!filename || typeof filename !== "string") return "";
+  // Defense: strip any leading directory the client might have shoved
+  // into "name", then take only the last extension.
+  const base = filename.split(/[\\/]/).pop() ?? "";
+  const match = base.match(/\.([A-Za-z0-9]{1,16})$/);
+  return match ? "." + match[1].toLowerCase() : "";
+}
+
+/**
+ * Build the per-upload storage path. The file_id and ext are validated
+ * against strict regexes; any path-separator or `..` segment from a
+ * malicious client gets rejected at the regex layer before any I/O.
+ */
+export function buildStoragePath(fileId: string, ext: string, opts: { uploadsRoot?: string; now?: Date } = {}): GeneratedFile {
+  if (!FILE_ID_REGEX.test(fileId)) throw new Error(`file_id "${fileId}" is invalid`);
+  if (ext && !/^\.[A-Za-z0-9]{1,16}$/.test(ext)) throw new Error(`ext "${ext}" is invalid`);
+  const dateBucket = getDateBucket(opts.now);
+  const root = opts.uploadsRoot ?? getUploadsRoot();
+  const relativePath = join(dateBucket, fileId + ext);
+  const absolutePath = join(root, relativePath);
+  return { fileId, ext, dateBucket, relativePath, absolutePath };
+}
+
+export type UploadIndexEntry = {
+  file_id: string;
+  date_bucket: string; // YYYY-MM-DD
+  ext: string;         // ".png" / ".pdf" / etc., including the leading dot, lowercase
+  name: string;        // original filename — stored as metadata only, NEVER used in paths
+  mime: string;        // best-effort mime from the multipart Content-Type
+  size: number;        // byte length of the binary
+  owner?: string;      // upload token id / username, for traceability
+  uploaded_at: string; // ISO 8601
+};
+
+/** Path to the index entry file for a given file_id. Lives in
+ * `<uploadsRoot>/.index/<file_id>.json` so /api/files/:file_id can do
+ * an O(1) lookup without scanning every date bucket. */
+export function indexEntryPath(fileId: string, opts: { uploadsRoot?: string } = {}): string | null {
+  if (!FILE_ID_REGEX.test(fileId)) return null;
+  const root = opts.uploadsRoot ?? getUploadsRoot();
+  return join(root, ".index", fileId + ".json");
+}
+
+/** Validate an index entry shape before serving a download. Defends
+ * against a malformed/tampered .index file by re-checking the same
+ * invariants the upload handler enforces. */
+export function validateIndexEntry(entry: unknown): entry is UploadIndexEntry {
+  if (!entry || typeof entry !== "object") return false;
+  const e: any = entry;
+  return (
+    typeof e.file_id === "string" && FILE_ID_REGEX.test(e.file_id) &&
+    typeof e.date_bucket === "string" && /^\d{4}-\d{2}-\d{2}$/.test(e.date_bucket) &&
+    typeof e.ext === "string" && (e.ext === "" || /^\.[A-Za-z0-9]{1,16}$/.test(e.ext)) &&
+    typeof e.size === "number" && e.size >= 0 && e.size <= MAX_UPLOAD_BYTES
+  );
+}
+
+/**
+ * Defence-in-depth: confirm a resolved absolute path falls within
+ * the uploads root. The HTTP handler calls this after constructing
+ * the download path; if it ever reports `false` we hard-fail the
+ * request rather than serve whatever the resolved path points at.
+ */
+export function isPathInsideUploadsRoot(absPath: string, uploadsRoot?: string): boolean {
+  const root = uploadsRoot ?? getUploadsRoot();
+  const rel = relative(root, absPath);
+  return rel !== "" && !rel.startsWith("..") && !isAbsolute(rel);
+}
+
+// ── Rate limit (in-memory, per token/IP) ────────────────────────────
+
+type UploadRateState = { count: number; resetAt: number };
+
+export class UploadRateLimiter {
+  private state = new Map<string, UploadRateState>();
+
+  constructor(
+    private readonly windowMs: number = UPLOAD_RATE_WINDOW_MS,
+    private readonly maxPerWindow: number = UPLOAD_RATE_MAX_PER_WINDOW,
+  ) {}
+
+  /**
+   * Attempt to consume one upload quota slot for `key` (typically the
+   * caller's token id, falling back to IP for legacy auth). Returns
+   * `{ allowed: true }` when the upload may proceed and
+   * `{ allowed: false, retryAfterMs }` when the caller is rate-limited.
+   * Window is rolling on first touch (not aligned to wall-clock hours).
+   */
+  check(key: string, nowMs: number = Date.now()): { allowed: boolean; remaining: number; resetAt: number; retryAfterMs?: number } {
+    const entry = this.state.get(key);
+    if (!entry || nowMs >= entry.resetAt) {
+      const resetAt = nowMs + this.windowMs;
+      this.state.set(key, { count: 1, resetAt });
+      return { allowed: true, remaining: this.maxPerWindow - 1, resetAt };
+    }
+    if (entry.count >= this.maxPerWindow) {
+      return { allowed: false, remaining: 0, resetAt: entry.resetAt, retryAfterMs: entry.resetAt - nowMs };
+    }
+    entry.count++;
+    return { allowed: true, remaining: this.maxPerWindow - entry.count, resetAt: entry.resetAt };
+  }
+
+  /** Visible for tests — clear all rate-limit state. */
+  reset(): void {
+    this.state.clear();
+  }
+
+  /** Visible for tests — current bucket count. */
+  size(): number {
+    return this.state.size;
+  }
+}
+
+/** Module-level singleton shared by the HTTP /api/upload handler. */
+export const sharedUploadRateLimiter = new UploadRateLimiter();
+
+// ── Attachment schema (shared between REST /api/task and MCP send_task) ───
+
+/**
+ * Canonical shape for the `attachments` field on tasks. Both the REST
+ * /api/task body and the MCP send_task tool accept this same array;
+ * server persistence routes through `tasks.meta_json` so the storage
+ * layout is identical regardless of transport.
+ */
+export type TaskAttachment = {
+  type: "file";
+  file_id: string;
+  name?: string;
+  mime?: string;
+  size?: number;
+};
+
+/** Light validator usable from the REST handler. Returns null on success or an error message string. */
+export function validateAttachments(input: unknown): { ok: true; attachments: TaskAttachment[] } | { ok: false; error: string } {
+  if (input === undefined || input === null) return { ok: true, attachments: [] };
+  if (!Array.isArray(input)) return { ok: false, error: "attachments must be an array" };
+  if (input.length > 20) return { ok: false, error: "too many attachments (max 20)" };
+  const out: TaskAttachment[] = [];
+  for (let i = 0; i < input.length; i++) {
+    const a: any = input[i];
+    if (!a || typeof a !== "object") return { ok: false, error: `attachments[${i}] is not an object` };
+    if (a.type !== "file") return { ok: false, error: `attachments[${i}].type must be "file"` };
+    if (typeof a.file_id !== "string" || !FILE_ID_REGEX.test(a.file_id)) {
+      return { ok: false, error: `attachments[${i}].file_id is invalid` };
+    }
+    if (a.name !== undefined && (typeof a.name !== "string" || a.name.length > 255)) {
+      return { ok: false, error: `attachments[${i}].name must be a string ≤ 255 chars` };
+    }
+    if (a.mime !== undefined && (typeof a.mime !== "string" || a.mime.length > 100)) {
+      return { ok: false, error: `attachments[${i}].mime must be a string ≤ 100 chars` };
+    }
+    if (a.size !== undefined && (typeof a.size !== "number" || a.size < 0 || a.size > MAX_UPLOAD_BYTES)) {
+      return { ok: false, error: `attachments[${i}].size must be a non-negative number ≤ ${MAX_UPLOAD_BYTES}` };
+    }
+    out.push({ type: "file", file_id: a.file_id, name: a.name, mime: a.mime, size: a.size });
+  }
+  return { ok: true, attachments: out };
+}