@3sln/trove 0.0.2

package/package.json

@@ -0,0 +1,75 @@
+{
+  "name": "@3sln/trove",
+  "version": "0.0.2",
+  "type": "module",
+  "description": "Trove — a self-hostable, plugin-extensible Google Drive. Semantic search, pluggable storage (S3 / filesystem / NAS), and a VS Code-style contribution system with sandboxed plugins.",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/3sln/trove.git"
+  },
+  "license": "MIT",
+  "author": "Ray Stubbs",
+  "keywords": [
+    "drive",
+    "file storage",
+    "semantic search",
+    "self-hosted",
+    "plugins",
+    "s3"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": "./packages/server/src/index.js",
+    "./server": "./packages/server/src/index.js",
+    "./server/*": "./packages/server/src/*",
+    "./core": "./packages/core/src/index.js",
+    "./core/*": "./packages/core/src/*",
+    "./plugin-sdk": "./packages/plugin-sdk/src/index.js",
+    "./plugin-sdk/*": "./packages/plugin-sdk/src/*",
+    "./web/*": "./packages/web/src/*",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "packages/core/src",
+    "packages/server/src",
+    "packages/plugin-sdk/src",
+    "packages/web/src",
+    "packages/web/dist"
+  ],
+  "scripts": {
+    "dev": "web-dev-server --config packages/web/web-dev-server.config.mjs",
+    "build:web": "cd packages/web && bun build.mjs",
+    "serve": "bun packages/server/src/adapters/bun.js",
+    "serve:node": "node packages/server/src/adapters/node.js",
+    "test": "bun test",
+    "test:browser": "web-test-runner --config packages/web/web-test-runner.config.mjs",
+    "backup": "bun scripts/backup.mjs",
+    "prepack": "bun run build:web",
+    "sync-version": "node scripts/sync-version.mjs",
+    "version": "npm run sync-version && git add packages/create-trove/package.json"
+  },
+  "engines": {
+    "node": ">=20",
+    "bun": ">=1.1.0"
+  },
+  "dependencies": {
+    "@3sln/ngin": "^0.0.4",
+    "fflate": "^0.8.3"
+  },
+  "optionalDependencies": {
+    "sqlite-vec": "^0.1.9"
+  },
+  "devDependencies": {
+    "@3sln/dodo": "^0.0.10",
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "@web/dev-server": "^1.0.0",
+    "@web/test-runner": "^1.0.0",
+    "aws4": "^1.13.2",
+    "es-module-lexer": "^1.7.0",
+    "playwright-core": "^1.61.1",
+    "s3rver": "^3.7.1",
+    "sql.js": "^1.14.1"
+  }
+}

package/packages/core/src/collections/index.js

@@ -0,0 +1,249 @@
+// Collections — the top-level ownership + permission boundary. Every file and
+// folder belongs to exactly one collection, and (almost) all authorization is at
+// the collection level: a principal holds some subset of { read, write, delete,
+// admin } capabilities on a collection, granted via its ACL.
+//
+// A collection is also a STORE CONFIG: it names a backing store (S3 bucket+prefix,
+// a filesystem path, memory…). Different collections can live on entirely
+// different backends, or share one bucket via distinct prefixes. Users who hold
+// the global "create collections" capability can spin up new collections by
+// supplying a store config — Trove instantiates the backend on demand.
+//
+// Records persist in the pluggable KeyValueStore. Store configs may contain
+// secrets (access keys); treat that store as sensitive.
+
+import { TroveError } from '../errors.js';
+import { PrefixedStorage } from '../storage/prefixed.js';
+import { newId } from '../util.js';
+
+export const CAPABILITIES = ['read', 'write', 'delete', 'admin'];
+const NS = 'collections';
+
+// admin implies everything; a simple implication table keeps checks declarative.
+function expand(caps) {
+  const set = new Set(caps);
+  if (set.has('admin')) for (const c of CAPABILITIES) set.add(c);
+  return set;
+}
+
+export class CollectionService {
+  /**
+   * @param {object} deps
+   * @param {import('../kv.js').KeyValueStore} deps.kv
+   * @param {(storeConfig: object) => import('../storage/interface.js').StorageBackend} deps.storageFactory
+   * @param {string[]} [deps.admins] global admin principal ids (can do anything, create collections)
+   * @param {string[]} [deps.creatorRoles] roles allowed to create collections
+   * @param {object} [deps.defaultStore] store config for the auto-created 'default' collection
+   * @param {boolean} [deps.defaultOpen] if true (default), 'default' grants everyone all caps (single-user friendly)
+   */
+  constructor({ kv, storageFactory, admins = [], creatorRoles = [], defaultStore, defaultOpen = true, storageOverrides }) {
+    if (!kv) throw TroveError.invalid('CollectionService requires a kv store');
+    if (!storageFactory) throw TroveError.invalid('CollectionService requires a storageFactory');
+    this.kv = kv;
+    this.storageFactory = storageFactory;
+    this.admins = new Set(admins);
+    this.creatorRoles = new Set(creatorRoles);
+    this.defaultStore = defaultStore || { driver: 'memory' };
+    this.defaultOpen = defaultOpen;
+    // Pre-built backends to reuse an existing instance (e.g. the server's primary
+    // storage for the 'default' collection, so data isn't split across two Maps).
+    this._storage = new Map(Object.entries(storageOverrides || {}));
+  }
+
+  async init() {
+    const existing = await this.kv.get(NS, 'default');
+    if (!existing) {
+      await this.kv.set(NS, 'default', {
+        id: 'default', name: 'My Drive', description: 'Default collection',
+        store: this.defaultStore,
+        acl: { grants: this.defaultOpen ? [{ type: 'anyone', capabilities: ['read', 'write', 'delete', 'admin'] }] : [] },
+        createdAt: Date.now(), createdBy: 'system', system: true,
+      });
+    }
+  }
+
+  async list(principal) {
+    const rows = await this.kv.list(NS);
+    return rows
+      .map((r) => r.value)
+      .filter((c) => this.can(principal, c, 'read'))
+      .map((c) => this.describe(c, principal))
+      .sort((a, b) => (a.name || '').localeCompare(b.name || ''));
+  }
+
+  async get(id) {
+    const c = await this.kv.get(NS, id);
+    if (!c) throw TroveError.notFound('Collection');
+    return c;
+  }
+
+  /** A safe, principal-scoped view (no secrets). */
+  describe(c, principal) {
+    const caps = [...this.capabilities(principal, c)];
+    return {
+      id: c.id, name: c.name, description: c.description || '',
+      driver: c.store?.driver, system: !!c.system,
+      capabilities: caps, createdAt: c.createdAt,
+    };
+  }
+
+  /** Build (and cache) the StorageBackend for a collection from its store config. */
+  async storageFor(collectionId) {
+    if (this._storage.has(collectionId)) return this._storage.get(collectionId);
+    const c = await this.get(collectionId);
+    let backend = this.storageFactory(c.store);
+    if (c.store?.prefix) backend = new PrefixedStorage(backend, c.store.prefix);
+    this._storage.set(collectionId, backend);
+    return backend;
+  }
+
+  // --- permissions -----------------------------------------------------------
+
+  /**
+   * Does `subject` — a string an operator or an ACL wrote down — name this principal?
+   *
+   * By id, or by email. The id is whatever the IdP put in `sub`, and for Cloudflare
+   * Access that is an internal user UUID, not an address: `TROVE_ADMINS=you@example.com`
+   * matched nothing and there was no way to discover the UUID short of decoding a JWT.
+   * A drive that looks administered and isn't is the worst version of this to ship, so
+   * the address people actually think of as their identity works too.
+   *
+   * Email compares case-insensitively, which is how mailboxes behave and how every IdP
+   * hands them over.
+   */
+  #names(subject, principal) {
+    if (!subject || !principal) return false;
+    if (subject === principal.id) return true;
+    const email = principal.email;
+    return !!email && String(subject).toLowerCase() === String(email).toLowerCase();
+  }
+
+  /** Is this principal on the global admin list (by id or by email)? */
+  #isNamedAdmin(principal) {
+    if (!principal) return false;
+    if (this.admins.has(principal.id)) return true;
+    for (const a of this.admins) if (this.#names(a, principal)) return true;
+    return false;
+  }
+
+  /** The set of capabilities `principal` holds on a collection record/id. */
+  capabilities(principal, collectionOrId) {
+    const c = typeof collectionOrId === 'string' ? null : collectionOrId;
+    if (!c) return new Set(); // callers pass the record; unknown → none
+    if (this.#isNamedAdmin(principal)) return expand(['admin']);
+    const roles = new Set(principal?.roles || []);
+    // Note: a creator role grants the ability to CREATE collections, not blanket
+    // access to every collection — per-collection access comes only from the ACL below.
+    const granted = new Set();
+    for (const g of c.acl?.grants || []) {
+      const match =
+        g.type === 'anyone' ||
+        // Same for a per-collection grant: sharing with someone by the address you know
+        // them by has to work, or an ACL written by a human never matches.
+        (g.type === 'user' && this.#names(g.subject, principal)) ||
+        (g.type === 'role' && roles.has(g.subject));
+      if (match) for (const cap of g.capabilities || []) granted.add(cap);
+    }
+    return expand([...granted]);
+  }
+
+  can(principal, collectionOrId, capability) {
+    return this.capabilities(principal, collectionOrId).has(capability);
+  }
+  async assert(principal, collectionId, capability) {
+    const c = await this.get(collectionId);
+    if (!this.can(principal, c, capability)) {
+      throw TroveError.forbidden(`You lack "${capability}" on collection "${c.name}"`);
+    }
+    return c;
+  }
+
+  /**
+   * May this principal act on the drive AS A WHOLE?
+   *
+   * True for a named admin, and also for anyone who can already read and write every
+   * collection that exists — which is not a new grant, it is naming what is already
+   * true. It matters because the default zero-config self-host is exactly that shape:
+   * one anonymous user with full access to an open default collection. Gating
+   * drive-wide maintenance (rebuilding the search index) behind a TROVE_ADMINS list
+   * nobody is on would make it unreachable in the configuration most people run first,
+   * while a locked-down multi-tenant drive still restricts it to real admins.
+   *
+   * Deliberately NOT used for anything that grants new power — installing a server
+   * indexer runs code, and stays `isAdmin` only.
+   */
+  async hasWholeDrive(principal) {
+    if (this.isAdmin(principal)) return true;
+    if (!principal) return false;
+    const all = (await this.kv.list(NS)).map((r) => r.value).filter(Boolean);
+    if (!all.length) return false;
+    return all.every((c) => this.can(principal, c, 'read') && this.can(principal, c, 'write'));
+  }
+
+  /** Global admin (can do anything, incl. grant admin-only plugin capabilities). */
+  isAdmin(principal) {
+    return this.#isNamedAdmin(principal);
+  }
+
+  /** Global capability to create new collections. */
+  canCreate(principal) {
+    if (!principal) return false;
+    if (this.#isNamedAdmin(principal)) return true;
+    return (principal.roles || []).some((r) => this.creatorRoles.has(r));
+  }
+
+  // --- CRUD ------------------------------------------------------------------
+
+  /**
+   * Create a collection from a store config. Requires the create capability.
+   * The creator is granted admin on the new collection.
+   */
+  async create({ name, description, store, acl }, principal) {
+    if (!this.canCreate(principal)) throw TroveError.forbidden('You cannot create collections');
+    if (!name?.trim()) throw TroveError.invalid('Collection name is required');
+    if (!store?.driver) throw TroveError.invalid('A backing store (driver + config) is required');
+    // Validate the store config actually builds.
+    try {
+      this.storageFactory(store);
+    } catch (err) {
+      throw TroveError.invalid(`Invalid store config: ${err.message}`, { cause: err });
+    }
+    const id = newId('col');
+    const grants = acl?.grants ? [...acl.grants] : [];
+    grants.push({ type: 'user', subject: principal.id, capabilities: ['admin'] });
+    const record = { id, name: name.trim(), description: description || '', store, acl: { grants }, createdAt: Date.now(), createdBy: principal.id };
+    await this.kv.set(NS, id, record);
+    return this.describe(record, principal);
+  }
+
+  async update(id, patch, principal) {
+    const c = await this.assert(principal, id, 'admin');
+    const next = { ...c };
+    if (patch.name != null) next.name = patch.name;
+    if (patch.description != null) next.description = patch.description;
+    if (patch.acl) next.acl = patch.acl; // full ACL replace (admin-only)
+    if (patch.store) {
+      next.store = patch.store;
+      this._storage.delete(id); // rebuild backend next use
+    }
+    await this.kv.set(NS, id, next);
+    return this.describe(next, principal);
+  }
+
+  /** Remove the collection record. Caller is responsible for its nodes/objects. */
+  async remove(id, principal) {
+    if (id === 'default') throw TroveError.invalid('The default collection cannot be deleted');
+    await this.assert(principal, id, 'admin');
+    await this.kv.delete(NS, id);
+    this._storage.delete(id);
+    return { ok: true };
+  }
+
+  /** Full ACL grant management (admin-only). */
+  async setGrant(id, grant, principal) {
+    const c = await this.assert(principal, id, 'admin');
+    const grants = (c.acl?.grants || []).filter((g) => !(g.type === grant.type && g.subject === grant.subject));
+    if (grant.capabilities?.length) grants.push(grant);
+    return this.update(id, { acl: { grants } }, principal);
+  }
+}

package/packages/core/src/errors.js

@@ -0,0 +1,186 @@
+// A small, deliberate error taxonomy. Everything that can fail in Trove throws
+// (or rejects with) a TroveError subclass so callers — and ultimately the user —
+// get a stable `code`, a human `message`, and a `retryable` signal that the
+// retry helper and the HTTP layer both key off of. Wrapping a lower-level cause
+// (an S3 SDK error, a fetch TypeError) preserves it under `.cause` for logs
+// while presenting a clean surface upward.
+
+/** Stable, machine-readable codes. Mirrored by the HTTP layer → status codes. */
+export const ErrorCode = Object.freeze({
+  NOT_FOUND: 'not_found',
+  ALREADY_EXISTS: 'already_exists',
+  INVALID: 'invalid', // bad request / validation
+  CONFLICT: 'conflict', // version/etag mismatch, concurrent edit
+  UNAUTHORIZED: 'unauthorized',
+  FORBIDDEN: 'forbidden',
+  UNSUPPORTED: 'unsupported', // backend can't do this (e.g. presign on fs)
+  QUOTA: 'quota', // out of space (507) / rate limited by capacity (429) — see below
+  TOO_LARGE: 'too_large', // this request is bigger than a configured limit allows
+  BAD_RANGE: 'bad_range', // the requested byte range doesn't exist in this object (416)
+  TRANSIENT: 'transient', // network blip, 5xx, throttle — safe to retry
+  TIMEOUT: 'timeout',
+  ABORTED: 'aborted', // caller cancelled (AbortSignal)
+  INTERNAL: 'internal', // unexpected; a bug
+});
+
+// Which codes are, by default, safe to retry with backoff.
+const RETRYABLE = new Set([ErrorCode.TRANSIENT, ErrorCode.TIMEOUT, ErrorCode.QUOTA]);
+
+// HTTP status for each code, so the server layer stays declarative.
+const HTTP_STATUS = {
+  [ErrorCode.NOT_FOUND]: 404,
+  [ErrorCode.ALREADY_EXISTS]: 409,
+  [ErrorCode.INVALID]: 400,
+  [ErrorCode.CONFLICT]: 412,
+  [ErrorCode.UNAUTHORIZED]: 401,
+  [ErrorCode.FORBIDDEN]: 403,
+  [ErrorCode.UNSUPPORTED]: 501,
+  [ErrorCode.QUOTA]: 429,
+  [ErrorCode.TOO_LARGE]: 413,
+  [ErrorCode.BAD_RANGE]: 416,
+  [ErrorCode.TRANSIENT]: 503,
+  [ErrorCode.TIMEOUT]: 504,
+  [ErrorCode.ABORTED]: 499,
+  [ErrorCode.INTERNAL]: 500,
+};
+
+export class TroveError extends Error {
+  /**
+   * @param {string} code one of ErrorCode
+   * @param {string} message human-readable, safe to surface to the user
+   * @param {{cause?: unknown, retryable?: boolean, details?: object}} [opts]
+   */
+  constructor(code, message, opts = {}) {
+    super(message, opts.cause !== undefined ? { cause: opts.cause } : undefined);
+    this.name = 'TroveError';
+    this.code = code || ErrorCode.INTERNAL;
+    // Explicit override wins; otherwise derive from the code.
+    this.retryable = opts.retryable ?? RETRYABLE.has(this.code);
+    this.details = opts.details ?? null;
+    this.status = HTTP_STATUS[this.code] ?? 500;
+    // QUOTA covers two failures that deserve different statuses. A rate limit is 429 —
+    // back off and try again. Being out of DISK is 507: retrying changes nothing, and
+    // telling a client to retry sends it into a loop against a condition only a human
+    // can clear. `retryable` is what tells them apart.
+    //
+    // "Your file is bigger than we allow" is neither, and has its own code (TOO_LARGE,
+    // 413): the store is not full, and nothing about waiting or freeing space helps.
+    if (this.code === ErrorCode.QUOTA && !this.retryable) this.status = 507;
+  }
+
+  /** Shape sent over the wire and logged. Never leaks the raw cause. */
+  toJSON() {
+    return {
+      error: {
+        code: this.code,
+        message: this.message,
+        retryable: this.retryable,
+        ...(this.details ? { details: this.details } : {}),
+      },
+    };
+  }
+
+  // Ergonomic constructors — read like prose at the call site.
+  static notFound(what, opts) {
+    return new TroveError(ErrorCode.NOT_FOUND, what ? `${what} not found` : 'Not found', opts);
+  }
+  static alreadyExists(what, opts) {
+    return new TroveError(ErrorCode.ALREADY_EXISTS, `${what} already exists`, opts);
+  }
+  static invalid(message, opts) {
+    return new TroveError(ErrorCode.INVALID, message, opts);
+  }
+  static conflict(message, opts) {
+    return new TroveError(ErrorCode.CONFLICT, message, opts);
+  }
+  static unsupported(message, opts) {
+    return new TroveError(ErrorCode.UNSUPPORTED, message, opts);
+  }
+  static transient(message, opts) {
+    return new TroveError(ErrorCode.TRANSIENT, message, { retryable: true, ...opts });
+  }
+  static timeout(message, opts) {
+    return new TroveError(ErrorCode.TIMEOUT, message, { retryable: true, ...opts });
+  }
+  static aborted(message = 'Operation aborted', opts) {
+    return new TroveError(ErrorCode.ABORTED, message, { retryable: false, ...opts });
+  }
+  static unauthorized(message = 'Unauthorized', opts) {
+    return new TroveError(ErrorCode.UNAUTHORIZED, message, opts);
+  }
+  static tooLarge(message = 'Too large', opts) {
+    return new TroveError(ErrorCode.TOO_LARGE, message, { retryable: false, ...opts });
+  }
+  /** The requested byte range doesn't exist in this object — 416, not 400 or 500. */
+  static badRange(message = 'Range not satisfiable', opts) {
+    return new TroveError(ErrorCode.BAD_RANGE, message, { retryable: false, ...opts });
+  }
+  static forbidden(message = 'Forbidden', opts) {
+    return new TroveError(ErrorCode.FORBIDDEN, message, opts);
+  }
+  static internal(message = 'Internal error', opts) {
+    return new TroveError(ErrorCode.INTERNAL, message, opts);
+  }
+}
+
+/**
+ * Normalize any thrown value into a TroveError. Recognises AbortError, common
+ * fetch/network failures, and Node fs errno codes so lower layers can just
+ * `throw wrapError(e)` and get sane classification for free.
+ */
+export function wrapError(err, fallbackMessage = 'Unexpected error') {
+  if (err instanceof TroveError) return err;
+
+  // Cancellation.
+  if (err?.name === 'AbortError' || err?.code === 'ABORT_ERR') {
+    return TroveError.aborted(err.message || 'Operation aborted', { cause: err });
+  }
+
+  // Node/browser network failures are transient by nature.
+  const netCodes = new Set([
+    'ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'EAI_AGAIN', 'EPIPE', 'ENOTFOUND', 'UND_ERR_SOCKET',
+  ]);
+  if (netCodes.has(err?.code) || (err instanceof TypeError && /fetch|network/i.test(err.message || ''))) {
+    return TroveError.transient(err.message || 'Network error', { cause: err });
+  }
+
+  // Node fs errnos.
+  switch (err?.code) {
+    case 'ENOENT':
+      return TroveError.notFound(null, { cause: err });
+    case 'EEXIST':
+      return TroveError.alreadyExists('Path', { cause: err });
+    case 'EACCES':
+    case 'EPERM':
+      return new TroveError(ErrorCode.FORBIDDEN, 'Permission denied', { cause: err });
+    // Out of room. NOT retryable, unlike the rate-limit sense of QUOTA that shares this
+    // code — retrying a full disk just burns the user's time to reach the same answer.
+    // The message is what someone can act on, rather than the kernel's phrasing.
+    case 'ENOSPC':
+      return new TroveError(ErrorCode.QUOTA, 'The storage volume is full — free some space and try again', { cause: err, retryable: false });
+    case 'EDQUOT':
+      return new TroveError(ErrorCode.QUOTA, 'The storage quota for this volume has been reached', { cause: err, retryable: false });
+    case 'EFBIG':
+      return new TroveError(ErrorCode.QUOTA, 'That file is larger than this filesystem can store', { cause: err, retryable: false });
+  }
+
+  return TroveError.internal(err?.message || fallbackMessage, { cause: err });
+}
+
+/**
+ * Did this fail because the store is out of room?
+ *
+ * QUOTA covers two different things — "no space left" and "you are being rate limited"
+ * — and only the first is a standing condition someone has to go and fix. Callers that
+ * want to raise a persistent, actionable problem need to tell them apart, and the
+ * distinguishing fact is that a full disk is not retryable.
+ */
+export function isOutOfSpace(err) {
+  return err instanceof TroveError && err.code === ErrorCode.QUOTA && err.retryable === false;
+}
+
+/** True if the thrown value should be retried. */
+export function isRetryable(err) {
+  if (err instanceof TroveError) return err.retryable;
+  return wrapError(err).retryable;
+}