lowlander 0.4.0 → 0.5.0

package/dashboard/server.ts

@@ -0,0 +1,255 @@
+import * as E from 'edinburgh';
+import { timingSafeEqual } from 'crypto';
+import { ServerProxy, createStreamType, getStreamTypesForModel, warpsocket } from '../server/server.js';
+import { getMainApi, getPassword } from '../server/wshandler.js';
+
+// `modelRegistry` is a documented export of edinburgh's models module but is
+// not re-exported from the package entry. Reach it by URL relative to the
+// resolved package main — works in both Bun (.ts) and Node (built .js).
+const _modelsModule = await import(new URL('./models.js', import.meta.resolve('edinburgh')).href) as {
+    modelRegistry: Record<string, E.AnyModelClass>;
+};
+const modelRegistry = _modelsModule.modelRegistry;
+
+
+function passwordOk(provided: string): boolean {
+    const expected = getPassword();
+    if (typeof provided !== 'string' || provided.length !== expected.length) return false;
+    return timingSafeEqual(Buffer.from(provided), Buffer.from(expected));
+}
+
+function describeType(type: E.TypeWrapper<unknown>): { kind: string; display: string; linkedModel?: string } {
+    const linked = type.getLinkedModel();
+    return {
+        kind: (type as any).kind,
+        display: type.toString(),
+        linkedModel: linked?.tableName,
+    };
+}
+
+function describeIndex(index: any) {
+    const fields = Array.from(index._indexFields.keys()) as string[];
+    return {
+        fields,
+        fieldTypes: fields.map(f => describeType(index._indexFields.get(f))),
+        computed: !!index._computeFn,
+        kind: index._getTypeName() as 'primary' | 'unique' | 'secondary',
+    };
+}
+
+function describeModel(Model: E.AnyModelClass) {
+    const fields: { name: string; type: ReturnType<typeof describeType>; description?: string; hasDefault: boolean }[] = [];
+    for (const [name, cfg] of Object.entries(Model.fields) as [string, E.FieldConfig<unknown>][]) {
+        fields.push({
+            name,
+            type: describeType(cfg.type),
+            description: cfg.description,
+            hasDefault: cfg.default !== undefined,
+        });
+    }
+    const indexes: { name: string; info: ReturnType<typeof describeIndex> }[] = [{
+        name: '(primary)',
+        info: describeIndex(Model),
+    }];
+    for (const [name, idx] of Object.entries((Model as any)._secondaries || {})) {
+        indexes.push({ name, info: describeIndex(idx) });
+    }
+    const streamTypes = getStreamTypesForModel(Model).map((ST: any) => ({
+        id: ST.id,
+        fields: ST.fields,
+        cache: ST.cache,
+    }));
+    return {
+        tableName: Model.tableName,
+        fields,
+        indexes,
+        streamTypes,
+    };
+}
+
+function serializeValue(v: any, depth = 0): any {
+    if (v === null || v === undefined) return v;
+    if (v instanceof E.Model) {
+        const cls: any = v.constructor;
+        const tn = cls.tableName || cls.name;
+        const pkBytes = (v as any).getPrimaryKey?.();
+        const pkArr = pkBytes && cls._pkToArray ? cls._pkToArray(pkBytes) : null;
+        const pk = pkArr ? (pkArr.length === 1 ? pkArr[0] : pkArr) : null;
+        return { __ref: tn, pk };
+    }
+    if (v instanceof Uint8Array) return Array.from(v.slice(0, 64)).map(b => b.toString(16).padStart(2, '0')).join('');
+    if (v instanceof Date) return v.toISOString();
+    if (v instanceof Set) return depth > 2 ? `Set(${v.size})` : Array.from(v).map(x => serializeValue(x, depth + 1));
+    if (Array.isArray(v)) return depth > 2 ? `Array(${v.length})` : v.map(x => serializeValue(x, depth + 1));
+    if (typeof v === 'object') {
+        if (depth > 2) return '...';
+        const out: Record<string, any> = {};
+        for (const k of Object.keys(v)) out[k] = serializeValue(v[k], depth + 1);
+        return out;
+    }
+    return v;
+}
+
+function instanceToPlain(instance: any): Record<string, any> {
+    const Model = instance.constructor as E.AnyModelClass;
+    const out: Record<string, any> = {};
+    for (const fieldName of Object.keys(Model.fields)) {
+        try {
+            out[fieldName] = serializeValue((instance as any)[fieldName]);
+        } catch (err: any) {
+            out[fieldName] = `<error: ${err?.message || err}>`;
+        }
+    }
+    return out;
+}
+
+function modelByName(name: string): E.AnyModelClass {
+    const Model = modelRegistry[name];
+    if (!Model) throw new Error(`Unknown model: ${name}`);
+    return Model;
+}
+
+function findIndex(Model: E.AnyModelClass, indexName: string): any {
+    if (indexName === '(primary)') return Model;
+    const idx = (Model as any)._secondaries?.[indexName];
+    if (!idx) throw new Error(`Unknown index ${indexName} on ${Model.tableName}`);
+    return idx;
+}
+
+// =====================================================================
+
+class DashboardAPI {
+    listModels() {
+        return Object.keys(modelRegistry).sort().map(name => ({
+            tableName: name,
+            indexCount: 1 + Object.keys((modelRegistry[name] as any)._secondaries || {}).length,
+            fieldCount: Object.keys(modelRegistry[name].fields).length,
+            streamTypeCount: getStreamTypesForModel(modelRegistry[name]).length,
+        }));
+    }
+
+    getModel(name: string) {
+        return describeModel(modelByName(name));
+    }
+
+    listApiMethods() {
+        const api = getMainApi();
+        if (!api) return [];
+        const out: { name: string; kind: 'function' | 'value' }[] = [];
+        for (const key of Object.keys(api).sort()) {
+            if (key === '_dashboard' || key.startsWith('_')) continue;
+            const v = (api as any)[key];
+            out.push({ name: key, kind: typeof v === 'function' ? 'function' : 'value' });
+        }
+        return out;
+    }
+
+    getApiMethodSource(name: string): string | undefined {
+        const api = getMainApi();
+        const fn = api && (api as any)[name];
+        if (typeof fn !== 'function') return undefined;
+        try {
+            // Bundlers usually preserve enough source for a useful preview
+            const s = String(fn);
+            return s.length > 4000 ? s.slice(0, 4000) + '\n/* … truncated */' : s;
+        } catch {
+            return undefined;
+        }
+    }
+
+    findRecords(modelName: string, indexName: string, opts: {
+        search?: any;
+        reverse?: boolean;
+        limit?: number;
+    } = {}) {
+        const Model = modelByName(modelName);
+        const idx = findIndex(Model, indexName);
+        const fieldNames = Array.from(idx._indexFields.keys()) as string[];
+
+        const findOpts: any = {};
+        if (opts.search !== undefined && opts.search !== '') {
+            findOpts.is = opts.search;
+        }
+        if (opts.reverse) findOpts.reverse = true;
+
+        const limit = Math.min(opts.limit ?? 10, 1000);
+        let iter: Iterable<any>;
+        try { iter = idx.find(findOpts); }
+        catch { return { rows: [], indexFields: fieldNames, scanned: 0 }; }
+        const results: { pk: any; values: Record<string, any> }[] = [];
+        let scanned = 0;
+        for (const row of iter) {
+            scanned++;
+            const plain = instanceToPlain(row);
+            const cls: any = (row as any).constructor;
+            const pkBytes = (row as any).getPrimaryKey ? (row as any).getPrimaryKey() : null;
+            const pkArr = pkBytes && cls._pkToArray ? cls._pkToArray(pkBytes) : null;
+            const pk = pkArr ? (pkArr.length === 1 ? pkArr[0] : pkArr) : null;
+            results.push({
+                pk: serializeValue(pk),
+                values: plain,
+            });
+            if (results.length >= limit) break;
+        }
+        return {
+            rows: results,
+            indexFields: fieldNames,
+            scanned,
+        };
+    }
+
+    getRecord(modelName: string, pk: any) {
+        const Model = modelByName(modelName);
+        const pkArgs = Array.isArray(pk) ? pk : [pk];
+        const instance = (Model as any).get(...pkArgs);
+        if (!instance) return undefined;
+        return instanceToPlain(instance);
+    }
+
+    streamRecord(modelName: string, streamTypeId: number, pk: any) {
+        const Model = modelByName(modelName);
+        const ST = getStreamTypesForModel(Model).find((s: any) => s.id === streamTypeId) as any;
+        if (!ST) throw new Error(`No stream type with id ${streamTypeId} for ${modelName}`);
+        const pkArgs = Array.isArray(pk) ? pk : [pk];
+        const instance = (Model as any).get(...pkArgs);
+        if (!instance) throw new Error('Record not found');
+        return new ST(instance);
+    }
+
+    getDebugState(mode: 'channels' | 'sockets' | 'workers' | 'kv') {
+        return warpsocket.getDebugState(mode as any);
+    }
+}
+
+const DashboardProxyValue = 'authenticated';
+
+/**
+ * RPC entry point. Re-export this from your top-level api file:
+ *
+ * ```ts
+ * export { _dashboard } from 'lowlander/dashboard';
+ * ```
+ *
+ * On first call, a password is generated (or reused from the
+ * `LOWLANDER_DASHBOARD_PASSWORD` env var) and printed to the server console.
+ */
+export function _dashboard(password: string) {
+    if (!passwordOk(password)) {
+        throw new Error('Invalid dashboard password');
+    }
+    return new ServerProxy(new DashboardAPI(), DashboardProxyValue);
+}
+
+// Re-export so callers don't need a separate import
+export type { DashboardAPI };
+
+// Convenience stream type used internally so live record viewing works
+// for arbitrary models without the developer having to pre-create one.
+// (Built lazily per model+selection because createStreamType is keyed on the
+// shape of the selection.)
+const allFieldsStreams = new Map<string, ReturnType<typeof createStreamType>>();
+export function _dashboardAllFieldsStream(modelName: string) {
+    // Currently unused; kept for future "view live" feature.
+    void allFieldsStreams;
+    void modelName;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lowlander",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "TypeScript framework for data persistence, type-safe RPCs and real-time (partial) client synchronization.",
   "type": "module",
   "main": "./build/server/server.js",
@@ -21,34 +21,45 @@
       "bun": "./server/server.ts",
       "import": "./build/server/server.js",
       "types": "./build/server/server.d.ts"
+    },
+    "./dashboard": {
+      "bun": "./dashboard/index.ts",
+      "import": "./build/dashboard/index.js",
+      "types": "./build/dashboard/index.d.ts"
     }
   },
   "files": [
     "build/",
     "server/",
     "client/",
+    "dashboard/",
     "skill/"
   ],
   "scripts": {
-    "build": "tsc -b && cp -rf examples/helloworld/client/assets examples/helloworld/client/index.html build/examples/helloworld/client/ && npm run build:docs",
+    "build": "tsc -b && node --import=tsx/esm dashboard/build-bundle.ts && node --import=tsx/esm examples/helloworld/build-client.ts && npm run build:docs",
     "build:docs": "rm -rf skill ; mkdir skill ; cat skill-header.md README.md > skill/SKILL.md && readme-tsdoc --split --file skill/SKILL.md && readme-tsdoc --repo-url https://github.com/vanviegen/lowlander",
-    "typecheck": "bun x tsc -b",
+    "typecheck": "npx tsc -b",
     "clean": "rm -rf build dist skill",
-    "prepack": "bun install && bun run build",
-    "test": "bun test",
+    "prepack": "npm install && npm run build",
+    "test": "vitest run",
+    "example": "npm run build && (cd examples/helloworld && npm install) && node build/examples/helloworld/server/main.js",
     "postinstall": "[ -f .local-post-install.sh ] && sh .local-post-install.sh || true"
   },
   "dependencies": {
-    "warpsocket": "^0.8.3"
+    "warpsocket": "^0.9.1"
   },
   "peerDependencies": {
-    "aberdeen": "^1.10.1",
-    "edinburgh": "^0.6.1"
+    "aberdeen": "^1.15.0",
+    "edinburgh": "^0.6.4"
   },
   "devDependencies": {
-    "@types/bun": "^1.3.10",
-    "aberdeen": "^1.10.1",
-    "readme-tsdoc": "^1.1.6"
+    "@types/node": "^20.19.41",
+    "aberdeen": "^1.15.0",
+    "esbuild": "^0.28.0",
+    "readme-tsdoc": "^1.1.6",
+    "sirv": "^3.0.2",
+    "tsx": "^4.0.0",
+    "vitest": "^4.1.7"
   },
   "workspaces": [
     "examples/*"

package/server/server.ts

@@ -1,6 +1,9 @@
 import * as E from "edinburgh";
 import DataPack from "edinburgh/datapack";
 import * as realWarpsocket from 'warpsocket';
+import { randomBytes } from 'crypto';
+import { existsSync, readFileSync, writeFileSync } from 'fs';
+import { join } from 'path';
 
 // Get log level from environment variable
 // 0: no logging (default)
@@ -22,6 +25,11 @@ const CHANNEL_TYPE_MODEL = 1;
 
 /** @internal Registry mapping model classes to their stream types */
 const streamTypesPerModel: Map<E.AnyModelClass, typeof StreamTypeBase<unknown>[]> = new Map();
+
+/** @internal Used by the dashboard module to inspect registered stream types. */
+export function getStreamTypesForModel(Model: E.AnyModelClass): readonly (typeof StreamTypeBase<unknown>)[] {
+    return streamTypesPerModel.get(Model) || [];
+}
  
 /**
  * Base class for stream types created by {@link createStreamType}.
@@ -209,7 +217,9 @@ export function createStreamType<T, S extends FieldSelection<T>>(
         static id = streamTypeId;
         static cache = options?.cache;
     }
-    streamTypes.push(StreamType);
+    if (!streamTypes.some(st => (st as any).id === streamTypeId)) {
+        streamTypes.push(StreamType);
+    }
     return StreamType as any;
 }
 
@@ -454,10 +464,16 @@ export async function start(mainApiFile: string, opts: {bind?: string, threads?:
     if (opts.injectWarpSocket) {
         warpsocket = opts.injectWarpSocket;
     }
+    const pwFile = join(process.cwd(), '.lowlander_dashboard_password');
+    if (!existsSync(pwFile)) try { writeFileSync(pwFile, randomBytes(24).toString('base64url'), { flag: 'wx' }); } catch {}
+    const password = process.env.LOWLANDER_DASHBOARD_PASSWORD ?? readFileSync(pwFile, 'utf8').trim();
+    if (!process.env.LOWLANDER_DASHBOARD_PASSWORD) {
+        console.log(`\n${'='.repeat(56)}\n  Lowlander dashboard password:  ${password}\n${'='.repeat(56)}\n`);
+    }
     await warpsocket.start({
         bind: opts.bind || '0.0.0.0:8080',
         threads: opts.threads,
         workerPath: WSHANDLER_FILE,
-        workerArg: mainApiFile,
+        workerArg: { apiFile: mainApiFile, password },
     });
 }

package/server/wshandler.ts

@@ -2,9 +2,13 @@ import DataPack from 'edinburgh/datapack';
 import { warpsocket, Socket, StreamTypeBase, pushModel, ServerProxy, logLevel } from './server.js';
 import { SERVER_MESSAGES, CLIENT_MESSAGES } from './protocol.js';
 import * as E from 'edinburgh';
+import type { HttpRequest, HttpResponse } from 'warpsocket';
 
 let mainApi: object | undefined;
 
+/** @internal Used by the dashboard module to introspect the user's API. */
+export function getMainApi(): object | undefined { return mainApi; }
+
 export const socketProxies = new Map<number, Map<number, any>>(); // {socketId: {proxyId: proxyObject}}
 
 export interface Request {
@@ -26,16 +30,40 @@ function sendError(socketId: number, requestId: number, message: string) {
     send(socketId, requestId, SERVER_MESSAGES.error, message);
 }
 
-export async function handleStart(apiFile: any) {
+let dashboardPassword = '';
+/** @internal Used by the dashboard module to authenticate requests. */
+export function getPassword(): string { return dashboardPassword; }
+
+export async function handleStart(arg: any) {
+    const { apiFile, password } = arg;
+    dashboardPassword = password;
     if (logLevel >= 1) console.log('[lowlander] Worker started, loading', apiFile);
     mainApi = await import(apiFile);
 }
 
+export async function handleHttpRequest(req: HttpRequest, res: HttpResponse): Promise<void> {
+    const handler = (mainApi as any)?.handleHttpRequest;
+    if (typeof handler === 'function') {
+        return handler(req, res);
+    }
+    res.statusCode = 404;
+    res.setHeader('content-type', 'text/plain');
+    res.end('Not found');
+}
+
 
 export async function handleBinaryMessage(message: Uint8Array, socketId: number) {
+    if (logLevel >= 2) console.log('[lowlander] handleBinaryMessage socket', socketId, 'bytes', message.length, 'hex', Array.from(message.slice(0, 16)).map(b => b.toString(16).padStart(2,'0')).join(' '));
     const pack = new DataPack(message);
-    const requestId = pack.readPositiveInt();
-    const type = pack.readNumber();
+    let requestId: number;
+    let type: number;
+    try {
+        requestId = pack.readPositiveInt();
+        type = pack.readNumber();
+    } catch (e: any) {
+        console.error('[lowlander] Failed to parse message header:', e.message, 'hex:', Array.from(message.slice(0, 16)).map(b => b.toString(16).padStart(2,'0')).join(' '));
+        return;
+    }
 
     if (type === CLIENT_MESSAGES.cancel) {
         // Delete server proxy object, if any
@@ -81,7 +109,16 @@ export async function handleBinaryMessage(message: Uint8Array, socketId: number)
         // Obtain function reference
         const methodName = pack.readString();
         let func = (api as any)[methodName];
-        if (typeof func !== 'function' || methodName.startsWith('_')) {
+        // `_dashboard` is the one underscore-prefixed name reserved for the
+        // dashboard module; everything else with a leading underscore is private.
+        // Also block: classes (typeof === 'function' but not callable as plain
+        // function), and Object.prototype methods accessible via prototype chain.
+        if (
+            typeof func !== 'function' ||
+            (methodName.startsWith('_') && methodName !== '_dashboard') ||
+            Function.prototype.toString.call(func).trimStart().startsWith('class ') ||
+            Object.prototype.hasOwnProperty.call(Object.prototype, methodName)
+        ) {
             return sendError(socketId, requestId, `Method not found: ${methodName}`);
         }
 
@@ -107,15 +144,21 @@ export async function handleBinaryMessage(message: Uint8Array, socketId: number)
                 // Result processing/serialization should be within the transaction, as it may involve (lazy) loading models.
                 // The actual socket send remains deferred until after commit.
 
-                if (response instanceof ServerProxy) {
+                // NOTE: Use duck-typing instead of instanceof for ServerProxy and StreamTypeBase,
+                // because api.js and wshandler.js each bundle their own copy of server.ts,
+                // so instanceof checks across those two bundles always return false.
+                // E.Model is safe to use because edinburgh is externalized (shared).
+                if (response !== null && typeof response === 'object' && 'api' in response && 'value' in response) {
+                    const serverProxyResponse = response as ServerProxy<any, any>;
                     let proxies = socketProxies.get(socketId);
                     if (!proxies) socketProxies.set(socketId, proxies = new Map());
                     if (logLevel >= 3) console.log('[lowlander] Setting proxy id', requestId, 'for socket', socketId);
-                    proxies.set(requestId, response.api);
+                    proxies.set(requestId, serverProxyResponse.api);
 
-                    if (response.value instanceof StreamTypeBase) {
-                        const StreamType = response.value.constructor as typeof StreamTypeBase<any>;
-                        const instance = response.value._instance;
+                    if (serverProxyResponse.value !== null && typeof serverProxyResponse.value === 'object' && (serverProxyResponse.value as any)._instance instanceof E.Model) {
+                        const streamValue = serverProxyResponse.value as unknown as StreamTypeBase<any>;
+                        const StreamType = streamValue.constructor as typeof StreamTypeBase<any>;
+                        const instance = streamValue._instance;
 
                         const virtualSocketId = warpsocket.createVirtualSocket(socketId, DataPack.createUint8Array(requestId, SERVER_MESSAGES.model_data));
                         virtualSocketIds.push(virtualSocketId);
@@ -124,12 +167,13 @@ export async function handleBinaryMessage(message: Uint8Array, socketId: number)
                         const cacheMs = StreamType.cache !== undefined ? StreamType.cache * 1000 : undefined;
                         pendingPacket = DataPack.createUint8Array(requestId, SERVER_MESSAGES.response_proxy_model, virtualSocketIds, instance.getPrimaryKeyHash() + StreamType.id, cacheMs);
                     } else {
-                        pendingPacket = DataPack.createUint8Array(requestId, SERVER_MESSAGES.response_proxy, response.value, virtualSocketIds);
+                        pendingPacket = DataPack.createUint8Array(requestId, SERVER_MESSAGES.response_proxy, serverProxyResponse.value, virtualSocketIds);
                     }
 
-                } else if (response instanceof StreamTypeBase) {
-                    const StreamType = response.constructor as typeof StreamTypeBase<any>;
-                    const instance = response._instance;
+                } else if (response !== null && typeof response === 'object' && (response as any)._instance instanceof E.Model) {
+                    const streamResponse = response as unknown as StreamTypeBase<any>;
+                    const StreamType = streamResponse.constructor as typeof StreamTypeBase<any>;
+                    const instance = streamResponse._instance;
 
                     // Create a virtual socket for the model updates, prefixed by requestId + 'd'
                     const virtualSocketId = warpsocket.createVirtualSocket(socketId, DataPack.createUint8Array(requestId, SERVER_MESSAGES.model_data));

package/skill/SKILL.md

@@ -25,17 +25,25 @@ This library is built on top of a number of libraries by the same author:
 
 Lowlander glues these together and adds real-time partial data synchronization and type-safe RPCs to provide a framework for rapidly building performant full-stack (database included!) web applications.
 
+## Example project
+
+An example project is included in `examples/helloworld`. To run it:
+
+```bash
+npm run example
+```
+
+Opens at http://localhost:8080 with the Aberdeen dashboard at http://localhost:8080/_dashboard (password printed to console on start).
+
 ## Tutorial
 
 ### Project Setup
 
 ```bash
-bun init
-bun add lowlander aberdeen edinburgh
+npm init
+npm add lowlander aberdeen edinburgh
 ```
 
-(npm should also work for all of this.)
-
 Create the project structure:
 
 ```
@@ -339,10 +347,50 @@ Set the `LOWLANDER_LOG_LEVEL` environment variable to a number from 0 to 3:
 
 Set `EDINBURGH_LOG_LEVEL` similarly for Edinburgh internals.
 
+### Dashboard
+
+Lowlander ships with an optional admin/developer dashboard for inspecting
+Edinburgh models, browsing index rows, listing RPC methods, viewing source
+code, and peeking at warpsocket debug state (channels, sockets, workers,
+KV). It's a single self-contained HTML bundle.
+
+To enable it:
+
+1. Re-export `_dashboard` from your top-level API module:
+
+   ```ts
+   // server/api.ts
+   export { _dashboard } from "lowlander/dashboard";
+   ```
+
+2. Serve the bundled HTML by calling `serveDashboard(res)` from a
+   `warpsocket` `handleHttpRequest` export:
+
+   ```ts
+   import type { HttpRequest, HttpResponse } from "warpsocket";
+   import { serveDashboard } from "lowlander/dashboard";
+
+   export function handleHttpRequest(req: HttpRequest, res: HttpResponse) {
+       if (req.url === '/_dashboard' || req.url.startsWith('/_dashboard?')) {
+           return serveDashboard(res);
+       }
+       // … serve your own static files …
+   }
+   ```
+
+3. On first server start (per warpsocket KV namespace), a random password
+   is generated and printed to the console. Override with the
+   `LOWLANDER_DASHBOARD_PASSWORD` env var.
+
+The dashboard prompts for the websocket URL (defaults to the current host)
+and password on first load, then stores them in localStorage.
+
 ## Server API Reference
 
 The following is auto-generated from `server/server.ts`:
 
+### [getStreamTypesForModel](getStreamTypesForModel.md) · function
+
 ### [createStreamType](createStreamType.md) · function
 
 Creates a stream type for reactive model streaming to clients with automatic updates.

package/skill/getStreamTypesForModel.md

@@ -0,0 +1,7 @@
+### getStreamTypesForModel · function
+
+**Signature:** `(Model: AnyModelClass) => readonly (typeof StreamTypeBase<unknown>)[]`
+
+**Parameters:**
+
+- `Model: E.AnyModelClass`