@protobase/cli 0.67.1 → 0.67.3

package/dist/cli/package.js

@@ -1,4 +1,4 @@
 //#region package.json
-var version = "0.67.1";
+var version = "0.67.3";
 //#endregion
 export { version };

package/dist/dev/entry.js

@@ -1,8 +1,7 @@
 import { buildProject } from "../config/build-project.js";
 import { projectIcons } from "../build/app-icons.js";
 import { projectErrorReporter } from "../project/error-reporter.js";
-import { projectAdmin } from "../project/project-admin.js";
-import { projectUserTables } from "../project/project-user-tables.js";
+import { servedAdmin } from "../project/served-admin.js";
 import { devJobs } from "./dev-jobs.js";
 import { devRealtimePrune } from "./dev-prune.js";
 import { devDatabase, loadDevConfig } from "./project.js";
@@ -56,7 +55,7 @@ const ui = {
 	fieldTypes: {},
 	icons: projectIcons(projectDir)
 };
-const { admin, jobs } = projectAdmin(project, db, ui, reporter, upgradeWebSocket, await projectUserTables(project, db));
+const { admin, jobs } = await servedAdmin(project, db, ui, reporter, upgradeWebSocket);
 if (process.env.PROTOBASE_DEV_JOBS !== "false") await devJobs(db, jobs, admin.decryptField, project.mailer);
 devRealtimePrune(admin);
 var entry_default = withDevServiceWorker(new Hono(), Boolean(project.notifications)).route("/", admin);

package/dist/project/project-admin.js

@@ -7,11 +7,15 @@ import { configExports, createAdmin } from "@protobase/server";
 * project's own and the delivery of its events and history. With a `reporter`, every error the API turns into a 500,
 * and every stopped query, goes to the error tracker too, besides the project's `onUnhandledError` or the console. With
 * the host's `upgradeWebSocket`, it serves `/api/sb/realtime/v1` too. With `userTables`, the tables users add serve
-* beside the config's resources (see `projectUserTables`).
+* beside the config's resources (see `readUserTables`). `report` is what the admin does with an unhandled error.
 */
 const projectAdmin = (project, db, ui, reporter, upgradeWebSocket, userTables) => {
 	const { resources, views, pages, userMenu, bottomBars, subscriptions } = configExports(project.config);
 	const unhandled = project.options?.onUnhandledError ?? ((error) => console.error(error));
+	const report = (error, request) => {
+		reporter?.report(error, request);
+		unhandled(error, request);
+	};
 	const admin = createAdmin({
 		resources,
 		views,
@@ -44,15 +48,13 @@ const projectAdmin = (project, db, ui, reporter, upgradeWebSocket, userTables) =
 		...upgradeWebSocket && { upgradeWebSocket },
 		options: {
 			...project.options,
-			onUnhandledError: (error, request) => {
-				reporter?.report(error, request);
-				unhandled(error, request);
-			}
+			onUnhandledError: report
 		}
 	});
 	return {
 		admin,
-		jobs: [...project.jobs, ...admin.jobs]
+		jobs: [...project.jobs, ...admin.jobs],
+		report
 	};
 };
 //#endregion

package/dist/project/project-user-tables.js

@@ -1,15 +1,10 @@
 import { userTableResources } from "@protobase/server";
 //#region src/project/project-user-tables.ts
-/**
-* The tables users add, read from their definitions when the project turns `userTables` on, nothing otherwise, even
-* when the definitions are in the database. Each definition that does not fit the database is printed as a warning and
-* left out, so the rest of the app serves.
-*/
-const projectUserTables = async (project, db) => {
-	if (!project.userTables) return;
+/** The tables users add, read from their definitions; each one that does not fit the database is printed as a warning and left out, so the rest of the app serves. */
+const readUserTables = async (db) => {
 	const tables = await userTableResources(db);
 	for (const problem of tables.problems) console.warn(problem);
 	return tables;
 };
 //#endregion
-export { projectUserTables };
+export { readUserTables };

package/dist/project/served-admin.js

@@ -0,0 +1,29 @@
+import { projectAdmin } from "./project-admin.js";
+import { readUserTables } from "./project-user-tables.js";
+import { readDefinitionsVersion, reloadingAdmin } from "@protobase/server";
+//#region src/project/served-admin.ts
+/**
+* The admin a serving process answers with (`projectAdmin`). With `userTables`, it is rebuilt when the tables users add
+* change (`reloadingAdmin`); the version is read before the tables, so a change between the two reads rebuilds again.
+*/
+const servedAdmin = async (project, db, ui, reporter, upgradeWebSocket) => {
+	if (!project.userTables) return projectAdmin(project, db, ui, reporter, upgradeWebSocket);
+	const version = await readDefinitionsVersion(db);
+	const tables = await readUserTables(db);
+	const built = projectAdmin(project, db, ui, reporter, upgradeWebSocket, tables);
+	const admin = reloadingAdmin({
+		first: built.admin,
+		version,
+		readVersion: () => readDefinitionsVersion(db),
+		readTables: () => readUserTables(db),
+		build: (changed) => projectAdmin(project, db, ui, reporter, upgradeWebSocket, changed).admin,
+		report: built.report,
+		...project.options?.statementTimeoutMs !== void 0 && { statementTimeoutMs: project.options.statementTimeoutMs }
+	});
+	return {
+		...built,
+		admin
+	};
+};
+//#endregion
+export { servedAdmin };

package/dist/serve/lifecycle.js

@@ -1,8 +1,7 @@
 import { protobaseVersion } from "../version/version.js";
 import { buildProject } from "../config/build-project.js";
 import { openAppDatabase } from "../project/pg-db.js";
-import { projectAdmin } from "../project/project-admin.js";
-import { projectUserTables } from "../project/project-user-tables.js";
+import { servedAdmin } from "../project/served-admin.js";
 import { apiWebSocketServer } from "../project/api-websocket-server.js";
 import { withClientAddress } from "./client-address.js";
 import { createServeApp } from "./app.js";
@@ -50,7 +49,7 @@ const startServe = async ({ config, env, openDb = (url) => openAppDatabase(pg, u
 		if (env.startupChecks && project.notifications) await checkNotificationsSchema(db, project.notifications, project.auth);
 		if (env.startupChecks && project.status) await checkStatusSchema(db, project.status, project.auth);
 		if (env.startupChecks && project.userTables) await checkUserTablesSchema(db);
-		const { admin, jobs: definitions } = projectAdmin(project, db, ui, reporter, upgradeWebSocket, await projectUserTables(project, db));
+		const { admin, jobs: definitions } = await servedAdmin(project, db, ui, reporter, upgradeWebSocket);
 		if (env.startupChecks && definitions.length > 0) await checkJobsSchema(db);
 		const jobs = definitions.length > 0 ? await startJobs({
 			db,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@protobase/cli",
-  "version": "0.67.1",
+  "version": "0.67.3",
   "description": "The protobase command: scaffold, doctor, dev, build and serve a Protobase project.",
   "license": "Apache-2.0",
   "repository": {
@@ -43,12 +43,12 @@
   "dependencies": {
     "@better-auth/core": "^1.7.12",
     "@hono/node-server": "^2.1.3",
-    "@protobase/config": "0.67.1",
-    "@protobase/layout": "0.67.1",
-    "@protobase/query": "0.67.1",
-    "@protobase/schema": "0.67.1",
-    "@protobase/server": "0.67.1",
-    "@protobase/ui": "0.67.1",
+    "@protobase/config": "0.67.3",
+    "@protobase/layout": "0.67.3",
+    "@protobase/query": "0.67.3",
+    "@protobase/schema": "0.67.3",
+    "@protobase/server": "0.67.3",
+    "@protobase/ui": "0.67.3",
     "@react-email/render": "^2.1.0",
     "@scalar/hono-api-reference": "^0.12.10",
     "@sentry/vite-plugin": "^11.10.0",

package/skills/protobase/references/features/user-tables.md

@@ -4,4 +4,5 @@
 
 Tables an app's users add from the admin UI, by hand or from a file, and how a developer takes them over as code.
 
-- [Tables defined in the database](../reference/user-tables.md#tables-defined-in-the-database) (in development): An app can keep tables its users add in its own database, beside the resources in its code. Each one is served like any other resource, at its own address with its list, record page, search and API.
+- [Tables defined in the database](../reference/user-tables.md#tables-defined-in-the-database) (beta): An app can keep tables its users add in its own database, beside the resources in its code. Each one is served like any other resource, at its own address with its list, record page, search and API. Every server process serves a new table within a second of it being added.
+- [Adding tables and fields](../reference/user-tables.md#adding-tables-and-fields) (beta): Someone with the right capability creates a table with its fields, or adds a field to it, through the app's API. It is created in the database at once, with an index for every field but JSON so its list can be filtered by it, and sorted by it unless it is long text, and only admins can see it until access is widened.

package/skills/protobase/references/reference/user-tables.md

@@ -8,17 +8,15 @@ An app's resources usually come from its code: a resource in `config/` over a ta
 
 ## Tables defined in the database
 
-:::caution[In development]
-The server serves the tables defined in the database, but no screen or API adds one yet, and a definition added while the server runs is served after its next start.
+:::caution[Beta]
+Tables are added through the API; the admin app has no screens for them yet, and an app with organizations cannot use them.
 :::
 
-An app can keep tables its users add in its own database, beside the resources in its code. Each one is served like any other resource, at its own address with its list, record page, search and API.
+An app can keep tables its users add in its own database, beside the resources in its code. Each one is served like any other resource, at its own address with its list, record page, search and API. Every server process serves a new table within a second of it being added.
 
 <gap>Tables users add cannot be used in an app with organizations because keeping their definitions and records per organization is not implemented.</gap>
 
-<gap>Users cannot add a table yet because the screens and the API that create one are not implemented.</gap>
-
-<gap>A definition added while the server runs cannot be served before the server starts again because picking up new definitions while serving is not implemented.</gap>
+<gap>Users cannot add a table from the admin app yet because its screens for tables and fields are not implemented.</gap>
 
 ### The switch
 
@@ -36,7 +34,7 @@ The tables users add and their definitions live in two schemas of the app's data
 
 | Schema | What it holds |
 | --- | --- |
-| `table_definitions` | `tables`, one row per table users add, with its singular and plural label, its title field and a version, and `fields`, each table's fields in order with their label, type and whether they are required |
+| `table_definitions` | `tables`, one row per table users add, with its singular and plural label, its title field and a version, 1 when it is added, and `fields`, each table's fields in order with their label, type and whether they are required |
 | `user_tables` | the tables themselves, each with an `id bigint generated always as identity` primary key and a column per field |
 
 With `userTables: true`, [`protobase migration`](cli.md#migration) writes a migration that adds both schemas, the two definition tables and the `pg_trgm` extension when it is missing, and grants the app's database role CREATE on `user_tables` alone, and writes nothing once the database has them. Apply it with the project's other migrations; the server never creates them itself. At start, `dev` and `serve` stop while one is missing; `PROTOBASE_STARTUP_CHECKS=false` skips that check in `serve`, as it skips the others.
@@ -45,7 +43,7 @@ A table's name and each field's name are lowercase letters, digits and `_`, star
 
 | Type | Column |
 | --- | --- |
-| `text` | `text` |
+| `text` | `text`, at most 500 characters |
 | `long_text` | `text`, edited in a box that grows with it |
 | `integer` | `integer` |
 | `decimal` | `numeric`, with the definition's precision and scale |
@@ -56,7 +54,7 @@ A table's name and each field's name are lowercase letters, digits and `_`, star
 
 ### How each one is served
 
-When the server starts, it builds a resource from each definition and serves it at `/<table>` and `/<table>/<key>`, with its list, record page, sidebar entry and REST API, and in `/meta`, as the resources in code are. The labels of the definition are the names the admin app shows.
+When the server starts, and whenever the definitions change, it builds a resource from each definition and serves it at `/<table>` and `/<table>/<key>`, with its list, record page, sidebar entry and REST API, and in `/meta`, as the resources in code are. The labels of the definition are the names the admin app shows.
 
 - **Admins only.** Each table starts readable and writable by admins alone, and is left out of role wildcards such as `*.read`, like [resources that need named grants](access.md#resources-that-need-named-grants). Anyone else does not see it.
 - **Filters, sorts and search follow the indexes.** A field can be filtered or sorted by only where an index of its table serves it, and the table is searched on its title field only where a `pg_trgm` index serves that field, as the [scan guard](config/api.md#scanguard) requires.
@@ -64,3 +62,61 @@ When the server starts, it builds a resource from each definition and serves it
 - **Code wins.** A resource in code with the name of a user table, or a resource Protobase adds, is served instead of the definition, which is then ignored with a warning. This is how a table users added is taken over as code.
 
 [`protobase doctor`](cli.md#doctor) checks each definition against the database as it checks the resources in code, and reports each one it leaves out as an error. [`protobase scaffold`](cli.md#scaffold) leaves out the definitions and the tables in `user_tables` that no resource in code claims, and writes and updates the config of one that a resource claims.
+
+### How a change reaches every process
+
+A definition added, changed or removed is served by every process of `serve` and `dev` within a second, with no restart or deploy. Each process rebuilds its API from the resources in code and the definitions as they are now, and the admin app reads `/meta` again once `X-Meta-Version` changes, as after any change of the app.
+
+- **When a process looks.** `/meta` reads the version of the definitions before it answers. Any other request reads it at most once a second, and once more when its answer would be not found or invalid, as for a resource or field the process does not know yet; it is then answered by the API built anew. The process that makes a change rebuilds at once. A process that receives no request reads nothing, so a database that scales to zero stays asleep.
+- **What a look costs.** The version is one read of `table_definitions.tables`, under a millisecond. Each change of a definition or its fields must raise the definition's `version`, as the [API](#adding-tables-and-fields) does; adding and removing a definition change the version on their own.
+- **What a rebuild costs.** A rebuild reads the definitions and the indexes of each table, then builds the API, once per process: requests that need it meanwhile wait for that one rebuild. Requests already running finish on the API they started with, and an open WebSocket keeps the API it opened with. <gap>A channel socket opened before a rebuild cannot hear a broadcast sent through a socket opened after it, in the same process and without a realtime relay, until it reconnects, because moving open sockets to the rebuilt API is not implemented.</gap> <gap>A request whose body is not JSON, such as a file upload, that names a table added under a second ago cannot be answered by the rebuilt API because replaying a streamed body is not implemented; it is answered not found, and works on the next try.</gap>
+
+Measured with six fields per table on one machine running Postgres and the processes:
+
+| Tables users add | Rebuild, one process | Rebuild, each of four processes at once |
+| --- | --- | --- |
+| 10 | under 0.1 s | under 0.1 s |
+| 50 | about 0.1 s | about 0.4 s |
+| 200 | about 0.7 s | about 1.4 s |
+| 500 | about 2.2 s | about 4.5 s |
+
+- **A rebuild that fails.** When the API does not build with the new definitions, the process reports why through `onUnhandledError`, naming what it refused, and keeps serving the API it had, with the resources in code. A definition that does not fit the database is left out with a warning, as at start.
+
+## Adding tables and fields
+
+:::caution[Beta]
+Tables and fields are added through the API alone, and are never renamed, retyped or removed.
+:::
+
+Someone with the right capability creates a table with its fields, or adds a field to it, through the app's API. It is created in the database at once, with an index for every field but JSON so its list can be filtered by it, and sorted by it unless it is long text, and only admins can see it until access is widened.
+
+With `userTables: true`, the API has two resources over the definitions, with the standard addresses and routes of the [API](api.md#routes):
+
+| Resource | A record | Fields |
+| --- | --- | --- |
+| `user_tables` | a table, at `/user_tables/<id>` | `name`, `singular_label`, `plural_label`, `title_field`; `id` and `version` are read-only |
+| `user_table_fields` | a field of a table, at `/user_table_fields/<table_id>,<name>` | `table_id`, `name`, `label`, `type` (one of the [types](#the-schemas-and-the-migration)), `position` (the order of the fields), `precision` and `scale` (a decimal's alone) |
+
+Make a table with its fields in one [batch write](api.md#batch-writes): the table, each field pointing at it, then the title field, which names a field that exists. Nothing is created unless all of it is:
+
+```json
+POST /api/v1:batchWrite
+{
+  "ops": [
+    { "op": "create", "resource": "user_tables", "ref": "notes", "data": { "name": "notes", "singular_label": "Note", "plural_label": "Notes" } },
+    { "op": "create", "resource": "user_table_fields", "data": { "table_id": { "$ref": "notes" }, "name": "title", "label": "Title", "type": "text", "position": 1 } },
+    { "op": "create", "resource": "user_table_fields", "data": { "table_id": { "$ref": "notes" }, "name": "price", "label": "Price", "type": "decimal", "precision": 12, "scale": 2, "position": 2 } },
+    { "op": "update", "resource": "user_tables", "key": { "$ref": "notes" }, "etag": "*", "data": { "title_field": "title" } }
+  ]
+}
+```
+
+The answer has the table's record, so a client links to the new table at `/notes`. Add a field later with `POST /api/v1/user_table_fields`.
+
+- **What is created.** A table is `user_tables.<name>` with its `id bigint generated always as identity` key. A field is a column of its type that may be empty and has no default, so the records already there read it as empty, with a btree index, or a `pg_trgm` one for `long_text`, whose values outgrow a btree entry; a `json` field gets none. The title field gets the `pg_trgm` index search needs. The rows, the table or column, its indexes and the version are written in one transaction: anything refused leaves none of them.
+- **What is refused.** A name outside the rules above, a word SQL reserves such as `select`, a field named `id` or `etag`, the name of a resource in code, of one Protobase adds or of a table `user_tables` already has, a field marked required, and a title that is not a text field of the table, each as a field error. A table whose name a resource in code has now, which is how a table is taken over as code, takes no new field.
+- **What changes later.** The labels, the title field and the fields' order. A name, a type, a decimal's precision and scale never change, and no table or field is deleted. <gap>A table or field cannot be renamed, retyped or removed because changing the columns of a table users add is not implemented.</gap>
+- **Who may.** With the project's [roles](access.md), the capabilities `user_tables.create` and `user_table_fields.create`, which only `*` or a grant that names them holds, never a wildcard such as `*.create`, like [resources that need named grants](access.md#resources-that-need-named-grants); reading and changing the labels need `.read` and `.update` the same way. Without roles, admins. Each new table is its admins' alone, as [each one served](#how-each-one-is-served) is.
+- **The rest of the app.** The process that made the change serves it before it answers, and every other process [within a second](#how-a-change-reaches-every-process). While a deploy freezes writes, a create answers as any write does then.
+
+Building an index holds the table's write lock, and every statement runs within the request's statement timeout (1 s by default). <gap>Adding a field to a table with very many records cannot finish when building its index takes longer than the statement timeout, because building it outside the request is not implemented; the field is then not added.</gap>