@protobase/cli 0.68.5 → 0.68.6

package/dist/cli/package.js

@@ -1,4 +1,4 @@
 //#region package.json
-var version = "0.68.5";
+var version = "0.68.6";
 //#endregion
 export { version };

package/dist/serve/app.js

@@ -1,13 +1,13 @@
 import { isApiPath } from "../bundle/manifest.js";
 import { serveStaticSite } from "./static-site.js";
-import { jobsDrainApp, webauthnPath } from "@protobase/server";
+import { jobsDrainApp, redactHookKeys, webauthnPath } from "@protobase/server";
 import { Hono } from "hono";
 import { mobileReleaseApp } from "@protobase/server/mobile-release";
 import { logger } from "hono/logger";
 //#region src/serve/app.ts
 const createServeApp = ({ admin, requestLog, site, jobs, drainToken, mobile }) => {
 	const app = new Hono();
-	if (requestLog) app.use(logger());
+	if (requestLog) app.use(logger((line) => console.log(redactHookKeys(line))));
 	app.get("/health", (c) => c.json({ status: "ok" }));
 	if (site) app.use("*", (c, next) => isApiPath(site.api, c.req.path) || c.req.path === webauthnPath ? next() : serveStaticSite(site, c.req.raw));
 	if (jobs && drainToken) app.route("/", jobsDrainApp({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@protobase/cli",
-  "version": "0.68.5",
+  "version": "0.68.6",
   "description": "The protobase command: scaffold, doctor, dev, build and serve a Protobase project.",
   "license": "Apache-2.0",
   "repository": {
@@ -43,13 +43,13 @@
   "dependencies": {
     "@better-auth/core": "^1.7.12",
     "@hono/node-server": "^2.1.3",
-    "@protobase/config": "0.68.5",
-    "@protobase/layout": "0.68.5",
-    "@protobase/query": "0.68.5",
-    "@protobase/schema": "0.68.5",
-    "@protobase/server": "0.68.5",
-    "@protobase/ui": "0.68.5",
-    "@protobase/workflows": "0.68.5",
+    "@protobase/config": "0.68.6",
+    "@protobase/layout": "0.68.6",
+    "@protobase/query": "0.68.6",
+    "@protobase/schema": "0.68.6",
+    "@protobase/server": "0.68.6",
+    "@protobase/ui": "0.68.6",
+    "@protobase/workflows": "0.68.6",
     "@react-email/render": "^2.1.0",
     "@scalar/hono-api-reference": "^0.12.10",
     "@sentry/vite-plugin": "^11.10.0",

package/skills/protobase/references/features/workflows.md

@@ -22,5 +22,5 @@ Graphs of steps the server runs along their edges, written in a language a model
 - [Triggers turned on while the app runs](../reference/workflows.md#turning-a-trigger-on) (beta): Turning a workflow on or changing it registers its trigger on every server process, and turning it off stops it, with no restart.
 - [Workflow plugins](../reference/workflow-plugins.md) (beta): A file of the app or a package adds trigger and step types through the same public interface the built-in types use.
 - [Guarded requests for plugins](../reference/workflow-plugins.md#sending-requests) (beta): A plugin's requests reach only public addresses, follow no redirect and read at most 10 MiB.
-- [Webhook trigger](../reference/workflow-plugins.md#polling-and-webhook-coming-soon) (coming soon): A workflow starts from a signed call to a URL of its own, limited in size and rate.
-- [Polling triggers](../reference/workflow-plugins.md#polling-and-webhook-coming-soon) (coming soon): A workflow starts once for each new item a plugin finds on a schedule.
+- [Webhook trigger](../reference/workflows.md#appwebhook) (beta): A workflow starts from a signed call to a URL of its own, limited in size and rate.
+- [Polling triggers](../reference/workflow-plugins.md#polling-and-webhook) (beta): A workflow starts once for each new item a plugin finds on a schedule, or at each time of a cron with `app:schedule`.

package/skills/protobase/references/reference/mcp.md

@@ -100,6 +100,8 @@ In an app with [workflows](workflows.md), the endpoint also offers tools to find
 | `create_workflow` | Creates a workflow from its `name`, `description` and `definition`, and returns its `id`, `version` and ETag. |
 | `update_workflow` | Changes a workflow's definition by an `edit` or whole, as `definition`, and its `name` or `description`, and returns its `id`, new `version` and ETag. With the `etag` that `get_workflow` gave, it fails when someone changed the workflow since; an edit without one is saved against the version it was applied to. |
 | `run_workflow` | Starts a run of a workflow's current version with an `input`, and returns its `run_id` and `status`. With `rerun_of`, a failed run's id, it [re-runs](workflows.md#re-running-a-failed-run) that run from its failed step instead, on its input. |
+| `enable_workflow` | Turns a workflow's trigger on (`enabled: true`) on the `version` it names, which must be the workflow's current version, or off, and returns its `enabled_version`. A workflow saved since that version is refused, so an approval never turns on a version the person did not see. |
+| `new_webhook_url` | Issues a new URL and secret for a workflow that starts from a [webhook trigger](workflows.md#appwebhook), keeping a `secret` the sender makes when given one, and returns `{ url, secret }`, shown this once; the old URL stops. |
 
 An `edit` is a [JSON Merge Patch](https://www.rfc-editor.org/rfc/rfc7396) of the saved definition: only what changes, at its place in the document. An object merges into the object there, `null` removes a member, and any other value, a list included, replaces it. Changing one argument of one state:
 
@@ -109,7 +111,7 @@ An `edit` is a [JSON Merge Patch](https://www.rfc-editor.org/rfc/rfc7396) of the
 
 A definition the save check refuses comes back as `{ "error": "invalid-workflow", "message": ..., "problems": [...] }`, each problem `{ state, field, message }` naming the state, the field and what was expected, as `validate_workflow` gives them; nothing is saved. A run goes on in the background on the app's jobs, so `run_workflow` returns while it is queued: the client reads it with `get_workflow_run` until it has `succeeded` or `failed`, fixes the step that failed and re-runs the run from that step.
 
-`create_workflow`, `update_workflow` and `run_workflow` come with [writes](#writes), and the person approves each first, as any write, unless they hold one of the [roles without approval](#writes-without-approval); with writes limited to some resources, they save on `workflows` and run on `workflow_runs` only when the list names them. A run does what its steps do, such as requests, mail and record writes, as the workflow's last editor.
+`create_workflow`, `update_workflow`, `run_workflow`, `enable_workflow` and `new_webhook_url` come with [writes](#writes), and the person approves each first, as any write, unless they hold one of the [roles without approval](#writes-without-approval); with writes limited to some resources, they save, turn on and issue URLs on `workflows` and run on `workflow_runs` only when the list names them. A run does what its steps do, such as requests, mail and record writes, as the workflow's last editor.
 
 ### SQL for admins
 

package/skills/protobase/references/reference/workflow-plugins.md

@@ -5,7 +5,7 @@
 Add trigger and step types to workflows from a file of the app or a package, through the same public interface the built-in types use.
 
 :::caution[Beta]
-Plugins add step types and record triggers. Polling and webhook triggers, and connections that sign in to other services, are declared by the interface but not run yet.
+Plugins add step types and record triggers. They also add polling and webhook triggers, which run within the limits named in their section. The interface declares connections that sign in to other services, but the app does not run them.
 :::
 
 A **workflow plugin** adds trigger and step types to an app's [workflows](workflows.md). Its types are named `<plugin>:<key>`, such as `acme:create_invoice`: a Task names a step type as its `Resource`, and a definition's `trigger` holds a trigger type with its input. The built-in types, `app:http`, `app:records.*`, `app:mail.send` and `app:record`, are a plugin like any other, written on this interface alone, so a plugin's types run, are checked and are listed exactly as they are.
@@ -106,9 +106,41 @@ triggers: {
 
 `ready(input)` returns the resource, the readiness filter `when` and the `key` fields, and the engine runs it as it runs [`app:record`](workflows.md#apprecord): one run per record that becomes ready, at the time its time terms say, read as the user who saved the enabled version.
 
-### Polling and webhook (coming soon)
+### Polling and webhook
 
-The interface declares `kind: 'polling'`, with `every`, `limit`, `start` and `poll`, and `kind: 'webhook'`, with `receive` and the optional `subscribe`, `unsubscribe` and `renew`. <gap>A plugin's polling or webhook trigger cannot start runs because the polling and webhook triggers are not implemented; the type list marks them unavailable.</gap>
+:::caution[Beta]
+A webhook trigger that registers its own URL cannot subscribe in an app without sign-in, since it reads the app's public address from the sign-in origin, and a poll job that failed all its tries does not start polling again by itself: retry it in the admin app's failed jobs.
+:::
+
+A polling trigger asks a service for new items on a schedule:
+
+```ts
+type Order = { id: number }
+
+triggers: {
+  new_order: {
+    kind: 'polling',
+    label: 'New order',
+    description: 'Starts a run for each new order.',
+    input: { type: 'object', properties: {} },
+    every: '1 minute',
+    limit: 50,
+    start: async ({ fetch }) => ((await (await fetch('https://shop.example/orders/latest')).json()) as Order).id,
+    // At most `limit` orders after the cursor, oldest first, so a backlog is read a page per poll.
+    poll: async ({ fetch, cursor, limit }) => {
+      const orders = (await (await fetch(`https://shop.example/orders?after=${cursor}&sort=asc&limit=${limit}`)).json()) as Order[]
+      return { items: orders.map((order) => ({ key: String(order.id), data: order })), cursor: orders.at(-1)?.id ?? cursor }
+    },
+  },
+},
+```
+
+- **`every`** is a duration such as `1 minute` (at least a second), or `{ cron, timezone }`, or a function of the trigger's input that returns either, as `app:schedule` takes its cron from its input. `poll` gets `due`, the time the poll was queued for.
+- **`start`** runs once, when the trigger has no cursor yet, and returns the first one. A new version of the workflow with the same trigger input keeps the cursor and does not call `start` again; turning the workflow off and on starts afresh.
+- **`poll`** gets `limit` and returns the items after `cursor`, at most `limit` (default and most 100), each with a `key` of at most 200 characters, and the next cursor, which must move forward. Each item starts one run, once per workflow and key, with `data` as its input, so an item seen on two polls, or by two processes, starts one run. `store` keeps a value of its own across polls; the cursor and `store` hold at most 64 KiB of JSON each.
+- **Errors:** a poll that throws, gives more items than its limit, or a cursor too large, starts no run: the trigger is `failing` with the error, and polls again after `every × 2ⁿ`, at most an hour; its next poll that works makes it `active`. A `ConnectionError('link_expired')` makes every trigger on that sign-in `needs_sign_in` and stops their polls. A poll may run for five minutes, when its `signal` aborts. A poll that was still running as its workflow was turned off or changed starts nothing.
+
+A webhook trigger takes calls of the workflow's URL, [as `app:webhook` does](workflows.md#appwebhook): the route checks the call's signature by the trigger's `auth` before any plugin code sees it, so spread `webhookInputProperties` from `@protobase/workflows/plugin` into the trigger's input schema. `receive({ request, event, idempotencyKey, input })` gets the call without its credentials and returns the runs it starts, at most 100, each `{ key?, data }`, and may return a `response` to answer with instead of `202`, such as a service's URL check. A trigger that registers its URL with its service itself declares `subscribe({ url, secret })`, which the app calls as the workflow is turned on with a URL it makes for the plugin alone, and returns the subscription's state; `renew` runs every `renewEvery` (default `1 day`) and `unsubscribe` as the workflow is turned off or deleted. Such a trigger has no URL to issue by hand. <gap>A webhook trigger that registers its own URL cannot subscribe in an app without sign-in because reading the app's public address from elsewhere is not implemented.</gap> <gap>A poll job that failed all its tries cannot start polling again by itself because a sweep of stopped triggers is not implemented; retrying the job in the admin app's failed jobs resumes it.</gap>
 
 ## Connections (coming soon)
 

package/skills/protobase/references/reference/workflows.md

@@ -5,7 +5,7 @@
 Workflows written as state machines in the States Language with JSONata, checked on every save, run step by step on the app's jobs, with every step recorded.
 
 :::caution[Beta]
-Workflows run when they are started through the API, over MCP, from the app's code or by a test run in the admin app, and from record triggers. Webhook and polling triggers and sign-in connections to other services are not built yet.
+Workflows run when they are started through the API, over MCP, from the app's code or by a test run in the admin app, and from record triggers. They also run from webhook and polling triggers, within the limits named at the start of their sections. Connections that sign in to other services are not built.
 :::
 
 A **workflow** is a graph of steps that the server runs one after another along its edges: fetch something, decide, run a step per item, merge the results, write records, send mail. It is written in the **States Language**, the JSON language of state machines, with **JSONata** as its expression language, in a profile of it that reads every value by name. A language model writes and changes it as easily as a person, and the same document is what the server checks and runs. Each run records every step it takes, with what the step read, gave and logged.
@@ -91,12 +91,14 @@ The document holds no layout: the [editor](#the-editor) lays the graph out from
 | --- | --- | --- |
 | `"manual": {}` | when someone runs the workflow by hand | the input they give |
 | `"app:record": { "resource", "when", "key" }` | once per record that becomes ready | `{ resource, key, record }` |
+| `"app:webhook": { "auth", "body_limit", "per_minute", "idempotency_header" }` | once per signed call of its [URL](#appwebhook) | `{ method, query, headers, body, received_at }` |
+| `"app:schedule": { "cron", "timezone" }` | at each time of the [cron](#appschedule) | `{ scheduled_at }` |
 
-A save checks a trigger's input against its type, and refuses a type the app does not have; besides triggers, runs start through the API, over MCP, from the app's code or by a [test run](#test-runs). <gap>A workflow cannot start from a webhook or a schedule because the webhook and polling triggers are not implemented.</gap>
+A save checks a trigger's input against its type, and refuses a type the app does not have; besides triggers, runs start through the API, over MCP, from the app's code or by a [test run](#test-runs). A plugin's [polling and webhook triggers](workflow-plugins.md#polling-and-webhook) run as these do.
 
 ### Turning a trigger on
 
-A trigger starts runs only while its workflow is enabled: set `enabled_version` to the workflow's current `version` through the API, in the admin app or with `update_workflow`, and clear it to turn the workflow off. The trigger is registered at once on every server process, with no restart, and its status, `active` or `failing` with its last error, is the workflow's `workflow_triggers` record. A save of an enabled workflow keeps it enabled on its new version; a new version whose trigger has the same type and input keeps the running trigger as it is, and any other change stops it and starts the new one. A trigger's runs run the enabled version and act as the user who saved it, as any run acts as its version's saver.
+A trigger starts runs only while its workflow is enabled: set `enabled_version` to the workflow's current `version` through the API, in the admin app or with `enable_workflow` over MCP, and clear it to turn the workflow off. The trigger is registered at once on every server process, with no restart, and its status, `active` or `failing` with its last error, is the workflow's `workflow_triggers` record. A save of an enabled workflow keeps it enabled on its new version; a new version whose trigger has the same type and input keeps the running trigger as it is, and any other change stops it and starts the new one. A trigger's runs run the enabled version and act as the user who saved it, as any run acts as its version's saver.
 
 ### app:record
 
@@ -113,6 +115,49 @@ Starts one run for each record of `resource` that becomes ready, with `{ resourc
 
 <gap>A workflow cannot start on every write of a record because an every-change trigger is not implemented; a record trigger starts once per ready key.</gap> <gap>An owner is not told when a trigger starts failing because trigger alerts are not implemented.</gap>
 
+### app:webhook
+
+:::caution[Beta]
+The rate limits count per server process, not across the app's processes, and `hmac` and `bearer` calls carry no signed time, so a captured call can be sent again while its secret holds.
+:::
+
+```json
+{ "app:webhook": { "auth": { "mode": "standard_webhooks" }, "body_limit": 1048576, "per_minute": 60 } }
+```
+
+Starts one run for each call of the workflow's own URL that proves it comes from the sender. Issue the URL with `POST /api/workflows/<id>/webhook`, or `new_webhook_url` over MCP: it answers `{ url, secret }` once, and the app keeps only a hash of the URL's key, so it cannot show them again. Send `{ "secret": "<secret>" }` to keep a secret the sender makes itself, such as a Standard Webhooks sender's `whsec_` secret. A new URL replaces the old one, which stops at once. Issuing needs the right to change that workflow in your tenant; anyone else gets 404.
+
+The sender POSTs to the URL. `auth.mode` says how a call proves it is theirs:
+
+| `auth.mode` | The call carries |
+| --- | --- |
+| `standard_webhooks` (default) | `webhook-id`, `webhook-timestamp` and `webhook-signature` headers, signed with the secret, at most five minutes off |
+| `hmac` | an HMAC of the body in `auth.header`, after `auth.prefix`, in `auth.encoding` (`hex` or `base64`) with `auth.hash` (`SHA-1`, `SHA-256` or `SHA-512`) |
+| `bearer` | `Authorization: Bearer <secret>`; the app keeps only the secret's hash |
+
+The run's input is `{ method, query, headers, body, received_at }`: `body` is the JSON, the form fields (a field sent more than once as the list of its values) or the text sent, and a body that makes the input over 1 MiB as JSON, as escaping can before the body itself is, is a [file of the run](#file-values), which needs the app's `private` file provider. `headers` leaves out `authorization`, `cookie`, every `webhook-*` header, the `hmac` signature header and `x-forwarded-*`, and the input never holds the URL, so no credential reaches a run, its steps or its logs. A `standard_webhooks` call starts one run per `webhook-id`, and an `hmac` or `bearer` call one per value of the `idempotency_header` you name, when it sends one; the id is at most 200 characters.
+
+The URL answers `202 { run_id }`, or:
+
+| Status | When |
+| --- | --- |
+| `404` | the key opens no running webhook trigger: unknown, turned off, deleted, or the enabled version has another trigger |
+| `405` | the method is not POST |
+| `413` | the body is over `body_limit` bytes (default 1 MiB, at most 10 MiB, counted as it streams) |
+| `401` | the call does not carry the secret or a valid signature |
+| `429` | over `per_minute` calls that passed the check (default 60), or the client's address was refused 30 times this minute |
+| `400` | the id is over 200 characters |
+
+Only calls that pass the check spend `per_minute`, so someone who knows the URL but not the secret cannot use up the sender's calls. Both limits count per server process, so an app on several processes takes up to that many times more. The client's address is the one [`protobase serve`](cli.md#serve) sets from the connection and its trusted proxies; behind another server, the limit is only as good as the `X-Forwarded-For` it sets. `hmac` and `bearer` calls carry no signed time, so a captured call can be sent again while its secret holds, unless the sender's idempotency header stops it; `standard_webhooks` refuses one more than five minutes old. Changing `auth.mode` to `standard_webhooks` or `hmac` after a `bearer` URL needs a new URL, since the app kept only the bearer secret's hash. With the request log on, the log shows the URL's key as `[key]`.
+
+### app:schedule
+
+```json
+{ "app:schedule": { "cron": "0 6 * * 1-5", "timezone": "Europe/Amsterdam" } }
+```
+
+Starts one run at each time of a five-field cron, in an IANA time zone (default UTC), with `{ scheduled_at }` as the run's input. A time starts one run however many server processes run. The first time missed while no worker ran starts one run, late, once a worker is back; the other times missed meanwhile start none.
+
 ### Reading values by name
 
 Every value a state reads has a name, so a step can be moved or added without the data around it shifting:
@@ -170,7 +215,7 @@ A workflow's record page in the admin app, `/workflows/<id>`, and its create for
 - **A step's form.** Choosing a step opens its form under the graph: its name, which renames every edge to it, then what its type takes. A Task's form has a field for each argument of its step type's input schema, the ones it needs marked, as listed under [Step types](#step-types) or by its plugin: one line of text for a string, one line for a number or `true` and `false`, a menu for an `enum`, names separated by commas for a list of strings, and JSON for anything else; a Choice's its rules and `Default`; a Map's `Items` and `MaxConcurrency`; a Wait's `Seconds`; a Pass's and a Succeed's `Output`; a Fail's `Error` and `Cause`. A field takes text, an expression (`{% ... %}`) or JSON; an emptied field is left out. *The step as JSON* edits every field of the step, such as `Retry`, `Assign` and `TimeoutSeconds`. The trigger's form picks among the app's triggers, its plugins' included, and draws a field for each input of its schema the same way, such as a record trigger's resource, `when` and `key`.
 - **Problems on their step.** The editor runs the same check as a save on every change, and shows each problem on the step it names, as a count on its box and a list and a marked field in its form; a problem with the trigger or the first step shows on the trigger, and one about the whole document above the graph. Saving writes the definition through the API, where the save hook checks it again: a definition with a problem is refused with the 400 above, whose `errors` hold each problem for the `definition` field, with its state and field in `params`.
 
-A save also refuses a Task that leaves out an argument its step type needs, such as `app:http` without `url`, unless its `Arguments` is one expression, whose value is known only when it runs. A user who may read a workflow but not change it sees the same graph and forms, locked. <gap>The editor cannot undo a single change because an undo history is not implemented.</gap> The editor reads the app's types from `GET /api/workflows/types`, `client.workflowTypes()` of `@protobase/client`: the same list `list_workflow_types` gives over MCP, `{ step_types, triggers }`, which only a user who may read workflows gets. A type the app cannot run, such as a polling or webhook trigger in this version, is not offered, and the check refuses it with the reason; a stored trigger of such a type, or of a type the app does not have, shows its type and input, read-only. <gap>A box on the graph cannot name a plugin's trigger or step type by its label because handing the type list to the canvas is not implemented; it shows the type's id, such as `acme:shout`.</gap>
+A save also refuses a Task that leaves out an argument its step type needs, such as `app:http` without `url`, unless its `Arguments` is one expression, whose value is known only when it runs. A user who may read a workflow but not change it sees the same graph and forms, locked. <gap>The editor cannot undo a single change because an undo history is not implemented.</gap> The editor reads the app's types from `GET /api/workflows/types`, `client.workflowTypes()` of `@protobase/client`: the same list `list_workflow_types` gives over MCP, `{ step_types, triggers }`, which only a user who may read workflows gets. A type the app cannot run, such as one that signs in to another service through a connection, is not offered, and the check refuses it with the reason; a stored trigger of such a type, or of a type the app does not have, shows its type and input, read-only. <gap>A box on the graph cannot name a plugin's trigger or step type by its label because handing the type list to the canvas is not implemented; it shows the type's id, such as `acme:shout`.</gap>
 
 ## Step types
 
@@ -260,7 +305,7 @@ A workflow's page offers *Test run*: the form for a new run of the workflow, who
 
 ## Over MCP
 
-An AI client signed in to the app's [MCP endpoint](mcp.md#workflows) finds, makes, changes and test-runs workflows as the person: it lists the step types and triggers with their arguments, reads and validates definitions, saves one whole or by a small edit, starts a run with an input, reads each of the run's steps with its input, output, logs and error, and re-runs a failed run from its failed step once the workflow is fixed. The endpoint's instructions describe this language, and a refused definition comes back with its problems, each naming the state and field.
+An AI client signed in to the app's [MCP endpoint](mcp.md#workflows) finds, makes, changes and test-runs workflows as the person: it lists the step types and triggers with their arguments, reads and validates definitions, saves one whole or by a small edit, starts a run with an input, reads each of the run's steps with its input, output, logs and error, and re-runs a failed run from its failed step once the workflow is fixed, turns a workflow's trigger on and off, and issues a webhook trigger's URL. The endpoint's instructions describe this language, and a refused definition comes back with its problems, each naming the state and field.
 
 ## Who may do what