@procedures/client 0.7.0 → 0.8.1

package/src/events/rows.ts

@@ -0,0 +1,48 @@
+/**
+ * The rows of a published value (roadmap CLI-15).
+ *
+ * A value on the bus can hold rows of data: the values of a collection change event, for example.
+ * The publisher of the remote events gives each row to the output filter of the host (`admitItem`)
+ * for each subscriber, and each subscriber gets only the rows that its filter admits. Thus a row
+ * reaches a subscriber only through the same filter as a read.
+ *
+ * The rows are on a symbol key, so they stay in this process: JSON does not take them.
+ */
+
+/** The rows of a published value, and the function that makes the value of the admitted rows. */
+export interface PublishedRows {
+  /** The rows that the output filter examines, one at a time. */
+  readonly rows: readonly unknown[];
+  /** This function gives the value for a subscriber: `admitted[i]` tells if row `i` stays. */
+  keep(admitted: readonly boolean[]): unknown;
+}
+
+/** The key of the rows of a published value. */
+const ROWS: unique symbol = Symbol.for("procedures.publishedRows");
+
+/**
+ * This function gives the value with its rows, for the filter of each subscriber. The rows are not
+ * enumerable: a copy or a comparison of the value does not see them.
+ */
+export function withRows<T extends object>(value: T, rows: PublishedRows): T {
+  return Object.defineProperty(value, ROWS, { value: rows, enumerable: false });
+}
+
+/** The rows of a published value, or undefined for a value without rows. */
+export function rowsOf(value: unknown): PublishedRows | undefined {
+  if (typeof value !== "object" || value === null) return undefined;
+  const rows = (value as { [ROWS]?: unknown })[ROWS];
+  return typeof rows === "object" && rows !== null ? (rows as PublishedRows) : undefined;
+}
+
+/**
+ * The value of a published value for one subscriber: each row passes the filter, and the value
+ * keeps the admitted rows. An error of the filter refuses the row. Without rows or a filter, the
+ * value stays as it is.
+ */
+export async function admittedValue(value: unknown, admit: ((item: unknown) => Promise<boolean>) | undefined): Promise<unknown> {
+  const rows = rowsOf(value);
+  if (!rows || !admit) return value;
+  const admitted = await Promise.all(rows.rows.map((row) => admit(row).catch(() => false)));
+  return rows.keep(admitted);
+}

package/src/events/types.ts

@@ -82,9 +82,10 @@ export interface EventBus {
    *
    * @param channel - Channel to subscribe to
    * @param handler - Handler function called for each event
+   * @param options - `onClear`: the function that runs when `clear()` or `clearAll()` removes the handler
    * @returns Unsubscribe function
    */
-  on<T>(channel: string, handler: EventHandler<T>): Unsubscribe;
+  on<T>(channel: string, handler: EventHandler<T>, options?: EventListenOptions): Unsubscribe;
 
   /**
    * Subscribe to a single event on a channel.
@@ -177,7 +178,8 @@ export interface TypedEventBus<TChannels extends ChannelMap> {
   emit<K extends keyof TChannels & string>(channel: K, data: TChannels[K]): void;
   on<K extends keyof TChannels & string>(
     channel: K,
-    handler: EventHandler<TChannels[K]>
+    handler: EventHandler<TChannels[K]>,
+    options?: EventListenOptions
   ): Unsubscribe;
   once<K extends keyof TChannels & string>(channel: K, options?: EventWaitOptions): Promise<TChannels[K]>;
   stream<K extends keyof TChannels & string>(channel: K, options?: EventWaitOptions): AsyncIterable<TChannels[K]>;
@@ -238,6 +240,17 @@ export interface EventWaitOptions {
   signal?: AbortSignal | undefined;
 }
 
+/**
+ * Options for a handler of `on()`.
+ */
+export interface EventListenOptions {
+  /**
+   * This function runs one time when `clear()` or `clearAll()` removes the handler. An unsubscribe
+   * does not run it. The publisher of the remote events uses it to end a subscription with notice.
+   */
+  onClear?: (() => void) | undefined;
+}
+
 /**
  * Options for creating an event bus.
  */

package/src/index.ts

@@ -27,7 +27,7 @@ export type {
   HttpUrlStrategy,
 } from "./server/index.js";
 export { WebSocketServerTransport } from "./server/index.js";
-export { checkBrowserRequest, isLoopbackHost, isLoopbackAddress, hostNameOf } from "./server/browser-guard.js";
+export { checkBrowserRequest, hasAmbientCredentials, isLoopbackHost, isLoopbackAddress, hostNameOf } from "./server/browser-guard.js";
 export type { BrowserGuardOptions, GuardedRequest, GuardResult } from "./server/browser-guard.js";
 export type {
   WebSocketServerTransportOptions,

package/src/node.ts

@@ -9,9 +9,30 @@
  * This entry also exports `writeNdjsonStream`: the writer of an NDJSON stream response of wire
  * protocol v1, for a `ServerResponse` of `node:http`. `HttpServerTransport` uses it, so a server
  * without Express writes the same bytes.
+ *
+ * The procedure faces of Node, without a framework (CLI-3): `procedureHandler(server, options)`
+ * gives a request handler `(req, res) => boolean`, and `upgradeHandler(server, options)` gives an
+ * upgrade handler `(req, socket, head) => boolean`. Each one takes the requests of its path and
+ * gives `false` for the others, so the routes of the application share the listener.
  */
 
 export { WebSocketTransport } from "./adapters/websocket/client/node.js";
 export type { WebSocketTransportOptions, WebSocketClass } from "./adapters/websocket/client/types.js";
 export { writeNdjsonStream } from "./adapters/http/server/ndjson.js";
 export type { WriteNdjsonStreamOptions, NdjsonError } from "./adapters/http/server/ndjson.js";
+export { procedureHandler } from "./adapters/http/server/handler.js";
+export type { ProcedureHandlerOptions, NodeProcedureHandler, NodeUrlStrategy, UrlRequest } from "./adapters/http/server/handler.js";
+export type { CorsOptions } from "./adapters/http/server/cors.js";
+export type { RoutedRequest } from "./adapters/http/server/strategies.js";
+export { defaultServerUrlStrategy, rpcServerUrlStrategy } from "./adapters/http/server/strategies.js";
+export { upgradeHandler } from "./adapters/websocket/server/upgrade.js";
+export type { UpgradeHandlerOptions, NodeUpgradeHandler } from "./adapters/websocket/server/upgrade.js";
+export { WebSocketServerTransport } from "./adapters/websocket/server/transport.js";
+export type {
+  WebSocketServerTransportOptions,
+  WebSocketAuthHandler,
+  WebSocketConnectionHandler,
+  WebSocketMessage,
+  TrackedConnection,
+  ConnectionEventHandler,
+} from "./adapters/websocket/server/types.js";

package/src/procedures/caller.ts

@@ -0,0 +1,45 @@
+/**
+ * A caller of the procedures of some modules (`callerOf`). An app that calls the procedures of a
+ * package in its own process repeats three steps: a registry of the module, `invokePath` and
+ * `outputValue`. A caller does the three steps in one call. jarvis had four copies of them.
+ */
+
+import type { ProcedurePath } from "./types.js";
+import type { ProcedureRegistry } from "./registry.js";
+import { invokePath, outputValue, type InvokeOptions } from "./invoke.js";
+import { createRegistry, type ModuleInput } from "./module.js";
+
+/** The options of one call of a caller: the options of `invokePath` without the registry. */
+export type CallerOptions = Omit<InvokeOptions, "registry">;
+
+/** A function that runs the procedures of its modules (`callerOf`). */
+export interface Caller {
+  /**
+   * This function runs the procedure at `path` with `input`, and gives one value: the value of
+   * the procedure, or the last item of its stream. An empty stream rejects with `NO_OUTPUT`, and
+   * a path that the modules do not have rejects with `NOT_FOUND`.
+   */
+  <T = unknown>(path: ProcedurePath, input: unknown, options?: CallerOptions): Promise<T>;
+  /** The registry of the caller: `clientModule` and the modules, with their required modules. The first use makes it. */
+  readonly registry: ProcedureRegistry;
+}
+
+/**
+ * This function makes a caller of the procedures of the modules. The registry of the modules is
+ * made at the first call, so a caller costs nothing until it calls. The caller does not use the
+ * global registry.
+ *
+ * @example
+ * const gh = callerOf(ghModule);
+ * const runs = await gh<RunList>(["gh", "run", "list"], { repo: "owner/name" });
+ */
+export function callerOf(...modules: ModuleInput[]): Caller {
+  let registry: ProcedureRegistry | undefined;
+  const registryOf = (): ProcedureRegistry => (registry ??= createRegistry(...modules));
+  const call = async <T = unknown>(path: ProcedurePath, input: unknown, options: CallerOptions = {}): Promise<T> => {
+    const output = await invokePath<T>(path, input, { ...options, registry: registryOf() });
+    return outputValue(output, path);
+  };
+  Object.defineProperty(call, "registry", { get: registryOf, enumerable: true });
+  return call as Caller;
+}

package/src/procedures/capability.ts

@@ -178,6 +178,27 @@ export function narrowCapability<C>(
   return Object.freeze(result) as Capability<C>;
 }
 
+/**
+ * This function joins two requests for a narrower capability into one request that asks for
+ * both: the meet of the clearances, the intersection of the effects and the shorter time. Two
+ * different clearances need an order.
+ */
+export function joinNarrowing<C>(first: CapabilityRequest<C>, second: CapabilityRequest<C>, order?: ClearanceOrder<C>): CapabilityRequest<C> {
+  const result: CapabilityRequest<C> = {};
+  if (first.clearance === undefined || second.clearance === undefined || Object.is(first.clearance, second.clearance)) {
+    const clearance = first.clearance ?? second.clearance;
+    if (clearance !== undefined) result.clearance = clearance;
+  } else {
+    if (!order) throw new Error("Two different clearances need a clearance order: give clearanceOrder to the host (hostPolicy)");
+    result.clearance = order.meet(first.clearance, second.clearance);
+  }
+  const effects = first.effects === undefined ? second.effects : narrowEffects(first.effects, second.effects);
+  if (effects !== undefined) result.effects = effects;
+  const times = [first.timeoutMs, second.timeoutMs].filter((time): time is number => typeof time === "number" && Number.isFinite(time) && time >= 0);
+  if (times.length > 0) result.timeoutMs = Math.min(...times);
+  return result;
+}
+
 // =============================================================================
 // Admission
 // =============================================================================

package/src/procedures/collection/changes.ts

@@ -0,0 +1,245 @@
+/**
+ * The change events of the collection procedures (roadmap CLI-15).
+ *
+ * Each write of a collection procedure publishes one `CollectionChangeEvent` on the bus of its
+ * context (`ctx.bus`), under the topic `collections.<name>`. A peer gets the events with
+ * `events.subscribe` (`documentation/EVENTS.md`). The delivery is at most once, so an event is a
+ * hint of a change, and a subscriber reads the changes when it misses an event.
+ *
+ * - `since` and `version` are versions of `getChangesSince()` of the storage. The events of one
+ *   storage and one topic make a chain: the `since` of an event is the `version` of the event
+ *   before it. A subscriber whose cursor is `since` applies the event without a read.
+ * - The changes of an event come from the change log of the storage, not from the input of the
+ *   write. Thus an event also has the writes that did not go through the procedures.
+ * - By default, an event has only the ids and the versions: a value reaches a caller only through
+ *   a read, with its checks. With the option `values`, an event has the values too, and each row
+ *   `{ id, value }` passes the output filter of the host for each subscriber (`withRows()`).
+ * - A subscriber of the topic must have a capability that reads `collection:<name>`, as for a
+ *   read (`collectionTopicReads()`, the default `reads` of `EventPublisher`).
+ * - An event without `since` only tells that the collection changed: the storage has no change
+ *   log, the chain starts, or the log dropped the version. The subscriber then reads the changes.
+ * - The chain starts at the version of a call of `changes` (a pull of a subscriber). While no one
+ *   listens to the topic, the procedures publish nothing, and the chain stops.
+ * - The publishes of one storage and one topic run one at a time, in the order of the writes. A
+ *   write resolves after its event is on the bus. A failed publish never fails the write.
+ */
+
+import type { EventBus } from "../../events/types.js";
+import { withRows } from "../../events/rows.js";
+import type { StorageChanges } from "./storage.js";
+
+/**
+ * The change event of the writes of a collection, on the topic `collections.<name>` (roadmap
+ * CLI-15). The data crosses the wire as JSON.
+ */
+export interface CollectionChangeEvent {
+  /**
+   * The version of the storage before the changes. A subscriber whose cursor is this version
+   * applies the changes without a read. Without it, the event is only a hint: read the changes.
+   */
+  readonly since?: string | undefined;
+  /** The version of the storage after the changes, when the storage has a change log. */
+  readonly version?: string | undefined;
+  /** The ids of the items that changed, in the order of the last change of each id. */
+  readonly changed: readonly string[];
+  /** The ids that were deleted. */
+  readonly deleted: readonly string[];
+  /**
+   * With the option `values`: the changed items that the output filter of the subscriber admits.
+   * Without the option, the event has no values: read them with `changes`.
+   */
+  readonly values?: ReadonlyArray<{ readonly id: string; readonly value: unknown }> | undefined;
+  /** With the option `values`: the changed ids whose values the filter refused. Read them with `changes`. */
+  readonly withheld?: readonly string[] | undefined;
+  /** The origin of the writes, when one origin made all the changes of the event. */
+  readonly origin?: string | undefined;
+}
+
+/**
+ * This function gives the topic of the change events of a collection: `collections.<name>`. The
+ * partition of roadmap CLI-17 comes after the name, with the option `topic` of the procedures.
+ */
+export function collectionChangeTopic(collection: string): string {
+  return `collections.${collection}`;
+}
+
+/**
+ * This function gives the resources that a subscriber of a topic reads: `collection:<name>` for
+ * the topic `collections.<name>` and for a topic of a partition, `collections.<name>.<partition>`.
+ * Another topic gives no resource. `EventPublisher` uses it as its default `reads`.
+ */
+export function collectionTopicReads(topic: string): readonly string[] {
+  const match = /^collections\.([^.]+)(?:\.|$)/.exec(topic);
+  return match ? [`collection:${match[1]}`] : [];
+}
+
+/** The part of a storage that the change events read. */
+interface ChangeLog {
+  getChangesSince?(since?: string): Promise<StorageChanges<unknown>>;
+}
+
+/** A write that ended, and that no event told yet. */
+interface EndedWrite {
+  /** The ids of the write, or "*" for a clear. */
+  readonly ids: readonly string[] | "*";
+  readonly origin: string | undefined;
+  /** True when the chain had a version when the write started. */
+  readonly anchored: boolean;
+}
+
+/** The chain of the events of one storage and one topic. */
+interface ChangeChain {
+  /** The `version` of the last event, or of a call of `changes`: the `since` of the next event. */
+  anchor: string | undefined;
+  /** The publishes, one at a time. */
+  tail: Promise<void>;
+  /** The writes that ended since the last publish. */
+  readonly ended: EndedWrite[];
+}
+
+/** The chains of each storage, by topic. A storage that goes away takes its chains with it. */
+const CHAINS = new WeakMap<object, Map<string, ChangeChain>>();
+
+function chainOf(storage: object, topic: string): ChangeChain {
+  let topics = CHAINS.get(storage);
+  if (!topics) {
+    topics = new Map();
+    CHAINS.set(storage, topics);
+  }
+  let chain = topics.get(topic);
+  if (!chain) {
+    chain = { anchor: undefined, tail: Promise.resolve(), ended: [] };
+    topics.set(topic, chain);
+  }
+  return chain;
+}
+
+/** The origin of an event: the one origin of its writes, when the changes are only of these writes. */
+function originOf(writes: readonly EndedWrite[], changes: StorageChanges<unknown> | undefined): string | undefined {
+  const origin = writes[0]?.origin;
+  if (origin === undefined || writes.some((write) => write.origin !== origin)) return undefined;
+  if (!changes || writes.some((write) => write.ids === "*")) return origin;
+  const ids = new Set<string>();
+  for (const write of writes) for (const id of write.ids as readonly string[]) ids.add(id);
+  const own = changes.changed.every(([id]) => ids.has(id)) && changes.deleted.every((id) => ids.has(id));
+  return own ? origin : undefined;
+}
+
+/** A hint: the collection changed, and the subscriber reads the changes. */
+function hint(version: string | undefined, origin: string | undefined): CollectionChangeEvent {
+  return {
+    ...(version !== undefined && { version }),
+    changed: [],
+    deleted: [],
+    ...(origin !== undefined && { origin }),
+  };
+}
+
+/**
+ * The event of a delta with the values of the changed items. Each row `{ id, value }` passes the
+ * output filter of each subscriber, and a refused row goes from `values` to `withheld`.
+ */
+function withValues(event: CollectionChangeEvent, entries: ReadonlyArray<{ readonly id: string; readonly value: unknown }>): CollectionChangeEvent {
+  return withRows(
+    { ...event, values: entries },
+    {
+      rows: entries,
+      keep: (admitted) => {
+        const withheld = entries.filter((_, index) => !admitted[index]).map(({ id }) => id);
+        return {
+          ...event,
+          values: entries.filter((_, index) => admitted[index]),
+          ...(withheld.length > 0 && { withheld }),
+        };
+      },
+    },
+  );
+}
+
+/** One publish: the event of the writes that ended since the last publish. */
+async function publish(chain: ChangeChain, storage: ChangeLog, bus: EventBus, topic: string, values: boolean): Promise<void> {
+  const writes = chain.ended.splice(0);
+  // An earlier publish told these writes
+  if (writes.length === 0) return;
+  if (!bus.hasSubscribers(topic)) {
+    // No one listens: the procedures do not read the log. The next call of `changes` starts the chain again.
+    chain.anchor = undefined;
+    return;
+  }
+  const since = chain.anchor;
+  if (!storage.getChangesSince || since === undefined) {
+    bus.emit(topic, hint(undefined, originOf(writes, undefined)));
+    return;
+  }
+  let changes: StorageChanges<unknown>;
+  try {
+    changes = await storage.getChangesSince(since);
+  } catch {
+    // The subscribers read the changes themselves. The chain keeps its version.
+    bus.emit(topic, hint(undefined, originOf(writes, undefined)));
+    return;
+  }
+  chain.anchor = changes.version;
+  // A full result: the log does not know the version of the chain any more
+  if (changes.full || writes.some((write) => !write.anchored)) {
+    bus.emit(topic, hint(changes.version, originOf(writes, undefined)));
+    return;
+  }
+  if (changes.changed.length === 0 && changes.deleted.length === 0) return;
+  const origin = originOf(writes, changes);
+  const event: CollectionChangeEvent = {
+    since,
+    version: changes.version,
+    changed: changes.changed.map(([id]) => id),
+    deleted: changes.deleted,
+    ...(origin !== undefined && { origin }),
+  };
+  bus.emit(topic, values ? withValues(event, changes.changed.map(([id, value]) => ({ id, value }))) : event);
+}
+
+/**
+ * This function runs a write of a collection procedure, and then publishes the change event of
+ * the write on the bus. Without a bus or a topic, it only runs the write. A failed write
+ * publishes too, because a batch can change a part of the storage. The error of the publish
+ * never reaches the caller.
+ *
+ * @param bus - The bus of the context, or undefined
+ * @param storage - The storage of the collection: the chain of the events belongs to it
+ * @param topic - The topic of the events, or undefined for no event
+ * @param ids - The ids of the write, or "*" for a clear
+ * @param origin - The origin of the write
+ * @param write - The write
+ * @param values - True: the events carry the values, for the output filter of each subscriber
+ */
+export async function writeWithChangeEvent<R>(
+  bus: EventBus | undefined,
+  storage: object & ChangeLog,
+  topic: string | undefined,
+  ids: readonly string[] | "*",
+  origin: string | undefined,
+  write: () => Promise<R>,
+  values = false,
+): Promise<R> {
+  if (bus === undefined || topic === undefined) return write();
+  const chain = chainOf(storage, topic);
+  const started: EndedWrite = { ids, origin, anchored: chain.anchor !== undefined };
+  try {
+    return await write();
+  } finally {
+    chain.ended.push(started);
+    const run = chain.tail.then(() => publish(chain, storage, bus, topic, values));
+    chain.tail = run.catch(() => undefined);
+    await chain.tail;
+  }
+}
+
+/**
+ * This function starts the chain of the events of a storage at the version of a call of
+ * `changes`, when the chain has no version. The subscriber that made the call then applies the
+ * next event without a read.
+ */
+export function startChangeChain(bus: EventBus | undefined, storage: object, topic: string | undefined, version: string): void {
+  if (bus === undefined || topic === undefined) return;
+  const chain = chainOf(storage, topic);
+  if (chain.anchor === undefined) chain.anchor = version;
+}

package/src/procedures/collection/index.ts

@@ -21,6 +21,12 @@ export {
   getAllOutputSchema,
   entriesInputSchema,
   entriesOutputSchema,
+  findInputSchema,
+  findOutputSchema,
+  patchInputSchema,
+  patchOutputSchema,
+  idsInputSchema,
+  idsOutputSchema,
   changesInputSchema,
   changesOutputSchema,
   sizeInputSchema,
@@ -44,6 +50,9 @@ export type {
   GetAllInput,
   EntriesInput,
   CollectionEntry,
+  FindInput,
+  PatchInput,
+  IdsInput,
   ChangesInput,
   ChangesOutput,
   SizeInput,
@@ -55,6 +64,52 @@ export type {
 
 // Procedures
 export { createCollectionProcedures } from "./procedures.js";
+export type { CollectionProceduresOptions, CollectionChangeEventsOptions } from "./procedures.js";
+
+// The change events of the writes (roadmap CLI-15)
+export { collectionChangeTopic, collectionTopicReads } from "./changes.js";
+export type { CollectionChangeEvent } from "./changes.js";
 
 // The storage interface of the core (the shape of client-collections' CollectionStorage)
-export type { CollectionStorage, StorageChanges, StorageMetadata } from "./storage.js";
+export type {
+  CollectionStorage,
+  StorageChanges,
+  StorageMetadata,
+  StorageWriteOptions,
+  StorageSetChange,
+  StorageDeleteChange,
+  StorageClearChange,
+  StorageChangeEvent,
+  StorageChangeBatch,
+  StorageObserver,
+} from "./storage.js";
+
+// The query language of the storages: the same JSON on the wire and in query() (roadmap CLI-34:
+// the one copy of the language)
+export {
+  DEFAULT_FIND_LIMIT,
+  DEFAULT_QUERY_OPERATORS,
+  DEFAULT_QUERY_POLICY,
+  QUERY_OPERATORS,
+  applyPatch,
+  compareStorageValues,
+  compileSelector,
+  isPatchOperations,
+  matchesSelector,
+  mergePatch,
+  patchProblems,
+  queryEntries,
+  queryOptionsProblems,
+  selectorProblems,
+} from "./query.js";
+export type {
+  StorageFieldOperators,
+  StoragePatch,
+  StoragePatchOperations,
+  StorageQueryOperator,
+  StorageQueryOptions,
+  StorageQueryPolicy,
+  StorageSelector,
+  StorageSort,
+  StorageSortDirection,
+} from "./query.js";