@procedures/client 0.8.0 → 0.9.0

package/src/procedures/collection/changes.ts

@@ -0,0 +1,245 @@
+/**
+ * The change events of the collection procedures (roadmap CLI-15).
+ *
+ * Each write of a collection procedure publishes one `CollectionChangeEvent` on the bus of its
+ * context (`ctx.bus`), under the topic `collections.<name>`. A peer gets the events with
+ * `events.subscribe` (`documentation/EVENTS.md`). The delivery is at most once, so an event is a
+ * hint of a change, and a subscriber reads the changes when it misses an event.
+ *
+ * - `since` and `version` are versions of `getChangesSince()` of the storage. The events of one
+ *   storage and one topic make a chain: the `since` of an event is the `version` of the event
+ *   before it. A subscriber whose cursor is `since` applies the event without a read.
+ * - The changes of an event come from the change log of the storage, not from the input of the
+ *   write. Thus an event also has the writes that did not go through the procedures.
+ * - By default, an event has only the ids and the versions: a value reaches a caller only through
+ *   a read, with its checks. With the option `values`, an event has the values too, and each row
+ *   `{ id, value }` passes the output filter of the host for each subscriber (`withRows()`).
+ * - A subscriber of the topic must have a capability that reads `collection:<name>`, as for a
+ *   read (`collectionTopicReads()`, the default `reads` of `EventPublisher`).
+ * - An event without `since` only tells that the collection changed: the storage has no change
+ *   log, the chain starts, or the log dropped the version. The subscriber then reads the changes.
+ * - The chain starts at the version of a call of `changes` (a pull of a subscriber). While no one
+ *   listens to the topic, the procedures publish nothing, and the chain stops.
+ * - The publishes of one storage and one topic run one at a time, in the order of the writes. A
+ *   write resolves after its event is on the bus. A failed publish never fails the write.
+ */
+
+import type { EventBus } from "../../events/types.js";
+import { withRows } from "../../events/rows.js";
+import type { StorageChanges } from "./storage.js";
+
+/**
+ * The change event of the writes of a collection, on the topic `collections.<name>` (roadmap
+ * CLI-15). The data crosses the wire as JSON.
+ */
+export interface CollectionChangeEvent {
+  /**
+   * The version of the storage before the changes. A subscriber whose cursor is this version
+   * applies the changes without a read. Without it, the event is only a hint: read the changes.
+   */
+  readonly since?: string | undefined;
+  /** The version of the storage after the changes, when the storage has a change log. */
+  readonly version?: string | undefined;
+  /** The ids of the items that changed, in the order of the last change of each id. */
+  readonly changed: readonly string[];
+  /** The ids that were deleted. */
+  readonly deleted: readonly string[];
+  /**
+   * With the option `values`: the changed items that the output filter of the subscriber admits.
+   * Without the option, the event has no values: read them with `changes`.
+   */
+  readonly values?: ReadonlyArray<{ readonly id: string; readonly value: unknown }> | undefined;
+  /** With the option `values`: the changed ids whose values the filter refused. Read them with `changes`. */
+  readonly withheld?: readonly string[] | undefined;
+  /** The origin of the writes, when one origin made all the changes of the event. */
+  readonly origin?: string | undefined;
+}
+
+/**
+ * This function gives the topic of the change events of a collection: `collections.<name>`, or
+ * `collections.<name>.<partition>` for a partition (roadmap CLI-17).
+ */
+export function collectionChangeTopic(collection: string, partition?: string): string {
+  return partition === undefined ? `collections.${collection}` : `collections.${collection}.${partition}`;
+}
+
+/**
+ * This function gives the resources that a subscriber of a topic reads: `collection:<name>` for
+ * the topic `collections.<name>` and for a topic of a partition, `collections.<name>.<partition>`.
+ * Another topic gives no resource. `EventPublisher` uses it as its default `reads`.
+ */
+export function collectionTopicReads(topic: string): readonly string[] {
+  const match = /^collections\.([^.]+)(?:\.|$)/.exec(topic);
+  return match ? [`collection:${match[1]}`] : [];
+}
+
+/** The part of a storage that the change events read. */
+interface ChangeLog {
+  getChangesSince?(since?: string): Promise<StorageChanges<unknown>>;
+}
+
+/** A write that ended, and that no event told yet. */
+interface EndedWrite {
+  /** The ids of the write, or "*" for a clear. */
+  readonly ids: readonly string[] | "*";
+  readonly origin: string | undefined;
+  /** True when the chain had a version when the write started. */
+  readonly anchored: boolean;
+}
+
+/** The chain of the events of one storage and one topic. */
+interface ChangeChain {
+  /** The `version` of the last event, or of a call of `changes`: the `since` of the next event. */
+  anchor: string | undefined;
+  /** The publishes, one at a time. */
+  tail: Promise<void>;
+  /** The writes that ended since the last publish. */
+  readonly ended: EndedWrite[];
+}
+
+/** The chains of each storage, by topic. A storage that goes away takes its chains with it. */
+const CHAINS = new WeakMap<object, Map<string, ChangeChain>>();
+
+function chainOf(storage: object, topic: string): ChangeChain {
+  let topics = CHAINS.get(storage);
+  if (!topics) {
+    topics = new Map();
+    CHAINS.set(storage, topics);
+  }
+  let chain = topics.get(topic);
+  if (!chain) {
+    chain = { anchor: undefined, tail: Promise.resolve(), ended: [] };
+    topics.set(topic, chain);
+  }
+  return chain;
+}
+
+/** The origin of an event: the one origin of its writes, when the changes are only of these writes. */
+function originOf(writes: readonly EndedWrite[], changes: StorageChanges<unknown> | undefined): string | undefined {
+  const origin = writes[0]?.origin;
+  if (origin === undefined || writes.some((write) => write.origin !== origin)) return undefined;
+  if (!changes || writes.some((write) => write.ids === "*")) return origin;
+  const ids = new Set<string>();
+  for (const write of writes) for (const id of write.ids as readonly string[]) ids.add(id);
+  const own = changes.changed.every(([id]) => ids.has(id)) && changes.deleted.every((id) => ids.has(id));
+  return own ? origin : undefined;
+}
+
+/** A hint: the collection changed, and the subscriber reads the changes. */
+function hint(version: string | undefined, origin: string | undefined): CollectionChangeEvent {
+  return {
+    ...(version !== undefined && { version }),
+    changed: [],
+    deleted: [],
+    ...(origin !== undefined && { origin }),
+  };
+}
+
+/**
+ * The event of a delta with the values of the changed items. Each row `{ id, value }` passes the
+ * output filter of each subscriber, and a refused row goes from `values` to `withheld`.
+ */
+function withValues(event: CollectionChangeEvent, entries: ReadonlyArray<{ readonly id: string; readonly value: unknown }>): CollectionChangeEvent {
+  return withRows(
+    { ...event, values: entries },
+    {
+      rows: entries,
+      keep: (admitted) => {
+        const withheld = entries.filter((_, index) => !admitted[index]).map(({ id }) => id);
+        return {
+          ...event,
+          values: entries.filter((_, index) => admitted[index]),
+          ...(withheld.length > 0 && { withheld }),
+        };
+      },
+    },
+  );
+}
+
+/** One publish: the event of the writes that ended since the last publish. */
+async function publish(chain: ChangeChain, storage: ChangeLog, bus: EventBus, topic: string, values: boolean): Promise<void> {
+  const writes = chain.ended.splice(0);
+  // An earlier publish told these writes
+  if (writes.length === 0) return;
+  if (!bus.hasSubscribers(topic)) {
+    // No one listens: the procedures do not read the log. The next call of `changes` starts the chain again.
+    chain.anchor = undefined;
+    return;
+  }
+  const since = chain.anchor;
+  if (!storage.getChangesSince || since === undefined) {
+    bus.emit(topic, hint(undefined, originOf(writes, undefined)));
+    return;
+  }
+  let changes: StorageChanges<unknown>;
+  try {
+    changes = await storage.getChangesSince(since);
+  } catch {
+    // The subscribers read the changes themselves. The chain keeps its version.
+    bus.emit(topic, hint(undefined, originOf(writes, undefined)));
+    return;
+  }
+  chain.anchor = changes.version;
+  // A full result: the log does not know the version of the chain any more
+  if (changes.full || writes.some((write) => !write.anchored)) {
+    bus.emit(topic, hint(changes.version, originOf(writes, undefined)));
+    return;
+  }
+  if (changes.changed.length === 0 && changes.deleted.length === 0) return;
+  const origin = originOf(writes, changes);
+  const event: CollectionChangeEvent = {
+    since,
+    version: changes.version,
+    changed: changes.changed.map(([id]) => id),
+    deleted: changes.deleted,
+    ...(origin !== undefined && { origin }),
+  };
+  bus.emit(topic, values ? withValues(event, changes.changed.map(([id, value]) => ({ id, value }))) : event);
+}
+
+/**
+ * This function runs a write of a collection procedure, and then publishes the change event of
+ * the write on the bus. Without a bus or a topic, it only runs the write. A failed write
+ * publishes too, because a batch can change a part of the storage. The error of the publish
+ * never reaches the caller.
+ *
+ * @param bus - The bus of the context, or undefined
+ * @param storage - The storage of the collection: the chain of the events belongs to it
+ * @param topic - The topic of the events, or undefined for no event
+ * @param ids - The ids of the write, or "*" for a clear
+ * @param origin - The origin of the write
+ * @param write - The write
+ * @param values - True: the events carry the values, for the output filter of each subscriber
+ */
+export async function writeWithChangeEvent<R>(
+  bus: EventBus | undefined,
+  storage: object & ChangeLog,
+  topic: string | undefined,
+  ids: readonly string[] | "*",
+  origin: string | undefined,
+  write: () => Promise<R>,
+  values = false,
+): Promise<R> {
+  if (bus === undefined || topic === undefined) return write();
+  const chain = chainOf(storage, topic);
+  const started: EndedWrite = { ids, origin, anchored: chain.anchor !== undefined };
+  try {
+    return await write();
+  } finally {
+    chain.ended.push(started);
+    const run = chain.tail.then(() => publish(chain, storage, bus, topic, values));
+    chain.tail = run.catch(() => undefined);
+    await chain.tail;
+  }
+}
+
+/**
+ * This function starts the chain of the events of a storage at the version of a call of
+ * `changes`, when the chain has no version. The subscriber that made the call then applies the
+ * next event without a read.
+ */
+export function startChangeChain(bus: EventBus | undefined, storage: object, topic: string | undefined, version: string): void {
+  if (bus === undefined || topic === undefined) return;
+  const chain = chainOf(storage, topic);
+  if (chain.anchor === undefined) chain.anchor = version;
+}

package/src/procedures/collection/index.ts

@@ -21,6 +21,10 @@ export {
   getAllOutputSchema,
   entriesInputSchema,
   entriesOutputSchema,
+  findInputSchema,
+  findOutputSchema,
+  patchInputSchema,
+  patchOutputSchema,
   idsInputSchema,
   idsOutputSchema,
   changesInputSchema,
@@ -46,6 +50,8 @@ export type {
   GetAllInput,
   EntriesInput,
   CollectionEntry,
+  FindInput,
+  PatchInput,
   IdsInput,
   ChangesInput,
   ChangesOutput,
@@ -58,6 +64,56 @@ export type {
 
 // Procedures
 export { createCollectionProcedures } from "./procedures.js";
+export type { CollectionProceduresOptions, CollectionChangeEventsOptions } from "./procedures.js";
+
+// The change events of the writes (roadmap CLI-15)
+export { collectionChangeTopic, collectionTopicReads } from "./changes.js";
+export type { CollectionChangeEvent } from "./changes.js";
+
+// The partitions of a collection (roadmap CLI-17)
+export { PARTITION_PATTERN, collectionTopicPartition, partitionOfCall, storageOfCall, withPartition } from "./partition.js";
+export type { CollectionPartitions, PartitionInput, PartitionResolver } from "./partition.js";
 
 // The storage interface of the core (the shape of client-collections' CollectionStorage)
-export type { CollectionStorage, StorageChanges, StorageMetadata } from "./storage.js";
+export type {
+  CollectionStorage,
+  StorageChanges,
+  StorageMetadata,
+  StorageWriteOptions,
+  StorageSetChange,
+  StorageDeleteChange,
+  StorageClearChange,
+  StorageChangeEvent,
+  StorageChangeBatch,
+  StorageObserver,
+} from "./storage.js";
+
+// The query language of the storages: the same JSON on the wire and in query() (roadmap CLI-34:
+// the one copy of the language)
+export {
+  DEFAULT_FIND_LIMIT,
+  DEFAULT_QUERY_OPERATORS,
+  DEFAULT_QUERY_POLICY,
+  QUERY_OPERATORS,
+  applyPatch,
+  compareStorageValues,
+  compileSelector,
+  isPatchOperations,
+  matchesSelector,
+  mergePatch,
+  patchProblems,
+  queryEntries,
+  queryOptionsProblems,
+  selectorProblems,
+} from "./query.js";
+export type {
+  StorageFieldOperators,
+  StoragePatch,
+  StoragePatchOperations,
+  StorageQueryOperator,
+  StorageQueryOptions,
+  StorageQueryPolicy,
+  StorageSelector,
+  StorageSort,
+  StorageSortDirection,
+} from "./query.js";

package/src/procedures/collection/partition.ts

@@ -0,0 +1,123 @@
+/**
+ * The partitions of a collection (roadmap CLI-17). One `collections.<name>.*` surface serves the
+ * items of several partitions, for example one per tenant, or one per tenant and scope. Each call
+ * reads and writes one partition.
+ *
+ * - The host registers the partitions of a collection with its repository: the storage of each
+ *   partition and the resolver (`ProcedureServer.usePartitions(name, { storageOf, resolve })`).
+ * - The input of a call can ask for a partition (`partition`, for example a scope). The resolver
+ *   gives the partition of the call from its context: the tenant comes from the principal, never
+ *   from the input alone. It can refuse the call.
+ * - A subscriber names the partition that it asks for in the topic, as an input does:
+ *   `collections.<name>.<requested>`. The publisher resolves it in the same way and listens to the
+ *   topic of the resolved partition, so a subscriber cannot name the topic of another tenant.
+ * - A collection without partitions refuses a call or a topic with a partition.
+ */
+import type { ZodLike } from "../../client/validation/types.js";
+import { InvocationError } from "../invoke.js";
+import type { ProcedureContext } from "../types.js";
+import type { CollectionStorage } from "./storage.js";
+
+/** The characters of a partition: letters, digits and `_ . : @ ~ + | / -`, 256 at most. */
+export const PARTITION_PATTERN: RegExp = /^[A-Za-z0-9_.:@~+|/-]{1,256}$/;
+
+/** The field of an input that asks for a partition. */
+export interface PartitionInput {
+  partition?: string;
+}
+
+/**
+ * The resolver of the partition of a call. It gets the context (with `ctx.principal`) and the
+ * partition that the input asks for. It gives the partition, `undefined` for the storage of the
+ * collection without a partition, or `null` to refuse the call (`FORBIDDEN`, reason
+ * `partition-not-allowed`). An error that it throws passes through.
+ */
+export type PartitionResolver = (ctx: ProcedureContext, requested: string | undefined) => string | undefined | null | Promise<string | undefined | null>;
+
+/**
+ * This function adds the field `partition` to the input schema of a collection procedure. The
+ * repository of the call decides at the call if the collection has partitions.
+ */
+export function withPartition<T>(schema: ZodLike<T>): ZodLike<T & PartitionInput> {
+  const validate = (data: unknown): { success: true; data: T & PartitionInput } | { success: false; error: { message: string; errors: { path: (string | number)[]; message: string }[] } } => {
+    const fail = (message: string) => ({ success: false as const, error: { message, errors: [{ path: ["partition"], message }] } });
+    const requested = typeof data === "object" && data !== null && !Array.isArray(data) ? (data as Record<string, unknown>)["partition"] : undefined;
+    if (requested !== undefined && (typeof requested !== "string" || !PARTITION_PATTERN.test(requested))) {
+      return fail("partition must have 1 to 256 letters, digits or the characters _ . : @ ~ + | / -");
+    }
+    const inner = schema.safeParse(data);
+    if (!inner.success) return inner as { success: false; error: { message: string; errors: { path: (string | number)[]; message: string }[] } };
+    const parsed = inner.data as T & PartitionInput;
+    return { success: true, data: requested === undefined ? parsed : { ...parsed, partition: requested as string } };
+  };
+  return {
+    parse(data: unknown) {
+      const result = validate(data);
+      if (result.success) return result.data;
+      throw new Error(result.error.message);
+    },
+    safeParse: validate,
+  } as ZodLike<T & PartitionInput>;
+}
+
+/** The partitions of a collection: the storage of each partition, and the resolver of a call. */
+export interface CollectionPartitions<T = unknown> {
+  /**
+   * The storage of a partition. Give the same storage for the same partition each time: the change
+   * events of a storage keep their order with it.
+   */
+  storageOf(partition: string): CollectionStorage<T>;
+  /** The resolver of the partition of a call. */
+  resolve: PartitionResolver;
+}
+
+/** The refusal of a partition. */
+function partitionRefused(ctx: ProcedureContext, message: string): InvocationError {
+  return new InvocationError("FORBIDDEN", message, ctx.path, "partition-not-allowed");
+}
+
+/**
+ * This function gives the partition of a call to a collection: the resolver of the repository
+ * gives it from the context and the requested partition. It throws `FORBIDDEN` (reason
+ * `partition-not-allowed`) when the resolver refuses, or when the collection has no partitions and
+ * the call asks for one.
+ */
+export async function partitionOfCall(ctx: ProcedureContext, collection: string, requested: string | undefined): Promise<string | undefined> {
+  const repository = ctx.repository;
+  if (!repository?.hasPartitions?.(collection) || !repository.resolvePartition) {
+    if (requested !== undefined) throw partitionRefused(ctx, `The collection ${collection} has no partitions`);
+    return undefined;
+  }
+  const partition = await repository.resolvePartition(collection, ctx, requested);
+  if (partition === null) {
+    throw partitionRefused(ctx, `The call cannot use the partition${requested === undefined ? "" : ` "${requested}"`} of ${collection}`);
+  }
+  if (partition !== undefined && !PARTITION_PATTERN.test(partition)) {
+    throw new Error(`The partition resolver gave a partition that is not valid: ${JSON.stringify(partition).slice(0, 80)}`);
+  }
+  return partition;
+}
+
+/**
+ * This function gives the storage of a call: of the collection, or of its partition. A repository
+ * without `getPartitionStorage` refuses a partition.
+ */
+export function storageOfCall<T>(ctx: ProcedureContext, collection: string, partition: string | undefined): CollectionStorage<T> {
+  const repository = ctx.repository;
+  if (!repository) throw new Error("Repository not available in procedure context");
+  if (partition === undefined) return repository.getStorage<T>(collection);
+  if (!repository.getPartitionStorage) {
+    throw new Error(`The repository has no partitions, so it cannot give the partition "${partition}" of ${collection}`);
+  }
+  return repository.getPartitionStorage<T>(collection, partition);
+}
+
+/**
+ * This function gives the collection and the partition part of a topic `collections.<name>` or
+ * `collections.<name>.<partition>`, or undefined for another topic.
+ */
+export function collectionTopicPartition(topic: string): { collection: string; partition: string | undefined } | undefined {
+  const match = /^collections\.([^.]+)(?:\.(.+))?$/.exec(topic);
+  if (!match) return undefined;
+  return { collection: match[1]!, partition: match[2] };
+}