@procedures/collections 0.8.0 → 0.9.0

package/src/storage/interface.ts

@@ -1,16 +1,21 @@
 /**
- * Collection Storage Interface
+ * The interface of the collection storages. These storages implement it:
+ * - `InMemoryStorage` (this package): a fast storage in a `Map`, in memory only.
+ * - `ApiStorage` (`@procedures/store`): a remote collection through a `Client`.
+ * - `HybridStorage` (`@procedures/store`): a local copy that syncs with a remote, with a
+ *   conflict policy and a queue of the local writes.
+ * - `MongoStorage` (`@procedures/mongo`).
  *
- * Abstract storage layer for collections, enabling different backends:
- * - InMemoryStorage: Fast, volatile storage using Map
- * - ApiStorage: Remote persistence via universal client
- * - HybridStorage: Local cache + remote sync with conflict resolution
- * - SyncedStorage: Periodic background sync with offline support
+ * `@procedures/client` declares a copy of this interface with the same shape, so that the core
+ * does not depend on this package. `SqliteStorage` (`@procedures/sqlite`) implements that copy.
  *
- * IMPORTANT: All methods return Promise to ensure client code is
- * transport-agnostic (cannot distinguish local from remote storage).
+ * Each method gives a `Promise`. Thus the code of a caller cannot tell a local storage from a
+ * remote storage.
  */
 
+// The query language is a part of the wire contract of the core: its one copy is there (roadmap CLI-34)
+import type { StoragePatch, StorageQueryOptions, StorageQueryPolicy, StorageSelector } from "@procedures/client/query";
+
 /**
  * Storage backend interface for collections.
  *
@@ -26,6 +31,11 @@
  * - A delta of `getChangesSince` follows the order of the last change of each id.
  * - After `close()`, each operation rejects with an error whose message says "closed". A second
  *   `close()` does nothing.
+ * - `query()` gives the result of `queryEntries()` for the entries of the storage.
+ * - `observe()` gives one batch for each write that changes the storage, before the write resolves.
+ * - `patch()` changes the item in one step: two patches at the same time keep each field.
+ *
+ * The optional methods are features: a caller tests for each one before it uses it.
  */
 export interface CollectionStorage<T> {
   //
@@ -82,6 +92,37 @@ export interface CollectionStorage<T> {
    */
   find(predicate: (item: T) => boolean): Promise<T[]>;
 
+  /**
+   * The entries whose items match a selector, with an order, a skip and a limit. The selector is
+   * JSON, so a remote storage sends it to its server, and a database can run it itself. The result
+   * is the result of `queryEntries()` for the entries of the storage. A storage without this method
+   * does not run selectors: a caller uses `queryEntries()` on the entries.
+   *
+   * @param selector - The selector. `{}` matches each item.
+   * @param options - The order, the skip and the limit
+   * @param policy - The query policy of the caller: the procedure `find` gives the policy of the
+   *   host. Without it, the storage uses its own policy. The default policy does not permit `$regex`.
+   * @returns Promise resolving to the [id, item] pairs, with copies of the items. A selector or
+   *   options with a problem, or that the policy does not permit, reject with a `TypeError`.
+   */
+  query?(selector: StorageSelector, options?: StorageQueryOptions, policy?: StorageQueryPolicy): Promise<Array<[string, T]>>;
+
+  /**
+   * This method starts to send the changes of the storage to an observer. Each write that changes
+   * the storage gives one batch, before the write resolves. A write that changes nothing gives no
+   * batch. A storage without this method does not tell its changes.
+   *
+   * - A set gives a `set` change with a copy of the value. A patch gives a `set` change with the new item.
+   * - A delete gives a `delete` change for each id that existed.
+   * - A clear of a storage with items gives one `clear` change.
+   * - Each change has the `origin` of the write options, or no origin.
+   *
+   * @param observer - The function that gets each batch. Its error does not change the write.
+   * @returns The function that stops the batches. A second use of it does nothing.
+   * @throws Error after `close()`
+   */
+  observe?(observer: StorageObserver<T>): () => void;
+
   /**
    * Check if item exists.
    *
@@ -106,24 +147,42 @@ export interface CollectionStorage<T> {
    *
    * @param id - Item identifier
    * @param value - Item to store
+   * @param options - The origin of the write, for the observers
    * @returns Promise resolving when operation completes
    */
-  set(id: string, value: T): Promise<void>;
+  set(id: string, value: T, options?: StorageWriteOptions): Promise<void>;
 
   /**
    * Delete item by ID.
    *
    * @param id - Item identifier
+   * @param options - The origin of the write, for the observers
    * @returns Promise resolving to true if item was deleted, false if not found
    */
-  delete(id: string): Promise<boolean>;
+  delete(id: string, options?: StorageWriteOptions): Promise<boolean>;
 
   /**
    * Clear all items from collection.
    *
+   * @param options - The origin of the write, for the observers
    * @returns Promise resolving when operation completes
    */
-  clear(): Promise<void>;
+  clear(options?: StorageWriteOptions): Promise<void>;
+
+  /**
+   * This method changes one item in one step. The fields are a JSON Merge Patch (RFC 7386): each
+   * field replaces the field of the item, an object field merges into an object field, and `null`
+   * removes the field. Or they are the operations `{ $set, $unset }`: `$set` can set `null` and
+   * nested field paths. Two patches of one item at the same time keep the fields of each. A storage
+   * without this method cannot change an item in one step.
+   *
+   * @param id - Item identifier
+   * @param fields - The fields: an object of one form. Another value rejects with a `TypeError`.
+   * @param options - The origin of the write, for the observers
+   * @returns Promise resolving to a copy of the new item, or undefined when the id has no item. A
+   *   missing id stays missing.
+   */
+  patch?(id: string, fields: StoragePatch, options?: StorageWriteOptions): Promise<T | undefined>;
 
   //
   // ═══ Bulk Operations ═══
@@ -133,17 +192,19 @@ export interface CollectionStorage<T> {
    * Set multiple items at once (more efficient than individual sets).
    *
    * @param items - Array of [id, value] tuples
+   * @param options - The origin of the write, for the observers
    * @returns Promise resolving when operation completes
    */
-  setBatch(items: Array<[string, T]>): Promise<void>;
+  setBatch(items: Array<[string, T]>, options?: StorageWriteOptions): Promise<void>;
 
   /**
    * Delete multiple items at once.
    *
    * @param ids - Array of item identifiers
+   * @param options - The origin of the write, for the observers
    * @returns Promise resolving to number of items actually deleted
    */
-  deleteBatch(ids: string[]): Promise<number>;
+  deleteBatch(ids: string[], options?: StorageWriteOptions): Promise<number>;
 
   /**
    * Get multiple items by ID at once.
@@ -183,6 +244,49 @@ export interface CollectionStorage<T> {
   getMetadata?(): Promise<StorageMetadata>;
 }
 
+/** The options of a write. */
+export interface StorageWriteOptions {
+  /**
+   * The tag of the writer. Each change that the write gives has it, so an observer knows its own
+   * writes: a sync does not send back the changes that it wrote itself.
+   */
+  readonly origin?: string | undefined;
+}
+
+/** A change of `observe()`: an id got a value. */
+export interface StorageSetChange<T> {
+  readonly type: "set";
+  readonly id: string;
+  /** A copy of the new item. */
+  readonly value: T;
+  /** The origin of the write, or undefined when the write had none. */
+  readonly origin?: string | undefined;
+}
+
+/** A change of `observe()`: an id lost its item. */
+export interface StorageDeleteChange {
+  readonly type: "delete";
+  readonly id: string;
+  /** The origin of the write, or undefined when the write had none. */
+  readonly origin?: string | undefined;
+}
+
+/** A change of `observe()`: the storage lost each item. */
+export interface StorageClearChange {
+  readonly type: "clear";
+  /** The origin of the write, or undefined when the write had none. */
+  readonly origin?: string | undefined;
+}
+
+/** One change of `observe()`. */
+export type StorageChangeEvent<T> = StorageSetChange<T> | StorageDeleteChange | StorageClearChange;
+
+/** The changes of one write, in the order of the write. The observers share it: do not change it. */
+export type StorageChangeBatch<T> = readonly StorageChangeEvent<T>[];
+
+/** The function that gets each batch of `observe()`. */
+export type StorageObserver<T> = (changes: StorageChangeBatch<T>) => void;
+
 /**
  * The changes of a storage after a version (roadmap R28).
  *

package/src/storage/listeners.ts

@@ -0,0 +1,72 @@
+/**
+ * A set of listeners for the events of a storage: the change batches of `observe()`, or the status
+ * of a sync. A storage package uses it, so each storage treats its listeners in the same way.
+ */
+
+/** The function that gets the error of a listener. */
+export type ListenerErrorHandler = (error: unknown) => void;
+
+/**
+ * The default handler of the error of a listener. It throws the error again in a microtask, as an
+ * `EventTarget` reports the error of a listener. Thus the error is not lost, and the write that
+ * sent the event does not fail.
+ */
+export const reportListenerError: ListenerErrorHandler = (error) => {
+  queueMicrotask(() => {
+    throw error;
+  });
+};
+
+/**
+ * The listeners of one kind of event.
+ *
+ * - `emit()` gives the event to each listener, in the order of `add()`.
+ * - A listener that `add()` gets during an `emit()` gets the next event, not this one.
+ * - The error of a listener goes to the error handler. The other listeners get the event.
+ * - The same function can be added two times. Each `add()` gives its own function that removes it.
+ */
+export class StorageListeners<E> {
+  private readonly listeners = new Set<{ readonly listener: (event: E) => void }>();
+  private readonly onError: ListenerErrorHandler;
+
+  /** @param onError - The function that gets the error of a listener. The default is `reportListenerError`. */
+  constructor(onError: ListenerErrorHandler = reportListenerError) {
+    this.onError = onError;
+  }
+
+  /** The number of listeners. */
+  get size(): number {
+    return this.listeners.size;
+  }
+
+  /**
+   * This method adds a listener.
+   *
+   * @returns The function that removes the listener. A second use of it does nothing.
+   */
+  add(listener: (event: E) => void): () => void {
+    const entry = { listener };
+    this.listeners.add(entry);
+    return () => {
+      this.listeners.delete(entry);
+    };
+  }
+
+  /** This method gives the event to each listener. */
+  emit(event: E): void {
+    for (const entry of [...this.listeners]) {
+      // A listener that a listener before it removed gets no event
+      if (!this.listeners.has(entry)) continue;
+      try {
+        entry.listener(event);
+      } catch (error) {
+        this.onError(error);
+      }
+    }
+  }
+
+  /** This method removes each listener. */
+  clear(): void {
+    this.listeners.clear();
+  }
+}

package/src/storage/memory.ts

@@ -5,10 +5,40 @@
  * All operations return Promises for interface consistency.
  */
 
-import type { CollectionStorage, StorageChanges, StorageMetadata } from "./interface.js";
+import type {
+  CollectionStorage,
+  StorageChangeBatch,
+  StorageChangeEvent,
+  StorageChanges,
+  StorageMetadata,
+  StorageObserver,
+  StorageWriteOptions,
+} from "./interface.js";
+import { StorageListeners, type ListenerErrorHandler } from "./listeners.js";
+import {
+  applyPatch,
+  patchProblems,
+  queryEntries,
+  type StoragePatch,
+  type StorageQueryOptions,
+  type StorageQueryPolicy,
+  type StorageSelector,
+} from "@procedures/client/query";
 
 /** Options of an in-memory storage. */
 export interface InMemoryStorageOptions {
+  /**
+   * The function that gets the error of an observer of `observe()`. The default throws the error
+   * again in a microtask (`reportListenerError`). The write and the other observers do not fail.
+   */
+  onObserverError?: ListenerErrorHandler | undefined;
+  /**
+   * The query policy of `query()` for a call without a policy: the operators and the limits. The
+   * procedure `find` gives the policy of its host. A storage that gets selectors of callers that it
+   * does not trust keeps the default.
+   * @default DEFAULT_QUERY_POLICY: no `$regex`
+   */
+  queryPolicy?: StorageQueryPolicy | undefined;
   /**
    * The number of deletes that the change log keeps (roadmap R28). After more deletes, the log
    * drops the oldest ones, and `getChangesSince` of an older version gives all items.
@@ -37,6 +67,11 @@ type Copy = <V>(value: V) => V;
 const same: Copy = (value) => value;
 const clone: Copy = (value) => structuredClone(value);
 
+/** A change with the origin of the write, when the write has one. */
+function withOrigin<T>(change: StorageChangeEvent<T>, options: StorageWriteOptions | undefined): StorageChangeEvent<T> {
+  return options?.origin === undefined ? change : { ...change, origin: options.origin };
+}
+
 /**
  * In-memory storage backed by JavaScript Map.
  *
@@ -50,6 +85,8 @@ const clone: Copy = (value) => structuredClone(value);
  * - Copies: a change of a written or a read value does not change the stored item (the option
  *   `copies: false` turns this off)
  * - `getIds()` gives the ids without a copy of the values
+ * - `query()` runs a selector, `observe()` tells each change with the origin of its write, and
+ *   `patch()` merges fields into an item in one step
  * - After `close()`, each operation rejects, and a second `close()` does nothing
  *
  * @example
@@ -75,10 +112,16 @@ export class InMemoryStorage<T> implements CollectionStorage<T> {
   private readonly maxTombstones: number;
   private readonly copy: Copy;
   private closed = false;
+  /** The observers of `observe()` */
+  private readonly observers: StorageListeners<StorageChangeBatch<T>>;
+  /** The query policy of a call without a policy */
+  private readonly queryPolicy: StorageQueryPolicy | undefined;
 
   constructor(options: InMemoryStorageOptions = {}) {
     this.maxTombstones = Math.max(0, options.maxTombstones ?? 10_000);
     this.copy = options.copies === false ? same : clone;
+    this.observers = new StorageListeners<StorageChangeBatch<T>>(options.onObserverError);
+    this.queryPolicy = options.queryPolicy;
   }
 
   //
@@ -142,6 +185,26 @@ export class InMemoryStorage<T> implements CollectionStorage<T> {
     return results;
   }
 
+  /**
+   * The entries whose items match the selector, with the order, the skip and the limit of
+   * `queryEntries()`, under the policy of the call or of the options. The result has copies of the
+   * items.
+   */
+  async query(selector: StorageSelector, options?: StorageQueryOptions, policy?: StorageQueryPolicy): Promise<Array<[string, T]>> {
+    this.checkOpen();
+    return queryEntries(this.data, selector, options, policy ?? this.queryPolicy).map(([id, value]): [string, T] => [id, this.copy(value)]);
+  }
+
+  /**
+   * This method starts to send the changes of each write to the observer, before the write resolves.
+   * The values of the changes are copies, unless the option `copies` is false. The observers share
+   * each batch.
+   */
+  observe(observer: StorageObserver<T>): () => void {
+    this.checkOpen();
+    return this.observers.add(observer);
+  }
+
   async has(id: string): Promise<boolean> {
     this.checkOpen();
     return this.data.has(id);
@@ -156,23 +219,47 @@ export class InMemoryStorage<T> implements CollectionStorage<T> {
   // ═══ Write Operations ═══
   //
 
-  async set(id: string, value: T): Promise<void> {
+  async set(id: string, value: T, options?: StorageWriteOptions): Promise<void> {
     this.checkOpen();
-    this.data.set(id, this.copy(value));
+    const stored = this.copy(value);
+    this.data.set(id, stored);
     this.noteSet(id);
+    if (this.observers.size > 0) this.observers.emit([this.setChange(id, stored, options)]);
   }
 
-  async delete(id: string): Promise<boolean> {
+  async delete(id: string, options?: StorageWriteOptions): Promise<boolean> {
     this.checkOpen();
     const existed = this.data.delete(id);
-    if (existed) this.noteDelete(id);
+    if (existed) {
+      this.noteDelete(id);
+      if (this.observers.size > 0) this.observers.emit([withOrigin<T>({ type: "delete", id }, options)]);
+    }
     return existed;
   }
 
-  async clear(): Promise<void> {
+  async clear(options?: StorageWriteOptions): Promise<void> {
     this.checkOpen();
+    const had = this.data.size > 0;
     for (const id of this.data.keys()) this.noteDelete(id);
     this.data.clear();
+    if (had && this.observers.size > 0) this.observers.emit([withOrigin<T>({ type: "clear" }, options)]);
+  }
+
+  /**
+   * This method changes the item of the id in one step (a JSON Merge Patch, or the operations
+   * `$set` and `$unset`), and gives a copy of the new item. A missing id stays missing, and the
+   * result is undefined. A patch that cannot apply to the item changes nothing.
+   */
+  async patch(id: string, fields: StoragePatch, options?: StorageWriteOptions): Promise<T | undefined> {
+    this.checkOpen();
+    const [problem] = patchProblems(fields);
+    if (problem !== undefined) throw new TypeError(`The patch of ${id} is not correct: ${problem}`);
+    if (!this.data.has(id)) return undefined;
+    const stored = this.copy(applyPatch(this.data.get(id), fields) as unknown as T);
+    this.data.set(id, stored);
+    this.noteSet(id);
+    if (this.observers.size > 0) this.observers.emit([this.setChange(id, stored, options)]);
+    return this.copy(stored);
   }
 
   //
@@ -180,25 +267,31 @@ export class InMemoryStorage<T> implements CollectionStorage<T> {
   //
 
   /** This method sets several items. When one value has no copy, nothing changes. */
-  async setBatch(items: Array<[string, T]>): Promise<void> {
+  async setBatch(items: Array<[string, T]>, options?: StorageWriteOptions): Promise<void> {
     this.checkOpen();
     const copies = items.map(([id, value]): [string, T] => [id, this.copy(value)]);
     for (const [id, value] of copies) {
       this.data.set(id, value);
       this.noteSet(id);
     }
+    if (copies.length > 0 && this.observers.size > 0) {
+      this.observers.emit(copies.map(([id, value]) => this.setChange(id, value, options)));
+    }
   }
 
-  async deleteBatch(ids: string[]): Promise<number> {
+  async deleteBatch(ids: string[], options?: StorageWriteOptions): Promise<number> {
     this.checkOpen();
-    let deleted = 0;
+    const deleted: string[] = [];
     for (const id of ids) {
       if (this.data.delete(id)) {
         this.noteDelete(id);
-        deleted++;
+        deleted.push(id);
       }
     }
-    return deleted;
+    if (deleted.length > 0 && this.observers.size > 0) {
+      this.observers.emit(deleted.map((id) => withOrigin<T>({ type: "delete", id }, options)));
+    }
+    return deleted.length;
   }
 
   async getBatch(ids: string[]): Promise<Map<string, T>> {
@@ -227,6 +320,7 @@ export class InMemoryStorage<T> implements CollectionStorage<T> {
     this.data.clear();
     this.changedAt.clear();
     this.deletedAt.clear();
+    this.observers.clear();
   }
 
   async getMetadata(): Promise<StorageMetadata> {
@@ -311,6 +405,11 @@ export class InMemoryStorage<T> implements CollectionStorage<T> {
     return Array.from(this.data, ([id, value]): [string, T] => [id, this.copy(value)]);
   }
 
+  /** The `set` change of a stored value: a copy of it for the observers. */
+  private setChange(id: string, stored: T, options: StorageWriteOptions | undefined): StorageChangeEvent<T> {
+    return withOrigin<T>({ type: "set", id, value: this.copy(stored) }, options);
+  }
+
   //
   // ═══ The change log ═══
   //