@ignex/ninox 0.1.1 → 0.1.3

package/README.md

@@ -185,7 +185,7 @@ A quick tour — each entry links to the full guide further down.
 | ⚡ | [Cache + dedup](#performance-by-default) | 0 round trips on repeat reads; N identical reads → 1 DB call |
 | 🔗 | [Relations + DataLoader](#relations--populate) | `belongsTo` / `hasMany` / `manyToMany` without N+1 |
 | 📄 | [Pagination](#pagination-offset--keyset) | `$facet` (data + count, 1 round trip) + keyset cursors |
-| 📊 | [Typed aggregation](#typed-aggregation-pipeline) | Chainable `pipeline()` with inferred result types |
+| 📊 | [Typed aggregation](#typed-aggregation-pipeline) | Chainable `pipeline()` with inferred result types — materialized results are cached |
 | 🔍 | [Search & joins](#search--joins) | `$text` / regex search + single-hop `lookupJoin` |
 | 🗺️ | [Geospatial](#geospatial) | `s.geoPoint()` + typed `$geoNear` with distances |
 | 📦 | [Repository layer](#repository-layer) | Optional domain-typed wrapper (`getById`, `create`, `page`, …) |
@@ -244,6 +244,17 @@ Terminals: `.toArray()`, `.first()`, `.cursor()`. The callback
 the schema); reach for `db.pipeline()` when you also want inferred result types.
 See [`examples/10-typed-pipeline.ts`](./examples/10-typed-pipeline.ts).
 
+> ⚡ **Aggregation results are cached too.** `pipeline().toArray()/.first()`,
+> `groupBy`, `dateRangeAnalysis`, `textSearch`, `lookupJoin`, and
+> `paginateFlexible` route through the same write-through `QueryCache` as flat
+> reads (warm reads = **0 driver calls**). Invalidation is write-through and
+> per-source: a `$lookup`/`$unionWith` join is registered under **every** source
+> collection, so a write to any of them drops it. Pipelines that write
+> (`$out`/`$merge`) or are non-deterministic (`$sample`) are never cached; use
+> `{ cache: false }` for other non-deterministic pipelines (`$rand`/`$function`).
+> Live cursors — `aggregate()` and `pipeline().cursor()` — stream and are not
+> cached.
+
 ## Query builder & full CRUD
 
 Fluent queries: `.where().or().sort().skip().limit().select().hint()` →
@@ -462,7 +473,10 @@ await runner.scaffold('add_tags'); // create a new NNN_add_tags.ts template
 The **default path is the fast path.** A `QueryCache` and in-flight dedup are
 created and enabled automatically — repeated identical reads become cache hits
 (0 driver calls) and identical concurrent reads coalesce into one driver call.
-Opt out when you need to:
+The same applies to **materializing aggregation results** (`pipeline().toArray()`
+/ `.first()`, `groupBy`, `dateRangeAnalysis`, `textSearch`, `lookupJoin`,
+`paginateFlexible`) — they share the same cache, with invalidation per source
+collection. Opt out when you need to:
 
 ```ts
 // Service-wide: disable caching, dedup, or everything at once.
@@ -571,6 +585,32 @@ Cached results are shared **by reference** (zero clone overhead); if a caller mi
 mutate a result, enable per-query `clone: true` (or `cache: { clone: true }`) so
 each read returns a fresh copy and can't poison the cache.
 
+### Failure semantics — the staleness window
+
+Freshness is best-effort under failure: while the background freshness
+mechanism is down, the HotCache keeps serving from memory, so **low latency can
+hide stale reads**. Here is exactly what happens in the failure cases that
+matter, and how the cache self-heals:
+
+| failure | what happens | recovery |
+| --- | --- | --- |
+| **Stream error / server kill** (network drop, replica failover) | watcher logs `hot cache change stream error`, closes the stream, retries with jittered backoff (1s → 5s) | on **reopen** the bound collection is invalidated once — entries that could have gone stale during the outage are re-fetched on the next read |
+| **Resume-token expiry** (`ChangeStreamHistoryLost` — the oplog pruned past the stream's token) | treated as a *recoverable* error, NOT "unsupported" → same backoff/retry path; the re-opened stream has no resume token and starts from "now" | invalidate-on-reopen drops any entries that missed changes during the gap; the next read re-fetches |
+| **Rollback / `invalidate` events** (collection dropped/renamed) | the stream emits `invalidate` and ends | the loop reconnects and invalidates on reopen |
+| **Consumer downtime** (the process hosting the watcher is down/paused) | writes are NOT observed, so the in-process LRU keeps serving stale values | once the watcher reconnects, invalidate-on-reopen drops the collection's entries and the next read is fresh; a consumer that never reconnects leaves entries stale forever unless `ttlMs`/`refreshIntervalMs` are set |
+
+To **bound the staleness window itself** (not just recover from it):
+
+- set per-query `ttlMs` (expiry) and/or `refreshIntervalMs` (background
+  refresh) — `refreshIntervalMs` also keeps the cache self-updating even if the
+  watcher never comes back;
+- watch `hot.stats()` (per-query `hits` / `misses` / `refreshes` / `loadErrors`)
+  and the `hot cache change stream reconnected` / `hot cache change stream
+  error` warnings — a rising `misses`-to-`hits` ratio or repeated reconnect
+  warnings mean the watcher is unhealthy;
+- in standalone mode, queries with neither `refreshIntervalMs` nor `ttlMs` are
+  served until manually invalidated (see the standalone warning below).
+
 ## Background refresh system
 
 The standalone-mode workhorse behind the HotCache: a single global background
@@ -620,7 +660,7 @@ mapping:
 
 | React Query | ninox |
 | --- | --- |
-| `queryKey` | the cache key — HotCache: **query name + call parameters**; query cache: **collection + filter hash** |
+| `queryKey` | the cache key — HotCache: **query name + call parameters**; query cache: **collection + filter/pipeline hash** |
 | `queryClient.invalidateQueries(['products'])` | `hot.invalidateCollection('products')` or `hot.invalidate('topProducts')` |
 | invalidating one exact entry | `hot.invalidateParams('topProducts', 3)` / `topProducts.invalidate(3)` |
 | `refetchInterval` (background refetch) | the standalone [background ticker](#background-refresh-system) (`refreshIntervalMs`) |
@@ -630,7 +670,8 @@ mapping:
 **What keeps the cache correct, by default:**
 
 1. **Write-through invalidation.** Every ORM insert/update/delete drops that
-   collection's cached reads immediately — like React Query invalidating on a
+   collection's cached reads — and every cached aggregation result that reads it
+   (via its source collections) — immediately, like React Query invalidating on a
    mutation, but you never have to call it.
 2. **External-writer protection.** Writes from other processes, the raw `client`
    escape hatch, or direct DB access can't invalidate on their own. Three ways to

package/dist/index.d.ts

@@ -273,7 +273,10 @@ declare const cacheCollectionKey: (dbName: string, physical: string) => string;
  * Read-through query cache with per-collection invalidation.
  *
  * Keys are `collection<sep>hash` so a single write can invalidate every cached
- * read for that collection. Invalidation is WRITE-THROUGH only: the ORM drops a
+ * read for that collection. `set()` can also register an entry under ADDITIONAL
+ * collection keys (see `collections`) so derived reads — e.g. an aggregation
+ * with `$lookup`/`$unionWith` — are invalidated when ANY of their source
+ * collections changes. Invalidation is WRITE-THROUGH only: the ORM drops a
  * collection's entries on every write it performs. Reads are NOT invalidated by
  * external writers (other processes, the raw `client` escape hatch, or direct
  * DB writes) — with `ttlMs: 0` (default) those reads stay stale indefinitely,
@@ -283,6 +286,8 @@ declare const cacheCollectionKey: (dbName: string, physical: string) => string;
 declare class QueryCache {
     private lru;
     private index;
+    /** Per-collection version — bumped on every invalidation/clear of that collection. */
+    private colVersions;
     private ttlMs;
     private clone;
     private hits;
@@ -297,8 +302,14 @@ declare class QueryCache {
     stats(): QueryCacheStats;
     /** Compose a cache key for a collection + filter/options hash. */
     key(collection: string, filterHash: string): string;
+    /**
+     * Current version of a collection key (db-namespaced, e.g. from
+     * `cacheCollectionKey`). Reads capture it BEFORE fetching and pass it to
+     * `set()` so a concurrent invalidation can never be hidden by a late write.
+     */
+    versionOf(collection: string): number;
     get(key: string): unknown | undefined;
-    set(key: string, value: unknown, ttlMs?: number): void;
+    set(key: string, value: unknown, ttlMs?: number, collections?: string[], versions?: Record<string, number>): void;
     delete(key: string): void;
     /** Drop every cached entry belonging to a physical collection. */
     invalidateByCollection(collection: string): void;
@@ -1350,6 +1361,84 @@ interface DbOpMeta {
     op: string;
 }
 
+/**
+ * Aggregation types + the shared context handed to the aggregation op-group
+ * factories (`./aggregate.ts`, `./group.ts`, `./text-search.ts`,
+ * `./lookup-join.ts`). Type-only plus the `AggregationCtx` interface.
+ */
+
+interface GroupByConfig {
+    /** Group key: a field name or a document of `$`-expressions. */
+    groupBy: string | Record<string, unknown>;
+    /** Accumulator expressions, e.g. `{ revenue: { $sum: '$total' } }`. */
+    accumulate?: Record<string, unknown>;
+    /** Sort the grouped output. */
+    sort?: Record<string, 1 | -1>;
+    /** Cap the number of groups returned. */
+    limit?: number;
+}
+interface DateRangeConfig {
+    /** Date field to bucket on (must exist on every doc). */
+    dateField: string;
+    /** Inclusive range start. */
+    startDate: Date;
+    /** Inclusive range end. */
+    endDate: Date;
+    /** Bucket granularity (default `day`). */
+    granularity?: 'hour' | 'day' | 'week' | 'month' | 'year';
+}
+interface LookupConfig {
+    /** Related (logical) collection to join. */
+    fromCollection: string;
+    /** Field on the source docs holding the foreign key. */
+    localField: string;
+    /** Field on the foreign docs to match. */
+    foreignField: string;
+    /** Output field name for the joined array. */
+    as: string;
+    /** Optional `$pipeline` limiting/filtering the joined docs. */
+    pipeline?: Document[];
+    /** When true, `$unwind` the joined array (preserving nulls). */
+    unwindSingle?: boolean;
+}
+interface PipelineCustomization {
+    /** Stages prepended before the generated aggregation. */
+    prePipeline?: Document[];
+    /** Stages appended after the generated aggregation. */
+    postPipeline?: Document[];
+}
+interface AggregationOpsOptions {
+    resolveCollectionName: (logical: string) => string;
+    wrapMongoErrors?: boolean;
+    /** Shared read cache (`undefined` = caching disabled). */
+    cache?: QueryCache;
+    /** Service default for in-flight read dedup. */
+    dedupeReads: boolean;
+    /** Shared in-flight dedup (`undefined` = no dedup). */
+    inFlight?: InFlight;
+}
+/**
+ * The cached-aggregation runner every materializing op uses (see
+ * `cached-read.ts`). Caches results in the shared `QueryCache` (write-through,
+ * invalidated per source collection) and dedupes identical in-flight calls.
+ */
+type CachedAggregate = <X extends string, T>(input: {
+    collection: X;
+    opName: string;
+    pipeline: Document[];
+    options?: AggregateOptions & QueryOptions;
+    /** Additional LOGICAL source collections (e.g. `lookupJoin` fromCollections). */
+    sources?: string[];
+    execute: (resolved: ResolvedQueryOptions) => Promise<T>;
+}) => Promise<T>;
+/** Pagination subset used by `textSearch`. */
+interface TextSearchPagination {
+    page?: number;
+    limit?: number;
+    sort?: Record<string, 1 | -1>;
+    maxLimit?: number;
+}
+
 interface CrudOpDeps<TCollection> {
     trace: <T>(meta: DbOpMeta, fn: () => T | Promise<T>) => Promise<T>;
     meta: (collection: TCollection, op: string) => DbOpMeta;
@@ -1570,6 +1659,12 @@ interface PipelineBuilderDeps {
         hint?: Hint;
     }) => AggregateOptions;
     options?: AggregateOptions & QueryOptions;
+    /**
+     * Cached-aggregation runner (write-through cache + dedup). Present on real
+     * builders so `.toArray()` / `.first()` are cached; absent on stage-only
+     * builders (sub-pipelines read via `.raw()` and never execute).
+     */
+    cachedAggregate?: CachedAggregate;
 }
 declare class PipelineBuilder<TClients extends DbClientsDefinition, TDb extends ExtractDbNames<TClients>, TDoc extends Document> {
     private readonly deps;
@@ -1959,68 +2054,14 @@ interface SearchConfig {
     sortByScore?: boolean;
 }
 
-/**
- * Aggregation types + the shared context handed to the aggregation op-group
- * factories (`./aggregate.ts`, `./group.ts`, `./text-search.ts`,
- * `./lookup-join.ts`). Type-only plus the `AggregationCtx` interface.
- */
-
-interface GroupByConfig {
-    /** Group key: a field name or a document of `$`-expressions. */
-    groupBy: string | Record<string, unknown>;
-    /** Accumulator expressions, e.g. `{ revenue: { $sum: '$total' } }`. */
-    accumulate?: Record<string, unknown>;
-    /** Sort the grouped output. */
-    sort?: Record<string, 1 | -1>;
-    /** Cap the number of groups returned. */
-    limit?: number;
-}
-interface DateRangeConfig {
-    /** Date field to bucket on (must exist on every doc). */
-    dateField: string;
-    /** Inclusive range start. */
-    startDate: Date;
-    /** Inclusive range end. */
-    endDate: Date;
-    /** Bucket granularity (default `day`). */
-    granularity?: 'hour' | 'day' | 'week' | 'month' | 'year';
-}
-interface LookupConfig {
-    /** Related (logical) collection to join. */
-    fromCollection: string;
-    /** Field on the source docs holding the foreign key. */
-    localField: string;
-    /** Field on the foreign docs to match. */
-    foreignField: string;
-    /** Output field name for the joined array. */
-    as: string;
-    /** Optional `$pipeline` limiting/filtering the joined docs. */
-    pipeline?: Document[];
-    /** When true, `$unwind` the joined array (preserving nulls). */
-    unwindSingle?: boolean;
-}
-interface PipelineCustomization {
-    /** Stages prepended before the generated aggregation. */
-    prePipeline?: Document[];
-    /** Stages appended after the generated aggregation. */
-    postPipeline?: Document[];
-}
-interface AggregationOpsOptions {
-    resolveCollectionName: (logical: string) => string;
-    wrapMongoErrors?: boolean;
-}
-/** Pagination subset used by `textSearch`. */
-interface TextSearchPagination {
-    page?: number;
-    limit?: number;
-    sort?: Record<string, 1 | -1>;
-    maxLimit?: number;
-}
-
 /**
  * Aggregation ops for one database: `aggregate`, `pipeline`, `groupBy`,
  * `textSearch`, `dateRangeAnalysis`, `lookupJoin`. All collection-name first
- * and schema-typed.
+ * and schema-typed. Materializing ops (`pipeline().toArray()/.first()`,
+ * `groupBy`, `dateRangeAnalysis`, `textSearch`, `lookupJoin`) route through the
+ * shared cached-aggregation runner (`cachedAggregate`) — write-through cache +
+ * in-flight dedup, invalidated per source collection. `aggregate()` and
+ * `pipeline().cursor()` stream live cursors and stay uncached.
  */
 declare const makeAggregationOps: <TClients extends DbClientsDefinition, TDb extends ExtractDbNames<TClients>>(client: Db, dbLabel: string, logger: LoggerLike, opts: AggregationOpsOptions) => {
     lookupJoin: <X extends ExtractCollectionNames<TClients, TDb>, TResult extends Document = CollectionDoc<TClients[TDb]["collections"][X]>>(collection: X, baseFilter: FilterInput<CollectionDoc<TClients[TDb]["collections"][X]>>, lookups: LookupConfig[], customization?: PipelineCustomization, options?: mongodb.AggregateOptions & QueryOptions) => Promise<TResult[]>;
@@ -2133,11 +2174,20 @@ interface PaginationOpsOptions {
     drift?: DriftMode;
     /** Resolve the declared schema for a logical collection (drift detection). */
     getSchema?: (logical: string) => ObjectField | undefined;
+    /** Shared read cache (`undefined` = caching disabled). */
+    cache?: QueryCache;
+    /** Service default for in-flight read dedup. */
+    dedupeReads: boolean;
+    /** Shared in-flight dedup (`undefined` = no dedup). */
+    inFlight?: InFlight;
 }
 
 /**
  * Pagination ops for one database: `paginateFlexible` (offset, `$facet`) and
  * `paginateCursor` (keyset). Both are collection-name first and schema-typed.
+ * `paginateFlexible` routes through the shared cached-aggregation runner
+ * (write-through cache + dedup); `paginateCursor` is a find-based keyset read
+ * and stays uncached (its opaque cursor pages are largely unique).
  */
 declare const makePaginationOps: <TClients extends DbClientsDefinition, TDb extends ExtractDbNames<TClients>>(client: Db, dbLabel: string, logger: LoggerLike, opts: PaginationOpsOptions) => {
     paginateCursor: <X extends ExtractCollectionNames<TClients, TDb>>(collection: X, baseFilter: FilterInput<CollectionDoc<TClients[TDb]["collections"][X]>>, config: CursorPaginationConfig<CollectionDoc<TClients[TDb]["collections"][X]>>) => Promise<CursorPage<CollectionDoc<TClients[TDb]["collections"][X]>>>;