@anteros/core 0.0.24 → 0.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/database/db.cache.ts +197 -0
- package/database/decorator.ts +32 -0
- package/database/mongodbadapter.ts +244 -32
- package/index.ts +2 -0
- package/package.json +4 -2
- package/server/api.ts +1 -1
- package/server/boot.ts +106 -17
- package/server/hono.ts +63 -0
- package/server/mcp.ts +34 -12
- package/server/routes.ts +10 -0
- package/tests/api.test.ts +217 -3
- package/tests/db-cache.test.ts +214 -0
- package/tests/logger.test.ts +123 -0
- package/types/collection.d.ts +6 -0
- package/types/config.d.ts +37 -0
- package/types/mongo.d.ts +10 -0
- package/types/rest.d.ts +37 -0
- package/utils/cache.ts +2 -2
- package/utils/func.ts +3 -0
- package/utils/logger.ts +212 -0
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import type { FindOptions, FindCallOptions } from "../types/mongo";
|
|
4
|
+
import { Cache, useFilesystemCache, useMemoryCache, useRedisCache } from "../utils/cache";
|
|
5
|
+
import { cfg } from "../server/config";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* DB query cache — cache-aside for `find` with `{ useCache: true }`.
|
|
9
|
+
* (`findOne` is never cached — point queries, no gain.)
|
|
10
|
+
*
|
|
11
|
+
* - Key = sha1(collection params + options) → stored in a per-collection namespace
|
|
12
|
+
* (`db:{tenant}:{collection}`).
|
|
13
|
+
* - Every cached query is **tagged with the `_id`s of its results** (including the
|
|
14
|
+
* `_id`s of `$include`d documents) via bentocache's client-side tagging
|
|
15
|
+
* (`ctx.setTags` — adaptive caching, tags known after the factory runs).
|
|
16
|
+
* - On a write with known ids (update/delete), `deleteByTag` removes exactly the
|
|
17
|
+
* cached queries whose result contains one of those `_id`s (targeted). Writes
|
|
18
|
+
* with unknown/new ids (insert, bulk, findOneAndUpdate) clear the namespace.
|
|
19
|
+
* - Relations are tracked: if `orders` caches a query with `$include: ['clients']`,
|
|
20
|
+
* its entries are tagged with the embedded client `_id`s too — so modifying a
|
|
21
|
+
* client invalidates exactly the `orders` queries that embed it.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
const NAMESPACE = 'db';
|
|
25
|
+
const REL_NS = 'db:rel';
|
|
26
|
+
|
|
27
|
+
/** Sanitary cap on tags per entry (bentocache high-cardinality guidance) — configurable via `server.cache.maxTags`. The cost is ~0.18µs per tag check on each hit, negligible vs the cost of the real query */
|
|
28
|
+
const DEFAULT_MAX_TAGS = 50_000;
|
|
29
|
+
|
|
30
|
+
let dbCache: Cache | null = null;
|
|
31
|
+
|
|
32
|
+
/** Lazy singleton — configured once from `cfg.server.cache` (default: memory) */
|
|
33
|
+
function resolveCache(): Cache {
|
|
34
|
+
if (!dbCache) {
|
|
35
|
+
const conf = cfg.server.cache ?? {};
|
|
36
|
+
const driver = conf.driver ?? 'memory';
|
|
37
|
+
if (driver === 'redis') {
|
|
38
|
+
dbCache = useRedisCache({
|
|
39
|
+
connection: {
|
|
40
|
+
host: conf.redis?.host ?? 'localhost',
|
|
41
|
+
port: conf.redis?.port ?? 6379,
|
|
42
|
+
password: conf.redis?.password ?? '',
|
|
43
|
+
},
|
|
44
|
+
pruneInterval: conf.pruneInterval,
|
|
45
|
+
});
|
|
46
|
+
} else if (driver === 'filesystem') {
|
|
47
|
+
dbCache = useFilesystemCache({
|
|
48
|
+
directory: conf.directory ?? path.join(process.cwd(), '.cache'),
|
|
49
|
+
pruneInterval: conf.pruneInterval ?? '1h',
|
|
50
|
+
});
|
|
51
|
+
} else {
|
|
52
|
+
dbCache = useMemoryCache();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return dbCache;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function isEnabled(): boolean {
|
|
59
|
+
// Opt-in: the DB query cache is disabled unless `cache.enabled: true`
|
|
60
|
+
return cfg.server.cache?.enabled === true;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function colNs(tenant: string, collection: string): string {
|
|
64
|
+
return `${NAMESPACE}:${tenant}:${collection}`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function relNs(tenant: string): string {
|
|
68
|
+
return `${REL_NS}:${tenant}`;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function hashQuery(params: FindOptions, options: FindCallOptions = {}): string {
|
|
72
|
+
return createHash('sha1').update(JSON.stringify({ params, options })).digest('hex');
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Effective TTL: query option (server-side only) > collection def > server config > default '5m' */
|
|
76
|
+
function ttlFor(collection: string, optionsTtl?: string | number): string | number {
|
|
77
|
+
const col = cfg.collections?.find((c) => c.slug === collection);
|
|
78
|
+
return optionsTtl ?? col?.cache?.ttl ?? cfg.server.cache?.ttl ?? '5m';
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** `$include` field names — used to tag embedded `_id`s for cross-collection invalidation */
|
|
82
|
+
function includeFields(params: FindOptions): string[] {
|
|
83
|
+
return (params.$include ?? []).map((inc) => (typeof inc === 'string' ? inc : inc?.localField ?? inc?.from ?? '')).filter(Boolean);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Collect the `_id`s of a result — main docs + `$include`d embedded docs */
|
|
87
|
+
function extractResultIds(result: any, includes: string[]): string[] {
|
|
88
|
+
const ids = new Set<string>();
|
|
89
|
+
const add = (v: any) => {
|
|
90
|
+
if (v && typeof v === 'object' && v._id !== undefined) ids.add(String(v._id));
|
|
91
|
+
};
|
|
92
|
+
for (const doc of Array.isArray(result) ? result : [result]) {
|
|
93
|
+
add(doc);
|
|
94
|
+
for (const inc of includes) {
|
|
95
|
+
const embedded = doc?.[inc];
|
|
96
|
+
if (Array.isArray(embedded)) embedded.forEach(add);
|
|
97
|
+
else add(embedded);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return [...ids];
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Cache-aside read-through: serve the cached result for this query, otherwise run
|
|
105
|
+
* the factory (the real query) and store it, tagged by the result `_id`s.
|
|
106
|
+
*/
|
|
107
|
+
export async function dbGetOrSet<T>(
|
|
108
|
+
collection: string,
|
|
109
|
+
tenant: string,
|
|
110
|
+
params: FindOptions,
|
|
111
|
+
options: FindCallOptions,
|
|
112
|
+
factory: () => Promise<T>,
|
|
113
|
+
): Promise<T> {
|
|
114
|
+
if (!isEnabled()) return factory();
|
|
115
|
+
const cache = resolveCache();
|
|
116
|
+
const key = hashQuery(params, options);
|
|
117
|
+
const ns = cache.namespace(colNs(tenant, collection));
|
|
118
|
+
const includes = includeFields(params);
|
|
119
|
+
|
|
120
|
+
// Adaptive caching: the tags (result `_id`s) are only known after the factory runs
|
|
121
|
+
const value = await ns.getOrSet({
|
|
122
|
+
key,
|
|
123
|
+
ttl: ttlFor(collection, options.ttl),
|
|
124
|
+
factory: async (ctx: any) => {
|
|
125
|
+
const fresh = await factory();
|
|
126
|
+
const maxTags = cfg.server.cache?.maxTags ?? DEFAULT_MAX_TAGS;
|
|
127
|
+
const tags = extractResultIds(fresh, includes).slice(0, maxTags).map((id) => `id:${id}`);
|
|
128
|
+
if (tags.length) ctx.setTags(tags);
|
|
129
|
+
return fresh;
|
|
130
|
+
},
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
await recordRelations(cache, tenant, collection, params);
|
|
134
|
+
return value as T;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Register the collections this query depends on (via `$include` / `$lookup`) so a
|
|
139
|
+
* write on the *included* collection invalidates the *parent* collection too.
|
|
140
|
+
*/
|
|
141
|
+
async function recordRelations(cache: Cache, tenant: string, collection: string, params: FindOptions) {
|
|
142
|
+
const col = cfg.collections?.find((c) => c.slug === collection && c._tenant_ === tenant);
|
|
143
|
+
const targets = new Set<string>();
|
|
144
|
+
for (const inc of params.$include ?? []) {
|
|
145
|
+
if (typeof inc === 'string') {
|
|
146
|
+
const f = col?.fields.find((f) => f.name === inc);
|
|
147
|
+
if (f?.type === 'relationship' && f.relation?.to) targets.add(f.relation.to);
|
|
148
|
+
} else if (inc?.from) {
|
|
149
|
+
targets.add(inc.from);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (!targets.size) return;
|
|
153
|
+
const rel = cache.namespace(relNs(tenant));
|
|
154
|
+
for (const target of targets) {
|
|
155
|
+
if (target === collection) continue;
|
|
156
|
+
const deps = ((await rel.get({ key: target })) as string[]) ?? [];
|
|
157
|
+
if (!deps.includes(collection)) {
|
|
158
|
+
deps.push(collection);
|
|
159
|
+
await rel.set({ key: target, value: deps, ttl: '7d' });
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Invalidate a collection's cached queries after a write.
|
|
166
|
+
* - `ids` provided (update/delete of known documents) → **targeted**: only the
|
|
167
|
+
* cached queries tagged with one of those `_id`s are removed (deleteByTag).
|
|
168
|
+
* - `ids` omitted (insert, bulk, unknown) → the whole collection namespace is cleared.
|
|
169
|
+
* Relations ($include) are invalidated the same way on the dependent collections.
|
|
170
|
+
*/
|
|
171
|
+
export async function dbInvalidate(collection: string, tenant: string, ids?: string[]) {
|
|
172
|
+
if (!isEnabled()) return;
|
|
173
|
+
const cache = resolveCache();
|
|
174
|
+
const ns = cache.namespace(colNs(tenant, collection));
|
|
175
|
+
|
|
176
|
+
if (ids?.length) {
|
|
177
|
+
// String(id) normalizes native ObjectIds to their 24-hex form
|
|
178
|
+
await ns.deleteByTag({ tags: ids.map((id) => `id:${String(id)}`) });
|
|
179
|
+
} else {
|
|
180
|
+
await ns.clear();
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Relations: collections that $include this one embed its docs → invalidate them too
|
|
184
|
+
const rel = cache.namespace(relNs(tenant));
|
|
185
|
+
const dependents = ((await rel.get({ key: collection })) as string[]) ?? [];
|
|
186
|
+
for (const dep of dependents) {
|
|
187
|
+
const depNs = cache.namespace(colNs(tenant, dep));
|
|
188
|
+
if (ids?.length) {
|
|
189
|
+
await depNs.deleteByTag({ tags: ids.map((id) => `id:${String(id)}`) });
|
|
190
|
+
} else {
|
|
191
|
+
await depNs.clear();
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
await rel.set({ key: collection, value: [] });
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export { resolveCache as getDbCache };
|
package/database/decorator.ts
CHANGED
|
@@ -1,6 +1,38 @@
|
|
|
1
1
|
import { ObjectId } from "mongodb";
|
|
2
2
|
import { getCollection } from "./collection";
|
|
3
3
|
import { AppError } from "../lib/error";
|
|
4
|
+
import { logger } from "../utils/logger";
|
|
5
|
+
import { cfg } from "../server/config";
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
// Logs operations slower than cfg.server.logging.slowQueryMs (default 200ms) —
|
|
9
|
+
// the MongoDB-style slow query log for Anteros
|
|
10
|
+
|
|
11
|
+
export function LogSlowQuery(): any {
|
|
12
|
+
return function (target: any, propertyKey: string, descriptor: PropertyDescriptor): any {
|
|
13
|
+
if (!descriptor || !descriptor.value) {
|
|
14
|
+
return descriptor;
|
|
15
|
+
}
|
|
16
|
+
const originalMethod = descriptor.value;
|
|
17
|
+
descriptor.value = async function (this: any, ...args: any[]) {
|
|
18
|
+
const start = performance.now();
|
|
19
|
+
try {
|
|
20
|
+
return await originalMethod.apply(this, args);
|
|
21
|
+
} finally {
|
|
22
|
+
const duration = performance.now() - start;
|
|
23
|
+
const slowMs = cfg.server.logging?.slowQueryMs ?? 200;
|
|
24
|
+
if (duration >= slowMs) {
|
|
25
|
+
logger.warn(`Slow ${propertyKey} (${Math.round(duration)}ms)`, {
|
|
26
|
+
collection: args[0],
|
|
27
|
+
tenant: this.tenant_id,
|
|
28
|
+
duration: Math.round(duration),
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
return descriptor;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
4
36
|
|
|
5
37
|
|
|
6
38
|
// decoration function checkCollectionExists
|
|
@@ -3,16 +3,18 @@ import type { Tenant } from "../types/tenant";
|
|
|
3
3
|
import type { MongoClientOptions, Db, ClientSession, UpdateOptions, UpdateFilter, Document, DeleteResult, ChangeStreamOptions, ChangeStream, ChangeStreamDocument, AnyBulkWriteOperation, BulkWriteOptions, BulkWriteResult, UpdateResult } from "mongodb";
|
|
4
4
|
import { MongoClient, ObjectId, AggregationCursor } from "mongodb";
|
|
5
5
|
import type { FindOptions, findOneOptions } from "../types/mongo";
|
|
6
|
+
import type { FindCallOptions } from "../types/mongo";
|
|
7
|
+
import { dbGetOrSet, dbInvalidate, getDbCache } from "./db.cache";
|
|
6
8
|
import { sessionCtxStorage, asyncContextStorage, requestCtxStorage } from "../lib/asyncContextStorage";
|
|
7
9
|
import * as func from "../utils/func"
|
|
8
10
|
import { AppError, fn } from "../lib/error";
|
|
9
11
|
import { createWorkflow } from "./workflow";
|
|
10
12
|
import { getTenant } from "./tenant";
|
|
11
13
|
import { getCollection, getCollectionKeys } from "./collection"
|
|
12
|
-
import type { RestActions, BulkUpdateOperation, RestOptions } from "../types/rest";
|
|
14
|
+
import type { RestActions, BulkUpdateOperation, RestOptions, CursorOptions, TenantCache } from "../types/rest";
|
|
13
15
|
import type Joi from "joi";
|
|
14
16
|
import type { Collection } from "../types/collection";
|
|
15
|
-
import { CheckBulkWriteOperations, CheckIfCollectionExists, CheckIfId, CheckInsertData, CheckIfArrayOfIds, CheckFilter } from "./decorator";
|
|
17
|
+
import { CheckBulkWriteOperations, CheckIfCollectionExists, CheckIfId, CheckInsertData, CheckIfArrayOfIds, CheckFilter, LogSlowQuery } from "./decorator";
|
|
16
18
|
import type { ActionsApiList, updateApiOptions } from "../types/api";
|
|
17
19
|
import type { ActivityInput } from "../types/activity";
|
|
18
20
|
import * as os from 'node:os';
|
|
@@ -214,13 +216,20 @@ class MongoRest {
|
|
|
214
216
|
action: string,
|
|
215
217
|
collection: string,
|
|
216
218
|
input: any,
|
|
217
|
-
fn: () => Promise<T
|
|
219
|
+
fn: () => Promise<T>,
|
|
220
|
+
options?: { invalidate?: boolean | string[] }
|
|
218
221
|
): Promise<T> {
|
|
219
222
|
const start = Date.now()
|
|
220
223
|
try {
|
|
221
224
|
const result = await fn()
|
|
222
225
|
const duration = Date.now() - start
|
|
223
226
|
this.#logActivity({ action, collection, input, result, duration })
|
|
227
|
+
// Writes invalidate the collection's query cache (and its $include dependents).
|
|
228
|
+
// An array of ids = targeted invalidation (only queries containing those ids),
|
|
229
|
+
// `true` = clear the whole collection namespace.
|
|
230
|
+
if (options?.invalidate) {
|
|
231
|
+
await this.#invalidateCache(collection, typeof options.invalidate === 'boolean' ? undefined : options.invalidate)
|
|
232
|
+
}
|
|
224
233
|
return result
|
|
225
234
|
} catch (err: any) {
|
|
226
235
|
const duration = Date.now() - start
|
|
@@ -229,6 +238,11 @@ class MongoRest {
|
|
|
229
238
|
}
|
|
230
239
|
}
|
|
231
240
|
|
|
241
|
+
/** Invalidate the DB query cache for a collection after a write (targeted by ids when known) */
|
|
242
|
+
#invalidateCache(collection: string, ids?: string[]) {
|
|
243
|
+
return dbInvalidate(collection, this.tenant_id, ids)
|
|
244
|
+
}
|
|
245
|
+
|
|
232
246
|
async connect(options?: {
|
|
233
247
|
uri?: string;
|
|
234
248
|
options?: MongoClientOptions;
|
|
@@ -269,6 +283,7 @@ class MongoRest {
|
|
|
269
283
|
})
|
|
270
284
|
}
|
|
271
285
|
|
|
286
|
+
@LogSlowQuery()
|
|
272
287
|
@CheckIfCollectionExists()
|
|
273
288
|
async aggregate(collection: string, pipeline: any[]): Promise<Document[]> {
|
|
274
289
|
const action = 'aggregate' as ActionsApiList
|
|
@@ -310,38 +325,192 @@ class MongoRest {
|
|
|
310
325
|
}
|
|
311
326
|
}
|
|
312
327
|
|
|
328
|
+
/**
|
|
329
|
+
* Apply cursor methods (batchSize, maxTimeMS). comment/hint are passed
|
|
330
|
+
* through the aggregate() options (not available on AggregationCursor in v7).
|
|
331
|
+
*/
|
|
332
|
+
#applyCursorOptions(cursor: AggregationCursor, options: CursorOptions) {
|
|
333
|
+
if (options.batchSize) cursor.batchSize(options.batchSize)
|
|
334
|
+
if (options.maxTimeMS) cursor.maxTimeMS(options.maxTimeMS)
|
|
335
|
+
}
|
|
313
336
|
|
|
337
|
+
/**
|
|
338
|
+
* Stream `find`-style results one by one — JSON-converted docs, cursor auto-closed.
|
|
339
|
+
* Iterates everything (no default $limit, unlike find() which caps at 100).
|
|
340
|
+
*
|
|
341
|
+
* With `{ withCount: true }`, a background `countDocuments` runs in parallel
|
|
342
|
+
* and each yielded item becomes `{ count, doc }` — the total + the document.
|
|
343
|
+
*
|
|
344
|
+
* @example
|
|
345
|
+
* for await (const doc of rest.findStream('orders', { $match: { status: 'paid' } }, { batchSize: 500 })) { … }
|
|
346
|
+
* for await (const d of rest.findStream('orders', { $match: { status: 'paid' } }, { withCount: true })) {
|
|
347
|
+
* console.log(d.count, d.doc); // total + document
|
|
348
|
+
* }
|
|
349
|
+
* for await (const u of rest.findStream<User>('users', { $match: { active: true } }, {
|
|
350
|
+
* map: (u) => ({ id: u._id, name: u.name }),
|
|
351
|
+
* signal: controller.signal,
|
|
352
|
+
* })) { … }
|
|
353
|
+
* // Page mode: { pageSize } yields { count, docs, hasNext() } batches
|
|
354
|
+
* for await (const page of rest.findStream('orders', { $match: { status: 'paid' } }, { pageSize: 50 })) {
|
|
355
|
+
* console.log(page.count, page.docs.length, page.hasNext());
|
|
356
|
+
* }
|
|
357
|
+
*/
|
|
314
358
|
@CheckIfCollectionExists()
|
|
315
|
-
async
|
|
316
|
-
const
|
|
359
|
+
async *findStream<T = any>(collection: string, params: FindOptions = {}, options: CursorOptions = {}): AsyncGenerator<any> {
|
|
360
|
+
const pageSize = options.pageSize;
|
|
361
|
+
// Page mode counts by default (opt-out with withCount: false); per-doc mode counts only when withCount: true
|
|
362
|
+
const withCount = pageSize ? options.withCount !== false : options.withCount === true;
|
|
363
|
+
// Background total count (runs in parallel with the iteration)
|
|
364
|
+
const countPromise = withCount
|
|
365
|
+
? this.countDocuments(collection, params.$match ?? {})
|
|
366
|
+
: null;
|
|
367
|
+
const col = getCollection(collection, this.#tenant.id) as Collection
|
|
368
|
+
let pipeline = func.buildPipeline({ ...params, $limit: params.$limit ?? 0 }, { col: col })
|
|
369
|
+
pipeline = await func.buildInput(pipeline, { rest: this })
|
|
370
|
+
pipeline = func.toBson(pipeline, { col })
|
|
371
|
+
const cursor = this.db.collection(collection).aggregate(pipeline, {
|
|
372
|
+
session: this.session,
|
|
373
|
+
allowDiskUse: true,
|
|
374
|
+
comment: options.comment,
|
|
375
|
+
hint: options.hint,
|
|
376
|
+
}) as AggregationCursor
|
|
377
|
+
this.#applyCursorOptions(cursor, options)
|
|
378
|
+
// Resolve the count ONCE before iterating — the value is reused on every yield
|
|
379
|
+
const count = countPromise ? await countPromise : null
|
|
317
380
|
try {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
381
|
+
if (pageSize) {
|
|
382
|
+
// ── Page mode: batches of `pageSize` → { count?, docs, hasNext() } ──
|
|
383
|
+
while (await cursor.hasNext()) {
|
|
384
|
+
if (options.signal?.aborted) break
|
|
385
|
+
const docs: any[] = []
|
|
386
|
+
for (let i = 0; i < pageSize && await cursor.hasNext(); i++) {
|
|
387
|
+
let value: any = func.toJson(await cursor.next())
|
|
388
|
+
if (options.map) value = options.map(value as T)
|
|
389
|
+
docs.push(value)
|
|
390
|
+
}
|
|
391
|
+
const hasMore = await cursor.hasNext()
|
|
392
|
+
yield {
|
|
393
|
+
...(withCount ? { count } : {}),
|
|
394
|
+
docs,
|
|
395
|
+
hasNext: () => hasMore,
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
} else {
|
|
399
|
+
// ── Per-doc mode: doc or { count, doc } ──
|
|
400
|
+
for await (const doc of cursor) {
|
|
401
|
+
if (options.signal?.aborted) break
|
|
402
|
+
let value: any = func.toJson(doc)
|
|
403
|
+
if (options.map) value = options.map(value as T)
|
|
404
|
+
if (options.withCount) {
|
|
405
|
+
yield { count, doc: value }
|
|
406
|
+
} else {
|
|
407
|
+
yield value
|
|
408
|
+
}
|
|
409
|
+
}
|
|
326
410
|
}
|
|
411
|
+
} finally {
|
|
412
|
+
await cursor.close().catch(() => {})
|
|
413
|
+
}
|
|
414
|
+
}
|
|
327
415
|
|
|
328
|
-
|
|
416
|
+
/**
|
|
417
|
+
* Stream `aggregate` results one by one (same safety checks as `aggregate`) —
|
|
418
|
+
* JSON-converted docs, cursor auto-closed.
|
|
419
|
+
*
|
|
420
|
+
* @example
|
|
421
|
+
* for await (const doc of rest.aggregateStream('orders', [{ $match: { status: 'paid' } }], { batchSize: 500 })) { … }
|
|
422
|
+
* for await (const d of rest.aggregateStream('orders', [{ $group: { _id: '$status', n: { $sum: 1 } } }], { withCount: true })) {
|
|
423
|
+
* console.log(d.count, d.doc); // total pipeline results + each doc
|
|
424
|
+
* }
|
|
425
|
+
*/
|
|
426
|
+
@CheckIfCollectionExists()
|
|
427
|
+
async *aggregateStream<T = any>(collection: string, pipeline: any[] = [], options: CursorOptions = {}): AsyncGenerator<any> {
|
|
428
|
+
const col = getCollection(collection, this.#tenant.id) as Collection
|
|
429
|
+
const allowedCollections = [
|
|
430
|
+
...(cfg.collections ?? []).filter((c) => c._tenant_ === this.tenant_id).map((c) => c.slug),
|
|
431
|
+
...(cfg.fileCollections ?? []).filter((fc) => fc._tenant_ === this.tenant_id).map((fc) => fc.slug),
|
|
432
|
+
'_activities_', '_locks_', '_workflows_',
|
|
433
|
+
];
|
|
434
|
+
const isSafe = func.isSafeAggregatePipeline(pipeline, allowedCollections);
|
|
435
|
+
if (!isSafe.isSafe) {
|
|
436
|
+
throw isSafe.error;
|
|
437
|
+
}
|
|
438
|
+
pipeline = func.toBson(pipeline, { col })
|
|
439
|
+
// Background total count of the pipeline OUTPUT (via a $count stage) —
|
|
440
|
+
// parallel with the iteration; best-effort (fails silently for $out/$merge)
|
|
441
|
+
const countPromise = options.withCount
|
|
442
|
+
? this.db.collection(collection).aggregate([...pipeline, { $count: 'count' }], {
|
|
329
443
|
session: this.session,
|
|
330
|
-
allowDiskUse: true
|
|
331
|
-
}).toArray()
|
|
332
|
-
|
|
333
|
-
|
|
444
|
+
allowDiskUse: true,
|
|
445
|
+
}).toArray().then((r) => (r[0]?.count as number) ?? 0).catch(() => 0)
|
|
446
|
+
: null;
|
|
447
|
+
const cursor = this.db.collection(collection).aggregate(pipeline, {
|
|
448
|
+
session: this.session,
|
|
449
|
+
allowDiskUse: true,
|
|
450
|
+
comment: options.comment,
|
|
451
|
+
hint: options.hint,
|
|
452
|
+
}) as AggregationCursor
|
|
453
|
+
this.#applyCursorOptions(cursor, options)
|
|
454
|
+
// Resolve the count ONCE before iterating — the value is reused on every yield
|
|
455
|
+
const count = countPromise ? await countPromise : null
|
|
456
|
+
try {
|
|
457
|
+
for await (const doc of cursor) {
|
|
458
|
+
if (options.signal?.aborted) break
|
|
459
|
+
let value: any = func.toJson(doc)
|
|
460
|
+
if (options.map) value = options.map(value as T)
|
|
461
|
+
if (options.withCount) {
|
|
462
|
+
yield { count, doc: value }
|
|
463
|
+
} else {
|
|
464
|
+
yield value
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
} finally {
|
|
468
|
+
await cursor.close().catch(() => {})
|
|
469
|
+
}
|
|
470
|
+
}
|
|
334
471
|
|
|
335
|
-
|
|
336
|
-
|
|
472
|
+
@LogSlowQuery()
|
|
473
|
+
@CheckIfCollectionExists()
|
|
474
|
+
async find(collection: string, params: FindOptions = {}, options: FindCallOptions = {}): Promise<Document[]> {
|
|
475
|
+
const action = 'find' as ActionsApiList
|
|
476
|
+
try {
|
|
477
|
+
// Real query — hooks + pipeline + toArray
|
|
478
|
+
const run = async (): Promise<Document[]> => {
|
|
479
|
+
const col = getCollection(collection, this.#tenant.id) as Collection
|
|
480
|
+
let pipeline = func.buildPipeline(params, { col: col })
|
|
481
|
+
pipeline = await func.buildInput(pipeline, { rest: this })
|
|
482
|
+
pipeline = func.toBson(pipeline, { col })
|
|
483
|
+
|
|
484
|
+
const meta: any = { action, collection, params, options }
|
|
485
|
+
if (col.hooks?.beforeOperation) {
|
|
486
|
+
await col.hooks.beforeOperation({ rest: this, io, action, meta })
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
let result = await this.db.collection(collection).aggregate(pipeline, {
|
|
490
|
+
session: this.session,
|
|
491
|
+
allowDiskUse: true
|
|
492
|
+
}).toArray()
|
|
493
|
+
result = func.toJson(result)
|
|
494
|
+
meta.result = result
|
|
495
|
+
|
|
496
|
+
if (col.hooks?.afterOperation) {
|
|
497
|
+
await col.hooks.afterOperation({ rest: this, io, action, meta })
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
return result as any[]
|
|
337
501
|
}
|
|
338
502
|
|
|
339
|
-
|
|
503
|
+
// Cache-first when `useCache: true` — served from the query cache, populated on miss
|
|
504
|
+
if (options.useCache === true) {
|
|
505
|
+
return await dbGetOrSet(collection, this.tenant_id, params, options, run)
|
|
506
|
+
}
|
|
507
|
+
return await run()
|
|
340
508
|
} catch (err: any) {
|
|
341
509
|
throw err instanceof AppError ? err : new AppError(err?.message || 'Internal server error', { code: 'INTERNAL_API_ERROR', status: 500 })
|
|
342
510
|
}
|
|
343
511
|
}
|
|
344
512
|
|
|
513
|
+
@LogSlowQuery()
|
|
345
514
|
@CheckIfCollectionExists()
|
|
346
515
|
async findOne(collection: string, _id: string, params?: findOneOptions): Promise<any> {
|
|
347
516
|
const action = 'findOne' as ActionsApiList
|
|
@@ -377,6 +546,7 @@ class MongoRest {
|
|
|
377
546
|
}
|
|
378
547
|
}
|
|
379
548
|
|
|
549
|
+
@LogSlowQuery()
|
|
380
550
|
@CheckIfCollectionExists()
|
|
381
551
|
@CheckInsertData('insertOne')
|
|
382
552
|
async insertOne<T>(collection: string, data: T): Promise<T & { _id: string }> {
|
|
@@ -406,9 +576,10 @@ class MongoRest {
|
|
|
406
576
|
}
|
|
407
577
|
|
|
408
578
|
return result
|
|
409
|
-
})
|
|
579
|
+
}, { invalidate: true })
|
|
410
580
|
}
|
|
411
581
|
|
|
582
|
+
@LogSlowQuery()
|
|
412
583
|
@CheckIfCollectionExists()
|
|
413
584
|
@CheckInsertData('insertMany')
|
|
414
585
|
async insertMany<T>(collection: string, data: T[]): Promise<(T & { _id: string })[]> {
|
|
@@ -445,9 +616,10 @@ class MongoRest {
|
|
|
445
616
|
}
|
|
446
617
|
|
|
447
618
|
return result as (T & { _id: string })[]
|
|
448
|
-
})
|
|
619
|
+
}, { invalidate: true })
|
|
449
620
|
}
|
|
450
621
|
|
|
622
|
+
@LogSlowQuery()
|
|
451
623
|
@CheckIfCollectionExists()
|
|
452
624
|
@CheckIfId('updateOne')
|
|
453
625
|
async updateOne(collection: string, _id: string, update: UpdateFilter<any>) {
|
|
@@ -482,9 +654,10 @@ class MongoRest {
|
|
|
482
654
|
}
|
|
483
655
|
|
|
484
656
|
return result
|
|
485
|
-
})
|
|
657
|
+
}, { invalidate: [_id] })
|
|
486
658
|
}
|
|
487
659
|
|
|
660
|
+
@LogSlowQuery()
|
|
488
661
|
@CheckIfCollectionExists()
|
|
489
662
|
@CheckFilter()
|
|
490
663
|
async findOneAndUpdate(collection: string, filter: Document, update: UpdateFilter<any>, options?: updateApiOptions): Promise<Document | null> {
|
|
@@ -549,9 +722,10 @@ class MongoRest {
|
|
|
549
722
|
}
|
|
550
723
|
|
|
551
724
|
return result as Document | null
|
|
552
|
-
})
|
|
725
|
+
}, { invalidate: true })
|
|
553
726
|
}
|
|
554
727
|
|
|
728
|
+
@LogSlowQuery()
|
|
555
729
|
@CheckIfCollectionExists()
|
|
556
730
|
@CheckIfArrayOfIds('updateMany')
|
|
557
731
|
async updateMany(collection: string, _ids: string[], update: UpdateFilter<any>): Promise<UpdateResult> {
|
|
@@ -583,9 +757,10 @@ class MongoRest {
|
|
|
583
757
|
}
|
|
584
758
|
|
|
585
759
|
return result as UpdateResult
|
|
586
|
-
})
|
|
760
|
+
}, { invalidate: _ids })
|
|
587
761
|
}
|
|
588
762
|
|
|
763
|
+
@LogSlowQuery()
|
|
589
764
|
@CheckIfCollectionExists()
|
|
590
765
|
@CheckIfId('deleteOne')
|
|
591
766
|
async deleteOne(collection: string, _id: string) {
|
|
@@ -610,10 +785,11 @@ class MongoRest {
|
|
|
610
785
|
}
|
|
611
786
|
|
|
612
787
|
return result
|
|
613
|
-
})
|
|
788
|
+
}, { invalidate: [_id] })
|
|
614
789
|
}
|
|
615
790
|
|
|
616
791
|
|
|
792
|
+
@LogSlowQuery()
|
|
617
793
|
@CheckIfCollectionExists()
|
|
618
794
|
@CheckIfArrayOfIds('deleteMany')
|
|
619
795
|
async deleteMany(collection: string, _ids: string[]): Promise<DeleteResult> {
|
|
@@ -638,10 +814,11 @@ class MongoRest {
|
|
|
638
814
|
}
|
|
639
815
|
|
|
640
816
|
return result
|
|
641
|
-
})
|
|
817
|
+
}, { invalidate: _ids })
|
|
642
818
|
}
|
|
643
819
|
|
|
644
820
|
|
|
821
|
+
@LogSlowQuery()
|
|
645
822
|
@CheckIfCollectionExists()
|
|
646
823
|
@CheckBulkWriteOperations()
|
|
647
824
|
async bulkWrite(collection: string, operations: AnyBulkWriteOperation[], options?: BulkWriteOptions): Promise<BulkWriteResult> {
|
|
@@ -704,10 +881,11 @@ class MongoRest {
|
|
|
704
881
|
})
|
|
705
882
|
|
|
706
883
|
return result
|
|
707
|
-
})
|
|
884
|
+
}, { invalidate: true })
|
|
708
885
|
}
|
|
709
886
|
|
|
710
887
|
|
|
888
|
+
@LogSlowQuery()
|
|
711
889
|
@CheckIfCollectionExists()
|
|
712
890
|
async bulkUpdate(collection: string, operations: BulkUpdateOperation[], options?: BulkWriteOptions): Promise<BulkWriteResult> {
|
|
713
891
|
const action = 'bulkUpdate' as ActionsApiList
|
|
@@ -737,16 +915,18 @@ class MongoRest {
|
|
|
737
915
|
})
|
|
738
916
|
|
|
739
917
|
return result
|
|
740
|
-
})
|
|
918
|
+
}, { invalidate: true })
|
|
741
919
|
}
|
|
742
920
|
|
|
743
921
|
|
|
744
922
|
|
|
923
|
+
@LogSlowQuery()
|
|
745
924
|
@CheckIfCollectionExists()
|
|
746
925
|
async countDocuments(collection: string, query: any = {}): Promise<number> {
|
|
747
926
|
const action = 'countDocuments' as ActionsApiList
|
|
748
927
|
return this.#executeWithAudit(action, collection, { query }, async () => {
|
|
749
|
-
|
|
928
|
+
// ⚠️ must pass the filter — countDocuments() without args counts the whole collection
|
|
929
|
+
return await this.db.collection(collection).countDocuments(query)
|
|
750
930
|
})
|
|
751
931
|
}
|
|
752
932
|
|
|
@@ -755,7 +935,7 @@ class MongoRest {
|
|
|
755
935
|
const action = 'dropCollection' as ActionsApiList
|
|
756
936
|
return this.#executeWithAudit(action, collection, undefined, async () => {
|
|
757
937
|
return await this.db.collection(collection).drop()
|
|
758
|
-
})
|
|
938
|
+
}, { invalidate: true })
|
|
759
939
|
}
|
|
760
940
|
|
|
761
941
|
@CheckIfCollectionExists()
|
|
@@ -921,6 +1101,38 @@ class MongoRest {
|
|
|
921
1101
|
}
|
|
922
1102
|
}
|
|
923
1103
|
|
|
1104
|
+
/**
|
|
1105
|
+
* Tenant-scoped cache — `rest.cache.*` stores under the `app:{tenant_id}`
|
|
1106
|
+
* namespace, so keys never collide across tenants. Uses the configured
|
|
1107
|
+
* cache driver (server.cache — default filesystem ./.cache).
|
|
1108
|
+
*
|
|
1109
|
+
* @example
|
|
1110
|
+
* await rest.cache.set('price:42', { total: 99 }, '5m');
|
|
1111
|
+
* const v = await rest.cache.getOrSet('stats', () => rest.aggregate('orders', [...]), '1h');
|
|
1112
|
+
* await rest.cache.delete('price:42');
|
|
1113
|
+
* await rest.cache.clear();
|
|
1114
|
+
*/
|
|
1115
|
+
get cache(): TenantCache {
|
|
1116
|
+
const tenantId = this.tenant_id;
|
|
1117
|
+
const cache = getDbCache();
|
|
1118
|
+
const ns = () => cache.namespace(`app:${tenantId}`);
|
|
1119
|
+
return {
|
|
1120
|
+
set: (key, value, ttl = '5m') => ns().set({ key, value, ttl }),
|
|
1121
|
+
setForever: (key, value) => ns().setForever({ key, value }),
|
|
1122
|
+
get: (key) => ns().get({ key }),
|
|
1123
|
+
getOrSet: (key, factory, ttl = '5m') => ns().getOrSet({ key, factory, ttl }),
|
|
1124
|
+
getOrSetForever: (key, factory) => ns().getOrSetForever({ key, factory }),
|
|
1125
|
+
has: (key) => ns().has({ key }),
|
|
1126
|
+
missing: (key) => ns().missing({ key }),
|
|
1127
|
+
delete: (key) => ns().delete({ key }),
|
|
1128
|
+
deleteMany: (keys) => ns().deleteMany({ keys }),
|
|
1129
|
+
pull: (key) => ns().pull(key),
|
|
1130
|
+
expire: (key, ttl) => ns().expire({ key, ttl }),
|
|
1131
|
+
clear: () => ns().clear(),
|
|
1132
|
+
namespace: (name) => cache.namespace(`app:${tenantId}:${name}`),
|
|
1133
|
+
};
|
|
1134
|
+
}
|
|
1135
|
+
|
|
924
1136
|
/**
|
|
925
1137
|
* Acquire a distributed lock for this tenant.
|
|
926
1138
|
* Uses MongoDB atomic findOneAndUpdate with upsert.
|