@snowtop/ent 0.2.5 → 0.2.7

package/core/loaders/assoc_edge_loader.js

@@ -15,32 +15,58 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.AssocEdgeLoaderFactory = exports.AssocDirectEdgeLoader = exports.AssocEdgeLoader = void 0;
-const dataloader_1 = __importDefault(require("dataloader"));
 const memoizee_1 = __importDefault(require("memoizee"));
 const clause = __importStar(require("../clause"));
 const ent_1 = require("../ent");
-const logger_1 = require("../logger");
+const cache_utils_1 = require("../cache_utils");
 const loader_1 = require("./loader");
-function createLoader(options, edgeType, edgeCtr, edgeData) {
-    const loaderOptions = {};
-    if ((0, logger_1.logEnabled)("query")) {
-        loaderOptions.cacheMap = new loader_1.CacheMap({
+function getDefaultOrderBy() {
+    return [
+        {
+            column: "time",
+            direction: "DESC",
+        },
+    ];
+}
+function getEffectiveOptions(options) {
+    return {
+        ...options,
+        orderby: options.orderby ?? getDefaultOrderBy(),
+        limit: options.limit || (0, ent_1.getDefaultLimit)(),
+        disableTransformations: options.disableTransformations ?? false,
+    };
+}
+function createLoader(options, edgeType, edgeCtr, edgeData, context) {
+    const loaderName = `assocEdgeLoader:${edgeType}`;
+    const loaderOptions = {
+        maxBatchSize: (0, loader_1.getLoaderMaxBatchSize)(),
+        cacheMap: (0, loader_1.createLoaderCacheMap)({
             tableName: edgeData.edgeTable,
-        });
-    }
-    return new dataloader_1.default(async (keys) => {
+        }),
+    };
+    return new loader_1.InstrumentedDataLoader(loaderName, async (keys) => {
         if (keys.length === 1) {
             // 1 key, just be simple and move on
             // same as AssocDirectEdgeLoader
@@ -48,6 +74,7 @@ function createLoader(options, edgeType, edgeCtr, edgeData) {
                 id1: keys[0],
                 edgeType: edgeType,
                 queryOptions: options,
+                context,
                 ctr: edgeCtr,
             });
             return [r];
@@ -59,14 +86,9 @@ function createLoader(options, edgeType, edgeCtr, edgeData) {
             // store the index....
             m.set(keys[i], i);
         }
-        options.orderby = options.orderby || [
-            {
-                column: "time",
-                direction: "DESC",
-            },
-        ];
+        const orderby = options.orderby ?? getDefaultOrderBy();
         // TODO defaultEdgeQueryOptions
-        options.limit = options.limit || (0, ent_1.getDefaultLimit)();
+        const limit = options.limit || (0, ent_1.getDefaultLimit)();
         const tableName = edgeData.edgeTable;
         const { cls: cls1, fields } = (0, ent_1.getEdgeClauseAndFields)(clause.Eq("edge_type", edgeType), {
             queryOptions: options,
@@ -75,8 +97,8 @@ function createLoader(options, edgeType, edgeCtr, edgeData) {
             tableName: tableName,
             fields,
             values: keys,
-            orderby: options.orderby,
-            limit: options.limit || (0, ent_1.getDefaultLimit)(),
+            orderby,
+            limit,
             groupColumn: "id1",
             clause: cls1,
         });
@@ -91,7 +113,7 @@ function createLoader(options, edgeType, edgeCtr, edgeData) {
             result[idx].push(new edgeCtr(row));
         }
         return result;
-    }, loaderOptions);
+    }, loaderOptions, edgeData.edgeTable);
 }
 class AssocEdgeLoader {
     constructor(edgeType, edgeCtr, options, context) {
@@ -106,7 +128,7 @@ class AssocEdgeLoader {
         if (!edgeData) {
             throw new Error(`error loading edge data for ${this.edgeType}`);
         }
-        this.loader = createLoader(this.options, this.edgeType, this.edgeCtr, edgeData);
+        this.loader = createLoader(this.options, this.edgeType, this.edgeCtr, edgeData, this.context);
         return this.loader;
     }
     async load(id) {
@@ -144,8 +166,40 @@ class AssocDirectEdgeLoader {
         this.edgeCtr = edgeCtr;
         this.options = options;
         this.context = context;
+        if (this.context) {
+            this.loaderFn = (0, memoizee_1.default)(this.getLoader);
+        }
+    }
+    async getLoader() {
+        if (this.loader) {
+            return this.loader;
+        }
+        const edgeData = await (0, ent_1.loadEdgeData)(this.edgeType);
+        if (!edgeData) {
+            throw new Error(`error loading edge data for ${this.edgeType}`);
+        }
+        const loaderName = `assocDirectEdgeLoader:${this.edgeType}`;
+        this.loader = new loader_1.InstrumentedDataLoader(loaderName, async (keys) => {
+            return Promise.all(keys.map((id) => (0, ent_1.loadCustomEdges)({
+                id1: id,
+                edgeType: this.edgeType,
+                context: this.context,
+                queryOptions: this.options,
+                ctr: this.edgeCtr,
+            })));
+        }, {
+            maxBatchSize: (0, loader_1.getLoaderMaxBatchSize)(),
+            cacheMap: (0, loader_1.createLoaderCacheMap)({
+                tableName: edgeData.edgeTable,
+            }),
+        }, edgeData.edgeTable);
+        return this.loader;
     }
     async load(id) {
+        if (this.loaderFn) {
+            const loader = await this.loaderFn();
+            return loader.load(id);
+        }
         return (0, ent_1.loadCustomEdges)({
             id1: id,
             edgeType: this.edgeType,
@@ -173,7 +227,9 @@ class AssocDirectEdgeLoader {
             ctr: this.edgeCtr,
         });
     }
-    clearAll() { }
+    clearAll() {
+        this.loader && this.loader.clearAll();
+    }
 }
 exports.AssocDirectEdgeLoader = AssocDirectEdgeLoader;
 class AssocEdgeLoaderFactory {
@@ -207,9 +263,9 @@ class AssocEdgeLoaderFactory {
         if (options?.clause || !context) {
             return new AssocDirectEdgeLoader(this.edgeType, edgeCtr, options, context);
         }
-        // we create a loader which can combine first X queries in the same fetch
-        const key = `${this.name}:limit:${options.limit}:orderby:${options.orderby?.map((orderBy) => JSON.stringify(orderBy))}:disableTransformations:${options.disableTransformations}`;
-        return (0, loader_1.getCustomLoader)(key, () => new AssocEdgeLoader(this.edgeType, ctr, options, context), context);
+        const effectiveOptions = getEffectiveOptions(options);
+        const key = `${this.name}:limit:${effectiveOptions.limit}:orderby:${(0, cache_utils_1.stableStringify)(effectiveOptions.orderby)}:disableTransformations:${effectiveOptions.disableTransformations}`;
+        return (0, loader_1.getCustomLoader)(key, () => new AssocEdgeLoader(this.edgeType, ctr, effectiveOptions, context), context);
     }
 }
 exports.AssocEdgeLoaderFactory = AssocEdgeLoaderFactory;

package/core/loaders/index.d.ts

@@ -3,3 +3,5 @@ export { RawCountLoader, RawCountLoaderFactory } from "./raw_count_loader";
 export { AssocEdgeCountLoader, AssocEdgeCountLoaderFactory, } from "./assoc_count_loader";
 export { AssocDirectEdgeLoader, AssocEdgeLoader, AssocEdgeLoaderFactory, } from "./assoc_edge_loader";
 export { QueryLoaderFactory } from "./query_loader";
+export { getLoaderCacheMaxEntries, getLoaderMaxBatchSize, setLoaderCacheMaxEntries, setLoaderMaxBatchSize, } from "./loader";
+export { setClauseLoaderConcurrency } from "./object_loader";

package/core/loaders/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.QueryLoaderFactory = exports.AssocEdgeLoaderFactory = exports.AssocEdgeLoader = exports.AssocDirectEdgeLoader = exports.AssocEdgeCountLoaderFactory = exports.AssocEdgeCountLoader = exports.RawCountLoaderFactory = exports.RawCountLoader = exports.ObjectLoaderFactory = exports.ObjectLoader = void 0;
+exports.setClauseLoaderConcurrency = exports.setLoaderMaxBatchSize = exports.setLoaderCacheMaxEntries = exports.getLoaderMaxBatchSize = exports.getLoaderCacheMaxEntries = exports.QueryLoaderFactory = exports.AssocEdgeLoaderFactory = exports.AssocEdgeLoader = exports.AssocDirectEdgeLoader = exports.AssocEdgeCountLoaderFactory = exports.AssocEdgeCountLoader = exports.RawCountLoaderFactory = exports.RawCountLoader = exports.ObjectLoaderFactory = exports.ObjectLoader = void 0;
 var object_loader_1 = require("./object_loader");
 Object.defineProperty(exports, "ObjectLoader", { enumerable: true, get: function () { return object_loader_1.ObjectLoader; } });
 Object.defineProperty(exports, "ObjectLoaderFactory", { enumerable: true, get: function () { return object_loader_1.ObjectLoaderFactory; } });
@@ -16,3 +16,10 @@ Object.defineProperty(exports, "AssocEdgeLoader", { enumerable: true, get: funct
 Object.defineProperty(exports, "AssocEdgeLoaderFactory", { enumerable: true, get: function () { return assoc_edge_loader_1.AssocEdgeLoaderFactory; } });
 var query_loader_1 = require("./query_loader");
 Object.defineProperty(exports, "QueryLoaderFactory", { enumerable: true, get: function () { return query_loader_1.QueryLoaderFactory; } });
+var loader_1 = require("./loader");
+Object.defineProperty(exports, "getLoaderCacheMaxEntries", { enumerable: true, get: function () { return loader_1.getLoaderCacheMaxEntries; } });
+Object.defineProperty(exports, "getLoaderMaxBatchSize", { enumerable: true, get: function () { return loader_1.getLoaderMaxBatchSize; } });
+Object.defineProperty(exports, "setLoaderCacheMaxEntries", { enumerable: true, get: function () { return loader_1.setLoaderCacheMaxEntries; } });
+Object.defineProperty(exports, "setLoaderMaxBatchSize", { enumerable: true, get: function () { return loader_1.setLoaderMaxBatchSize; } });
+var object_loader_2 = require("./object_loader");
+Object.defineProperty(exports, "setClauseLoaderConcurrency", { enumerable: true, get: function () { return object_loader_2.setClauseLoaderConcurrency; } });

package/core/loaders/loader.d.ts

@@ -1,6 +1,38 @@
+import DataLoader from "dataloader";
 import { Loader, LoaderFactory, Context, DataOptions } from "../base";
+export declare function getLoaderMaxBatchSize(): number;
+export declare function setLoaderMaxBatchSize(size?: number | null): void;
+export declare function getLoaderCacheMaxEntries(): number;
+export declare function setLoaderCacheMaxEntries(size?: number | null): void;
 export declare function getLoader<K, V>(factory: LoaderFactory<K, V>, create: () => Loader<K, V>, context?: Context): Loader<K, V>;
 export declare function getCustomLoader<K, V>(key: string, create: () => Loader<K, V>, context?: Context): Loader<K, V>;
+export type CacheMapLike<K, V> = {
+    get(key: K): V | undefined;
+    set(key: K, value: V): any;
+    delete(key: K): any;
+    clear(): any;
+};
+type CacheKeyFn<K> = (key: K) => any;
+type BatchLoadFn<K, V> = (keys: readonly K[]) => PromiseLike<ArrayLike<V | Error>>;
+export declare class InstrumentedDataLoader<K, V> extends DataLoader<K, V> {
+    constructor(loaderName: string, batchLoadFn: BatchLoadFn<K, V>, options: DataLoader.Options<K, V>, tableName?: string, cacheKeyFn?: (key: K) => unknown);
+}
+export declare class BoundedCacheMap<K, V> {
+    private cacheMap;
+    private maxEntries;
+    private cacheKeyFn?;
+    private order;
+    constructor(cacheMap: CacheMapLike<K, V>, maxEntries: number, cacheKeyFn?: CacheKeyFn<K> | undefined);
+    private normalizeKey;
+    private touch;
+    private evictIfNeeded;
+    get(key: K): V | undefined;
+    set(key: K, value: V): any;
+    delete(key: K): any;
+    clear(): any;
+}
+export declare function createBoundedCacheMap<K, V>(cacheMap: CacheMapLike<K, V>, cacheKeyFn?: CacheKeyFn<K>): CacheMapLike<K, V>;
+export declare function createLoaderCacheMap<K, V>(options: DataOptions): CacheMapLike<K, V>;
 export declare class CacheMap {
     private options;
     private m;
@@ -10,3 +42,4 @@ export declare class CacheMap {
     delete(key: any): boolean;
     clear(): void;
 }
+export {};

package/core/loaders/loader.js

@@ -1,7 +1,50 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CacheMap = exports.getCustomLoader = exports.getLoader = void 0;
+exports.CacheMap = exports.BoundedCacheMap = exports.InstrumentedDataLoader = void 0;
+exports.getLoaderMaxBatchSize = getLoaderMaxBatchSize;
+exports.setLoaderMaxBatchSize = setLoaderMaxBatchSize;
+exports.getLoaderCacheMaxEntries = getLoaderCacheMaxEntries;
+exports.setLoaderCacheMaxEntries = setLoaderCacheMaxEntries;
+exports.getLoader = getLoader;
+exports.getCustomLoader = getCustomLoader;
+exports.createBoundedCacheMap = createBoundedCacheMap;
+exports.createLoaderCacheMap = createLoaderCacheMap;
+const dataloader_1 = __importDefault(require("dataloader"));
 const logger_1 = require("../logger");
+const metrics_1 = require("../metrics");
+const DEFAULT_MAX_BATCH_SIZE = 1000;
+let loaderMaxBatchSize = DEFAULT_MAX_BATCH_SIZE;
+const DEFAULT_MAX_CACHE_ENTRIES = 1000;
+let loaderCacheMaxEntries = DEFAULT_MAX_CACHE_ENTRIES;
+function getLoaderMaxBatchSize() {
+    return loaderMaxBatchSize;
+}
+function setLoaderMaxBatchSize(size) {
+    if (size === undefined || size === null) {
+        loaderMaxBatchSize = DEFAULT_MAX_BATCH_SIZE;
+        return;
+    }
+    if (!Number.isFinite(size) || size <= 0) {
+        throw new Error(`maxBatchSize must be a positive number`);
+    }
+    loaderMaxBatchSize = Math.floor(size);
+}
+function getLoaderCacheMaxEntries() {
+    return loaderCacheMaxEntries;
+}
+function setLoaderCacheMaxEntries(size) {
+    if (size === undefined || size === null) {
+        loaderCacheMaxEntries = DEFAULT_MAX_CACHE_ENTRIES;
+        return;
+    }
+    if (!Number.isFinite(size) || size < 0) {
+        throw new Error(`maxCacheEntries must be a non-negative number`);
+    }
+    loaderCacheMaxEntries = Math.floor(size);
+}
 // this is like factory factory FML
 // helper function to handle context vs not
 // and to keep the API clean for clients who shouldn't have to worry about this
@@ -14,7 +57,6 @@ function getLoader(factory, create, context) {
     // g|set from context cache
     return context.cache.getLoader(factory.name, create);
 }
-exports.getLoader = getLoader;
 function getCustomLoader(key, create, context) {
     // just create a new one every time if no context cache
     if (!context?.cache) {
@@ -24,7 +66,121 @@ function getCustomLoader(key, create, context) {
     // g|set from context cache
     return context.cache.getLoader(key, create);
 }
-exports.getCustomLoader = getCustomLoader;
+function instrumentCacheMap(cacheMap, tableName, cacheKeyFn) {
+    if (!cacheMap || !tableName) {
+        return cacheMap;
+    }
+    return {
+        get(key) {
+            const value = cacheMap.get(key);
+            if (value !== undefined) {
+                const hook = (0, metrics_1.getOnDataLoaderCacheHit)();
+                if (hook) {
+                    hook({
+                        tableName,
+                        key: cacheKeyFn ? cacheKeyFn(key) : key,
+                    });
+                }
+            }
+            return value;
+        },
+        set(key, value) {
+            return cacheMap.set(key, value);
+        },
+        delete(key) {
+            return cacheMap.delete(key);
+        },
+        clear() {
+            return cacheMap.clear();
+        },
+    };
+}
+class InstrumentedDataLoader extends dataloader_1.default {
+    constructor(loaderName, batchLoadFn, options, tableName, cacheKeyFn) {
+        const wrappedBatchFn = async (keys) => {
+            if (keys.length) {
+                const hook = (0, metrics_1.getOnDataLoaderBatch)();
+                if (hook) {
+                    hook({
+                        loaderName,
+                        batchSize: keys.length,
+                    });
+                }
+            }
+            return batchLoadFn(keys);
+        };
+        const cacheMap = instrumentCacheMap(options.cacheMap, tableName, cacheKeyFn);
+        const loaderOptions = cacheMap === options.cacheMap ? options : { ...options, cacheMap };
+        super(wrappedBatchFn, loaderOptions);
+    }
+}
+exports.InstrumentedDataLoader = InstrumentedDataLoader;
+class BoundedCacheMap {
+    constructor(cacheMap, maxEntries, cacheKeyFn) {
+        this.cacheMap = cacheMap;
+        this.maxEntries = maxEntries;
+        this.cacheKeyFn = cacheKeyFn;
+        this.order = new Map();
+    }
+    normalizeKey(key) {
+        return this.cacheKeyFn ? this.cacheKeyFn(key) : key;
+    }
+    touch(normalizedKey, key) {
+        if (this.order.has(normalizedKey)) {
+            this.order.delete(normalizedKey);
+        }
+        this.order.set(normalizedKey, key);
+    }
+    evictIfNeeded() {
+        while (this.order.size > this.maxEntries) {
+            const oldest = this.order.entries().next().value;
+            if (!oldest) {
+                return;
+            }
+            const [normalizedKey, key] = oldest;
+            this.order.delete(normalizedKey);
+            this.cacheMap.delete(key);
+        }
+    }
+    get(key) {
+        const value = this.cacheMap.get(key);
+        if (value !== undefined) {
+            const normalizedKey = this.normalizeKey(key);
+            this.touch(normalizedKey, key);
+        }
+        return value;
+    }
+    set(key, value) {
+        const normalizedKey = this.normalizeKey(key);
+        this.touch(normalizedKey, key);
+        const result = this.cacheMap.set(key, value);
+        this.evictIfNeeded();
+        return result;
+    }
+    delete(key) {
+        const normalizedKey = this.normalizeKey(key);
+        this.order.delete(normalizedKey);
+        return this.cacheMap.delete(key);
+    }
+    clear() {
+        this.order.clear();
+        return this.cacheMap.clear();
+    }
+}
+exports.BoundedCacheMap = BoundedCacheMap;
+function createBoundedCacheMap(cacheMap, cacheKeyFn) {
+    const maxEntries = getLoaderCacheMaxEntries();
+    if (maxEntries <= 0) {
+        return cacheMap;
+    }
+    return new BoundedCacheMap(cacheMap, maxEntries, cacheKeyFn);
+}
+function createLoaderCacheMap(options) {
+    const baseMap = (0, logger_1.logEnabled)("query")
+        ? new CacheMap(options)
+        : new Map();
+    return createBoundedCacheMap(baseMap);
+}
 class CacheMap {
     constructor(options) {
         this.options = options;

package/core/loaders/object_loader.d.ts

@@ -1,14 +1,17 @@
 import { ID, Data, SelectDataOptions, Context, Loader, LoaderFactory } from "../base";
+import { mapWithConcurrency } from "../async_utils";
 import * as clause from "../clause";
+export declare function setClauseLoaderConcurrency(limit: number): void;
+export { mapWithConcurrency };
 export declare class ObjectLoader<TQueryData extends Data = Data, TResultData extends Data = TQueryData, K = keyof TQueryData> implements Loader<ID, TResultData | null>, Loader<clause.Clause<TQueryData, K>, TResultData[] | null> {
     private options;
-    context?: Context<import("../base").Viewer<import("../base").Ent<any> | null, ID | null>> | undefined;
+    context?: Context | undefined;
     private toPrime?;
     private idLoader;
     private clauseLoader;
     private primedLoaders;
     private memoizedInitPrime;
-    constructor(options: SelectDataOptions, context?: Context<import("../base").Viewer<import("../base").Ent<any> | null, ID | null>> | undefined, toPrime?: ObjectLoaderFactory<TResultData>[] | undefined);
+    constructor(options: SelectDataOptions, context?: Context | undefined, toPrime?: ObjectLoaderFactory<TResultData>[] | undefined);
     getOptions(): SelectDataOptions;
     private initPrime;
     load(key: ID): Promise<TResultData | null>;
@@ -25,9 +28,9 @@ export declare class ObjectLoader<TQueryData extends Data = Data, TResultData ex
 }
 export declare class ObjectCountLoader<V extends Data = Data, K = keyof V> implements Loader<clause.Clause<V, K>, number> {
     private options;
-    context?: Context<import("../base").Viewer<import("../base").Ent<any> | null, ID | null>> | undefined;
+    context?: Context | undefined;
     private loader;
-    constructor(options: SelectDataOptions, context?: Context<import("../base").Viewer<import("../base").Ent<any> | null, ID | null>> | undefined);
+    constructor(options: SelectDataOptions, context?: Context | undefined);
     getOptions(): SelectDataOptions;
     load(key: clause.Clause<V, K>): Promise<number>;
     clearAll(): void;
@@ -46,4 +49,3 @@ export declare class ObjectLoaderFactory<V extends Data = Data> implements Loade
     createCountLoader<K = keyof V>(context?: Context): ObjectCountLoader<V, K>;
     addToPrime(factory: ObjectLoaderFactory<V>): this;
 }
-export {};