@snowtop/ent 0.2.6 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/async_utils.d.ts +1 -0
- package/core/async_utils.js +29 -0
- package/core/base.d.ts +1 -1
- package/core/config.d.ts +2 -0
- package/core/config.js +7 -0
- package/core/context.d.ts +3 -1
- package/core/context.js +96 -25
- package/core/ent.d.ts +2 -2
- package/core/ent.js +11 -41
- package/core/loaders/assoc_count_loader.js +1 -1
- package/core/loaders/assoc_edge_loader.d.ts +3 -0
- package/core/loaders/assoc_edge_loader.js +45 -13
- package/core/loaders/index.d.ts +2 -1
- package/core/loaders/index.js +5 -1
- package/core/loaders/loader.d.ts +31 -0
- package/core/loaders/loader.js +140 -1
- package/core/loaders/object_loader.d.ts +2 -2
- package/core/loaders/object_loader.js +36 -51
- package/core/loaders/query_loader.js +36 -16
- package/core/loaders/raw_count_loader.d.ts +2 -2
- package/core/loaders/raw_count_loader.js +12 -14
- package/core/metrics.d.ts +22 -0
- package/core/metrics.js +31 -0
- package/index.d.ts +2 -1
- package/index.js +5 -2
- package/package.json +1 -1
- /package/core/{loaders/cache_utils.d.ts → cache_utils.d.ts} +0 -0
- /package/core/{loaders/cache_utils.js → cache_utils.js} +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function mapWithConcurrency<T, U>(items: T[], limit: number, mapper: (item: T, index: number) => Promise<U>): Promise<U[]>;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.mapWithConcurrency = mapWithConcurrency;
|
|
4
|
+
async function mapWithConcurrency(items, limit, mapper) {
|
|
5
|
+
if (!items.length) {
|
|
6
|
+
return [];
|
|
7
|
+
}
|
|
8
|
+
const results = new Array(items.length);
|
|
9
|
+
const concurrency = limit === Infinity
|
|
10
|
+
? items.length
|
|
11
|
+
: Number.isFinite(limit) && limit > 0
|
|
12
|
+
? Math.max(1, Math.floor(limit))
|
|
13
|
+
: 1;
|
|
14
|
+
let nextIndex = 0;
|
|
15
|
+
const workers = new Array(Math.min(concurrency, items.length))
|
|
16
|
+
.fill(null)
|
|
17
|
+
.map(async () => {
|
|
18
|
+
while (true) {
|
|
19
|
+
const currentIndex = nextIndex;
|
|
20
|
+
if (currentIndex >= items.length) {
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
nextIndex += 1;
|
|
24
|
+
results[currentIndex] = await mapper(items[currentIndex], currentIndex);
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
await Promise.all(workers);
|
|
28
|
+
return results;
|
|
29
|
+
}
|
package/core/base.d.ts
CHANGED
|
@@ -26,7 +26,7 @@ export interface PrimableLoader<K, V> extends Loader<K, V> {
|
|
|
26
26
|
prime(d: V): void;
|
|
27
27
|
primeAll?(d: V): void;
|
|
28
28
|
}
|
|
29
|
-
export type QueryOptions = Required<Pick<LoadRowsOptions, "clause" | "fields" | "tableName">> & Pick<LoadRowsOptions, "orderby" | "join">;
|
|
29
|
+
export type QueryOptions = Required<Pick<LoadRowsOptions, "clause" | "fields" | "tableName">> & Pick<LoadRowsOptions, "distinct" | "alias" | "fieldsAlias" | "disableFieldsAlias" | "disableDefaultOrderByAlias" | "groupby" | "orderby" | "join" | "limit" | "offset">;
|
|
30
30
|
interface cache {
|
|
31
31
|
getLoader<K, V>(name: string, create: () => Loader<K, V>): Loader<K, V>;
|
|
32
32
|
getLoaderWithLoadMany<K, V>(name: string, create: () => LoaderWithLoadMany<K, V>): LoaderWithLoadMany<K, V>;
|
package/core/config.d.ts
CHANGED
|
@@ -20,6 +20,8 @@ export interface Config {
|
|
|
20
20
|
logQueryWithError?: boolean;
|
|
21
21
|
defaultConnectionLimit?: number;
|
|
22
22
|
loaderMaxBatchSize?: number;
|
|
23
|
+
clauseLoaderConcurrency?: number;
|
|
24
|
+
entLoaderPrivacyConcurrencyLimit?: number;
|
|
23
25
|
}
|
|
24
26
|
export interface ConfigWithCodegen extends Config {
|
|
25
27
|
codegen?: CodegenConfig;
|
package/core/config.js
CHANGED
|
@@ -43,6 +43,7 @@ const db_1 = __importDefault(require("./db"));
|
|
|
43
43
|
const path = __importStar(require("path"));
|
|
44
44
|
const logger_1 = require("./logger");
|
|
45
45
|
const ent_1 = require("./ent");
|
|
46
|
+
const object_loader_1 = require("./loaders/object_loader");
|
|
46
47
|
const loader_1 = require("./loaders/loader");
|
|
47
48
|
// ent.config.ts eventually. for now ent.yml
|
|
48
49
|
// or ent.yml?
|
|
@@ -79,6 +80,12 @@ function setConfig(cfg) {
|
|
|
79
80
|
if (cfg.loaderMaxBatchSize !== undefined) {
|
|
80
81
|
(0, loader_1.setLoaderMaxBatchSize)(cfg.loaderMaxBatchSize);
|
|
81
82
|
}
|
|
83
|
+
if (cfg.clauseLoaderConcurrency !== undefined) {
|
|
84
|
+
(0, object_loader_1.setClauseLoaderConcurrency)(cfg.clauseLoaderConcurrency);
|
|
85
|
+
}
|
|
86
|
+
if (cfg.entLoaderPrivacyConcurrencyLimit !== undefined) {
|
|
87
|
+
(0, ent_1.setEntLoaderPrivacyConcurrencyLimit)(cfg.entLoaderPrivacyConcurrencyLimit);
|
|
88
|
+
}
|
|
82
89
|
}
|
|
83
90
|
function isBuffer(b) {
|
|
84
91
|
return b.write !== undefined;
|
package/core/context.d.ts
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import { IncomingMessage, ServerResponse } from "http";
|
|
2
2
|
import { Data, Loader, LoaderWithLoadMany, QueryOptions, Viewer } from "./base";
|
|
3
3
|
import { Context } from "./base";
|
|
4
|
+
export declare function getContextCacheMaxDiscardedLoaders(): number;
|
|
5
|
+
export declare function setContextCacheMaxDiscardedLoaders(size?: number | null): void;
|
|
4
6
|
export interface RequestContext<TViewer extends Viewer = Viewer> extends Context<TViewer> {
|
|
5
7
|
authViewer(viewer: TViewer): Promise<void>;
|
|
6
8
|
logout(): Promise<void>;
|
|
7
9
|
request: IncomingMessage;
|
|
8
10
|
response: ServerResponse;
|
|
9
11
|
}
|
|
12
|
+
export declare function getContextCacheKey(options: QueryOptions): string;
|
|
10
13
|
export declare class ContextCache {
|
|
11
14
|
loaders: Map<string, Loader<any, any>>;
|
|
12
15
|
loaderWithLoadMany: Map<string, LoaderWithLoadMany<any, any>>;
|
|
@@ -15,7 +18,6 @@ export declare class ContextCache {
|
|
|
15
18
|
getLoaderWithLoadMany<K, V>(name: string, create: () => LoaderWithLoadMany<K, V>): LoaderWithLoadMany<K, V>;
|
|
16
19
|
private itemMap;
|
|
17
20
|
private listMap;
|
|
18
|
-
private getkey;
|
|
19
21
|
getCachedRows(options: QueryOptions): Data[] | null;
|
|
20
22
|
getCachedRow(options: QueryOptions): Data | null;
|
|
21
23
|
primeCache(options: QueryOptions, rows: Data[]): void;
|
package/core/context.js
CHANGED
|
@@ -1,8 +1,78 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ContextCache = void 0;
|
|
4
|
+
exports.getContextCacheMaxDiscardedLoaders = getContextCacheMaxDiscardedLoaders;
|
|
5
|
+
exports.setContextCacheMaxDiscardedLoaders = setContextCacheMaxDiscardedLoaders;
|
|
6
|
+
exports.getContextCacheKey = getContextCacheKey;
|
|
4
7
|
const logger_1 = require("./logger");
|
|
5
|
-
const
|
|
8
|
+
const cache_utils_1 = require("./cache_utils");
|
|
9
|
+
const metrics_1 = require("./metrics");
|
|
10
|
+
const DEFAULT_MAX_DISCARDED_LOADERS = 1000;
|
|
11
|
+
let maxDiscardedLoaders = DEFAULT_MAX_DISCARDED_LOADERS;
|
|
12
|
+
function getContextCacheMaxDiscardedLoaders() {
|
|
13
|
+
return maxDiscardedLoaders;
|
|
14
|
+
}
|
|
15
|
+
function setContextCacheMaxDiscardedLoaders(size) {
|
|
16
|
+
if (size === undefined || size === null) {
|
|
17
|
+
maxDiscardedLoaders = DEFAULT_MAX_DISCARDED_LOADERS;
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
if (!Number.isFinite(size) || size < 0) {
|
|
21
|
+
throw new Error(`maxDiscardedLoaders must be a non-negative number`);
|
|
22
|
+
}
|
|
23
|
+
maxDiscardedLoaders = Math.floor(size);
|
|
24
|
+
}
|
|
25
|
+
function getContextCacheKey(options) {
|
|
26
|
+
const fields = options.fields
|
|
27
|
+
.map((f) => {
|
|
28
|
+
if (typeof f === "object") {
|
|
29
|
+
return `${f.alias}.${f.column}`;
|
|
30
|
+
}
|
|
31
|
+
return f;
|
|
32
|
+
})
|
|
33
|
+
.join(",");
|
|
34
|
+
const parts = [
|
|
35
|
+
`fields:${fields}`,
|
|
36
|
+
`clause:${options.clause.instanceKey()}`,
|
|
37
|
+
];
|
|
38
|
+
if (options.distinct !== undefined) {
|
|
39
|
+
parts.push(`distinct:${options.distinct}`);
|
|
40
|
+
}
|
|
41
|
+
if (options.alias !== undefined) {
|
|
42
|
+
parts.push(`alias:${options.alias}`);
|
|
43
|
+
}
|
|
44
|
+
if (options.fieldsAlias !== undefined) {
|
|
45
|
+
parts.push(`fieldsAlias:${options.fieldsAlias}`);
|
|
46
|
+
}
|
|
47
|
+
if (options.disableFieldsAlias !== undefined) {
|
|
48
|
+
parts.push(`disableFieldsAlias:${options.disableFieldsAlias}`);
|
|
49
|
+
}
|
|
50
|
+
if (options.disableDefaultOrderByAlias !== undefined) {
|
|
51
|
+
parts.push(`disableDefaultOrderByAlias:${options.disableDefaultOrderByAlias}`);
|
|
52
|
+
}
|
|
53
|
+
if (options.groupby !== undefined) {
|
|
54
|
+
parts.push(`groupby:${options.groupby}`);
|
|
55
|
+
}
|
|
56
|
+
if (options.orderby) {
|
|
57
|
+
parts.push(`orderby:${(0, cache_utils_1.stableStringify)(options.orderby)}`);
|
|
58
|
+
}
|
|
59
|
+
if (options.join) {
|
|
60
|
+
const joinKey = options.join.map((join) => ({
|
|
61
|
+
type: join.type ?? "inner",
|
|
62
|
+
tableName: join.tableName,
|
|
63
|
+
alias: join.alias,
|
|
64
|
+
clause: join.clause.instanceKey(),
|
|
65
|
+
}));
|
|
66
|
+
parts.push(`join:${(0, cache_utils_1.stableStringify)(joinKey)}`);
|
|
67
|
+
}
|
|
68
|
+
if (options.limit !== undefined) {
|
|
69
|
+
parts.push(`limit:${options.limit}`);
|
|
70
|
+
}
|
|
71
|
+
if (options.offset !== undefined) {
|
|
72
|
+
parts.push(`offset:${options.offset}`);
|
|
73
|
+
}
|
|
74
|
+
return parts.join(",");
|
|
75
|
+
}
|
|
6
76
|
class ContextCache {
|
|
7
77
|
constructor() {
|
|
8
78
|
this.loaders = new Map();
|
|
@@ -37,34 +107,21 @@ class ContextCache {
|
|
|
37
107
|
}
|
|
38
108
|
// tableName is ignored bcos already indexed on that
|
|
39
109
|
// maybe we just want to store sql queries???
|
|
40
|
-
getkey(options) {
|
|
41
|
-
let parts = [
|
|
42
|
-
options.fields
|
|
43
|
-
.map((f) => {
|
|
44
|
-
if (typeof f === "object") {
|
|
45
|
-
return `${f.alias}.${f.column}`;
|
|
46
|
-
}
|
|
47
|
-
return f;
|
|
48
|
-
})
|
|
49
|
-
.join(","),
|
|
50
|
-
options.clause.instanceKey(),
|
|
51
|
-
];
|
|
52
|
-
if (options.orderby) {
|
|
53
|
-
parts.push((0, query_impl_1.getOrderByPhrase)(options.orderby));
|
|
54
|
-
}
|
|
55
|
-
if (options.join) {
|
|
56
|
-
parts.push((0, query_impl_1.getJoinInfo)(options.join).phrase);
|
|
57
|
-
}
|
|
58
|
-
return parts.join(",");
|
|
59
|
-
}
|
|
60
110
|
getCachedRows(options) {
|
|
61
111
|
let m = this.listMap.get(options.tableName);
|
|
62
112
|
if (!m) {
|
|
63
113
|
return null;
|
|
64
114
|
}
|
|
65
|
-
const key =
|
|
115
|
+
const key = getContextCacheKey(options);
|
|
66
116
|
let rows = m.get(key);
|
|
67
117
|
if (rows) {
|
|
118
|
+
const hook = (0, metrics_1.getOnQueryCacheHit)();
|
|
119
|
+
if (hook) {
|
|
120
|
+
hook({
|
|
121
|
+
tableName: options.tableName,
|
|
122
|
+
key,
|
|
123
|
+
});
|
|
124
|
+
}
|
|
68
125
|
(0, logger_1.log)("cache", {
|
|
69
126
|
"cache-hit": key,
|
|
70
127
|
"tableName": options.tableName,
|
|
@@ -77,9 +134,16 @@ class ContextCache {
|
|
|
77
134
|
if (!m) {
|
|
78
135
|
return null;
|
|
79
136
|
}
|
|
80
|
-
const key =
|
|
137
|
+
const key = getContextCacheKey(options);
|
|
81
138
|
let row = m.get(key);
|
|
82
139
|
if (row) {
|
|
140
|
+
const hook = (0, metrics_1.getOnQueryCacheHit)();
|
|
141
|
+
if (hook) {
|
|
142
|
+
hook({
|
|
143
|
+
tableName: options.tableName,
|
|
144
|
+
key,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
83
147
|
(0, logger_1.log)("cache", {
|
|
84
148
|
"cache-hit": key,
|
|
85
149
|
"tableName": options.tableName,
|
|
@@ -90,12 +154,12 @@ class ContextCache {
|
|
|
90
154
|
primeCache(options, rows) {
|
|
91
155
|
if (Array.isArray(rows)) {
|
|
92
156
|
let m = this.listMap.get(options.tableName) || new Map();
|
|
93
|
-
m.set(
|
|
157
|
+
m.set(getContextCacheKey(options), rows);
|
|
94
158
|
this.listMap.set(options.tableName, m);
|
|
95
159
|
}
|
|
96
160
|
else {
|
|
97
161
|
let m = this.itemMap.get(options.tableName) || new Map();
|
|
98
|
-
m.set(
|
|
162
|
+
m.set(getContextCacheKey(options), rows);
|
|
99
163
|
this.itemMap.set(options.tableName, m);
|
|
100
164
|
}
|
|
101
165
|
}
|
|
@@ -113,6 +177,13 @@ class ContextCache {
|
|
|
113
177
|
this.loaderWithLoadMany.clear();
|
|
114
178
|
this.itemMap.clear();
|
|
115
179
|
this.listMap.clear();
|
|
180
|
+
if (maxDiscardedLoaders === 0) {
|
|
181
|
+
this.discardedLoaders = [];
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
if (this.discardedLoaders.length > maxDiscardedLoaders) {
|
|
185
|
+
this.discardedLoaders = this.discardedLoaders.slice(-maxDiscardedLoaders);
|
|
186
|
+
}
|
|
116
187
|
}
|
|
117
188
|
/**
|
|
118
189
|
* reset clears the cache and resets the discarded loaders
|
package/core/ent.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Context, CreateRowOptions, Data, DataOptions, EdgeQueryableDataOptions, EditRowOptions, Ent, ID, LoadCustomEntOptions, LoadEntOptions, LoadRowOptions, LoadRowsOptions, LoaderWithLoadMany, QueryDataOptions, SelectCustomDataOptions, Viewer } from "./base";
|
|
2
2
|
import { Queryer, SyncQueryer } from "./db";
|
|
3
|
-
import DataLoader from "dataloader";
|
|
4
3
|
import * as clause from "./clause";
|
|
4
|
+
import { InstrumentedDataLoader } from "./loaders/loader";
|
|
5
5
|
import { OrderBy } from "./query_impl";
|
|
6
6
|
declare class entCacheMap<TViewer extends Viewer, TEnt extends Ent<TViewer>> {
|
|
7
7
|
private viewer;
|
|
@@ -160,7 +160,7 @@ export declare class AssocEdgeData {
|
|
|
160
160
|
edgeTable: string;
|
|
161
161
|
constructor(data: Data);
|
|
162
162
|
}
|
|
163
|
-
export declare const assocEdgeLoader:
|
|
163
|
+
export declare const assocEdgeLoader: InstrumentedDataLoader<ID, Data | null>;
|
|
164
164
|
export declare function loadEdgeData(edgeType: string): Promise<AssocEdgeData | null>;
|
|
165
165
|
export declare function loadEdgeDatas(...edgeTypes: string[]): Promise<Map<string, AssocEdgeData>>;
|
|
166
166
|
export interface AssocEdgeConstructor<T extends AssocEdge> {
|
package/core/ent.js
CHANGED
|
@@ -32,9 +32,6 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
32
32
|
return result;
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
-
};
|
|
38
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
36
|
exports.assocEdgeLoader = exports.AssocEdgeData = exports.AssocEdge = void 0;
|
|
40
37
|
exports.rowIsError = rowIsError;
|
|
@@ -90,7 +87,7 @@ exports.applyPrivacyPolicyForRows = applyPrivacyPolicyForRows;
|
|
|
90
87
|
exports.getEdgeTypeInGroup = getEdgeTypeInGroup;
|
|
91
88
|
const db_1 = __importStar(require("./db"));
|
|
92
89
|
const privacy_1 = require("./privacy");
|
|
93
|
-
const
|
|
90
|
+
const async_utils_1 = require("./async_utils");
|
|
94
91
|
const clause = __importStar(require("./clause"));
|
|
95
92
|
const global_schema_1 = require("./global_schema");
|
|
96
93
|
const loader_1 = require("./loaders/loader");
|
|
@@ -125,15 +122,13 @@ class entCacheMap {
|
|
|
125
122
|
}
|
|
126
123
|
}
|
|
127
124
|
function createAssocEdgeConfigLoader(options) {
|
|
125
|
+
const loaderName = `assocEdgeConfigLoader:${options.tableName}`;
|
|
128
126
|
const loaderOptions = {
|
|
129
127
|
maxBatchSize: (0, loader_1.getLoaderMaxBatchSize)(),
|
|
128
|
+
cacheMap: (0, loader_1.createLoaderCacheMap)(options),
|
|
130
129
|
};
|
|
131
|
-
// if query logging is enabled, we should log what's happening with loader
|
|
132
|
-
if ((0, logger_1.logEnabled)("query")) {
|
|
133
|
-
loaderOptions.cacheMap = new loader_1.CacheMap(options);
|
|
134
|
-
}
|
|
135
130
|
// something here brokwn with strict:true
|
|
136
|
-
return new
|
|
131
|
+
return new loader_1.InstrumentedDataLoader(loaderName, async (ids) => {
|
|
137
132
|
if (!ids.length) {
|
|
138
133
|
return [];
|
|
139
134
|
}
|
|
@@ -155,7 +150,7 @@ function createAssocEdgeConfigLoader(options) {
|
|
|
155
150
|
}
|
|
156
151
|
}
|
|
157
152
|
return ids.map((id) => rowMap.get(id) ?? null);
|
|
158
|
-
}, loaderOptions);
|
|
153
|
+
}, loaderOptions, options.tableName);
|
|
159
154
|
}
|
|
160
155
|
// used to wrap errors that would eventually be thrown in ents
|
|
161
156
|
// not an Error because DataLoader automatically rejects that
|
|
@@ -177,48 +172,23 @@ function setEntLoaderPrivacyConcurrencyLimit(limit) {
|
|
|
177
172
|
function getEntLoaderPrivacyConcurrencyLimit() {
|
|
178
173
|
return entLoaderPrivacyConcurrencyLimit;
|
|
179
174
|
}
|
|
180
|
-
async function mapWithConcurrency(items, limit, mapper) {
|
|
181
|
-
if (!items.length) {
|
|
182
|
-
return [];
|
|
183
|
-
}
|
|
184
|
-
const results = new Array(items.length);
|
|
185
|
-
const concurrency = limit === Infinity
|
|
186
|
-
? items.length
|
|
187
|
-
: Number.isFinite(limit) && limit > 0
|
|
188
|
-
? Math.floor(limit)
|
|
189
|
-
: 1;
|
|
190
|
-
let nextIndex = 0;
|
|
191
|
-
const workers = new Array(Math.min(concurrency, items.length))
|
|
192
|
-
.fill(null)
|
|
193
|
-
.map(async () => {
|
|
194
|
-
while (true) {
|
|
195
|
-
const currentIndex = nextIndex;
|
|
196
|
-
if (currentIndex >= items.length) {
|
|
197
|
-
return;
|
|
198
|
-
}
|
|
199
|
-
nextIndex += 1;
|
|
200
|
-
results[currentIndex] = await mapper(items[currentIndex], currentIndex);
|
|
201
|
-
}
|
|
202
|
-
});
|
|
203
|
-
await Promise.all(workers);
|
|
204
|
-
return results;
|
|
205
|
-
}
|
|
206
175
|
function createEntLoader(viewer, options, map) {
|
|
176
|
+
const loaderName = `entLoader:${options.loaderFactory.name}`;
|
|
177
|
+
const tableName = options.loaderFactory.options?.tableName;
|
|
207
178
|
// share the cache across loaders even if we create a new instance
|
|
208
179
|
const loaderOptions = {
|
|
209
180
|
maxBatchSize: (0, loader_1.getLoaderMaxBatchSize)(),
|
|
210
181
|
};
|
|
211
|
-
loaderOptions.cacheMap = map;
|
|
212
|
-
return new
|
|
182
|
+
loaderOptions.cacheMap = (0, loader_1.createBoundedCacheMap)(map);
|
|
183
|
+
return new loader_1.InstrumentedDataLoader(loaderName, async (ids) => {
|
|
213
184
|
if (!ids.length) {
|
|
214
185
|
return [];
|
|
215
186
|
}
|
|
216
|
-
const tableName = options.loaderFactory.options?.tableName;
|
|
217
187
|
const loader = options.loaderFactory.createLoader(viewer.context);
|
|
218
188
|
const rows = await loader.loadMany(ids);
|
|
219
189
|
// this is a loader which should return the same order based on passed-in ids
|
|
220
190
|
// so let's depend on that...
|
|
221
|
-
return mapWithConcurrency(rows, getEntLoaderPrivacyConcurrencyLimit(), async (row, idx) => {
|
|
191
|
+
return (0, async_utils_1.mapWithConcurrency)(rows, getEntLoaderPrivacyConcurrencyLimit(), async (row, idx) => {
|
|
222
192
|
// db error
|
|
223
193
|
if (rowIsError(row)) {
|
|
224
194
|
if (row instanceof Error) {
|
|
@@ -239,7 +209,7 @@ function createEntLoader(viewer, options, map) {
|
|
|
239
209
|
}
|
|
240
210
|
return r;
|
|
241
211
|
});
|
|
242
|
-
}, loaderOptions);
|
|
212
|
+
}, loaderOptions, tableName);
|
|
243
213
|
}
|
|
244
214
|
class EntLoader {
|
|
245
215
|
constructor(viewer, options) {
|
|
@@ -23,7 +23,10 @@ export declare class AssocDirectEdgeLoader<T extends AssocEdge> implements Loade
|
|
|
23
23
|
private edgeCtr;
|
|
24
24
|
private options?;
|
|
25
25
|
context?: Context | undefined;
|
|
26
|
+
private loader;
|
|
27
|
+
private loaderFn;
|
|
26
28
|
constructor(edgeType: string, edgeCtr: AssocEdgeConstructor<T>, options?: EdgeQueryableDataOptions | undefined, context?: Context | undefined);
|
|
29
|
+
private getLoader;
|
|
27
30
|
load(id: ID): Promise<T[]>;
|
|
28
31
|
loadTwoWay(id: ID): Promise<T[]>;
|
|
29
32
|
loadEdgeForID2(id: ID, id2: ID): Promise<T | undefined>;
|
|
@@ -37,12 +37,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
39
|
exports.AssocEdgeLoaderFactory = exports.AssocDirectEdgeLoader = exports.AssocEdgeLoader = void 0;
|
|
40
|
-
const dataloader_1 = __importDefault(require("dataloader"));
|
|
41
40
|
const memoizee_1 = __importDefault(require("memoizee"));
|
|
42
41
|
const clause = __importStar(require("../clause"));
|
|
43
42
|
const ent_1 = require("../ent");
|
|
44
|
-
const
|
|
45
|
-
const cache_utils_1 = require("./cache_utils");
|
|
43
|
+
const cache_utils_1 = require("../cache_utils");
|
|
46
44
|
const loader_1 = require("./loader");
|
|
47
45
|
function getDefaultOrderBy() {
|
|
48
46
|
return [
|
|
@@ -60,16 +58,15 @@ function getEffectiveOptions(options) {
|
|
|
60
58
|
disableTransformations: options.disableTransformations ?? false,
|
|
61
59
|
};
|
|
62
60
|
}
|
|
63
|
-
function createLoader(options, edgeType, edgeCtr, edgeData) {
|
|
61
|
+
function createLoader(options, edgeType, edgeCtr, edgeData, context) {
|
|
62
|
+
const loaderName = `assocEdgeLoader:${edgeType}`;
|
|
64
63
|
const loaderOptions = {
|
|
65
64
|
maxBatchSize: (0, loader_1.getLoaderMaxBatchSize)(),
|
|
66
|
-
|
|
67
|
-
if ((0, logger_1.logEnabled)("query")) {
|
|
68
|
-
loaderOptions.cacheMap = new loader_1.CacheMap({
|
|
65
|
+
cacheMap: (0, loader_1.createLoaderCacheMap)({
|
|
69
66
|
tableName: edgeData.edgeTable,
|
|
70
|
-
})
|
|
71
|
-
}
|
|
72
|
-
return new
|
|
67
|
+
}),
|
|
68
|
+
};
|
|
69
|
+
return new loader_1.InstrumentedDataLoader(loaderName, async (keys) => {
|
|
73
70
|
if (keys.length === 1) {
|
|
74
71
|
// 1 key, just be simple and move on
|
|
75
72
|
// same as AssocDirectEdgeLoader
|
|
@@ -77,6 +74,7 @@ function createLoader(options, edgeType, edgeCtr, edgeData) {
|
|
|
77
74
|
id1: keys[0],
|
|
78
75
|
edgeType: edgeType,
|
|
79
76
|
queryOptions: options,
|
|
77
|
+
context,
|
|
80
78
|
ctr: edgeCtr,
|
|
81
79
|
});
|
|
82
80
|
return [r];
|
|
@@ -115,7 +113,7 @@ function createLoader(options, edgeType, edgeCtr, edgeData) {
|
|
|
115
113
|
result[idx].push(new edgeCtr(row));
|
|
116
114
|
}
|
|
117
115
|
return result;
|
|
118
|
-
}, loaderOptions);
|
|
116
|
+
}, loaderOptions, edgeData.edgeTable);
|
|
119
117
|
}
|
|
120
118
|
class AssocEdgeLoader {
|
|
121
119
|
constructor(edgeType, edgeCtr, options, context) {
|
|
@@ -130,7 +128,7 @@ class AssocEdgeLoader {
|
|
|
130
128
|
if (!edgeData) {
|
|
131
129
|
throw new Error(`error loading edge data for ${this.edgeType}`);
|
|
132
130
|
}
|
|
133
|
-
this.loader = createLoader(this.options, this.edgeType, this.edgeCtr, edgeData);
|
|
131
|
+
this.loader = createLoader(this.options, this.edgeType, this.edgeCtr, edgeData, this.context);
|
|
134
132
|
return this.loader;
|
|
135
133
|
}
|
|
136
134
|
async load(id) {
|
|
@@ -168,8 +166,40 @@ class AssocDirectEdgeLoader {
|
|
|
168
166
|
this.edgeCtr = edgeCtr;
|
|
169
167
|
this.options = options;
|
|
170
168
|
this.context = context;
|
|
169
|
+
if (this.context) {
|
|
170
|
+
this.loaderFn = (0, memoizee_1.default)(this.getLoader);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
async getLoader() {
|
|
174
|
+
if (this.loader) {
|
|
175
|
+
return this.loader;
|
|
176
|
+
}
|
|
177
|
+
const edgeData = await (0, ent_1.loadEdgeData)(this.edgeType);
|
|
178
|
+
if (!edgeData) {
|
|
179
|
+
throw new Error(`error loading edge data for ${this.edgeType}`);
|
|
180
|
+
}
|
|
181
|
+
const loaderName = `assocDirectEdgeLoader:${this.edgeType}`;
|
|
182
|
+
this.loader = new loader_1.InstrumentedDataLoader(loaderName, async (keys) => {
|
|
183
|
+
return Promise.all(keys.map((id) => (0, ent_1.loadCustomEdges)({
|
|
184
|
+
id1: id,
|
|
185
|
+
edgeType: this.edgeType,
|
|
186
|
+
context: this.context,
|
|
187
|
+
queryOptions: this.options,
|
|
188
|
+
ctr: this.edgeCtr,
|
|
189
|
+
})));
|
|
190
|
+
}, {
|
|
191
|
+
maxBatchSize: (0, loader_1.getLoaderMaxBatchSize)(),
|
|
192
|
+
cacheMap: (0, loader_1.createLoaderCacheMap)({
|
|
193
|
+
tableName: edgeData.edgeTable,
|
|
194
|
+
}),
|
|
195
|
+
}, edgeData.edgeTable);
|
|
196
|
+
return this.loader;
|
|
171
197
|
}
|
|
172
198
|
async load(id) {
|
|
199
|
+
if (this.loaderFn) {
|
|
200
|
+
const loader = await this.loaderFn();
|
|
201
|
+
return loader.load(id);
|
|
202
|
+
}
|
|
173
203
|
return (0, ent_1.loadCustomEdges)({
|
|
174
204
|
id1: id,
|
|
175
205
|
edgeType: this.edgeType,
|
|
@@ -197,7 +227,9 @@ class AssocDirectEdgeLoader {
|
|
|
197
227
|
ctr: this.edgeCtr,
|
|
198
228
|
});
|
|
199
229
|
}
|
|
200
|
-
clearAll() {
|
|
230
|
+
clearAll() {
|
|
231
|
+
this.loader && this.loader.clearAll();
|
|
232
|
+
}
|
|
201
233
|
}
|
|
202
234
|
exports.AssocDirectEdgeLoader = AssocDirectEdgeLoader;
|
|
203
235
|
class AssocEdgeLoaderFactory {
|
package/core/loaders/index.d.ts
CHANGED
|
@@ -3,4 +3,5 @@ export { RawCountLoader, RawCountLoaderFactory } from "./raw_count_loader";
|
|
|
3
3
|
export { AssocEdgeCountLoader, AssocEdgeCountLoaderFactory, } from "./assoc_count_loader";
|
|
4
4
|
export { AssocDirectEdgeLoader, AssocEdgeLoader, AssocEdgeLoaderFactory, } from "./assoc_edge_loader";
|
|
5
5
|
export { QueryLoaderFactory } from "./query_loader";
|
|
6
|
-
export { getLoaderMaxBatchSize, setLoaderMaxBatchSize } from "./loader";
|
|
6
|
+
export { getLoaderCacheMaxEntries, getLoaderMaxBatchSize, setLoaderCacheMaxEntries, setLoaderMaxBatchSize, } from "./loader";
|
|
7
|
+
export { setClauseLoaderConcurrency } from "./object_loader";
|
package/core/loaders/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.setLoaderMaxBatchSize = exports.getLoaderMaxBatchSize = exports.QueryLoaderFactory = exports.AssocEdgeLoaderFactory = exports.AssocEdgeLoader = exports.AssocDirectEdgeLoader = exports.AssocEdgeCountLoaderFactory = exports.AssocEdgeCountLoader = exports.RawCountLoaderFactory = exports.RawCountLoader = exports.ObjectLoaderFactory = exports.ObjectLoader = void 0;
|
|
3
|
+
exports.setClauseLoaderConcurrency = exports.setLoaderMaxBatchSize = exports.setLoaderCacheMaxEntries = exports.getLoaderMaxBatchSize = exports.getLoaderCacheMaxEntries = exports.QueryLoaderFactory = exports.AssocEdgeLoaderFactory = exports.AssocEdgeLoader = exports.AssocDirectEdgeLoader = exports.AssocEdgeCountLoaderFactory = exports.AssocEdgeCountLoader = exports.RawCountLoaderFactory = exports.RawCountLoader = exports.ObjectLoaderFactory = exports.ObjectLoader = void 0;
|
|
4
4
|
var object_loader_1 = require("./object_loader");
|
|
5
5
|
Object.defineProperty(exports, "ObjectLoader", { enumerable: true, get: function () { return object_loader_1.ObjectLoader; } });
|
|
6
6
|
Object.defineProperty(exports, "ObjectLoaderFactory", { enumerable: true, get: function () { return object_loader_1.ObjectLoaderFactory; } });
|
|
@@ -17,5 +17,9 @@ Object.defineProperty(exports, "AssocEdgeLoaderFactory", { enumerable: true, get
|
|
|
17
17
|
var query_loader_1 = require("./query_loader");
|
|
18
18
|
Object.defineProperty(exports, "QueryLoaderFactory", { enumerable: true, get: function () { return query_loader_1.QueryLoaderFactory; } });
|
|
19
19
|
var loader_1 = require("./loader");
|
|
20
|
+
Object.defineProperty(exports, "getLoaderCacheMaxEntries", { enumerable: true, get: function () { return loader_1.getLoaderCacheMaxEntries; } });
|
|
20
21
|
Object.defineProperty(exports, "getLoaderMaxBatchSize", { enumerable: true, get: function () { return loader_1.getLoaderMaxBatchSize; } });
|
|
22
|
+
Object.defineProperty(exports, "setLoaderCacheMaxEntries", { enumerable: true, get: function () { return loader_1.setLoaderCacheMaxEntries; } });
|
|
21
23
|
Object.defineProperty(exports, "setLoaderMaxBatchSize", { enumerable: true, get: function () { return loader_1.setLoaderMaxBatchSize; } });
|
|
24
|
+
var object_loader_2 = require("./object_loader");
|
|
25
|
+
Object.defineProperty(exports, "setClauseLoaderConcurrency", { enumerable: true, get: function () { return object_loader_2.setClauseLoaderConcurrency; } });
|
package/core/loaders/loader.d.ts
CHANGED
|
@@ -1,8 +1,38 @@
|
|
|
1
|
+
import DataLoader from "dataloader";
|
|
1
2
|
import { Loader, LoaderFactory, Context, DataOptions } from "../base";
|
|
2
3
|
export declare function getLoaderMaxBatchSize(): number;
|
|
3
4
|
export declare function setLoaderMaxBatchSize(size?: number | null): void;
|
|
5
|
+
export declare function getLoaderCacheMaxEntries(): number;
|
|
6
|
+
export declare function setLoaderCacheMaxEntries(size?: number | null): void;
|
|
4
7
|
export declare function getLoader<K, V>(factory: LoaderFactory<K, V>, create: () => Loader<K, V>, context?: Context): Loader<K, V>;
|
|
5
8
|
export declare function getCustomLoader<K, V>(key: string, create: () => Loader<K, V>, context?: Context): Loader<K, V>;
|
|
9
|
+
export type CacheMapLike<K, V> = {
|
|
10
|
+
get(key: K): V | undefined;
|
|
11
|
+
set(key: K, value: V): any;
|
|
12
|
+
delete(key: K): any;
|
|
13
|
+
clear(): any;
|
|
14
|
+
};
|
|
15
|
+
type CacheKeyFn<K> = (key: K) => any;
|
|
16
|
+
type BatchLoadFn<K, V> = (keys: readonly K[]) => PromiseLike<ArrayLike<V | Error>>;
|
|
17
|
+
export declare class InstrumentedDataLoader<K, V> extends DataLoader<K, V> {
|
|
18
|
+
constructor(loaderName: string, batchLoadFn: BatchLoadFn<K, V>, options: DataLoader.Options<K, V>, tableName?: string, cacheKeyFn?: (key: K) => unknown);
|
|
19
|
+
}
|
|
20
|
+
export declare class BoundedCacheMap<K, V> {
|
|
21
|
+
private cacheMap;
|
|
22
|
+
private maxEntries;
|
|
23
|
+
private cacheKeyFn?;
|
|
24
|
+
private order;
|
|
25
|
+
constructor(cacheMap: CacheMapLike<K, V>, maxEntries: number, cacheKeyFn?: CacheKeyFn<K> | undefined);
|
|
26
|
+
private normalizeKey;
|
|
27
|
+
private touch;
|
|
28
|
+
private evictIfNeeded;
|
|
29
|
+
get(key: K): V | undefined;
|
|
30
|
+
set(key: K, value: V): any;
|
|
31
|
+
delete(key: K): any;
|
|
32
|
+
clear(): any;
|
|
33
|
+
}
|
|
34
|
+
export declare function createBoundedCacheMap<K, V>(cacheMap: CacheMapLike<K, V>, cacheKeyFn?: CacheKeyFn<K>): CacheMapLike<K, V>;
|
|
35
|
+
export declare function createLoaderCacheMap<K, V>(options: DataOptions): CacheMapLike<K, V>;
|
|
6
36
|
export declare class CacheMap {
|
|
7
37
|
private options;
|
|
8
38
|
private m;
|
|
@@ -12,3 +42,4 @@ export declare class CacheMap {
|
|
|
12
42
|
delete(key: any): boolean;
|
|
13
43
|
clear(): void;
|
|
14
44
|
}
|
|
45
|
+
export {};
|