@snowtop/ent 0.1.0-alpha74 → 0.1.0-alpha75

package/core/base.d.ts

@@ -5,16 +5,23 @@ export interface Loader<T, V> {
     loadMany?(keys: T[]): Promise<(V | null)[]>;
     clearAll(): any;
 }
+interface LoaderWithLoadMany<T, V> extends Loader<T, V> {
+    loadMany(keys: T[]): Promise<(V | null)[]>;
+}
 export interface LoaderFactory<T, V> {
     name: string;
     createLoader(context?: Context): Loader<T, V>;
 }
+interface LoaderFactoryWithLoaderMany<T, V> extends LoaderFactory<T, V> {
+    createLoader(context?: Context): LoaderWithLoadMany<T, V>;
+}
 export interface ConfigurableLoaderFactory<T, V> extends LoaderFactory<T, V> {
     createConfigurableLoader(options: EdgeQueryableDataOptions, context?: Context): Loader<T, V>;
 }
 export declare type EdgeQueryableDataOptions = Partial<Pick<QueryableDataOptions, "limit" | "orderby" | "clause">>;
 export interface PrimableLoader<T, V> extends Loader<T, V> {
     prime(d: Data): void;
+    primeAll?(d: Data): void;
 }
 interface cache {
     getLoader<T, V>(name: string, create: () => Loader<T, V>): Loader<T, V>;
@@ -23,6 +30,7 @@ interface cache {
     primeCache(options: queryOptions, rows: Data[]): void;
     primeCache(options: queryOptions, rows: Data): void;
     clearCache(): void;
+    getEntCache(): Map<string, Ent | Error | null>;
 }
 interface queryOptions {
     fields: string[];
@@ -89,15 +97,15 @@ interface LoadableEntOptions<TEnt extends Ent, TViewer extends Viewer = Viewer>
     loaderFactory: LoaderFactoryWithOptions;
     ent: EntConstructor<TEnt, TViewer>;
 }
-interface LoaderFactoryWithOptions extends LoaderFactory<any, Data | null> {
+interface LoaderFactoryWithOptions extends LoaderFactoryWithLoaderMany<any, Data | null> {
     options?: SelectDataOptions;
 }
 export interface LoadEntOptions<TEnt extends Ent, TViewer extends Viewer = Viewer> extends LoadableEntOptions<TEnt, TViewer>, SelectBaseDataOptions {
     fieldPrivacy?: Map<string, PrivacyPolicy>;
 }
 export interface SelectCustomDataOptions extends SelectBaseDataOptions {
-    clause?: clause.Clause;
-    loaderFactory?: LoaderFactoryWithOptions;
+    loaderFactory: LoaderFactoryWithOptions;
+    prime?: boolean;
 }
 export interface LoadCustomEntOptions<TEnt extends Ent, TViewer extends Viewer = Viewer> extends SelectCustomDataOptions {
     ent: EntConstructor<TEnt, TViewer>;

package/core/clause.d.ts

@@ -22,6 +22,18 @@ declare class simpleClause implements Clause {
     logValues(): any[];
     instanceKey(): string;
 }
+export declare class inClause implements Clause {
+    private col;
+    private value;
+    private type;
+    static getPostgresInClauseValuesThreshold(): number;
+    constructor(col: string, value: any[], type?: string);
+    clause(idx: number): string;
+    columns(): string[];
+    values(): any[];
+    logValues(): any[];
+    instanceKey(): string;
+}
 declare class compositeClause implements Clause {
     private clauses;
     private sep;
@@ -86,6 +98,7 @@ export declare function And(...args: Clause[]): compositeClause;
 export declare function AndOptional(...args: (Clause | undefined)[]): Clause;
 export declare function Or(...args: Clause[]): compositeClause;
 export declare function In(col: string, ...values: any): Clause;
+export declare function In(col: string, values: any[], type?: string): Clause;
 interface TsQuery {
     language: "english" | "french" | "german" | "simple";
     value: string;

package/core/clause.js

@@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.JSONPathValuePredicate = exports.JSONObjectFieldKeyAsText = exports.JSONObjectFieldKeyASJSON = exports.sensitiveValue = exports.TsVectorWebsearchToTsQuery = exports.TsVectorPhraseToTsQuery = exports.TsVectorPlainToTsQuery = exports.TsVectorColTsQuery = exports.WebsearchToTsQuery = exports.PhraseToTsQuery = exports.PlainToTsQuery = exports.TsQuery = exports.In = exports.Or = exports.AndOptional = exports.And = exports.LessEq = exports.GreaterEq = exports.Less = exports.Greater = exports.NotEq = exports.Eq = exports.ArrayNotEq = exports.ArrayEq = exports.PostgresArrayNotOverlaps = exports.PostgresArrayOverlaps = exports.PostgresArrayNotContains = exports.PostgresArrayNotContainsValue = exports.PostgresArrayContains = exports.PostgresArrayContainsValue = void 0;
+exports.JSONPathValuePredicate = exports.JSONObjectFieldKeyAsText = exports.JSONObjectFieldKeyASJSON = exports.sensitiveValue = exports.TsVectorWebsearchToTsQuery = exports.TsVectorPhraseToTsQuery = exports.TsVectorPlainToTsQuery = exports.TsVectorColTsQuery = exports.WebsearchToTsQuery = exports.PhraseToTsQuery = exports.PlainToTsQuery = exports.TsQuery = exports.In = exports.Or = exports.AndOptional = exports.And = exports.LessEq = exports.GreaterEq = exports.Less = exports.Greater = exports.NotEq = exports.Eq = exports.ArrayNotEq = exports.ArrayEq = exports.PostgresArrayNotOverlaps = exports.PostgresArrayOverlaps = exports.PostgresArrayNotContains = exports.PostgresArrayNotContainsValue = exports.PostgresArrayContains = exports.PostgresArrayContainsValue = exports.inClause = void 0;
 const db_1 = __importStar(require("./db"));
 function isSensitive(val) {
     return (val !== null &&
@@ -213,17 +213,37 @@ class postgresArrayOperatorList extends postgresArrayOperator {
     }
 }
 class inClause {
-    constructor(col, value) {
+    constructor(col, value, type = "uuid") {
         this.col = col;
         this.value = value;
+        this.type = type;
+    }
+    static getPostgresInClauseValuesThreshold() {
+        return 70;
     }
     clause(idx) {
-        const dialect = db_1.default.getDialect();
+        // do a simple = when only one item
+        if (this.value.length === 1) {
+            return new simpleClause(this.col, this.value[0], "=").clause(idx);
+        }
+        const postgres = db_1.default.getDialect() === db_1.Dialect.Postgres;
+        const postgresValuesList = postgres &&
+            this.value.length >= inClause.getPostgresInClauseValuesThreshold();
         let indices;
-        if (dialect === db_1.Dialect.Postgres) {
+        if (postgres) {
             indices = [];
             for (let i = 0; i < this.value.length; i++) {
-                indices.push(`$${idx}`);
+                if (postgresValuesList) {
+                    if (i === 0) {
+                        indices.push(`($${idx}::${this.type})`);
+                    }
+                    else {
+                        indices.push(`($${idx})`);
+                    }
+                }
+                else {
+                    indices.push(`$${idx}`);
+                }
                 idx++;
             }
         }
@@ -231,7 +251,11 @@ class inClause {
             indices = new Array(this.value.length);
             indices.fill("?", 0);
         }
-        const inValue = indices.join(", ");
+        let inValue = indices.join(", ");
+        // wrap in VALUES list for postgres...
+        if (postgresValuesList) {
+            inValue = `VALUES${inValue}`;
+        }
         return `${this.col} IN (${inValue})`;
         // TODO we need to return idx at end to query builder...
         // or anything that's doing a composite query so next clause knows where to start
@@ -243,25 +267,15 @@ class inClause {
     }
     values() {
         const result = [];
-        for (const value of this.value) {
-            if (isSensitive(value)) {
-                result.push(value.value());
-            }
-            else {
-                result.push(value);
-            }
+        for (let value of this.value) {
+            result.push(rawValue(value));
         }
         return result;
     }
     logValues() {
         const result = [];
-        for (const value of this.value) {
-            if (isSensitive(value)) {
-                result.push(value.logValue());
-            }
-            else {
-                result.push(value);
-            }
+        for (let value of this.value) {
+            result.push(isSensitive(value) ? value.logValue() : value);
         }
         return result;
     }
@@ -269,6 +283,7 @@ class inClause {
         return `in:${this.col}:${this.values().join(",")}`;
     }
 }
+exports.inClause = inClause;
 class compositeClause {
     constructor(clauses, sep) {
         this.clauses = clauses;
@@ -486,9 +501,15 @@ function Or(...args) {
     return new compositeClause(args, " OR ");
 }
 exports.Or = Or;
-// TODO this breaks if values.length ===1 and array. todo fix
-function In(col, ...values) {
-    return new inClause(col, values);
+function In(...args) {
+    if (args.length < 2) {
+        throw new Error(`invalid args passed to In`);
+    }
+    // 2nd overload
+    if (Array.isArray(args[1])) {
+        return new inClause(args[0], args[1], args[2]);
+    }
+    return new inClause(args[0], args.slice(1));
 }
 exports.In = In;
 // if string defaults to english

package/core/context.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { Viewer, Data, Loader } from "./base";
+import { Viewer, Data, Loader, Ent } from "./base";
 import { IncomingMessage, ServerResponse } from "http";
 import * as clause from "./clause";
 import { Context } from "./base";
@@ -14,12 +14,14 @@ export declare class ContextCache {
     getLoader<T, V>(name: string, create: () => Loader<T, V>): Loader<T, V>;
     private itemMap;
     private listMap;
+    private entCache;
     private getkey;
     getCachedRows(options: queryOptions): Data[] | null;
     getCachedRow(options: queryOptions): Data | null;
     primeCache(options: queryOptions, rows: Data[]): void;
     primeCache(options: queryOptions, rows: Data): void;
     clearCache(): void;
+    getEntCache(): Map<string, Error | Ent<any> | null>;
 }
 interface queryOptions {
     fields: string[];

package/core/context.js

@@ -8,6 +8,7 @@ class ContextCache {
         // we have a per-table map to make it easier to purge and have less things to compare with
         this.itemMap = new Map();
         this.listMap = new Map();
+        this.entCache = new Map();
     }
     getLoader(name, create) {
         let l = this.loaders.get(name);
@@ -82,6 +83,10 @@ class ContextCache {
         this.loaders.clear();
         this.itemMap.clear();
         this.listMap.clear();
+        this.entCache.clear();
+    }
+    getEntCache() {
+        return this.entCache;
     }
 }
 exports.ContextCache = ContextCache;

package/core/ent.d.ts

@@ -5,14 +5,24 @@ import * as clause from "./clause";
 import { Builder } from "../action";
 import DataLoader from "dataloader";
 import { GlobalSchema } from "../schema/";
+export declare function getEntKey<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, id: ID, options: LoadEntOptions<TEnt, TViewer>): string;
 export declare function loadEnt<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, id: ID, options: LoadEntOptions<TEnt, TViewer>): Promise<TEnt | null>;
 export declare function loadEntViaKey<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, key: any, options: LoadEntOptions<TEnt, TViewer>): Promise<TEnt | null>;
 export declare function loadEntX<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, id: ID, options: LoadEntOptions<TEnt, TViewer>): Promise<TEnt>;
 export declare function loadEntXViaKey<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, key: any, options: LoadEntOptions<TEnt, TViewer>): Promise<TEnt>;
+/**
+ * @deprecated use loadCustomEnts
+ */
 export declare function loadEntFromClause<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, options: LoadEntOptions<TEnt, TViewer>, clause: clause.Clause): Promise<TEnt | null>;
+/**
+ * @deprecated use loadCustomEnts
+ */
 export declare function loadEntXFromClause<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, options: LoadEntOptions<TEnt, TViewer>, clause: clause.Clause): Promise<TEnt>;
 export declare function loadEnts<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, options: LoadEntOptions<TEnt, TViewer>, ...ids: ID[]): Promise<Map<ID, TEnt>>;
 export declare function loadEntsList<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, options: LoadEntOptions<TEnt, TViewer>, ...ids: ID[]): Promise<TEnt[]>;
+/**
+ * @deperecated use loadCustomEnts
+ */
 export declare function loadEntsFromClause<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, clause: clause.Clause, options: LoadEntOptions<TEnt, TViewer>): Promise<Map<ID, TEnt>>;
 export declare function loadCustomEnts<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, options: LoadCustomEntOptions<TEnt, TViewer>, query: CustomQuery): Promise<TEnt[]>;
 interface parameterizedQueryOptions {
@@ -210,7 +220,6 @@ interface loadEdgeForIDOptions<T extends AssocEdge> extends loadCustomEdgesOptio
 export declare function loadEdgeForID2<T extends AssocEdge>(options: loadEdgeForIDOptions<T>): Promise<T | undefined>;
 export declare function loadNodesByEdge<T extends Ent>(viewer: Viewer, id1: ID, edgeType: string, options: LoadEntOptions<T>): Promise<T[]>;
 export declare function applyPrivacyPolicyForRow<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, options: LoadEntOptions<TEnt, TViewer>, row: Data | null): Promise<TEnt | null>;
-export declare function applyPrivacyPolicyForRowX<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, options: LoadEntOptions<TEnt, TViewer>, row: Data): Promise<TEnt>;
 export declare function applyPrivacyPolicyForRows<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, rows: Data[], options: LoadEntOptions<TEnt, TViewer>): Promise<Map<ID, TEnt>>;
 export declare function getEdgeTypeInGroup<T extends string>(viewer: Viewer, id1: ID, id2: ID, m: Map<T, string>): Promise<[T, AssocEdge] | undefined>;
 export {};

package/core/ent.js

@@ -22,8 +22,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.applyPrivacyPolicyForRow = exports.loadNodesByEdge = exports.loadEdgeForID2 = exports.loadRawEdgeCountX = exports.loadUniqueNode = exports.loadUniqueEdge = exports.loadCustomEdges = exports.getEdgeClauseAndFields = exports.loadEdges = exports.defaultEdgeQueryOptions = exports.DefaultLimit = exports.loadEdgeDatas = exports.loadEdgeData = exports.assocEdgeLoader = exports.AssocEdgeData = exports.getCursor = exports.AssocEdge = exports.DeleteNodeOperation = exports.deleteRowsSync = exports.deleteRows = exports.editRowSync = exports.editRow = exports.buildUpdateQuery = exports.createRowSync = exports.createRow = exports.buildInsertQuery = exports.EdgeOperation = exports.__hasGlobalSchema = exports.clearGlobalSchema = exports.setGlobalSchema = exports.EditNodeOperation = exports.buildGroupQuery = exports.buildQuery = exports.loadRows = exports.performRawQuery = exports.loadRow = exports.loadRowX = exports.loadDerivedEntX = exports.loadDerivedEnt = exports.loadCustomData = exports.loadCustomEnts = exports.loadEntsFromClause = exports.loadEntsList = exports.loadEnts = exports.loadEntXFromClause = exports.loadEntFromClause = exports.loadEntXViaKey = exports.loadEntX = exports.loadEntViaKey = exports.loadEnt = void 0;
-exports.getEdgeTypeInGroup = exports.applyPrivacyPolicyForRows = exports.applyPrivacyPolicyForRowX = void 0;
+exports.loadNodesByEdge = exports.loadEdgeForID2 = exports.loadRawEdgeCountX = exports.loadUniqueNode = exports.loadUniqueEdge = exports.loadCustomEdges = exports.getEdgeClauseAndFields = exports.loadEdges = exports.defaultEdgeQueryOptions = exports.DefaultLimit = exports.loadEdgeDatas = exports.loadEdgeData = exports.assocEdgeLoader = exports.AssocEdgeData = exports.getCursor = exports.AssocEdge = exports.DeleteNodeOperation = exports.deleteRowsSync = exports.deleteRows = exports.editRowSync = exports.editRow = exports.buildUpdateQuery = exports.createRowSync = exports.createRow = exports.buildInsertQuery = exports.EdgeOperation = exports.__hasGlobalSchema = exports.clearGlobalSchema = exports.setGlobalSchema = exports.EditNodeOperation = exports.buildGroupQuery = exports.buildQuery = exports.loadRows = exports.performRawQuery = exports.loadRow = exports.loadRowX = exports.loadDerivedEntX = exports.loadDerivedEnt = exports.loadCustomData = exports.loadCustomEnts = exports.loadEntsFromClause = exports.loadEntsList = exports.loadEnts = exports.loadEntXFromClause = exports.loadEntFromClause = exports.loadEntXViaKey = exports.loadEntX = exports.loadEntViaKey = exports.loadEnt = exports.getEntKey = void 0;
+exports.getEdgeTypeInGroup = exports.applyPrivacyPolicyForRows = exports.applyPrivacyPolicyForRow = void 0;
 const db_1 = __importStar(require("./db"));
 const privacy_1 = require("./privacy");
 const clause = __importStar(require("./clause"));
@@ -87,10 +87,58 @@ function createDataLoader(options) {
         return result;
     }, loaderOptions);
 }
+function getEntKey(viewer, id, options) {
+    return `${viewer.instanceKey()}:${options.loaderFactory.name}:${id}`;
+}
+exports.getEntKey = getEntKey;
+// fetches the ent from cache if cache exists.
+function entFromCacheMaybe(viewer, id, options) {
+    const cache = viewer.context?.cache?.getEntCache();
+    if (!cache) {
+        return {};
+    }
+    const key = getEntKey(viewer, id, options);
+    const r = cache.get(key);
+    if (r !== undefined) {
+        (0, logger_1.log)("cache", {
+            "ent-cache-hit": key,
+        });
+    }
+    return {
+        key,
+        ent: r instanceof Error ? undefined : r,
+        error: r instanceof Error ? r : undefined,
+        cache,
+    };
+}
 // Ent accessors
+async function applyPrivacyPolicyForRowAndStoreInCache(viewer, options, row, info) {
+    const ent = await applyPrivacyPolicyForRow(viewer, options, row);
+    if (info.cache && info.key) {
+        info.cache.set(info.key, ent);
+    }
+    return ent instanceof Error ? null : ent;
+}
+async function applyPrivacyPolicyForRowAndStoreInCacheX(viewer, options, row, info) {
+    const ent = await applyPrivacyPolicyForRowImpl(viewer, options, row);
+    if (info.cache && info.key) {
+        info.cache.set(info.key, ent);
+    }
+    if (ent instanceof Error) {
+        throw ent;
+    }
+    if (ent === null) {
+        throw new Error(`TODO`);
+    }
+    return ent;
+}
 async function loadEnt(viewer, id, options) {
+    const info = entFromCacheMaybe(viewer, id, options);
+    if (info.ent !== undefined) {
+        return info.ent;
+    }
     const row = await options.loaderFactory.createLoader(viewer.context).load(id);
-    return await applyPrivacyPolicyForRow(viewer, options, row);
+    return applyPrivacyPolicyForRowAndStoreInCache(viewer, options, row, info);
 }
 exports.loadEnt = loadEnt;
 // this is the same implementation-wise (right now) as loadEnt. it's just clearer that it's not loaded via ID.
@@ -99,18 +147,36 @@ async function loadEntViaKey(viewer, key, options) {
     const row = await options.loaderFactory
         .createLoader(viewer.context)
         .load(key);
-    return await applyPrivacyPolicyForRow(viewer, options, row);
+    if (!row) {
+        return null;
+    }
+    // TODO every row.id needs to be audited...
+    const info = entFromCacheMaybe(viewer, row.id, options);
+    if (info.ent !== undefined) {
+        return info.ent;
+    }
+    return applyPrivacyPolicyForRowAndStoreInCache(viewer, options, row, info);
 }
 exports.loadEntViaKey = loadEntViaKey;
+// need a cached error...
 async function loadEntX(viewer, id, options) {
+    const info = entFromCacheMaybe(viewer, id, options);
+    if (info.error !== undefined) {
+        throw info.error;
+    }
+    if (info.ent !== undefined && info.ent !== null) {
+        return info.ent;
+    }
     const row = await options.loaderFactory.createLoader(viewer.context).load(id);
     if (!row) {
         // todo make this better
         throw new Error(`${options.loaderFactory.name}: couldn't find row for value ${id}`);
     }
-    return await applyPrivacyPolicyForRowX(viewer, options, row);
+    return applyPrivacyPolicyForRowAndStoreInCacheX(viewer, options, row, info);
 }
 exports.loadEntX = loadEntX;
+// TODO test this and loadEntViaKey
+// replace loadEntViaClause??
 async function loadEntXViaKey(viewer, key, options) {
     const row = await options.loaderFactory
         .createLoader(viewer.context)
@@ -119,9 +185,19 @@ async function loadEntXViaKey(viewer, key, options) {
         // todo make this better
         throw new Error(`${options.loaderFactory.name}: couldn't find row for value ${key}`);
     }
-    return await applyPrivacyPolicyForRowX(viewer, options, row);
+    const info = entFromCacheMaybe(viewer, row.id, options);
+    if (info.error !== undefined) {
+        throw info.error;
+    }
+    if (info.ent !== undefined && info.ent !== null) {
+        return info.ent;
+    }
+    return applyPrivacyPolicyForRowAndStoreInCacheX(viewer, options, row, info);
 }
 exports.loadEntXViaKey = loadEntXViaKey;
+/**
+ * @deprecated use loadCustomEnts
+ */
 async function loadEntFromClause(viewer, options, clause) {
     const rowOptions = {
         ...options,
@@ -129,12 +205,15 @@ async function loadEntFromClause(viewer, options, clause) {
         context: viewer.context,
     };
     const row = await loadRow(rowOptions);
-    return await applyPrivacyPolicyForRow(viewer, options, row);
+    return applyPrivacyPolicyForRow(viewer, options, row);
 }
 exports.loadEntFromClause = loadEntFromClause;
 // same as loadEntFromClause
 // only works for ents where primary key is "id"
 // use loadEnt with a loaderFactory if different
+/**
+ * @deprecated use loadCustomEnts
+ */
 async function loadEntXFromClause(viewer, options, clause) {
     const rowOptions = {
         ...options,
@@ -149,33 +228,62 @@ async function loadEnts(viewer, options, ...ids) {
     if (!ids.length) {
         return new Map();
     }
-    let loaded = false;
-    let rows = [];
-    // TODO loadMany everywhere
-    const l = options.loaderFactory.createLoader(viewer.context);
-    if (l.loadMany) {
-        loaded = true;
-        rows = await l.loadMany(ids);
-    }
-    // TODO rewrite all of this
+    // result
     let m = new Map();
-    if (loaded) {
-        let rows2 = [];
-        for (const row of rows) {
-            if (!row) {
-                continue;
+    const cache = viewer.context?.cache?.getEntCache();
+    let toFetch = [];
+    if (cache) {
+        for (const id of ids) {
+            const key = getEntKey(viewer, id, options);
+            const ent = cache.get(key);
+            if (ent !== undefined) {
+                (0, logger_1.log)("cache", {
+                    "ent-cache-hit": key,
+                });
+                if (ent === null) {
+                    // TODO this should return null if not loadable...
+                    continue;
+                }
+                // @ts-ignore
+                m.set(id, ent);
             }
-            if (row instanceof Error) {
-                throw row;
+            else {
+                toFetch.push(id);
             }
-            rows2.push(row);
         }
-        m = await applyPrivacyPolicyForRows(viewer, rows2, options);
     }
     else {
-        m = await loadEntsFromClause(viewer, 
-        // this is always "id" if not using an ObjectLoaderFactory
-        clause.In("id", ...ids), options);
+        toFetch = ids;
+    }
+    // all in ent cache!
+    if (!toFetch.length) {
+        return m;
+    }
+    const l = options.loaderFactory.createLoader(viewer.context);
+    const rows = await l.loadMany(toFetch);
+    let rows2 = [];
+    for (const row of rows) {
+        if (!row) {
+            continue;
+        }
+        if (row instanceof Error) {
+            throw row;
+        }
+        rows2.push(row);
+    }
+    const m2 = await applyPrivacyPolicyForRows(viewer, rows2, options);
+    for (const row of rows2) {
+        const id = row[options.loaderFactory.options?.key || "id"];
+        const ent = m2.get(id);
+        if (cache) {
+            // put back in cache...
+            // store null for rows that can't be seen
+            cache.set(getEntKey(viewer, id, options), ent ?? null);
+        }
+        if (ent !== undefined) {
+            // TODO this should return null if not loadable...?
+            m.set(id, ent);
+        }
     }
     return m;
     // TODO do we want to change this to be a map not a list so that it's easy to check for existence?
@@ -198,6 +306,9 @@ async function loadEntsList(viewer, options, ...ids) {
 exports.loadEntsList = loadEntsList;
 // we return a map here so that any sorting for queries that exist
 // can be done in O(N) time
+/**
+ * @deperecated use loadCustomEnts
+ */
 async function loadEntsFromClause(viewer, clause, options) {
     const rowOptions = {
         ...options,
@@ -212,8 +323,18 @@ async function loadCustomEnts(viewer, options, query) {
     const rows = await loadCustomData(options, query, viewer.context);
     const result = new Array(rows.length);
     await Promise.all(rows.map(async (row, idx) => {
-        const ent = new options.ent(viewer, row);
-        let privacyEnt = await applyPrivacyPolicyForEnt(viewer, ent, row, options);
+        // TODO what if key is different
+        const info = entFromCacheMaybe(viewer, row.id, options);
+        if (info.ent !== undefined) {
+            if (info.ent === null) {
+                // we're done here
+                return;
+            }
+            // @ts-ignore
+            result[idx] = info.ent;
+            return;
+        }
+        const privacyEnt = await applyPrivacyPolicyForRowAndStoreInCache(viewer, options, row, info);
         if (privacyEnt) {
             result[idx] = privacyEnt;
         }
@@ -254,11 +375,25 @@ function isParameterizedQuery(opts) {
  *  }) // doesn't change the query
  */
 async function loadCustomData(options, query, context) {
-    function getClause(cls) {
-        if (options.clause && options.loaderFactory?.options?.clause) {
-            throw new Error(`cannot pass both options.clause && optsions.loaderFactory.options.clause`);
+    const rows = await loadCustomDataImpl(options, query, context);
+    // prime the data so that subsequent fetches of the row with this id are a cache hit.
+    if (options.prime) {
+        const loader = options.loaderFactory.createLoader(context);
+        if (isPrimableLoader(loader) && loader.primeAll !== undefined) {
+            for (const row of rows) {
+                loader.primeAll(row);
+            }
         }
-        let optClause = options.clause || options.loaderFactory?.options?.clause;
+    }
+    return rows;
+}
+exports.loadCustomData = loadCustomData;
+function isPrimableLoader(loader) {
+    return loader != undefined;
+}
+async function loadCustomDataImpl(options, query, context) {
+    function getClause(cls) {
+        let optClause = options.loaderFactory?.options?.clause;
         if (typeof optClause === "function") {
             optClause = optClause();
         }
@@ -269,7 +404,7 @@ async function loadCustomData(options, query, context) {
     }
     if (typeof query === "string") {
         // no caching, perform raw query
-        return await performRawQuery(query, [], []);
+        return performRawQuery(query, [], []);
     }
     else if (isClause(query)) {
         // if a Clause is passed in and we have a default clause
@@ -277,7 +412,7 @@ async function loadCustomData(options, query, context) {
         // if we want to disableTransformations, need to indicate that with
         // disableTransformations option
         // this will have rudimentary caching but nothing crazy
-        return await loadRows({
+        return loadRows({
             ...options,
             clause: getClause(query),
             context: context,
@@ -285,7 +420,7 @@ async function loadCustomData(options, query, context) {
     }
     else if (isParameterizedQuery(query)) {
         // no caching, perform raw query
-        return await performRawQuery(query.query, query.values || [], query.logValues);
+        return performRawQuery(query.query, query.values || [], query.logValues);
     }
     else {
         let cls = query.clause;
@@ -293,7 +428,7 @@ async function loadCustomData(options, query, context) {
             cls = getClause(cls);
         }
         // this will have rudimentary caching but nothing crazy
-        return await loadRows({
+        return loadRows({
             ...query,
             ...options,
             context: context,
@@ -301,15 +436,20 @@ async function loadCustomData(options, query, context) {
         });
     }
 }
-exports.loadCustomData = loadCustomData;
 // Derived ents
+// no ent caching
 async function loadDerivedEnt(viewer, data, loader) {
     const ent = new loader(viewer, data);
-    return await applyPrivacyPolicyForEnt(viewer, ent, data, {
+    const r = await applyPrivacyPolicyForEnt(viewer, ent, data, {
         ent: loader,
     });
+    if (r instanceof Error) {
+        return null;
+    }
+    return r;
 }
 exports.loadDerivedEnt = loadDerivedEnt;
+// won't have caching yet either
 async function loadDerivedEntX(viewer, data, loader) {
     const ent = new loader(viewer, data);
     return await applyPrivacyPolicyForEntX(viewer, ent, data, { ent: loader });
@@ -319,18 +459,23 @@ exports.loadDerivedEntX = loadDerivedEntX;
 // TODO is there a smarter way to not instantiate two objects here?
 async function applyPrivacyPolicyForEnt(viewer, ent, data, fieldPrivacyOptions) {
     if (ent) {
-        const visible = await (0, privacy_1.applyPrivacyPolicy)(viewer, ent.getPrivacyPolicy(), ent);
-        if (!visible) {
-            return null;
+        const error = await (0, privacy_1.applyPrivacyPolicyImpl)(viewer, ent.getPrivacyPolicy(), ent);
+        if (error === null) {
+            return doFieldPrivacy(viewer, ent, data, fieldPrivacyOptions);
         }
-        return doFieldPrivacy(viewer, ent, data, fieldPrivacyOptions);
+        return error;
     }
     return null;
 }
 async function applyPrivacyPolicyForEntX(viewer, ent, data, options) {
-    // this will throw
-    await (0, privacy_1.applyPrivacyPolicyX)(viewer, ent.getPrivacyPolicy(), ent);
-    return doFieldPrivacy(viewer, ent, data, options);
+    const r = await applyPrivacyPolicyForEnt(viewer, ent, data, options);
+    if (r instanceof Error) {
+        throw r;
+    }
+    if (r === null) {
+        throw new Error(`couldn't apply privacyPoliy for ent ${ent.id}`);
+    }
+    return r;
 }
 async function doFieldPrivacy(viewer, ent, data, options) {
     if (!options.fieldPrivacy) {
@@ -339,12 +484,12 @@ async function doFieldPrivacy(viewer, ent, data, options) {
     const promises = [];
     let somethingChanged = false;
     for (const [k, policy] of options.fieldPrivacy) {
+        const curr = data[k];
+        if (curr === null || curr === undefined) {
+            continue;
+        }
         promises.push((async () => {
             // don't do anything if key is null or for some reason missing
-            const curr = data[k];
-            if (curr === null || curr === undefined) {
-                return;
-            }
             const r = await (0, privacy_1.applyPrivacyPolicy)(viewer, policy, ent);
             if (!r) {
                 data[k] = null;
@@ -547,7 +692,7 @@ class EditNodeOperation {
                 optionClause = opts.clause;
             }
             if (optionClause) {
-                cls = clause.And(optionClause, cls);
+                cls = clause.And(cls, optionClause);
             }
         }
         const query = buildQuery({
@@ -1062,6 +1207,9 @@ function buildUpdateQuery(options, suffix) {
     let query = `UPDATE ${options.tableName} SET ${vals} WHERE `;
     query = query + options.whereClause.clause(idx);
     values.push(...options.whereClause.values());
+    if (options.fieldsToLog) {
+        logValues.push(...options.whereClause.logValues());
+    }
     if (suffix) {
         query = query + " " + suffix;
     }
@@ -1352,18 +1500,22 @@ async function loadNodesByEdge(viewer, id1, edgeType, options) {
 }
 exports.loadNodesByEdge = loadNodesByEdge;
 async function applyPrivacyPolicyForRow(viewer, options, row) {
+    const r = await applyPrivacyPolicyForRowImpl(viewer, options, row);
+    return r instanceof Error ? null : r;
+}
+exports.applyPrivacyPolicyForRow = applyPrivacyPolicyForRow;
+async function applyPrivacyPolicyForRowImpl(viewer, options, row) {
     if (!row) {
         return null;
     }
     const ent = new options.ent(viewer, row);
-    return await applyPrivacyPolicyForEnt(viewer, ent, row, options);
+    return applyPrivacyPolicyForEnt(viewer, ent, row, options);
 }
-exports.applyPrivacyPolicyForRow = applyPrivacyPolicyForRow;
 async function applyPrivacyPolicyForRowX(viewer, options, row) {
     const ent = new options.ent(viewer, row);
     return await applyPrivacyPolicyForEntX(viewer, ent, row, options);
 }
-exports.applyPrivacyPolicyForRowX = applyPrivacyPolicyForRowX;
+// TODO this needs to be changed to use ent cache as needed...
 async function applyPrivacyPolicyForRows(viewer, rows, options) {
     let m = new Map();
     // apply privacy logic

package/core/loaders/object_loader.d.ts

@@ -13,6 +13,7 @@ export declare class ObjectLoader<T> implements Loader<T, Data | null> {
     clearAll(): void;
     loadMany(keys: T[]): Promise<Data[]>;
     prime(data: Data): void;
+    primeAll(data: Data): void;
 }
 interface ObjectLoaderOptions extends SelectDataOptions {
     instanceKey?: string;

package/core/loaders/object_loader.js

@@ -53,7 +53,7 @@ function createDataLoader(options) {
                 optionClause = options.clause;
             }
             if (optionClause) {
-                cls = clause.And(optionClause, cls);
+                cls = clause.And(cls, optionClause);
             }
         }
         const rowOptions = {
@@ -136,7 +136,7 @@ class ObjectLoader {
                 optionClause = this.options.clause;
             }
             if (optionClause) {
-                cls = clause.And(optionClause, cls);
+                cls = clause.And(cls, optionClause);
             }
         }
         const rowOptions = {
@@ -163,7 +163,7 @@ class ObjectLoader {
                 optionClause = this.options.clause;
             }
             if (optionClause) {
-                cls = clause.And(optionClause, cls);
+                cls = clause.And(cls, optionClause);
             }
         }
         const rowOptions = {
@@ -181,6 +181,18 @@ class ObjectLoader {
             this.loader.prime(key, data);
         }
     }
+    // prime this loader and any other loaders it's aware of
+    primeAll(data) {
+        this.prime(data);
+        if (this.primedLoaders) {
+            for (const [key, loader] of this.primedLoaders) {
+                const value = data[key];
+                if (value !== undefined) {
+                    loader.prime(data);
+                }
+            }
+        }
+    }
 }
 exports.ObjectLoader = ObjectLoader;
 class ObjectLoaderFactory {

package/core/privacy.d.ts

@@ -183,6 +183,7 @@ export declare class AllowIfSubPolicyAllowsRule implements PrivacyPolicyRule {
 }
 export declare function applyPrivacyPolicy(v: Viewer, policy: PrivacyPolicy, ent: Ent | undefined): Promise<boolean>;
 export declare function applyPrivacyPolicyX(v: Viewer, policy: PrivacyPolicy, ent: Ent | undefined, throwErr?: () => Error): Promise<boolean>;
+export declare function applyPrivacyPolicyImpl(v: Viewer, policy: PrivacyPolicy, ent: Ent | undefined, throwErr?: () => Error): Promise<Error | null>;
 export declare const AlwaysAllowPrivacyPolicy: PrivacyPolicy;
 export declare const AlwaysDenyPrivacyPolicy: PrivacyPolicy;
 export declare const AllowIfViewerPrivacyPolicy: PrivacyPolicy;

package/core/privacy.js

@@ -1,9 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AllowIfViewerHasIdentityPrivacyPolicy = exports.AllowIfViewerPrivacyPolicy = exports.AlwaysDenyPrivacyPolicy = exports.AlwaysAllowPrivacyPolicy = exports.applyPrivacyPolicyX = exports.applyPrivacyPolicy = exports.AllowIfSubPolicyAllowsRule = exports.DelayedResultRule = exports.AllowIfConditionAppliesRule = exports.DenyIfViewerOutboundEdgeDoesNotExistRule = exports.DenyIfViewerInboundEdgeDoesNotExistRule = exports.DenyIfEdgeDoesNotExistRule = exports.DenyIfViewerOutboundEdgeExistsRule = exports.DenyIfViewerInboundEdgeExistsRule = exports.DenyIfEdgeExistsRule = exports.AllowIfViewerOutboundEdgeExistsRule = exports.AllowIfViewerInboundEdgeExistsRule = exports.AllowIfEdgeExistsRule = exports.DenyIfEntIsNotVisibleRule = exports.DenyIfEntIsVisibleRule = exports.DenyIfEntIsVisiblePolicy = exports.AllowIfEntIsVisiblePolicy = exports.AllowIfEntIsNotVisibleRule = exports.AllowIfEntIsVisibleRule = exports.DenyIfEntPropertyIsRule = exports.AllowIfEntPropertyIsRule = exports.AllowIfViewerIsEntPropertyRule = exports.AllowIfViewerIsRule = exports.DenyIfFuncRule = exports.AllowIfFuncRule = exports.DenyIfViewerEqualsRule = exports.AllowIfViewerEqualsRule = exports.AllowIfViewerRule = exports.AllowIfHasIdentity = exports.DenyIfLoggedInRule = exports.DenyIfLoggedOutRule = exports.AlwaysDenyRule = exports.AlwaysAllowRule = exports.EntPrivacyError = void 0;
+exports.AllowIfViewerHasIdentityPrivacyPolicy = exports.AllowIfViewerPrivacyPolicy = exports.AlwaysDenyPrivacyPolicy = exports.AlwaysAllowPrivacyPolicy = exports.applyPrivacyPolicyImpl = exports.applyPrivacyPolicyX = exports.applyPrivacyPolicy = exports.AllowIfSubPolicyAllowsRule = exports.DelayedResultRule = exports.AllowIfConditionAppliesRule = exports.DenyIfViewerOutboundEdgeDoesNotExistRule = exports.DenyIfViewerInboundEdgeDoesNotExistRule = exports.DenyIfEdgeDoesNotExistRule = exports.DenyIfViewerOutboundEdgeExistsRule = exports.DenyIfViewerInboundEdgeExistsRule = exports.DenyIfEdgeExistsRule = exports.AllowIfViewerOutboundEdgeExistsRule = exports.AllowIfViewerInboundEdgeExistsRule = exports.AllowIfEdgeExistsRule = exports.DenyIfEntIsNotVisibleRule = exports.DenyIfEntIsVisibleRule = exports.DenyIfEntIsVisiblePolicy = exports.AllowIfEntIsVisiblePolicy = exports.AllowIfEntIsNotVisibleRule = exports.AllowIfEntIsVisibleRule = exports.DenyIfEntPropertyIsRule = exports.AllowIfEntPropertyIsRule = exports.AllowIfViewerIsEntPropertyRule = exports.AllowIfViewerIsRule = exports.DenyIfFuncRule = exports.AllowIfFuncRule = exports.DenyIfViewerEqualsRule = exports.AllowIfViewerEqualsRule = exports.AllowIfViewerRule = exports.AllowIfHasIdentity = exports.DenyIfLoggedInRule = exports.DenyIfLoggedOutRule = exports.AlwaysDenyRule = exports.AlwaysAllowRule = exports.EntPrivacyError = void 0;
 const base_1 = require("./base");
 const ent_1 = require("./ent");
-const logger_1 = require("./logger");
 // copied from ./base
 var privacyResult;
 (function (privacyResult) {
@@ -437,42 +436,42 @@ class AllowIfSubPolicyAllowsRule {
 }
 exports.AllowIfSubPolicyAllowsRule = AllowIfSubPolicyAllowsRule;
 async function applyPrivacyPolicy(v, policy, ent) {
-    try {
-        return await applyPrivacyPolicyX(v, policy, ent);
-    }
-    catch (e) {
-        // TODO privacy errors should not throw
-        // but other expected errors should throw...
-        // we shouldn't just hide them
-        (0, logger_1.log)("debug", e);
-        return false;
-    }
+    const err = await applyPrivacyPolicyImpl(v, policy, ent);
+    return err === null;
 }
 exports.applyPrivacyPolicy = applyPrivacyPolicy;
-// this will throw an exception if fails or return error | null?
 async function applyPrivacyPolicyX(v, policy, ent, throwErr) {
+    const err = await applyPrivacyPolicyImpl(v, policy, ent, throwErr);
+    if (err !== null) {
+        throw err;
+    }
+    return true;
+}
+exports.applyPrivacyPolicyX = applyPrivacyPolicyX;
+// this will throw an exception if fails or return error | null?
+async function applyPrivacyPolicyImpl(v, policy, ent, throwErr) {
     for (const rule of policy.rules) {
         const res = await rule.apply(v, ent);
         if (res.result == privacyResult.Allow) {
-            return true;
+            return null;
         }
         else if (res.result == privacyResult.Deny) {
             // specific error throw that
             if (res.error) {
-                throw res.error;
+                return res.error;
             }
             if (res.getError) {
-                throw res.getError(policy, rule, ent);
+                return res.getError(policy, rule, ent);
             }
             if (throwErr) {
-                throw throwErr();
+                return throwErr();
             }
-            throw new EntPrivacyError(policy, rule, ent);
+            return new EntPrivacyError(policy, rule, ent);
         }
     }
-    throw new EntInvalidPrivacyPolicyError(policy, ent);
+    return new EntInvalidPrivacyPolicyError(policy, ent);
 }
-exports.applyPrivacyPolicyX = applyPrivacyPolicyX;
+exports.applyPrivacyPolicyImpl = applyPrivacyPolicyImpl;
 exports.AlwaysAllowPrivacyPolicy = {
     rules: [exports.AlwaysAllowRule],
 };

package/core/viewer.js

@@ -35,7 +35,7 @@ class IDViewer {
         return this.ent;
     }
     instanceKey() {
-        return `idViewer: ${this.viewerID}`;
+        return `idViewer:${this.viewerID}`;
     }
 }
 exports.IDViewer = IDViewer;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snowtop/ent",
-  "version": "0.1.0-alpha74",
+  "version": "0.1.0-alpha75",
   "description": "snowtop ent framework",
   "main": "index.js",
   "types": "index.d.ts",

package/schema/schema.d.ts

@@ -205,6 +205,7 @@ export interface FieldOptions {
     privacyPolicy?: PrivacyPolicy | (() => PrivacyPolicy);
     getDerivedFields?(name: string): FieldMap;
     convert?: ConvertType;
+    fetchOnDemand?: boolean;
     [x: string]: any;
 }
 export interface PolymorphicOptions {