@snowtop/ent 0.1.0-alpha13 → 0.1.0-alpha130

package/core/ent.js

@@ -1,7 +1,11 @@
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
@@ -22,13 +26,16 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getEdgeTypeInGroup = exports.applyPrivacyPolicyForRows = exports.applyPrivacyPolicyForRowX = exports.applyPrivacyPolicyForRow = exports.loadNodesByEdge = exports.loadEdgeForID2 = exports.loadRawEdgeCountX = exports.loadUniqueNode = exports.loadUniqueEdge = exports.loadCustomEdges = exports.loadEdges = exports.defaultEdgeQueryOptions = exports.DefaultLimit = exports.loadEdgeDatas = exports.loadEdgeData = exports.assocEdgeLoader = exports.AssocEdgeData = exports.getCursor = exports.AssocEdge = exports.DeleteNodeOperation = exports.deleteRowsSync = exports.deleteRows = exports.editRowSync = exports.editRow = exports.buildUpdateQuery = exports.createRowSync = exports.createRow = exports.buildInsertQuery = exports.EdgeOperation = exports.EditNodeOperation = exports.buildGroupQuery = exports.buildQuery = exports.loadRows = exports.performRawQuery = exports.loadRow = exports.loadRowX = exports.loadDerivedEntX = exports.loadDerivedEnt = exports.loadCustomData = exports.loadCustomEnts = exports.loadEntsFromClause = exports.loadEntsList = exports.loadEnts = exports.loadEntXFromClause = exports.loadEntFromClause = exports.loadEntXViaKey = exports.loadEntX = exports.loadEntViaKey = exports.loadEnt = void 0;
+exports.loadNodesByEdge = exports.loadEdgeForID2 = exports.loadRawEdgeCountX = exports.loadUniqueNode = exports.loadUniqueEdge = exports.loadCustomEdges = exports.getEdgeClauseAndFields = exports.loadEdges = exports.DefaultLimit = exports.loadEdgeDatas = exports.loadEdgeData = exports.assocEdgeLoader = exports.AssocEdgeData = exports.getCursor = exports.AssocEdge = exports.DeleteNodeOperation = exports.deleteRowsSync = exports.deleteRows = exports.editRowSync = exports.editRow = exports.buildUpdateQuery = exports.createRowSync = exports.createRow = exports.buildInsertQuery = exports.EdgeOperation = exports.EditNodeOperation = exports.RawQueryOperation = exports.buildGroupQuery = exports.buildQuery = exports.loadRows = exports.performRawQuery = exports.___setLogQueryErrorWithError = exports.loadRow = exports.loadRowX = exports.logQuery = exports.loadDerivedEntX = exports.loadDerivedEnt = exports.loadCustomCount = exports.loadCustomData = exports.loadCustomEnts = exports.loadEntsFromClause = exports.loadEntsList = exports.loadEnts = exports.loadEntXFromClause = exports.loadEntFromClause = exports.loadEntXViaKey = exports.loadEntX = exports.loadEntViaKey = exports.loadEnt = exports.getEntKey = void 0;
+exports.getEdgeTypeInGroup = exports.applyPrivacyPolicyForRows = exports.applyPrivacyPolicyForRow = void 0;
 const db_1 = __importStar(require("./db"));
 const privacy_1 = require("./privacy");
 const clause = __importStar(require("./clause"));
 const action_1 = require("../action");
 const logger_1 = require("./logger");
 const dataloader_1 = __importDefault(require("dataloader"));
+const schema_1 = require("../schema/schema");
+const global_schema_1 = require("./global_schema");
 // TODO kill this and createDataLoader
 class cacheMap {
     constructor(options) {
@@ -38,7 +45,7 @@ class cacheMap {
     get(key) {
         const ret = this.m.get(key);
         if (ret) {
-            (0, logger_1.log)("query", {
+            (0, logger_1.log)("cache", {
                 "dataloader-cache-hit": key,
                 "tableName": this.options.tableName,
             });
@@ -55,12 +62,41 @@ class cacheMap {
         return this.m.clear();
     }
 }
+class entCacheMap {
+    constructor(viewer, options) {
+        this.viewer = viewer;
+        this.options = options;
+        this.m = new Map();
+        this.logEnabled = false;
+        this.logEnabled = (0, logger_1.logEnabled)("cache");
+    }
+    get(id) {
+        const ret = this.m.get(id);
+        if (this.logEnabled && ret) {
+            const key = getEntKey(this.viewer, id, this.options);
+            (0, logger_1.log)("cache", {
+                "ent-cache-hit": key,
+            });
+        }
+        return ret;
+    }
+    set(key, value) {
+        return this.m.set(key, value);
+    }
+    delete(key) {
+        return this.m.delete(key);
+    }
+    clear() {
+        return this.m.clear();
+    }
+}
 function createDataLoader(options) {
     const loaderOptions = {};
     // if query logging is enabled, we should log what's happening with loader
     if ((0, logger_1.logEnabled)("query")) {
         loaderOptions.cacheMap = new cacheMap(options);
     }
+    // something here brokwn with strict:true
     return new dataloader_1.default(async (ids) => {
         if (!ids.length) {
             return [];
@@ -84,28 +120,151 @@ function createDataLoader(options) {
         return result;
     }, loaderOptions);
 }
-// Ent accessors
+// used to wrap errors that would eventually be thrown in ents
+// not an Error because DataLoader automatically rejects that
+class ErrorWrapper {
+    constructor(error) {
+        this.error = error;
+    }
+}
+function createEntLoader(viewer, options, map) {
+    // share the cache across loaders even if we create a new instance
+    const loaderOptions = {};
+    loaderOptions.cacheMap = map;
+    return new dataloader_1.default(async (ids) => {
+        if (!ids.length) {
+            return [];
+        }
+        let result = [];
+        const tableName = options.loaderFactory.options?.tableName;
+        const loader = options.loaderFactory.createLoader(viewer.context);
+        const rows = await loader.loadMany(ids);
+        // this is a loader which should return the same order based on passed-in ids
+        // so let's depend on that...
+        for (let idx = 0; idx < rows.length; idx++) {
+            const row = rows[idx];
+            // db error
+            if (row instanceof Error) {
+                result[idx] = row;
+                continue;
+            }
+            else if (!row) {
+                if (tableName) {
+                    result[idx] = new ErrorWrapper(new Error(`couldn't find row for value ${ids[idx]} in table ${tableName}`));
+                }
+                else {
+                    result[idx] = new ErrorWrapper(new Error(`couldn't find row for value ${ids[idx]}`));
+                }
+            }
+            else {
+                const r = await applyPrivacyPolicyForRowImpl(viewer, options, row);
+                if (r instanceof Error) {
+                    result[idx] = new ErrorWrapper(r);
+                }
+                else {
+                    result[idx] = r;
+                }
+            }
+        }
+        return result;
+    }, loaderOptions);
+}
+class EntLoader {
+    constructor(viewer, options) {
+        this.viewer = viewer;
+        this.options = options;
+        this.map = new entCacheMap(viewer, options);
+        this.loader = createEntLoader(this.viewer, this.options, this.map);
+    }
+    getMap() {
+        return this.map;
+    }
+    async load(id) {
+        return this.loader.load(id);
+    }
+    async loadMany(ids) {
+        return this.loader.loadMany(ids);
+    }
+    prime(id, ent) {
+        this.loader.prime(id, ent);
+    }
+    clear(id) {
+        this.loader.clear(id);
+    }
+    clearAll() {
+        this.loader.clearAll();
+    }
+}
+function getEntLoader(viewer, options) {
+    if (!viewer.context?.cache) {
+        return new EntLoader(viewer, options);
+    }
+    const name = `ent-loader:${viewer.instanceKey()}:${options.loaderFactory.name}`;
+    return viewer.context.cache.getLoaderWithLoadMany(name, () => new EntLoader(viewer, options));
+}
+function getEntKey(viewer, id, options) {
+    return `${viewer.instanceKey()}:${options.loaderFactory.name}:${id}`;
+}
+exports.getEntKey = getEntKey;
 async function loadEnt(viewer, id, options) {
-    const row = await options.loaderFactory.createLoader(viewer.context).load(id);
-    return await applyPrivacyPolicyForRow(viewer, options, row);
+    if (typeof id !== "string" &&
+        typeof id !== "number" &&
+        typeof id !== "bigint") {
+        throw new Error(`invalid id ${id} passed to loadEnt`);
+    }
+    const r = await getEntLoader(viewer, options).load(id);
+    return r instanceof ErrorWrapper ? null : r;
 }
 exports.loadEnt = loadEnt;
+async function applyPrivacyPolicyForRowAndStoreInEntLoader(viewer, row, options, 
+// can pass in loader when calling this for multi-id cases...
+loader) {
+    if (!loader) {
+        loader = getEntLoader(viewer, options);
+    }
+    // TODO every row.id needs to be audited...
+    // https://github.com/lolopinto/ent/issues/1064
+    const id = row.id;
+    // we should check the ent loader cache to see if this is already there
+    // TODO hmm... we eventually need a custom data-loader for this too so that it's all done correctly if there's a complicated fetch deep down in graphql
+    const result = loader.getMap().get(id);
+    if (result !== undefined) {
+        return result;
+    }
+    const r = await applyPrivacyPolicyForRowImpl(viewer, options, row);
+    if (r instanceof Error) {
+        loader.prime(id, new ErrorWrapper(r));
+        return new ErrorWrapper(r);
+    }
+    else {
+        loader.prime(id, r);
+        return r;
+    }
+}
 // this is the same implementation-wise (right now) as loadEnt. it's just clearer that it's not loaded via ID.
 // used for load via email address etc
 async function loadEntViaKey(viewer, key, options) {
     const row = await options.loaderFactory
         .createLoader(viewer.context)
         .load(key);
-    return await applyPrivacyPolicyForRow(viewer, options, row);
+    if (!row) {
+        return null;
+    }
+    const r = await applyPrivacyPolicyForRowAndStoreInEntLoader(viewer, row, options);
+    return r instanceof ErrorWrapper ? null : r;
 }
 exports.loadEntViaKey = loadEntViaKey;
 async function loadEntX(viewer, id, options) {
-    const row = await options.loaderFactory.createLoader(viewer.context).load(id);
-    if (!row) {
-        // todo make this better
-        throw new Error(`${options.loaderFactory.name}: couldn't find row for value ${id}`);
+    if (typeof id !== "string" &&
+        typeof id !== "number" &&
+        typeof id !== "bigint") {
+        throw new Error(`invalid id ${id} passed to loadEntX`);
     }
-    return await applyPrivacyPolicyForRowX(viewer, options, row);
+    const r = await getEntLoader(viewer, options).load(id);
+    if (r instanceof ErrorWrapper) {
+        throw r.error;
+    }
+    return r;
 }
 exports.loadEntX = loadEntX;
 async function loadEntXViaKey(viewer, key, options) {
@@ -116,9 +275,16 @@ async function loadEntXViaKey(viewer, key, options) {
         // todo make this better
         throw new Error(`${options.loaderFactory.name}: couldn't find row for value ${key}`);
     }
-    return await applyPrivacyPolicyForRowX(viewer, options, row);
+    const r = await applyPrivacyPolicyForRowAndStoreInEntLoader(viewer, row, options);
+    if (r instanceof ErrorWrapper) {
+        throw r.error;
+    }
+    return r;
 }
 exports.loadEntXViaKey = loadEntXViaKey;
+/**
+ * @deprecated use loadCustomEnts
+ */
 async function loadEntFromClause(viewer, options, clause) {
     const rowOptions = {
         ...options,
@@ -126,12 +292,18 @@ async function loadEntFromClause(viewer, options, clause) {
         context: viewer.context,
     };
     const row = await loadRow(rowOptions);
-    return await applyPrivacyPolicyForRow(viewer, options, row);
+    if (row === null) {
+        return null;
+    }
+    return applyPrivacyPolicyForRow(viewer, options, row);
 }
 exports.loadEntFromClause = loadEntFromClause;
 // same as loadEntFromClause
 // only works for ents where primary key is "id"
 // use loadEnt with a loaderFactory if different
+/**
+ * @deprecated use loadCustomEnts
+ */
 async function loadEntXFromClause(viewer, options, clause) {
     const rowOptions = {
         ...options,
@@ -146,37 +318,19 @@ async function loadEnts(viewer, options, ...ids) {
     if (!ids.length) {
         return new Map();
     }
-    let loaded = false;
-    let rows = [];
-    // TODO loadMany everywhere
-    const l = options.loaderFactory.createLoader(viewer.context);
-    if (l.loadMany) {
-        loaded = true;
-        rows = await l.loadMany(ids);
-    }
-    // TODO rewrite all of this
+    // result
     let m = new Map();
-    if (loaded) {
-        let rows2 = [];
-        for (const row of rows) {
-            if (!row) {
-                continue;
-            }
-            if (row instanceof Error) {
-                throw row;
-            }
-            rows2.push(row);
+    const ret = await getEntLoader(viewer, options).loadMany(ids);
+    for (const r of ret) {
+        if (r instanceof Error) {
+            throw r;
         }
-        m = await applyPrivacyPolicyForRows(viewer, rows2, options);
-    }
-    else {
-        m = await loadEntsFromClause(viewer, 
-        // this is always "id" if not using an ObjectLoaderFactory
-        clause.In("id", ...ids), options);
+        if (r instanceof ErrorWrapper) {
+            continue;
+        }
+        m.set(r.id, r);
     }
     return m;
-    // TODO do we want to change this to be a map not a list so that it's easy to check for existence?
-    // TODO eventually this should be doing a cache then db queyr and maybe depend on dataloader to get all the results at once
 }
 exports.loadEnts = loadEnts;
 // calls loadEnts and returns the results sorted in the order they were passed in
@@ -195,6 +349,9 @@ async function loadEntsList(viewer, options, ...ids) {
 exports.loadEntsList = loadEntsList;
 // we return a map here so that any sorting for queries that exist
 // can be done in O(N) time
+/**
+ * @deperecated use loadCustomEnts
+ */
 async function loadEntsFromClause(viewer, clause, options) {
     const rowOptions = {
         ...options,
@@ -202,65 +359,129 @@ async function loadEntsFromClause(viewer, clause, options) {
         context: viewer.context,
     };
     const rows = await loadRows(rowOptions);
-    return await applyPrivacyPolicyForRows(viewer, rows, options);
+    return applyPrivacyPolicyForRowsDeprecated(viewer, rows, options);
 }
 exports.loadEntsFromClause = loadEntsFromClause;
 async function loadCustomEnts(viewer, options, query) {
     const rows = await loadCustomData(options, query, viewer.context);
-    const result = new Array(rows.length);
-    await Promise.all(rows.map(async (row, idx) => {
-        const ent = new options.ent(viewer, row);
-        let privacyEnt = await applyPrivacyPolicyForEnt(viewer, ent, row, options);
-        if (privacyEnt) {
-            result[idx] = privacyEnt;
-        }
-    }));
-    // filter ents that aren't visible because of privacy
-    return result.filter((r) => r !== undefined);
+    return applyPrivacyPolicyForRows(viewer, rows, options);
 }
 exports.loadCustomEnts = loadCustomEnts;
 function isClause(opts) {
     const cls = opts;
     return cls.clause !== undefined && cls.values !== undefined;
 }
-function isRawQuery(opts) {
+function isParameterizedQuery(opts) {
     return opts.query !== undefined;
 }
+/**
+ * Note that if there's default read transformations (e.g. soft delete) and a clause is passed in
+ * either as Clause or QueryDataOptions without {disableTransformations: true}, the default transformation
+ * (e.g. soft delete) is applied.
+ *
+ * Passing a full SQL string or Paramterized SQL string doesn't apply it and the given string is sent to the
+ * database as written.
+ *
+ * e.g.
+ * Foo.loadCustom(opts, 'SELECT * FROM foo') // doesn't change the query
+ * Foo.loadCustom(opts, { query: 'SELECT * FROM foo WHERE id = ?', values: [1]}) // doesn't change the query
+ * Foo.loadCustom(opts, query.Eq('time', Date.now())) // changes the query
+ * Foo.loadCustom(opts, {
+ *   clause:  query.LessEq('time', Date.now()),
+ *   limit: 100,
+ *   orderby: 'time',
+ *  }) // changes the query
+ * Foo.loadCustom(opts, {
+ *   clause:  query.LessEq('time', Date.now()),
+ *   limit: 100,
+ *   orderby: 'time',
+ *   disableTransformations: false
+ *  }) // doesn't change the query
+ *
+ * For queries that pass in a clause, we batch them with an underlying dataloader so that multiple queries with the same clause
+ * or parallel queries with the same clause are batched together.
+ *
+ * If a raw or parameterized query is passed in, we don't attempt to batch them together and they're executed as is.
+ * If you end up with a scenario where you may need to coalesce or batch (non-clause) queries here, you should use some kind of memoization here.
+ */
 async function loadCustomData(options, query, context) {
+    const rows = await loadCustomDataImpl(options, query, context);
+    // prime the data so that subsequent fetches of the row with this id are a cache hit.
+    if (options.prime) {
+        const loader = options.loaderFactory.createLoader(context);
+        if (isPrimableLoader(loader) && loader.primeAll !== undefined) {
+            for (const row of rows) {
+                loader.primeAll(row);
+            }
+        }
+    }
+    return rows;
+}
+exports.loadCustomData = loadCustomData;
+// NOTE: if you use a raw query or paramterized query with this,
+// you should use `SELECT count(*) as count...`
+async function loadCustomCount(options, query, context) {
+    // if clause, we'll use the loader and strong typing/coalescing it provides
+    if (typeof query !== "string" && isClause(query)) {
+        return options.loaderFactory.createCountLoader(context).load(query);
+    }
+    const rows = await loadCustomDataImpl({
+        ...options,
+        fields: ["count(1) as count"],
+    }, query, context);
+    if (rows.length) {
+        return parseInt(rows[0].count);
+    }
+    return 0;
+}
+exports.loadCustomCount = loadCustomCount;
+function isPrimableLoader(loader) {
+    return loader != undefined;
+}
+async function loadCustomDataImpl(options, query, context) {
     if (typeof query === "string") {
         // no caching, perform raw query
-        return await performRawQuery(query, [], []);
+        return performRawQuery(query, [], []);
     }
     else if (isClause(query)) {
-        // this will have rudimentary caching but nothing crazy
-        return await loadRows({
-            ...options,
-            clause: query,
-            context: context,
-        });
+        const r = await options.loaderFactory
+            .createTypedLoader(context)
+            .load(query);
+        return r;
     }
-    else if (isRawQuery(query)) {
+    else if (isParameterizedQuery(query)) {
         // no caching, perform raw query
-        return await performRawQuery(query.query, query.values || [], query.logValues);
+        return performRawQuery(query.query, query.values || [], query.logValues);
     }
     else {
         // this will have rudimentary caching but nothing crazy
-        return await loadRows({
+        let cls = query.clause;
+        if (!query.disableTransformations) {
+            cls = clause.getCombinedClause(options.loaderFactory.options, query.clause);
+        }
+        return loadRows({
             ...query,
             ...options,
             context: context,
+            // @ts-expect-error
+            clause: cls,
         });
     }
 }
-exports.loadCustomData = loadCustomData;
 // Derived ents
+// no ent caching
 async function loadDerivedEnt(viewer, data, loader) {
     const ent = new loader(viewer, data);
-    return await applyPrivacyPolicyForEnt(viewer, ent, data, {
+    const r = await applyPrivacyPolicyForEnt(viewer, ent, data, {
         ent: loader,
     });
+    if (r instanceof Error) {
+        return null;
+    }
+    return r;
 }
 exports.loadDerivedEnt = loadDerivedEnt;
+// won't have caching yet either
 async function loadDerivedEntX(viewer, data, loader) {
     const ent = new loader(viewer, data);
     return await applyPrivacyPolicyForEntX(viewer, ent, data, { ent: loader });
@@ -269,19 +490,21 @@ exports.loadDerivedEntX = loadDerivedEntX;
 // everything calls into this two so should be fine
 // TODO is there a smarter way to not instantiate two objects here?
 async function applyPrivacyPolicyForEnt(viewer, ent, data, fieldPrivacyOptions) {
-    if (ent) {
-        const visible = await (0, privacy_1.applyPrivacyPolicy)(viewer, ent.getPrivacyPolicy(), ent);
-        if (!visible) {
-            return null;
-        }
+    const error = await (0, privacy_1.applyPrivacyPolicyImpl)(viewer, ent.getPrivacyPolicy(), ent);
+    if (error === null) {
         return doFieldPrivacy(viewer, ent, data, fieldPrivacyOptions);
     }
-    return null;
+    return error;
 }
 async function applyPrivacyPolicyForEntX(viewer, ent, data, options) {
-    // this will throw
-    await (0, privacy_1.applyPrivacyPolicyX)(viewer, ent.getPrivacyPolicy(), ent);
-    return doFieldPrivacy(viewer, ent, data, options);
+    const r = await applyPrivacyPolicyForEnt(viewer, ent, data, options);
+    if (r instanceof Error) {
+        throw r;
+    }
+    if (r === null) {
+        throw new Error(`couldn't apply privacyPoliy for ent ${ent.id}`);
+    }
+    return r;
 }
 async function doFieldPrivacy(viewer, ent, data, options) {
     if (!options.fieldPrivacy) {
@@ -289,13 +512,16 @@ async function doFieldPrivacy(viewer, ent, data, options) {
     }
     const promises = [];
     let somethingChanged = false;
+    const origData = {
+        ...data,
+    };
     for (const [k, policy] of options.fieldPrivacy) {
+        const curr = data[k];
+        if (curr === null || curr === undefined) {
+            continue;
+        }
         promises.push((async () => {
             // don't do anything if key is null or for some reason missing
-            const curr = data[k];
-            if (curr === null || curr === undefined) {
-                return;
-            }
             const r = await (0, privacy_1.applyPrivacyPolicy)(viewer, policy, ent);
             if (!r) {
                 data[k] = null;
@@ -306,8 +532,11 @@ async function doFieldPrivacy(viewer, ent, data, options) {
     await Promise.all(promises);
     if (somethingChanged) {
         // have to create new instance
-        return new options.ent(viewer, data);
+        const ent = new options.ent(viewer, data);
+        ent.__setRawDBData(origData);
+        return ent;
     }
+    ent.__setRawDBData(origData);
     return ent;
 }
 function logQuery(query, logValues) {
@@ -317,6 +546,7 @@ function logQuery(query, logValues) {
     });
     (0, logger_1.logTrace)();
 }
+exports.logQuery = logQuery;
 // TODO long term figure out if this API should be exposed
 async function loadRowX(options) {
     const result = await loadRow(options);
@@ -339,29 +569,26 @@ async function loadRow(options) {
     }
     const query = buildQuery(options);
     logQuery(query, options.clause.logValues());
-    try {
-        const pool = db_1.default.getInstance().getPool();
-        const res = await pool.query(query, options.clause.values());
-        if (res.rowCount != 1) {
-            if (res.rowCount > 1) {
-                (0, logger_1.log)("error", "got more than one row for query " + query);
-            }
-            return null;
-        }
-        // put the row in the cache...
-        if (cache) {
-            cache.primeCache(options, res.rows[0]);
+    const pool = db_1.default.getInstance().getPool();
+    const res = await pool.query(query, options.clause.values());
+    if (res.rowCount != 1) {
+        if (res.rowCount > 1) {
+            (0, logger_1.log)("error", "got more than one row for query " + query);
         }
-        return res.rows[0];
-    }
-    catch (e) {
-        // an example of an error being suppressed
-        // another one. TODO https://github.com/lolopinto/ent/issues/862
-        (0, logger_1.log)("error", e);
         return null;
     }
+    // put the row in the cache...
+    if (cache) {
+        cache.primeCache(options, res.rows[0]);
+    }
+    return res.rows[0];
 }
 exports.loadRow = loadRow;
+var _logQueryWithError = false;
+function ___setLogQueryErrorWithError(val) {
+    _logQueryWithError = val || false;
+}
+exports.___setLogQueryErrorWithError = ___setLogQueryErrorWithError;
 // this always goes to the db, no cache, nothing
 async function performRawQuery(query, values, logValues) {
     const pool = db_1.default.getInstance().getPool();
@@ -371,9 +598,11 @@ async function performRawQuery(query, values, logValues) {
         return res.rows;
     }
     catch (e) {
-        // TODO need to change every query to catch an error!
-        (0, logger_1.log)("error", e);
-        return [];
+        if (_logQueryWithError) {
+            const msg = e.message;
+            throw new Error(`error \`${msg}\` running query: \`${query}\` with values: \`${logValues}\``);
+        }
+        throw e;
     }
 }
 exports.performRawQuery = performRawQuery;
@@ -432,10 +661,41 @@ function buildGroupQuery(options) {
     ];
 }
 exports.buildGroupQuery = buildGroupQuery;
+class RawQueryOperation {
+    constructor(queries) {
+        this.queries = queries;
+    }
+    async performWrite(queryer, context) {
+        for (const q of this.queries) {
+            if (typeof q === "string") {
+                logQuery(q, []);
+                await queryer.query(q);
+            }
+            else {
+                logQuery(q.query, q.logValues || []);
+                await queryer.query(q.query, q.values);
+            }
+        }
+    }
+    performWriteSync(queryer, context) {
+        for (const q of this.queries) {
+            if (typeof q === "string") {
+                logQuery(q, []);
+                queryer.execSync(q);
+            }
+            else {
+                logQuery(q.query, q.logValues || []);
+                queryer.execSync(q.query, q.values);
+            }
+        }
+    }
+}
+exports.RawQueryOperation = RawQueryOperation;
 class EditNodeOperation {
     constructor(options, existingEnt = null) {
         this.options = options;
         this.existingEnt = existingEnt;
+        this.row = null;
         this.placeholderID = options.placeholderID;
     }
     resolve(executor) {
@@ -471,9 +731,10 @@ class EditNodeOperation {
             if (this.hasData(options.fields)) {
                 // even this with returning * may not always work if transformed...
                 // we can have a transformed flag to see if it should be returned?
-                this.row = await editRow(queryer, options, this.existingEnt.id, "RETURNING *");
+                this.row = await editRow(queryer, options, "RETURNING *");
             }
             else {
+                // @ts-ignore
                 this.row = this.existingEnt["data"];
             }
         }
@@ -496,7 +757,8 @@ class EditNodeOperation {
                 optionClause = opts.clause;
             }
             if (optionClause) {
-                cls = clause.And(optionClause, cls);
+                // @ts-expect-error ID|string mismatch
+                cls = clause.And(cls, optionClause);
             }
         }
         const query = buildQuery({
@@ -520,10 +782,11 @@ class EditNodeOperation {
         };
         if (this.existingEnt) {
             if (this.hasData(this.options.fields)) {
-                editRowSync(queryer, options, this.existingEnt.id, "RETURNING *");
+                editRowSync(queryer, options, "RETURNING *");
                 this.reloadRow(queryer, this.existingEnt.id, options);
             }
             else {
+                // @ts-ignore
                 this.row = this.existingEnt["data"];
             }
         }
@@ -545,7 +808,8 @@ class EditNodeOperation {
 }
 exports.EditNodeOperation = EditNodeOperation;
 class EdgeOperation {
-    constructor(edgeInput, options) {
+    constructor(builder, edgeInput, options) {
+        this.builder = builder;
         this.edgeInput = edgeInput;
         this.options = options;
     }
@@ -581,7 +845,32 @@ class EdgeOperation {
         }
     }
     getDeleteRowParams(edgeData, edge, context) {
+        let transformed = null;
+        let op = schema_1.SQLStatementOperation.Delete;
+        let updateData = null;
+        // TODO respect disableTransformations
+        const transformedEdgeWrite = (0, global_schema_1.__getGlobalSchema)()?.transformEdgeWrite;
+        if (transformedEdgeWrite) {
+            transformed = transformedEdgeWrite({
+                op: schema_1.SQLStatementOperation.Delete,
+                edge,
+            });
+            if (transformed) {
+                op = transformed.op;
+                if (transformed.op === schema_1.SQLStatementOperation.Insert) {
+                    throw new Error(`cannot currently transform a delete into an insert`);
+                }
+                if (transformed.op === schema_1.SQLStatementOperation.Update) {
+                    if (!transformed.data) {
+                        throw new Error(`cannot transform a delete into an update without providing data`);
+                    }
+                    updateData = transformed.data;
+                }
+            }
+        }
         return {
+            op,
+            updateData,
             options: {
                 tableName: edgeData.edgeTable,
                 context,
@@ -591,11 +880,36 @@ class EdgeOperation {
     }
     async performDeleteWrite(q, edgeData, edge, context) {
         const params = this.getDeleteRowParams(edgeData, edge, context);
-        return deleteRows(q, params.options, params.clause);
+        if (params.op === schema_1.SQLStatementOperation.Delete) {
+            return deleteRows(q, params.options, params.clause);
+        }
+        else {
+            if (params.op !== schema_1.SQLStatementOperation.Update) {
+                throw new Error(`invalid operation ${params.op}`);
+            }
+            await editRow(q, {
+                tableName: params.options.tableName,
+                whereClause: params.clause,
+                fields: params.updateData,
+                fieldsToLog: params.updateData,
+            });
+        }
     }
     performDeleteWriteSync(q, edgeData, edge, context) {
         const params = this.getDeleteRowParams(edgeData, edge, context);
-        return deleteRowsSync(q, params.options, params.clause);
+        if (params.op === schema_1.SQLStatementOperation.Delete) {
+            return deleteRowsSync(q, params.options, params.clause);
+        }
+        else {
+            if (params.op !== schema_1.SQLStatementOperation.Update) {
+                throw new Error(`invalid operation ${params.op}`);
+            }
+            editRowSync(q, {
+                tableName: params.options.tableName,
+                whereClause: params.clause,
+                fields: params.updateData,
+            });
+        }
     }
     getInsertRowParams(edgeData, edge, context) {
         const fields = {
@@ -614,6 +928,32 @@ class EdgeOperation {
             // maybe when actions exist?
             fields["time"] = new Date().toISOString();
         }
+        const onConflictFields = ["data"];
+        const extraEdgeFields = (0, global_schema_1.__getGlobalSchema)()?.extraEdgeFields;
+        if (extraEdgeFields) {
+            for (const name in extraEdgeFields) {
+                const f = extraEdgeFields[name];
+                if (f.defaultValueOnCreate) {
+                    const storageKey = (0, schema_1.getStorageKey)(f, name);
+                    fields[storageKey] = f.defaultValueOnCreate(this.builder, {});
+                    // onconflict make sure we override the default values
+                    // e.g. setting deleted_at = null for soft delete
+                    onConflictFields.push(storageKey);
+                }
+            }
+        }
+        // TODO respect disableTransformations
+        let transformed = null;
+        const transformEdgeWrite = (0, global_schema_1.__getGlobalSchema)()?.transformEdgeWrite;
+        if (transformEdgeWrite) {
+            transformed = transformEdgeWrite({
+                op: schema_1.SQLStatementOperation.Insert,
+                edge,
+            });
+            if (transformed) {
+                throw new Error(`transforming an insert edge not currently supported`);
+            }
+        }
         return [
             {
                 tableName: edgeData.edgeTable,
@@ -621,7 +961,9 @@ class EdgeOperation {
                 fieldsToLog: fields,
                 context,
             },
-            "ON CONFLICT(id1, edge_type, id2) DO UPDATE SET data = EXCLUDED.data",
+            `ON CONFLICT(id1, edge_type, id2) DO UPDATE SET ${onConflictFields
+                .map((f) => `${f} = EXCLUDED.${f}`)
+                .join(", ")}`,
         ];
     }
     async performInsertWrite(q, edgeData, edge, context) {
@@ -658,7 +1000,7 @@ class EdgeOperation {
         }
     }
     symmetricEdge() {
-        return new EdgeOperation({
+        return new EdgeOperation(this.builder, {
             id1: this.edgeInput.id2,
             id1Type: this.edgeInput.id2Type,
             id2: this.edgeInput.id1,
@@ -674,7 +1016,7 @@ class EdgeOperation {
         });
     }
     inverseEdge(edgeData) {
-        return new EdgeOperation({
+        return new EdgeOperation(this.builder, {
             id1: this.edgeInput.id2,
             id1Type: this.edgeInput.id2Type,
             id2: this.edgeInput.id1,
@@ -742,7 +1084,7 @@ class EdgeOperation {
         if (data) {
             edge.data = data;
         }
-        return new EdgeOperation(edge, {
+        return new EdgeOperation(builder, edge, {
             operation: action_1.WriteOperation.Insert,
             id2Placeholder,
             id1Placeholder,
@@ -763,7 +1105,7 @@ class EdgeOperation {
         if (data) {
             edge.data = data;
         }
-        return new EdgeOperation(edge, {
+        return new EdgeOperation(builder, edge, {
             operation: action_1.WriteOperation.Insert,
             id1Placeholder,
             id2Placeholder,
@@ -781,7 +1123,7 @@ class EdgeOperation {
             id2Type: "",
             id1Type: "",
         };
-        return new EdgeOperation(edge, {
+        return new EdgeOperation(builder, edge, {
             operation: action_1.WriteOperation.Delete,
         });
     }
@@ -796,7 +1138,7 @@ class EdgeOperation {
             id2Type: "",
             id1Type: "",
         };
-        return new EdgeOperation(edge, {
+        return new EdgeOperation(builder, edge, {
             operation: action_1.WriteOperation.Delete,
         });
     }
@@ -808,24 +1150,26 @@ function isSyncQueryer(queryer) {
 async function mutateRow(queryer, query, values, logValues, options) {
     logQuery(query, logValues);
     let cache = options.context?.cache;
+    let res;
     try {
-        let res;
         if (isSyncQueryer(queryer)) {
             res = queryer.execSync(query, values);
         }
         else {
             res = await queryer.exec(query, values);
         }
-        if (cache) {
-            cache.clearCache();
+    }
+    catch (e) {
+        if (_logQueryWithError) {
+            const msg = e.message;
+            throw new Error(`error \`${msg}\` running query: \`${query}\``);
         }
-        return res;
+        throw e;
     }
-    catch (err) {
-        // TODO:::why is this not rethrowing?
-        (0, logger_1.log)("error", err);
-        throw err;
+    if (cache) {
+        cache.clearCache();
     }
+    return res;
 }
 function mutateRowSync(queryer, query, values, logValues, options) {
     logQuery(query, logValues);
@@ -837,10 +1181,12 @@ function mutateRowSync(queryer, query, values, logValues, options) {
         }
         return res;
     }
-    catch (err) {
-        // TODO:::why is this not rethrowing?
-        (0, logger_1.log)("error", err);
-        throw err;
+    catch (e) {
+        if (_logQueryWithError) {
+            const msg = e.message;
+            throw new Error(`error \`${msg}\` running query: \`${query}\``);
+        }
+        throw e;
     }
 }
 function buildInsertQuery(options, suffix) {
@@ -893,32 +1239,47 @@ function createRowSync(queryer, options, suffix) {
     return null;
 }
 exports.createRowSync = createRowSync;
-function buildUpdateQuery(options, id, suffix) {
+function buildUpdateQuery(options, suffix) {
     let valsString = [];
     let values = [];
     let logValues = [];
     const dialect = db_1.default.getDialect();
     let idx = 1;
     for (const key in options.fields) {
-        values.push(options.fields[key]);
-        if (options.fieldsToLog) {
-            logValues.push(options.fieldsToLog[key]);
-        }
-        if (dialect === db_1.Dialect.Postgres) {
-            valsString.push(`${key} = $${idx}`);
-            idx++;
+        if (options.expressions && options.expressions.has(key)) {
+            const cls = options.expressions.get(key);
+            valsString.push(`${key} = ${cls.clause(idx)}`);
+            // TODO need to test a clause with more than one value...
+            const newVals = cls.values();
+            idx += newVals.length;
+            values.push(...newVals);
+            logValues.push(...cls.logValues());
         }
         else {
-            valsString.push(`${key} = ?`);
+            const val = options.fields[key];
+            values.push(val);
+            if (options.fieldsToLog) {
+                logValues.push(options.fieldsToLog[key]);
+            }
+            // TODO would be nice to use clause here. need update version of the queries so that
+            // we don't have to handle dialect specifics here
+            // can't use clause because of IS NULL
+            // valsString.push(clause.Eq(key, val).clause(idx));
+            if (dialect === db_1.Dialect.Postgres) {
+                valsString.push(`${key} = $${idx}`);
+            }
+            else {
+                valsString.push(`${key} = ?`);
+            }
+            idx++;
         }
     }
     const vals = valsString.join(", ");
     let query = `UPDATE ${options.tableName} SET ${vals} WHERE `;
-    if (dialect === db_1.Dialect.Postgres) {
-        query = query + `${options.key} = $${idx}`;
-    }
-    else {
-        query = query + `${options.key} = ?`;
+    query = query + options.whereClause.clause(idx);
+    values.push(...options.whereClause.values());
+    if (options.fieldsToLog) {
+        logValues.push(...options.whereClause.logValues());
     }
     if (suffix) {
         query = query + " " + suffix;
@@ -926,10 +1287,8 @@ function buildUpdateQuery(options, id, suffix) {
     return [query, values, logValues];
 }
 exports.buildUpdateQuery = buildUpdateQuery;
-async function editRow(queryer, options, id, suffix) {
-    const [query, values, logValues] = buildUpdateQuery(options, id, suffix);
-    // add id as value to prepared query
-    values.push(id);
+async function editRow(queryer, options, suffix) {
+    const [query, values, logValues] = buildUpdateQuery(options, suffix);
     const res = await mutateRow(queryer, query, values, logValues, options);
     if (res?.rowCount == 1) {
         // for now assume id primary key
@@ -940,10 +1299,8 @@ async function editRow(queryer, options, id, suffix) {
     return null;
 }
 exports.editRow = editRow;
-function editRowSync(queryer, options, id, suffix) {
-    const [query, values, logValues] = buildUpdateQuery(options, id, suffix);
-    // add id as value to prepared query
-    values.push(id);
+function editRowSync(queryer, options, suffix) {
+    const [query, values, logValues] = buildUpdateQuery(options, suffix);
     const res = mutateRowSync(queryer, query, values, logValues, options);
     if (res?.rowCount == 1) {
         // for now assume id primary key
@@ -994,21 +1351,22 @@ class AssocEdge {
         this.edgeType = data.edge_type;
         this.time = data.time;
         this.data = data.data;
+        this.rawData = data;
+    }
+    __getRawData() {
+        // incase there's extra db fields. useful for tests
+        // in production, a subclass of this should be in use so we won't need this...
+        return this.rawData;
     }
     getCursor() {
         return getCursor({
             row: this,
-            col: "time",
-            conv: (t) => {
-                if (typeof t === "string") {
-                    return Date.parse(t);
-                }
-                return t.getTime();
-            },
+            col: "id2",
         });
     }
 }
 exports.AssocEdge = AssocEdge;
+// TODO eventually update this for sortCol time unique keys
 function getCursor(opts) {
     const { row, col, conv } = opts;
     //  row: Data, col: string, conv?: (any) => any) {
@@ -1088,52 +1446,80 @@ const edgeFields = [
 ];
 exports.DefaultLimit = 1000;
 // TODO default limit from somewhere
-function defaultEdgeQueryOptions(id1, edgeType) {
+function defaultEdgeQueryOptions(id1, edgeType, id2) {
+    let cls = clause.And(clause.Eq("id1", id1), clause.Eq("edge_type", edgeType));
+    if (id2) {
+        cls = clause.And(cls, clause.Eq("id2", id2));
+    }
     return {
-        clause: clause.And(clause.Eq("id1", id1), clause.Eq("edge_type", edgeType)),
+        clause: cls,
         orderby: "time DESC",
         limit: exports.DefaultLimit,
     };
 }
-exports.defaultEdgeQueryOptions = defaultEdgeQueryOptions;
 async function loadEdges(options) {
     return loadCustomEdges({ ...options, ctr: AssocEdge });
 }
 exports.loadEdges = loadEdges;
+function getEdgeClauseAndFields(cls, options) {
+    let fields = edgeFields;
+    const transformEdgeRead = (0, global_schema_1.__getGlobalSchema)()?.transformEdgeRead;
+    if (transformEdgeRead) {
+        const transformClause = transformEdgeRead();
+        if (!options.disableTransformations) {
+            cls = clause.And(cls, transformClause);
+        }
+        fields = edgeFields.concat(transformClause.columns());
+    }
+    return {
+        cls,
+        fields,
+    };
+}
+exports.getEdgeClauseAndFields = getEdgeClauseAndFields;
 async function loadCustomEdges(options) {
-    const { id1, edgeType, context } = options;
+    const { cls: actualClause, fields, defaultOptions, tableName, } = await loadEgesInfo(options);
+    const rows = await loadRows({
+        tableName,
+        fields: fields,
+        clause: actualClause,
+        orderby: options.queryOptions?.orderby || defaultOptions.orderby,
+        limit: options.queryOptions?.limit || defaultOptions.limit,
+        context: options.context,
+    });
+    return rows.map((row) => {
+        return new options.ctr(row);
+    });
+}
+exports.loadCustomEdges = loadCustomEdges;
+async function loadEgesInfo(options, id2) {
+    const { id1, edgeType } = options;
     const edgeData = await loadEdgeData(edgeType);
     if (!edgeData) {
         throw new Error(`error loading edge data for ${edgeType}`);
     }
-    const defaultOptions = defaultEdgeQueryOptions(id1, edgeType);
+    const defaultOptions = defaultEdgeQueryOptions(id1, edgeType, id2);
     let cls = defaultOptions.clause;
     if (options.queryOptions?.clause) {
         cls = clause.And(cls, options.queryOptions.clause);
     }
-    const rows = await loadRows({
+    return {
+        ...getEdgeClauseAndFields(cls, options),
+        defaultOptions,
         tableName: edgeData.edgeTable,
-        fields: edgeFields,
-        clause: cls,
-        orderby: options.queryOptions?.orderby || defaultOptions.orderby,
-        limit: options.queryOptions?.limit || defaultOptions.limit,
-        context,
-    });
-    return rows.map((row) => {
-        return new options.ctr(row);
-    });
+    };
 }
-exports.loadCustomEdges = loadCustomEdges;
 async function loadUniqueEdge(options) {
     const { id1, edgeType, context } = options;
     const edgeData = await loadEdgeData(edgeType);
     if (!edgeData) {
         throw new Error(`error loading edge data for ${edgeType}`);
     }
+    const { cls, fields } = getEdgeClauseAndFields(clause.And(clause.Eq("id1", id1), clause.Eq("edge_type", edgeType)), options);
     const row = await loadRow({
         tableName: edgeData.edgeTable,
-        fields: edgeFields,
-        clause: clause.And(clause.Eq("id1", id1), clause.Eq("edge_type", edgeType)),
+        fields: fields,
+        clause: cls,
         context,
     });
     if (!row) {
@@ -1160,21 +1546,28 @@ async function loadRawEdgeCountX(options) {
     if (!edgeData) {
         throw new Error(`error loading edge data for ${edgeType}`);
     }
+    const { cls } = getEdgeClauseAndFields(clause.And(clause.Eq("id1", id1), clause.Eq("edge_type", edgeType)), options);
     const row = await loadRowX({
         tableName: edgeData.edgeTable,
         // sqlite needs as count otherwise it returns count(1)
         fields: ["count(1) as count"],
-        clause: clause.And(clause.Eq("id1", id1), clause.Eq("edge_type", edgeType)),
+        clause: cls,
         context,
     });
     return parseInt(row["count"], 10) || 0;
 }
 exports.loadRawEdgeCountX = loadRawEdgeCountX;
 async function loadEdgeForID2(options) {
-    // TODO at some point, same as in go, we can be smart about this and have heuristics to determine if we fetch everything here or not
-    // we're assuming a cache here but not always true and this can be expensive if not...
-    const edges = await loadCustomEdges(options);
-    return edges.find((edge) => edge.id2 == options.id2);
+    const { cls: actualClause, fields, tableName, } = await loadEgesInfo(options, options.id2);
+    const row = await loadRow({
+        tableName,
+        fields: fields,
+        clause: actualClause,
+        context: options.context,
+    });
+    if (row) {
+        return new options.ctr(row);
+    }
 }
 exports.loadEdgeForID2 = loadEdgeForID2;
 async function loadNodesByEdge(viewer, id1, edgeType, options) {
@@ -1190,19 +1583,20 @@ async function loadNodesByEdge(viewer, id1, edgeType, options) {
 }
 exports.loadNodesByEdge = loadNodesByEdge;
 async function applyPrivacyPolicyForRow(viewer, options, row) {
-    if (!row) {
-        return null;
-    }
-    const ent = new options.ent(viewer, row);
-    return await applyPrivacyPolicyForEnt(viewer, ent, row, options);
+    const r = await applyPrivacyPolicyForRowImpl(viewer, options, row);
+    return r instanceof Error ? null : r;
 }
 exports.applyPrivacyPolicyForRow = applyPrivacyPolicyForRow;
+async function applyPrivacyPolicyForRowImpl(viewer, options, row) {
+    const ent = new options.ent(viewer, row);
+    return applyPrivacyPolicyForEnt(viewer, ent, row, options);
+}
 async function applyPrivacyPolicyForRowX(viewer, options, row) {
     const ent = new options.ent(viewer, row);
     return await applyPrivacyPolicyForEntX(viewer, ent, row, options);
 }
-exports.applyPrivacyPolicyForRowX = applyPrivacyPolicyForRowX;
-async function applyPrivacyPolicyForRows(viewer, rows, options) {
+// deprecated. doesn't use entcache
+async function applyPrivacyPolicyForRowsDeprecated(viewer, rows, options) {
     let m = new Map();
     // apply privacy logic
     await Promise.all(rows.map(async (row) => {
@@ -1213,27 +1607,62 @@ async function applyPrivacyPolicyForRows(viewer, rows, options) {
     }));
     return m;
 }
-exports.applyPrivacyPolicyForRows = applyPrivacyPolicyForRows;
-async function loadEdgeWithConst(viewer, id1, id2, edgeEnum, edgeType) {
-    const edge = await loadEdgeForID2({
-        id1: id1,
-        id2: id2,
-        edgeType: edgeType,
-        context: viewer.context,
-        ctr: AssocEdge,
-    });
-    return [edgeEnum, edge];
+async function applyPrivacyPolicyForRows(viewer, rows, options) {
+    const result = new Array(rows.length);
+    if (!rows.length) {
+        return [];
+    }
+    const entLoader = getEntLoader(viewer, options);
+    await Promise.all(rows.map(async (row, idx) => {
+        const r = await applyPrivacyPolicyForRowAndStoreInEntLoader(viewer, row, options, entLoader);
+        if (r instanceof ErrorWrapper) {
+            return;
+        }
+        result[idx] = r;
+    }));
+    // filter ents that aren't visible because of privacy
+    return result.filter((r) => r !== undefined);
 }
+exports.applyPrivacyPolicyForRows = applyPrivacyPolicyForRows;
 // given a viewer, an id pair, and a map of edgeEnum to EdgeType
 // return the edgeEnum that's set in the group
 async function getEdgeTypeInGroup(viewer, id1, id2, m) {
     let promises = [];
-    for (const [k, v] of m) {
-        promises.push(loadEdgeWithConst(viewer, id1, id2, k, v));
+    const edgeDatas = await loadEdgeDatas(...Array.from(m.values()));
+    let tableToEdgeEnumMap = new Map();
+    for (const [edgeEnum, edgeType] of m) {
+        const edgeData = edgeDatas.get(edgeType);
+        if (!edgeData) {
+            throw new Error(`could not load edge data for '${edgeType}'`);
+        }
+        const l = tableToEdgeEnumMap.get(edgeData.edgeTable) ?? [];
+        l.push(edgeEnum);
+        tableToEdgeEnumMap.set(edgeData.edgeTable, l);
     }
+    tableToEdgeEnumMap.forEach((edgeEnums, tableName) => {
+        promises.push((async () => {
+            const edgeTypes = edgeEnums.map((edgeEnum) => m.get(edgeEnum));
+            const { cls, fields } = getEdgeClauseAndFields(clause.And(clause.Eq("id1", id1), clause.In("edge_type", edgeTypes), clause.Eq("id2", id2)), {});
+            const rows = await loadRows({
+                tableName,
+                fields,
+                clause: cls,
+                context: viewer.context,
+            });
+            const row = rows[0];
+            if (row) {
+                const edgeType = row.edge_type;
+                for (const [k, v] of m) {
+                    if (v === edgeType) {
+                        return [k, new AssocEdge(row)];
+                    }
+                }
+            }
+        })());
+    });
     const results = await Promise.all(promises);
     for (const res of results) {
-        if (res[1]) {
+        if (res && res[1]) {
             return [res[0], res[1]];
         }
     }