@snowtop/ent 0.1.0-alpha61 → 0.1.0-alpha68

package/core/base.d.ts

@@ -72,6 +72,7 @@ export interface QueryDataOptions {
     orderby?: string;
     groupby?: string;
     limit?: number;
+    disableTransformations?: boolean;
 }
 export interface LoadRowOptions extends QueryableDataOptions {
 }
@@ -85,13 +86,20 @@ export interface EditRowOptions extends CreateRowOptions {
     whereClause: clause.Clause;
 }
 interface LoadableEntOptions<TEnt extends Ent, TViewer extends Viewer = Viewer> {
-    loaderFactory: LoaderFactory<any, Data | null>;
+    loaderFactory: LoaderFactoryWithOptions;
     ent: EntConstructor<TEnt, TViewer>;
 }
+interface LoaderFactoryWithOptions extends LoaderFactory<any, Data | null> {
+    options?: SelectDataOptions;
+}
 export interface LoadEntOptions<TEnt extends Ent, TViewer extends Viewer = Viewer> extends LoadableEntOptions<TEnt, TViewer>, SelectBaseDataOptions {
     fieldPrivacy?: Map<string, PrivacyPolicy>;
 }
-export interface LoadCustomEntOptions<TEnt extends Ent, TViewer extends Viewer = Viewer> extends SelectBaseDataOptions {
+export interface SelectCustomDataOptions extends SelectBaseDataOptions {
+    clause?: clause.Clause;
+    loaderFactory?: LoaderFactoryWithOptions;
+}
+export interface LoadCustomEntOptions<TEnt extends Ent, TViewer extends Viewer = Viewer> extends SelectCustomDataOptions {
     ent: EntConstructor<TEnt, TViewer>;
     fieldPrivacy?: Map<string, PrivacyPolicy>;
 }

package/core/clause.d.ts

@@ -56,6 +56,18 @@ export declare function PostgresArrayNotContainsValue(col: string, value: any):
  * https://www.postgresql.org/docs/current/indexes-types.html#INDEXES-TYPES-GIN
  */
 export declare function PostgresArrayNotContains(col: string, value: any[]): Clause;
+/**
+ * creates a clause to determine if the arrays overlap, that is, do they have any elements in common
+ * only works with postgres gin indexes
+ * https://www.postgresql.org/docs/current/indexes-types.html#INDEXES-TYPES-GIN
+ */
+export declare function PostgresArrayOverlaps(col: string, value: any[]): Clause;
+/**
+ * creates a clause to determine if the arrays do not overlap, that is, do they have any elements in common
+ * only works with postgres gin indexes
+ * https://www.postgresql.org/docs/current/indexes-types.html#INDEXES-TYPES-GIN
+ */
+export declare function PostgresArrayNotOverlaps(col: string, value: any[]): Clause;
 /**
  * @deprecated use PostgresArrayContainsValue
  */
@@ -87,4 +99,8 @@ export declare function TsVectorPlainToTsQuery(col: string, val: string | TsQuer
 export declare function TsVectorPhraseToTsQuery(col: string, val: string | TsQuery): Clause;
 export declare function TsVectorWebsearchToTsQuery(col: string, val: string | TsQuery): Clause;
 export declare function sensitiveValue(val: any): SensitiveValue;
+export declare function JSONObjectFieldKeyASJSON(col: string, field: string): string;
+export declare function JSONObjectFieldKeyAsText(col: string, field: string): string;
+declare type predicate = "==" | ">" | "<" | "!=" | ">=" | "<=";
+export declare function JSONPathValuePredicate(dbCol: string, path: string, val: any, pred: predicate): Clause;
 export {};

package/core/clause.js

@@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sensitiveValue = exports.TsVectorWebsearchToTsQuery = exports.TsVectorPhraseToTsQuery = exports.TsVectorPlainToTsQuery = exports.TsVectorColTsQuery = exports.WebsearchToTsQuery = exports.PhraseToTsQuery = exports.PlainToTsQuery = exports.TsQuery = exports.In = exports.Or = exports.AndOptional = exports.And = exports.LessEq = exports.GreaterEq = exports.Less = exports.Greater = exports.NotEq = exports.Eq = exports.ArrayNotEq = exports.ArrayEq = exports.PostgresArrayNotContains = exports.PostgresArrayNotContainsValue = exports.PostgresArrayContains = exports.PostgresArrayContainsValue = void 0;
+exports.JSONPathValuePredicate = exports.JSONObjectFieldKeyAsText = exports.JSONObjectFieldKeyASJSON = exports.sensitiveValue = exports.TsVectorWebsearchToTsQuery = exports.TsVectorPhraseToTsQuery = exports.TsVectorPlainToTsQuery = exports.TsVectorColTsQuery = exports.WebsearchToTsQuery = exports.PhraseToTsQuery = exports.PlainToTsQuery = exports.TsQuery = exports.In = exports.Or = exports.AndOptional = exports.And = exports.LessEq = exports.GreaterEq = exports.Less = exports.Greater = exports.NotEq = exports.Eq = exports.ArrayNotEq = exports.ArrayEq = exports.PostgresArrayNotOverlaps = exports.PostgresArrayOverlaps = exports.PostgresArrayNotContains = exports.PostgresArrayNotContainsValue = exports.PostgresArrayContains = exports.PostgresArrayContainsValue = void 0;
 const db_1 = __importStar(require("./db"));
 function isSensitive(val) {
     return (val !== null &&
@@ -157,18 +157,19 @@ class arraySimpleClause {
         return `${this.col}${this.op}${rawValue(this.value)}`;
     }
 }
-class postgresArrayContains {
-    constructor(col, value, not) {
+class postgresArrayOperator {
+    constructor(col, value, op, not) {
         this.col = col;
         this.value = value;
+        this.op = op;
         this.not = not;
     }
     clause(idx) {
         if (db_1.default.getDialect() === db_1.Dialect.Postgres) {
             if (this.not) {
-                return `NOT ${this.col} @> $${idx}`;
+                return `NOT ${this.col} ${this.op} $${idx}`;
             }
-            return `${this.col} @> $${idx}`;
+            return `${this.col} ${this.op} $${idx}`;
         }
         throw new Error(`not supported`);
     }
@@ -189,14 +190,14 @@ class postgresArrayContains {
     }
     instanceKey() {
         if (this.not) {
-            return `NOT:${this.col}@>${rawValue(this.value)}`;
+            return `NOT:${this.col}${this.op}${rawValue(this.value)}`;
         }
-        return `${this.col}@>${rawValue(this.value)}`;
+        return `${this.col}${this.op}${rawValue(this.value)}`;
     }
 }
-class postgresArrayContainsList extends postgresArrayContains {
-    constructor(col, value, not) {
-        super(col, value, not);
+class postgresArrayOperatorList extends postgresArrayOperator {
+    constructor(col, value, op, not) {
+        super(col, value, op, not);
     }
     values() {
         return [
@@ -374,13 +375,15 @@ class websearchTosQueryClause extends tsQueryClause {
         return "websearch_to_tsquery";
     }
 }
+// postgres array operators
+// https://www.postgresql.org/docs/current/functions-array.html
 /**
  * creates a clause to determine if the given value is contained in the array stored in the column in the db
  * only works with postgres gin indexes
  * https://www.postgresql.org/docs/current/indexes-types.html#INDEXES-TYPES-GIN
  */
 function PostgresArrayContainsValue(col, value) {
-    return new postgresArrayContains(col, value);
+    return new postgresArrayOperator(col, value, "@>");
 }
 exports.PostgresArrayContainsValue = PostgresArrayContainsValue;
 /**
@@ -389,7 +392,7 @@ exports.PostgresArrayContainsValue = PostgresArrayContainsValue;
  * https://www.postgresql.org/docs/current/indexes-types.html#INDEXES-TYPES-GIN
  */
 function PostgresArrayContains(col, value) {
-    return new postgresArrayContainsList(col, value);
+    return new postgresArrayOperatorList(col, value, "@>");
 }
 exports.PostgresArrayContains = PostgresArrayContains;
 /**
@@ -398,7 +401,7 @@ exports.PostgresArrayContains = PostgresArrayContains;
  * https://www.postgresql.org/docs/current/indexes-types.html#INDEXES-TYPES-GIN
  */
 function PostgresArrayNotContainsValue(col, value) {
-    return new postgresArrayContains(col, value, true);
+    return new postgresArrayOperator(col, value, "@>", true);
 }
 exports.PostgresArrayNotContainsValue = PostgresArrayNotContainsValue;
 /**
@@ -407,9 +410,27 @@ exports.PostgresArrayNotContainsValue = PostgresArrayNotContainsValue;
  * https://www.postgresql.org/docs/current/indexes-types.html#INDEXES-TYPES-GIN
  */
 function PostgresArrayNotContains(col, value) {
-    return new postgresArrayContainsList(col, value, true);
+    return new postgresArrayOperatorList(col, value, "@>", true);
 }
 exports.PostgresArrayNotContains = PostgresArrayNotContains;
+/**
+ * creates a clause to determine if the arrays overlap, that is, do they have any elements in common
+ * only works with postgres gin indexes
+ * https://www.postgresql.org/docs/current/indexes-types.html#INDEXES-TYPES-GIN
+ */
+function PostgresArrayOverlaps(col, value) {
+    return new postgresArrayOperatorList(col, value, "&&");
+}
+exports.PostgresArrayOverlaps = PostgresArrayOverlaps;
+/**
+ * creates a clause to determine if the arrays do not overlap, that is, do they have any elements in common
+ * only works with postgres gin indexes
+ * https://www.postgresql.org/docs/current/indexes-types.html#INDEXES-TYPES-GIN
+ */
+function PostgresArrayNotOverlaps(col, value) {
+    return new postgresArrayOperatorList(col, value, "&&", true);
+}
+exports.PostgresArrayNotOverlaps = PostgresArrayNotOverlaps;
 /**
  * @deprecated use PostgresArrayContainsValue
  */
@@ -538,3 +559,55 @@ function sensitiveValue(val) {
     };
 }
 exports.sensitiveValue = sensitiveValue;
+// These don't return Clauses but return helpful things that can be passed to clauses
+// https://www.postgresql.org/docs/12/functions-json.html#FUNCTIONS-JSON-OP-TABLE
+// see test in db_clause.test.ts
+// unclear best time to use this...
+function JSONObjectFieldKeyASJSON(col, field) {
+    return `${col}->'${field}'`;
+}
+exports.JSONObjectFieldKeyASJSON = JSONObjectFieldKeyASJSON;
+function JSONObjectFieldKeyAsText(col, field) {
+    return `${col}->>'${field}'`;
+}
+exports.JSONObjectFieldKeyAsText = JSONObjectFieldKeyAsText;
+class jSONPathValuePredicateClause {
+    constructor(col, path, value, pred) {
+        this.col = col;
+        this.path = path;
+        this.value = value;
+        this.pred = pred;
+    }
+    clause(idx) {
+        if (db_1.default.getDialect() !== db_1.Dialect.Postgres) {
+            throw new Error(`not supported`);
+        }
+        return `${this.col} @@ $${idx}`;
+    }
+    columns() {
+        return [this.col];
+    }
+    wrap(val) {
+        return `${this.path} ${this.pred} ${JSON.stringify(val)}`;
+    }
+    values() {
+        if (isSensitive(this.value)) {
+            return [this.wrap(this.value.value())];
+        }
+        return [this.wrap(this.value)];
+    }
+    logValues() {
+        if (isSensitive(this.value)) {
+            return [this.wrap(this.value.logValue())];
+        }
+        return [this.wrap(this.value)];
+    }
+    instanceKey() {
+        return `${this.col}${this.path}${rawValue(this.value)}${this.pred}`;
+    }
+}
+// https://www.postgresql.org/docs/12/functions-json.html#FUNCTIONS-JSON-OP-TABLE
+function JSONPathValuePredicate(dbCol, path, val, pred) {
+    return new jSONPathValuePredicateClause(dbCol, path, val, pred);
+}
+exports.JSONPathValuePredicate = JSONPathValuePredicate;

package/core/ent.d.ts

@@ -1,5 +1,5 @@
 import { Queryer, SyncQueryer } from "./db";
-import { Viewer, Ent, ID, LoadRowsOptions, LoadRowOptions, Data, DataOptions, QueryableDataOptions, EditRowOptions, LoadEntOptions, LoadCustomEntOptions, EdgeQueryableDataOptions, Context, SelectBaseDataOptions, CreateRowOptions, QueryDataOptions } from "./base";
+import { Viewer, Ent, ID, LoadRowsOptions, LoadRowOptions, Data, DataOptions, QueryableDataOptions, EditRowOptions, LoadEntOptions, LoadCustomEntOptions, EdgeQueryableDataOptions, Context, CreateRowOptions, QueryDataOptions, SelectCustomDataOptions } from "./base";
 import { Executor } from "../action/action";
 import * as clause from "./clause";
 import { Builder } from "../action";
@@ -15,13 +15,37 @@ export declare function loadEnts<TEnt extends Ent<TViewer>, TViewer extends View
 export declare function loadEntsList<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, options: LoadEntOptions<TEnt, TViewer>, ...ids: ID[]): Promise<TEnt[]>;
 export declare function loadEntsFromClause<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, clause: clause.Clause, options: LoadEntOptions<TEnt, TViewer>): Promise<Map<ID, TEnt>>;
 export declare function loadCustomEnts<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, options: LoadCustomEntOptions<TEnt, TViewer>, query: CustomQuery): Promise<TEnt[]>;
-interface rawQueryOptions {
+interface parameterizedQueryOptions {
     query: string;
     values?: any[];
     logValues?: any[];
 }
-export declare type CustomQuery = string | rawQueryOptions | clause.Clause | QueryDataOptions;
-export declare function loadCustomData(options: SelectBaseDataOptions, query: CustomQuery, context: Context | undefined): Promise<Data[]>;
+export declare type CustomQuery = string | parameterizedQueryOptions | clause.Clause | QueryDataOptions;
+/**
+ * Note that if there's default read transformations (e.g. soft delete) and a clause is passed in
+ * either as Clause or QueryDataOptions without {disableTransformations: true}, the default transformation
+ * (e.g. soft delete) is applied.
+ *
+ * Passing a full SQL string or Paramterized SQL string doesn't apply it and the given string is sent to the
+ * database as written.
+ *
+ * e.g.
+ * Foo.loadCustom(opts, 'SELECT * FROM foo') // doesn't change the query
+ * Foo.loadCustom(opts, { query: 'SELECT * FROM foo WHERE id = ?', values: [1]}) // doesn't change the query
+ * Foo.loadCustom(opts, query.Eq('time', Date.now())) // changes the query
+ * Foo.loadCustom(opts, {
+ *   clause:  query.LessEq('time', Date.now()),
+ *   limit: 100,
+ *   orderby: 'time',
+ *  }) // changes the query
+ * Foo.loadCustom(opts, {
+ *   clause:  query.LessEq('time', Date.now()),
+ *   limit: 100,
+ *   orderby: 'time',
+ *   disableTransformations: false
+ *  }) // doesn't change the query
+ */
+export declare function loadCustomData(options: SelectCustomDataOptions, query: CustomQuery, context: Context | undefined): Promise<Data[]>;
 export declare function loadDerivedEnt<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, data: Data, loader: new (viewer: TViewer, data: Data) => TEnt): Promise<TEnt | null>;
 export declare function loadDerivedEntX<TEnt extends Ent<TViewer>, TViewer extends Viewer>(viewer: TViewer, data: Data, loader: new (viewer: TViewer, data: Data) => TEnt): Promise<TEnt>;
 export declare function loadRowX(options: LoadRowOptions): Promise<Data>;

package/core/ent.js

@@ -226,32 +226,78 @@ function isClause(opts) {
     const cls = opts;
     return cls.clause !== undefined && cls.values !== undefined;
 }
-function isRawQuery(opts) {
+function isParameterizedQuery(opts) {
     return opts.query !== undefined;
 }
+/**
+ * Note that if there's default read transformations (e.g. soft delete) and a clause is passed in
+ * either as Clause or QueryDataOptions without {disableTransformations: true}, the default transformation
+ * (e.g. soft delete) is applied.
+ *
+ * Passing a full SQL string or Paramterized SQL string doesn't apply it and the given string is sent to the
+ * database as written.
+ *
+ * e.g.
+ * Foo.loadCustom(opts, 'SELECT * FROM foo') // doesn't change the query
+ * Foo.loadCustom(opts, { query: 'SELECT * FROM foo WHERE id = ?', values: [1]}) // doesn't change the query
+ * Foo.loadCustom(opts, query.Eq('time', Date.now())) // changes the query
+ * Foo.loadCustom(opts, {
+ *   clause:  query.LessEq('time', Date.now()),
+ *   limit: 100,
+ *   orderby: 'time',
+ *  }) // changes the query
+ * Foo.loadCustom(opts, {
+ *   clause:  query.LessEq('time', Date.now()),
+ *   limit: 100,
+ *   orderby: 'time',
+ *   disableTransformations: false
+ *  }) // doesn't change the query
+ */
 async function loadCustomData(options, query, context) {
+    function getClause(cls) {
+        if (options.clause && options.loaderFactory?.options?.clause) {
+            throw new Error(`cannot pass both options.clause && optsions.loaderFactory.options.clause`);
+        }
+        let optClause = options.clause || options.loaderFactory?.options?.clause;
+        if (typeof optClause === "function") {
+            optClause = optClause();
+        }
+        if (!optClause) {
+            return cls;
+        }
+        return clause.And(cls, optClause);
+    }
     if (typeof query === "string") {
         // no caching, perform raw query
         return await performRawQuery(query, [], []);
     }
     else if (isClause(query)) {
+        // if a Clause is passed in and we have a default clause
+        // associated with the query, pass that in
+        // if we want to disableTransformations, need to indicate that with
+        // disableTransformations option
         // this will have rudimentary caching but nothing crazy
         return await loadRows({
             ...options,
-            clause: query,
+            clause: getClause(query),
             context: context,
         });
     }
-    else if (isRawQuery(query)) {
+    else if (isParameterizedQuery(query)) {
         // no caching, perform raw query
         return await performRawQuery(query.query, query.values || [], query.logValues);
     }
     else {
+        let cls = query.clause;
+        if (!query.disableTransformations) {
+            cls = getClause(cls);
+        }
         // this will have rudimentary caching but nothing crazy
         return await loadRows({
             ...query,
             ...options,
             context: context,
+            clause: cls,
         });
     }
 }

package/core/privacy.js

@@ -451,11 +451,8 @@ async function applyPrivacyPolicy(v, policy, ent) {
 exports.applyPrivacyPolicy = applyPrivacyPolicy;
 // this will throw an exception if fails or return error | null?
 async function applyPrivacyPolicyX(v, policy, ent, throwErr) {
-    // right now we apply all at same time. todo: be smart about this in the future
-    const results = await Promise.all(policy.rules.map((rule) => rule.apply(v, ent)));
-    for (let i = 0; i < results.length; i++) {
-        const res = results[i];
-        const rule = policy.rules[i];
+    for (const rule of policy.rules) {
+        const res = await rule.apply(v, ent);
         if (res.result == privacyResult.Allow) {
             return true;
         }

package/index.d.ts

@@ -24,6 +24,11 @@ declare const query: {
     PostgresArrayContains: typeof q.PostgresArrayContains;
     PostgresArrayNotContainsValue: typeof q.PostgresArrayNotContainsValue;
     PostgresArrayNotContains: typeof q.PostgresArrayNotContains;
+    PostgresArrayOverlaps: typeof q.PostgresArrayOverlaps;
+    PostgresArrayNotOverlaps: typeof q.PostgresArrayNotOverlaps;
+    JSONPathValuePredicate: typeof q.JSONPathValuePredicate;
+    JSONObjectFieldKeyASJSON: typeof q.JSONObjectFieldKeyASJSON;
+    JSONObjectFieldKeyAsText: typeof q.JSONObjectFieldKeyAsText;
     TsQuery: typeof q.TsQuery;
     PlainToTsQuery: typeof q.PlainToTsQuery;
     PhraseToTsQuery: typeof q.PhraseToTsQuery;

package/index.js

@@ -121,6 +121,11 @@ const query = {
     PostgresArrayContains: q.PostgresArrayContains,
     PostgresArrayNotContainsValue: q.PostgresArrayNotContainsValue,
     PostgresArrayNotContains: q.PostgresArrayNotContains,
+    PostgresArrayOverlaps: q.PostgresArrayOverlaps,
+    PostgresArrayNotOverlaps: q.PostgresArrayNotOverlaps,
+    JSONPathValuePredicate: q.JSONPathValuePredicate,
+    JSONObjectFieldKeyASJSON: q.JSONObjectFieldKeyASJSON,
+    JSONObjectFieldKeyAsText: q.JSONObjectFieldKeyAsText,
     TsQuery: q.TsQuery,
     PlainToTsQuery: q.PlainToTsQuery,
     PhraseToTsQuery: q.PhraseToTsQuery,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snowtop/ent",
-  "version": "0.1.0-alpha61",
+  "version": "0.1.0-alpha68",
   "description": "snowtop ent framework",
   "main": "index.js",
   "types": "index.d.ts",

package/schema/schema.d.ts

@@ -133,6 +133,10 @@ export interface ImportType {
     type: string;
     [x: string]: any;
 }
+export interface ConvertType {
+    path: string;
+    function: string;
+}
 declare type EnumMap = {
     [key: string]: string;
 };
@@ -200,6 +204,7 @@ export interface FieldOptions {
     polymorphic?: boolean | PolymorphicOptions;
     privacyPolicy?: PrivacyPolicy | (() => PrivacyPolicy);
     getDerivedFields?(name: string): FieldMap;
+    convert?: ConvertType;
     [x: string]: any;
 }
 export interface PolymorphicOptions {

package/testutils/db/temp_db.js

@@ -527,6 +527,7 @@ function setupSqlite(connString, tables, opts) {
     afterAll(async () => {
         await tdb.afterAll();
         fs.rmSync(tdb.getSqliteClient().name);
+        delete process.env.DB_CONNECTION_STRING;
     });
     return tdb;
 }

package/testutils/parse_sql.d.ts

@@ -9,6 +9,12 @@ export declare class EqOp {
     constructor(col: string, value: any);
     apply(data: Data): boolean;
 }
+export declare class NotEqOp {
+    private col;
+    private value;
+    constructor(col: string, value: any);
+    apply(data: Data): boolean;
+}
 export declare class GreaterOp {
     private col;
     private value;

package/testutils/parse_sql.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.performQuery = exports.OrOp = exports.AndOp = exports.InOp = exports.LessEqOp = exports.GreaterEqOp = exports.LessOp = exports.GreaterOp = exports.EqOp = exports.getDataToReturn = void 0;
+exports.performQuery = exports.OrOp = exports.AndOp = exports.InOp = exports.LessEqOp = exports.GreaterEqOp = exports.LessOp = exports.GreaterOp = exports.NotEqOp = exports.EqOp = exports.getDataToReturn = void 0;
 const node_sql_parser_1 = require("node-sql-parser");
 function getTableName(table) {
     if (Array.isArray(table)) {
@@ -184,6 +184,16 @@ class EqOp {
     }
 }
 exports.EqOp = EqOp;
+class NotEqOp {
+    constructor(col, value) {
+        this.col = col;
+        this.value = value;
+    }
+    apply(data) {
+        return data[this.col] !== this.value;
+    }
+}
+exports.NotEqOp = NotEqOp;
 class GreaterOp {
     constructor(col, value) {
         this.col = col;
@@ -308,11 +318,11 @@ function getOp(where, values) {
         case "OR":
             return new OrOp([getOp(where.left, values), getOp(where.right, values)]);
         case "IS":
-            if (where.right?.value === null) {
-                return new EqOp(getColumnFromRef(where.left), null);
-            }
+            return new EqOp(getColumnFromRef(where.left), where.right?.value);
+        case "IS NOT":
+            return new NotEqOp(getColumnFromRef(where.left), where.right?.value);
         default:
-            console.log(where);
+            console.debug(where);
             throw new Error(`unsupported op ${where.operator}`);
     }
 }