@snowtop/ent 0.1.0-alpha160-test4 → 0.1.0-alpha160-test6

package/src/core/loaders/query_loader.ts

@@ -0,0 +1,314 @@
+import DataLoader from "dataloader";
+import {
+  Context,
+  ID,
+  EdgeQueryableDataOptions,
+  Loader,
+  LoaderFactory,
+  Data,
+  PrimableLoader,
+} from "../base";
+import {
+  getDefaultLimit,
+  performRawQuery,
+  buildGroupQuery,
+  loadRows,
+} from "../ent";
+import * as clause from "../clause";
+import { logEnabled } from "../logger";
+import { CacheMap, getCustomLoader, getLoader } from "./loader";
+import memoizee from "memoizee";
+import { ObjectLoaderFactory } from "./object_loader";
+import { OrderBy, getOrderByPhrase } from "../query_impl";
+
+function getOrderByLocal(
+  options: QueryOptions,
+  queryOptions?: EdgeQueryableDataOptions,
+): OrderBy {
+  return (
+    options.orderby ??
+    queryOptions?.orderby ?? [
+      {
+        column: "created_at",
+        direction: "DESC",
+      },
+    ]
+  );
+}
+
+async function simpleCase<K extends any>(
+  options: QueryOptions,
+  id: K,
+  queryOptions?: EdgeQueryableDataOptions,
+) {
+  let cls: clause.Clause;
+  if (options.groupCol) {
+    cls = clause.Eq(options.groupCol, id);
+    if (options.clause) {
+      cls = clause.And(cls, options.clause);
+    }
+  } else if (options.clause) {
+    cls = options.clause;
+  } else {
+    throw new Error(`need options.groupCol or options.clause`);
+  }
+  if (queryOptions?.clause) {
+    cls = clause.And(cls, queryOptions.clause);
+  }
+
+  return await loadRows({
+    ...options,
+    clause: cls,
+    orderby: getOrderByLocal(options, queryOptions),
+    limit: queryOptions?.limit || getDefaultLimit(),
+  });
+}
+
+function createLoader<K extends any>(
+  options: QueryOptions,
+  queryOptions?: EdgeQueryableDataOptions,
+): DataLoader<K, Data[]> {
+  const loaderOptions: DataLoader.Options<K, Data[]> = {};
+
+  // if query logging is enabled, we should log what's happening with loader
+  if (logEnabled("query")) {
+    loaderOptions.cacheMap = new CacheMap(options);
+  }
+
+  return new DataLoader(async (keys: K[]) => {
+    if (!keys.length) {
+      return [];
+    }
+
+    // keep query simple if we're only fetching for one id
+    // or can't group because no groupCol...
+    if (keys.length == 1 || !options.groupCol) {
+      const rows = await simpleCase(options, keys[0], queryOptions);
+      return [rows];
+    }
+
+    let m = new Map<K, number>();
+    let result: Data[][] = [];
+    for (let i = 0; i < keys.length; i++) {
+      result.push([]);
+      // store the index....
+      m.set(keys[i], i);
+    }
+
+    const col = options.groupCol;
+    let extraClause: clause.Clause | undefined;
+    if (options.clause && queryOptions?.clause) {
+      extraClause = clause.And(options.clause, queryOptions.clause);
+    } else if (options.clause) {
+      extraClause = options.clause;
+    } else if (queryOptions?.clause) {
+      extraClause = queryOptions.clause;
+    }
+
+    const [query, cls2] = buildGroupQuery({
+      tableName: options.tableName,
+      fields: options.fields,
+      values: keys,
+      orderby: getOrderByLocal(options, queryOptions),
+      limit: queryOptions?.limit || getDefaultLimit(),
+      groupColumn: col,
+      clause: extraClause,
+    });
+
+    const rows = await performRawQuery(query, cls2.values(), cls2.logValues());
+    for (const row of rows) {
+      const srcID = row[col];
+      const idx = m.get(srcID);
+      delete row.row_num;
+      if (idx === undefined) {
+        throw new Error(
+          `malformed query. got ${srcID} back but didn't query for it`,
+        );
+      }
+      result[idx].push(row);
+    }
+    return result;
+  }, loaderOptions);
+}
+
+class QueryDirectLoader<K extends any> implements Loader<K, Data[]> {
+  private memoizedInitPrime: () => void;
+  private primedLoaders:
+    | Map<string, PrimableLoader<any, Data | null>>
+    | undefined;
+  constructor(
+    private options: QueryOptions,
+    private queryOptions?: EdgeQueryableDataOptions,
+    public context?: Context,
+  ) {
+    this.memoizedInitPrime = memoizee(this.initPrime.bind(this));
+  }
+
+  private initPrime() {
+    if (!this.context || !this.options?.toPrime) {
+      return;
+    }
+    let primedLoaders = new Map();
+    this.options.toPrime.forEach((prime) => {
+      const l2 = prime.createLoader(this.context);
+      if ((l2 as PrimableLoader<any, Data | null>).prime === undefined) {
+        return;
+      }
+      primedLoaders.set(prime.options.key, l2);
+    });
+    this.primedLoaders = primedLoaders;
+  }
+
+  async load(id: K): Promise<Data[]> {
+    const rows = await simpleCase(this.options, id, this.queryOptions);
+    if (this.context) {
+      this.memoizedInitPrime();
+      if (this.primedLoaders) {
+        for (const row of rows) {
+          for (const [key, loader] of this.primedLoaders) {
+            const value = row[key];
+            if (value !== undefined) {
+              loader.prime(row);
+            }
+          }
+        }
+      }
+    }
+    return rows;
+  }
+
+  clearAll() {}
+}
+
+// note, you should never call this directly
+// there's scenarios where QueryDirectLoader is needed instead of this...
+class QueryLoader<K extends any> implements Loader<K, Data[]> {
+  private loader: DataLoader<K, Data[]> | undefined;
+  private primedLoaders:
+    | Map<string, PrimableLoader<any, Data | null>>
+    | undefined;
+  private memoizedInitPrime: () => void;
+  constructor(
+    private options: QueryOptions,
+    public context?: Context,
+    private queryOptions?: EdgeQueryableDataOptions,
+  ) {
+    if (context) {
+      this.loader = createLoader(options, queryOptions);
+    }
+    this.memoizedInitPrime = memoizee(this.initPrime.bind(this));
+  }
+
+  private initPrime() {
+    if (!this.context || !this.options?.toPrime) {
+      return;
+    }
+    let primedLoaders = new Map();
+    this.options.toPrime.forEach((prime) => {
+      const l2 = prime.createLoader(this.context);
+      if ((l2 as PrimableLoader<any, Data | null>).prime === undefined) {
+        return;
+      }
+      primedLoaders.set(prime.options.key, l2);
+    });
+    this.primedLoaders = primedLoaders;
+  }
+
+  async load(id: K): Promise<Data[]> {
+    if (this.loader) {
+      this.memoizedInitPrime();
+      const rows = await this.loader.load(id);
+      if (this.primedLoaders) {
+        for (const row of rows) {
+          for (const [key, loader] of this.primedLoaders) {
+            const value = row[key];
+            if (value !== undefined) {
+              loader.prime(row);
+            }
+          }
+        }
+      }
+      return rows;
+    }
+
+    return simpleCase(this.options, id, this.queryOptions);
+  }
+
+  clearAll() {
+    this.loader && this.loader.clearAll();
+  }
+}
+
+interface QueryOptions {
+  fields: string[];
+  tableName: string; // or function for assoc_edge. come back to it
+  // if provided, we'll group queries to the database via this key and this will be the unique id we're querying for
+  // using window functions or not
+  groupCol?: string;
+  // will be combined with groupCol to make a simple or complex query
+  // if no groupCol, this is required
+  // if no clause and groupCol, we'll just use groupCol to make the query
+  clause?: clause.Clause;
+  // order by
+  orderby?: OrderBy;
+
+  // if provided, will be used to prime data in this object...
+  toPrime?: ObjectLoaderFactory<Data>[];
+}
+
+export class QueryLoaderFactory<K extends any>
+  implements LoaderFactory<K, Data[]>
+{
+  name: string;
+  constructor(private options: QueryOptions) {
+    if (options.groupCol) {
+      this.name = `queryLoader:${options.tableName}:${options.groupCol}`;
+    } else if (options.clause) {
+      this.name = `queryLoader:${
+        options.tableName
+      }:${options.clause.instanceKey()}`;
+    } else {
+      throw new Error(
+        `must pass at least one of groupCol and clause to QueryLoaderFactory`,
+      );
+    }
+  }
+
+  createLoader(context?: Context) {
+    return getLoader(
+      this,
+      () => new QueryLoader(this.options, context),
+      context,
+    );
+  }
+
+  createConfigurableLoader(
+    options: EdgeQueryableDataOptions,
+    context?: Context,
+  ) {
+    return QueryLoaderFactory.createConfigurableLoader(
+      this.name,
+      this.options,
+      options,
+      context,
+    );
+  }
+
+  static createConfigurableLoader(
+    name: string,
+    queryOptions: QueryOptions,
+    options: EdgeQueryableDataOptions,
+    context?: Context,
+  ) {
+    if (options.clause || !context) {
+      return new QueryDirectLoader(queryOptions, options, context);
+    }
+
+    const key = `${name}:limit:${options.limit}:orderby:${options.orderby}`;
+    return getCustomLoader(
+      key,
+      () => new QueryLoader(queryOptions, context, options),
+      context,
+    );
+  }
+}

package/src/core/loaders/raw_count_loader.ts

@@ -0,0 +1,175 @@
+import DataLoader from "dataloader";
+import {
+  LoadRowOptions,
+  ID,
+  Context,
+  Loader,
+  LoaderFactory,
+  SelectBaseDataOptions,
+} from "../base";
+import { loadRow, loadRows } from "../ent";
+import * as clause from "../clause";
+import { logEnabled } from "../logger";
+import { CacheMap, getLoader } from "./loader";
+
+interface QueryCountOptions {
+  tableName: string;
+
+  // must provide at least one or both of these
+  // if groupCol provided, we can do group by queries for multiple
+  // if not, can't group. similar philosophy to QueryLoader
+  groupCol?: string;
+  // if postgres and using an integer primary key, we need to pass this so that when we do an In query,
+  // we can cast accurately
+  // TODO https://github.com/lolopinto/ent/issues/1431
+  groupColType?: string;
+  clause?: clause.Clause;
+}
+
+async function simpleCase<K extends any>(
+  options: QueryCountOptions,
+  key: K,
+  context?: Context,
+) {
+  let cls: clause.Clause;
+  if (options.groupCol && options.clause) {
+    cls = clause.And(clause.Eq(options.groupCol, key), options.clause);
+  } else if (options.groupCol) {
+    cls = clause.Eq(options.groupCol, key);
+  } else if (options.clause) {
+    cls = options.clause;
+  } else {
+    throw new Error(`need options.groupCol or options.clause`);
+  }
+
+  const row = await loadRow({
+    ...options,
+    // sqlite needs as count otherwise it returns count(1)
+    fields: ["count(1) as count"],
+    clause: cls,
+    context,
+  });
+  return [parseInt(row?.count, 10) || 0];
+}
+
+export function createCountDataLoader<K extends any>(
+  options: QueryCountOptions,
+) {
+  const loaderOptions: DataLoader.Options<K, number> = {};
+
+  // if query logging is enabled, we should log what's happening with loader
+  if (logEnabled("query")) {
+    loaderOptions.cacheMap = new CacheMap(options);
+  }
+
+  return new DataLoader(async (keys: K[]) => {
+    if (!keys.length) {
+      return [];
+    }
+
+    // keep query simple if we're only fetching for one id
+    if (keys.length == 1 || !options.groupCol) {
+      return simpleCase(options, keys[0]);
+    }
+
+    let typ = options.groupColType || "uuid";
+    let cls: clause.Clause = clause.DBTypeIn(options.groupCol, keys, typ);
+    if (options.clause) {
+      cls = clause.And(cls, options.clause);
+    }
+
+    let m = new Map<K, number>();
+    let result: number[] = [];
+    for (let i = 0; i < keys.length; i++) {
+      result.push(0);
+      // store the index....
+      m.set(keys[i], i);
+    }
+
+    const rowOptions: LoadRowOptions = {
+      ...options,
+      fields: ["count(1) as count", options.groupCol],
+      groupby: options.groupCol,
+      clause: cls,
+    };
+
+    const rows = await loadRows(rowOptions);
+
+    for (const row of rows) {
+      const id = row[options.groupCol];
+      const idx = m.get(id);
+      if (idx === undefined) {
+        throw new Error(
+          `malformed query. got ${id} back but didn't query for it`,
+        );
+      }
+      result[idx] = parseInt(row.count, 10);
+    }
+    return result;
+  }, loaderOptions);
+}
+
+// for now this only works for single column counts
+// e.g. foreign key count
+export class RawCountLoader<K extends any> implements Loader<K, number> {
+  private loader: DataLoader<K, number> | undefined;
+  // tableName, columns
+  constructor(private options: QueryCountOptions, public context?: Context) {
+    if (context && options.groupCol) {
+      this.loader = createCountDataLoader(options);
+    }
+  }
+
+  async load(id: K): Promise<number> {
+    if (this.loader) {
+      return await this.loader.load(id);
+    }
+
+    const rows = await simpleCase(this.options, id, this.context);
+    return rows[0];
+  }
+
+  clearAll() {
+    this.loader && this.loader.clearAll();
+  }
+}
+
+export class RawCountLoaderFactory implements LoaderFactory<ID, number> {
+  name: string;
+  private options: QueryCountOptions;
+  constructor(options: SelectBaseDataOptions, col: string);
+  constructor(options: QueryCountOptions);
+  constructor(
+    options: SelectBaseDataOptions | QueryCountOptions,
+    col?: string,
+  ) {
+    if (typeof col === "string") {
+      // old API
+      this.options = {
+        ...options,
+        groupCol: col,
+      };
+    } else {
+      this.options = options;
+    }
+    if (this.options.groupCol) {
+      this.name = `${this.options.tableName}:${this.options.groupCol}`;
+    } else if (this.options.clause) {
+      this.name = `${
+        this.options.tableName
+      }:${this.options.clause.instanceKey()}`;
+    } else {
+      throw new Error(
+        `must pass at least one of groupCol and clause to QueryLoaderFactory`,
+      );
+    }
+  }
+
+  createLoader(context?: Context) {
+    return getLoader(
+      this,
+      () => new RawCountLoader(this.options, context),
+      context,
+    );
+  }
+}

package/src/core/logger.ts

@@ -0,0 +1,49 @@
+// query are things that go in database
+type logType = "query" | "warn" | "info" | "error" | "debug" | "cache";
+var m: { [key in logType]: string } = {
+  query: "log",
+  warn: "warn",
+  info: "log",
+  error: "error",
+  debug: "debug",
+  cache: "log",
+};
+var logLevels = new Map<logType, boolean>();
+
+export function setLogLevels(levels: logType | logType[]) {
+  if (!Array.isArray(levels)) {
+    levels = [levels];
+  }
+  levels.forEach((level) => logLevels.set(level, true));
+}
+
+export function clearLogLevels() {
+  logLevels.clear();
+}
+
+export function log(level: logType, msg: any) {
+  if (logLevels.has(level)) {
+    // mostly for sqlite error but fine for any type of error
+    if (level == "error" && msg instanceof Error && msg.message !== undefined) {
+      console.error(msg.message);
+      return;
+    }
+    console[m[level]](msg);
+  }
+}
+
+export function logIf(level: logType, logFn: () => any) {
+  if (logLevels.has(level)) {
+    console[m[level]](logFn());
+  }
+}
+
+export function logTrace() {
+  if (logLevels.has("debug")) {
+    console.trace();
+  }
+}
+
+export function logEnabled(level: logType) {
+  return logLevels.has(level);
+}