@technicity/data-service-generator 0.23.0-next.13 → 0.23.0-next.15

package/dist/generation/generate.js

@@ -106,17 +106,19 @@ async function generate(input) {
             tables = tables.filter((x) => !input.excludeTables?.includes(x));
         }
         ctx.log.debug({ tableCount: tables.length, tables }, "tables after filter");
-        // Process table-by-table to avoid exhausting DB connection pool (e.g. Cloud SQL limits)
-        const data = (await Promise.all(tables.map((x) => Promise.all([
-            getGetOneData(x, includeMappedFields, tables),
+        ctx.log.debug("getJunctionTables() warming cache (used by getRelationInfo)");
+        await getJunctionTables();
+        ctx.log.debug("getJunctionTables() cache warmed");
+        const data = await Promise.all(tables.flatMap((x) => [
+            getGetOneData(x, includeMappedFields),
             getGetListData(x),
             getGetListPaginatedData(x),
-            getPostOneData(x, specialCaseUuidColumn, includeMappedFields, tables),
+            getPostOneData(x, specialCaseUuidColumn, includeMappedFields),
             getPatchOneData(x, specialCaseUuidColumn, includeMappedFields),
             getPatchListData(x),
             getDeleteOneData(x),
             getDeleteListData(x)
-        ])))).flat();
+        ]));
         ctx.log.debug({ inputLength: data.length }, "SDK input data collected");
         const artifacts = await getArtifacts(tables, includeMappedFields, specialCaseUuidColumn);
         ctx.log.debug("getArtifacts() completed");
@@ -885,20 +887,22 @@ function getTypeOrderByName(table) {
 function getTypeDataPostName(table) {
     return "DataPost" + changeCase.pascalCase(table);
 }
-async function getGetOneData(table, includeMappedFields, tables) {
+async function getGetOneData(table, includeMappedFields) {
+    getCtx().log.debug({ table }, "getGetOneData() start");
     const typeFieldsName = getTypeFieldsName(table);
     const typeReturnBaseName = getTypeReturnBaseName(table);
     return {
         kind: "getOne",
         table,
         methodName: "get" + changeCase.pascalCase(table),
-        typeFields: await getTypeFields(table, typeFieldsName, includeMappedFields, tables),
+        typeFields: await getTypeFields(table, typeFieldsName, includeMappedFields),
         typeFieldsName,
-        typeReturnBase: await getTypeReturnBase(table, typeReturnBaseName, includeMappedFields, tables),
+        typeReturnBase: await getTypeReturnBase(table, typeReturnBaseName, includeMappedFields),
         typeReturnBaseName
     };
 }
 async function getGetListData(table) {
+    getCtx().log.debug({ table }, "getGetListData() start");
     const typeFieldsName = getTypeFieldsName(table);
     const typeReturnBaseName = getTypeReturnBaseName(table);
     const typeWhereName = getTypeWhereName(table);
@@ -918,6 +922,7 @@ async function getGetListData(table) {
     };
 }
 async function getGetListPaginatedData(table) {
+    getCtx().log.debug({ table }, "getGetListPaginatedData() start");
     const typeFieldsName = getTypeFieldsName(table);
     const typeReturnBaseName = getTypeReturnBaseName(table);
     const typeWhereName = getTypeWhereName(table);
@@ -932,7 +937,8 @@ async function getGetListPaginatedData(table) {
         typeOrderByName
     };
 }
-async function getPostOneData(table, specialCaseUuidColumn, includeMappedFields, tables) {
+async function getPostOneData(table, specialCaseUuidColumn, includeMappedFields) {
+    getCtx().log.debug({ table }, "getPostOneData() start");
     const typeFieldsName = getTypeFieldsName(table);
     const typeReturnBaseName = getTypeReturnBaseName(table);
     const typeDataName = getTypeDataPostName(table);
@@ -942,11 +948,12 @@ async function getPostOneData(table, specialCaseUuidColumn, includeMappedFields,
         methodName: "post" + changeCase.pascalCase(table),
         typeFieldsName,
         typeReturnBaseName,
-        typeData: await getTypeDataPost(table, typeDataName, specialCaseUuidColumn, includeMappedFields, tables),
+        typeData: await getTypeDataPost(table, typeDataName, specialCaseUuidColumn, includeMappedFields),
         typeDataName
     };
 }
 async function getPatchOneData(table, specialCaseUuidColumn, includeMappedFields) {
+    getCtx().log.debug({ table }, "getPatchOneData() start");
     const typeFieldsName = getTypeFieldsName(table);
     const typeReturnBaseName = getTypeReturnBaseName(table);
     const typeDataName = "DataPatch" + changeCase.pascalCase(table);
@@ -961,6 +968,7 @@ async function getPatchOneData(table, specialCaseUuidColumn, includeMappedFields
     };
 }
 async function getPatchListData(table) {
+    getCtx().log.debug({ table }, "getPatchListData() start");
     const typeFieldsName = getTypeFieldsName(table);
     const typeReturnBaseName = getTypeReturnBaseName(table);
     const typeWhereName = getTypeWhereName(table);
@@ -978,6 +986,7 @@ async function getPatchListData(table) {
     };
 }
 function getDeleteOneData(table) {
+    getCtx().log.debug({ table }, "getDeleteOneData() start");
     return {
         kind: "deleteOne",
         table,
@@ -985,6 +994,7 @@ function getDeleteOneData(table) {
     };
 }
 function getDeleteListData(table) {
+    getCtx().log.debug({ table }, "getDeleteListData() start");
     const typeWhereName = getTypeWhereName(table);
     return {
         kind: "deleteList",
@@ -994,10 +1004,12 @@ function getDeleteListData(table) {
     };
 }
 async function getTypeWhere(table, name) {
+    getCtx().log.debug({ table, name }, "getTypeWhere() start");
     const jsonSchemaWhere = await getJSONSchemaWhere(table);
     return (0, json_schema_to_typescript_1.compile)(jsonSchemaWhere, name, json2TsOpts);
 }
-async function getTypeDataPost(table, name, specialCaseUuidColumn, includeMappedFields, tables) {
+async function getTypeDataPost(table, name, specialCaseUuidColumn, includeMappedFields) {
+    getCtx().log.debug({ table, name }, "getTypeDataPost() start");
     const primaryColumn = await getPrimaryColumn(table);
     const tableMeta = (await getTableMeta(table)).filter((x) => x.Field !== primaryColumn.name);
     const nullable = tableMeta.reduce((acc, m) => ({
@@ -1010,7 +1022,7 @@ async function getTypeDataPost(table, name, specialCaseUuidColumn, includeMapped
     }), {});
     let properties = getJSONSchemaObjProperties(tableMeta);
     let notRequiredList = [];
-    const oneToManyRelations = (await getRelationInfo(table, tables)).filter((x) => x.type === "one-to-many__many-to-one" && x.kind === "one-to-many");
+    const oneToManyRelations = (await getRelationInfo(table)).filter((x) => x.type === "one-to-many__many-to-one" && x.kind === "one-to-many");
     const mappedFields = includeMappedFields ? await getMappedFields(table) : [];
     let mappedFieldsMap = new Map();
     if (includeMappedFields) {
@@ -1068,6 +1080,7 @@ async function getTypeDataPost(table, name, specialCaseUuidColumn, includeMapped
     return type;
 }
 async function getTypeDataPatch(table, name, specialCaseUuidColumn, includeMappedFields) {
+    getCtx().log.debug({ table, name }, "getTypeDataPatch() start");
     const primaryColumn = await getPrimaryColumn(table);
     let tableMeta = (await getTableMeta(table)).filter((x) => x.Field !== primaryColumn.name);
     if (specialCaseUuidColumn) {
@@ -1134,6 +1147,7 @@ function unwrapJSONType(type) {
     return type;
 }
 async function getMappedFields(table) {
+    getCtx().log.debug({ table }, "getMappedFields() start");
     const relationsManyToOne = await getRelationsManyToOne(table).then((xs) => xs.filter((x) => x.foreignKey.endsWith("Id")));
     let out = [];
     for (let x of relationsManyToOne) {
@@ -1157,6 +1171,7 @@ async function getMappedFields(table) {
     return out;
 }
 async function getJSONSchemaWhere(table) {
+    getCtx().log.debug({ table }, "getJSONSchemaWhere() start");
     const whereSchemaName = `_Where${changeCase.pascalCase(table)}`;
     const defWhere = {
         oneOf: [
@@ -1310,9 +1325,11 @@ async function getJSONSchemaWhere(table) {
     };
 }
 async function getTypeOrderBy(table, name) {
+    getCtx().log.debug({ table, name }, "getTypeOrderBy() start");
     return (0, json_schema_to_typescript_1.compile)((await getJSONSchemaOrderBy(table, name)), name, json2TsOpts);
 }
 async function getJSONSchemaOrderBy(table, name) {
+    getCtx().log.debug({ table, name }, "getJSONSchemaOrderBy() start");
     const fieldNames = await getTableMeta(table).then((xs) => xs.map((x) => x.Field));
     const def = {
         oneOf: fieldNames.map((k) => ({
@@ -1391,9 +1408,10 @@ export type TUpdateOperationsNumber = {$increment: number} | {$decrement: number
 `;
     return prettier.format(src, { parser: "typescript" });
 }
-async function getTypeFields(table, name, includeMappedFields, tables) {
+async function getTypeFields(table, name, includeMappedFields) {
+    getCtx().log.debug({ table, name }, "getTypeFields() start");
     const scalarKeys = Object.keys(getJSONSchemaObjProperties(await getTableMeta(table)));
-    const relations = await getRelationInfo(table, tables);
+    const relations = await getRelationInfo(table);
     const mappedFields = includeMappedFields ? await getMappedFields(table) : [];
     const keyWhere = "$where";
     const keyOrderBy = "$orderBy";
@@ -1453,10 +1471,11 @@ async function getTypeFields(table, name, includeMappedFields, tables) {
     type = imports + "\n\n" + type;
     return type;
 }
-async function getTypeReturnBase(table, name, includeMappedFields, tables) {
+async function getTypeReturnBase(table, name, includeMappedFields) {
+    getCtx().log.debug({ table, name }, "getTypeReturnBase() start");
     const tableMeta = await getTableMeta(table);
     const scalarProperties = getJSONSchemaObjProperties(tableMeta);
-    const relations = await getRelationInfo(table, tables);
+    const relations = await getRelationInfo(table);
     const mappedFields = includeMappedFields ? await getMappedFields(table) : [];
     const jsonSchemaReturn = {
         type: "object",
@@ -1515,7 +1534,7 @@ async function getArtifacts(tables, includeMappedFields, specialCaseUuidColumn)
             getShowCreateTable(table)
         ]);
         const scalarFields = tableMeta.map((x) => x.Field);
-        const relationInfo = await getRelationInfo(table, tables);
+        const relationInfo = await getRelationInfo(table);
         const relationFields = relationInfo.reduce((acc, x) => {
             if (x.type === "one-to-many__many-to-one") {
                 acc[x.name] = {
@@ -1618,9 +1637,13 @@ async function getArtifacts(tables, includeMappedFields, specialCaseUuidColumn)
     }, {});
     return artifacts;
 }
-const getRelationInfo = (0, memoize_1.default)(async function getRelationInfo(table, tables) {
+const getRelationInfo = (0, memoize_1.default)(async function getRelationInfo(table) {
+    const ctx = getCtx();
+    ctx.log.debug({ table }, "getRelationInfo() start");
     const relationsManyToOne = await getRelationsManyToOne(table);
+    ctx.log.debug({ table, count: relationsManyToOne.length }, "getRelationInfo() getRelationsManyToOne done");
     const relationsOneToMany = await getRelationsOneToMany(table);
+    ctx.log.debug({ table, count: relationsOneToMany.length }, "getRelationInfo() getRelationsOneToMany done");
     let out = [];
     out = out.concat(relationsManyToOne.reduce((acc, x) => {
         if (!x.foreignKey.endsWith("Id")) {
@@ -1638,6 +1661,7 @@ const getRelationInfo = (0, memoize_1.default)(async function getRelationInfo(ta
         });
         return acc;
     }, []));
+    ctx.log.debug({ table, manyToOneOutCount: out.length }, "getRelationInfo() many-to-one concat done");
     const relationsOneToManyDuplicates = (0, getDuplicates_1.getDuplicates)(relationsOneToMany.map((x) => x.referencedTable));
     out = out.concat(relationsOneToMany.reduce((acc, x) => {
         if (!x.referencedKey.endsWith("Id")) {
@@ -1665,7 +1689,11 @@ const getRelationInfo = (0, memoize_1.default)(async function getRelationInfo(ta
         });
         return acc;
     }, []));
-    const relationsManyToMany = (await getJunctionTables(tables)).reduce((acc, x) => {
+    ctx.log.debug({ table, afterOneToManyCount: out.length }, "getRelationInfo() one-to-many concat done");
+    ctx.log.debug({ table }, "getRelationInfo() getJunctionTables() calling");
+    const junctionTables = await getJunctionTables();
+    ctx.log.debug({ table, junctionTableCount: junctionTables.length }, "getRelationInfo() getJunctionTables() done");
+    const relationsManyToMany = junctionTables.reduce((acc, x) => {
         const dataForParentTable = x.relations.find((r) => r.referencedTable === table);
         if (dataForParentTable == null) {
             return acc;
@@ -1685,15 +1713,62 @@ const getRelationInfo = (0, memoize_1.default)(async function getRelationInfo(ta
         });
         return acc;
     }, []);
+    ctx.log.debug({ table, manyToManyCount: relationsManyToMany.length }, "getRelationInfo() many-to-many reduce done");
     out = out.concat(relationsManyToMany);
     out = _.sortBy([(x) => x.table, (x) => x.name], out);
+    ctx.log.debug({ table, totalCount: out.length }, "getRelationInfo() done");
     return out;
 }, (table) => getCtx().runId + ":" + table);
 function getRelationManyToOneFieldName(x) {
     return changeCase.camelCase(x.foreignKey.replace(new RegExp(x.referencedKey + "$", "i"), ""));
 }
 // TODO: not sure if this logic is correct
-const getJunctionTables = (0, memoize_1.default)(async function getJunctionTables(tables) {
+// Note: intentionally not the filtered tables
+const getJunctionTables = (0, memoize_1.default)(async function getJunctionTables() {
+    const { dialect, query } = getCtx();
+    if (dialect === "postgresql") {
+        // Single bulk query instead of getRelationsManyToOne(table) per table (avoids N-way pool contention)
+        const rows = await query(`SELECT kcu.table_name AS t1, kcu.column_name AS "t1Field",
+                ccu.table_name AS t2, ccu.column_name AS "t2Field",
+                col.is_nullable
+         FROM information_schema.key_column_usage kcu
+         JOIN information_schema.referential_constraints rc ON kcu.constraint_name = rc.constraint_name AND kcu.table_schema = rc.constraint_schema
+         JOIN information_schema.constraint_column_usage ccu ON rc.unique_constraint_name = ccu.constraint_name AND rc.unique_constraint_schema = ccu.table_schema
+         LEFT JOIN information_schema.columns col ON col.table_schema = kcu.table_schema AND col.table_name = kcu.table_name AND col.column_name = kcu.column_name
+         WHERE kcu.table_schema = 'public'
+         ORDER BY kcu.table_name, ccu.table_name, ccu.column_name`);
+        const byTable = new Map();
+        for (const r of rows) {
+            const list = byTable.get(r.t1) ?? [];
+            list.push({
+                t1Field: r.t1Field,
+                t2: r.t2,
+                t2Field: r.t2Field,
+                nullable: r.is_nullable === "YES"
+            });
+            byTable.set(r.t1, list);
+        }
+        const result = [];
+        for (const [table, fks] of byTable) {
+            if (fks.length === 2 &&
+                fks[0].t2 !== fks[1].t2 &&
+                [
+                    fks[0].t2 + fks[1].t2,
+                    fks[1].t2 + fks[0].t2
+                ].includes(table)) {
+                const relations = _.sortBy([(x) => x.referencedTable, (x) => x.referencedKey, (x) => x.foreignKey], fks.map((f) => ({
+                    table,
+                    foreignKey: f.t1Field,
+                    referencedTable: f.t2,
+                    referencedKey: f.t2Field,
+                    nullable: f.nullable
+                })));
+                result.push({ table, relations });
+            }
+        }
+        return result;
+    }
+    const tables = await getTableNames();
     return (await Promise.all(tables.map(async (table) => {
         const relations = await getRelationsManyToOne(table);
         if (relations.length === 2 &&
@@ -1711,6 +1786,7 @@ const getJunctionTables = (0, memoize_1.default)(async function getJunctionTable
 // `from` relations
 // https://stackoverflow.com/a/54732547
 const getRelationsManyToOne = (0, memoize_1.default)(async function getRelationsManyToOne(table) {
+    getCtx().log.debug({ table }, "getRelationsManyToOne() start");
     const { dialect, query } = getCtx();
     const tableMeta = await getTableMeta(table);
     let rs;
@@ -1756,6 +1832,7 @@ const getRelationsManyToOne = (0, memoize_1.default)(async function getRelations
 }, (table) => getCtx().runId + ":" + table);
 // `to` relations
 const getRelationsOneToMany = (0, memoize_1.default)(async function getRelationsOneToMany(table) {
+    getCtx().log.debug({ table }, "getRelationsOneToMany() start");
     const { dialect, query } = getCtx();
     let rs;
     if (dialect === "mysql") {
@@ -1799,6 +1876,7 @@ const getRelationsOneToMany = (0, memoize_1.default)(async function getRelations
     return _.sortBy([(x) => x.referencedTable, (x) => x.referencedKey, (x) => x.foreignKey], xs);
 }, (table) => getCtx().runId + ":" + table);
 async function getPrimaryColumn(table) {
+    getCtx().log.debug({ table }, "getPrimaryColumn() start");
     const tableMeta = await getTableMeta(table);
     const columns = tableMeta.filter((x) => x.Key === "PRI");
     if (columns.length !== 1) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@technicity/data-service-generator",
-  "version": "0.23.0-next.13",
+  "version": "0.23.0-next.15",
   "main": "./dist/index.js",
   "files": [
     "dist"