@backstage/plugin-catalog-backend 1.29.0-next.1 → 1.29.0-next.2

package/CHANGELOG.md

@@ -1,5 +1,36 @@
 # @backstage/plugin-catalog-backend
 
+## 1.29.0-next.2
+
+### Minor Changes
+
+- c1307b4: Implement `/entities` in terms of `queryEntities` to not run into memory and performance problems on large catalogs
+
+### Patch Changes
+
+- dfc8b41: Updated dependency `@opentelemetry/api` to `^1.9.0`.
+- 8013c9c: Perform the by-query count inlined with the main query
+- feba9ee: Use a join based strategy for filtering, when having small page sizes
+- 1fdb48e: Use a faster count method on pg when computing some metrics
+- 0c33465: Implement `/entities/by-name/:kind/:namespace/:name` using `getEntitiesByRefs`
+- d93390d: When parsing filters, do not make redundant `anyOf` and `allOf` nodes when there's only a single entry within them
+- 24ecea8: Avoid extra ordering in by-query when the user doesn't ask for it
+- Updated dependencies
+  - @backstage/backend-plugin-api@1.1.0-next.2
+  - @backstage/plugin-permission-node@0.8.6-next.2
+  - @backstage/backend-openapi-utils@0.4.0-next.2
+  - @backstage/errors@1.2.6-next.0
+  - @backstage/plugin-catalog-node@1.15.0-next.2
+  - @backstage/plugin-events-node@0.4.6-next.2
+  - @backstage/plugin-search-backend-module-catalog@0.2.6-next.2
+  - @backstage/catalog-client@1.9.0-next.2
+  - @backstage/catalog-model@1.7.2-next.0
+  - @backstage/config@1.3.1-next.0
+  - @backstage/integration@1.16.0-next.1
+  - @backstage/types@1.2.0
+  - @backstage/plugin-catalog-common@1.1.2-next.0
+  - @backstage/plugin-permission-common@0.8.3-next.0
+
 ## 1.29.0-next.1
 
 ### Minor Changes

package/dist/database/metrics.cjs.js

@@ -64,18 +64,36 @@ function initDatabaseMetrics(knex) {
     registered_locations: meter.createObservableGauge("catalog_registered_locations_count", {
       description: "Total amount of registered locations in the catalog"
     }).addCallback(async (gauge) => {
-      const total = await knex("locations").count({
-        count: "*"
-      });
-      gauge.observe(Number(total[0].count));
+      if (knex.client.config.client === "pg") {
+        const total = await knex.raw(`
+            SELECT reltuples::bigint AS estimate
+            FROM   pg_class
+            WHERE  oid = 'locations'::regclass;
+          `);
+        gauge.observe(Number(total.rows[0].estimate));
+      } else {
+        const total = await knex("locations").count({
+          count: "*"
+        });
+        gauge.observe(Number(total[0].count));
+      }
     }),
     relations: meter.createObservableGauge("catalog_relations_count", {
       description: "Total amount of relations between entities"
     }).addCallback(async (gauge) => {
-      const total = await knex("relations").count({
-        count: "*"
-      });
-      gauge.observe(Number(total[0].count));
+      if (knex.client.config.client === "pg") {
+        const total = await knex.raw(`
+            SELECT reltuples::bigint AS estimate
+            FROM   pg_class
+            WHERE  oid = 'relations'::regclass;
+          `);
+        gauge.observe(Number(total.rows[0].estimate));
+      } else {
+        const total = await knex("relations").count({
+          count: "*"
+        });
+        gauge.observe(Number(total[0].count));
+      }
     })
   };
 }

package/dist/database/metrics.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.cjs.js","sources":["../../src/database/metrics.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { createGaugeMetric } from '../util/metrics';\nimport { DbRelationsRow, DbLocationsRow, DbSearchRow } from './tables';\nimport { metrics } from '@opentelemetry/api';\n\nexport function initDatabaseMetrics(knex: Knex) {\n  const seenProm = new Set<string>();\n  const seen = new Set<string>();\n  const meter = metrics.getMeter('default');\n  return {\n    entities_count_prom: createGaugeMetric({\n      name: 'catalog_entities_count',\n      help: 'Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.',\n      labelNames: ['kind'],\n      async collect() {\n        const results = await knex<DbSearchRow>('search')\n          .where('key', '=', 'kind')\n          .whereNotNull('value')\n          .select({ kind: 'value', count: knex.raw('count(*)') })\n          .groupBy('value');\n\n        results.forEach(({ kind, count }) => {\n          seenProm.add(kind);\n          this.set({ kind }, Number(count));\n        });\n\n        // Set all the entities that were not seenProm to 0 and delete them from the seenProm set.\n        seenProm.forEach(kind => {\n          if (!results.some(r => r.kind === kind)) {\n            this.set({ kind }, 0);\n            seenProm.delete(kind);\n          }\n        });\n      },\n    }),\n    registered_locations_prom: createGaugeMetric({\n      name: 'catalog_registered_locations_count',\n      help: 'Total amount of registered locations in the catalog. DEPRECATED: Please use opentelemetry metrics instead.',\n      async collect() {\n        const total = await knex<DbLocationsRow>('locations').count({\n          count: '*',\n        });\n        this.set(Number(total[0].count));\n      },\n    }),\n    relations_prom: createGaugeMetric({\n      name: 'catalog_relations_count',\n      help: 'Total amount of relations between entities. DEPRECATED: Please use opentelemetry metrics instead.',\n      async collect() {\n        const total = await knex<DbRelationsRow>('relations').count({\n          count: '*',\n        });\n        this.set(Number(total[0].count));\n      },\n    }),\n    entities_count: meter\n      .createObservableGauge('catalog_entities_count', {\n        description: 'Total amount of entities in the catalog',\n      })\n      .addCallback(async gauge => {\n        const results = await knex<DbSearchRow>('search')\n          .where('key', '=', 'kind')\n          .whereNotNull('value')\n          .select({ kind: 'value', count: knex.raw('count(*)') })\n          .groupBy('value');\n\n        results.forEach(({ kind, count }) => {\n          seen.add(kind);\n          gauge.observe(Number(count), { kind });\n        });\n\n        // Set all the entities that were not seen to 0 and delete them from the seen set.\n        seen.forEach(kind => {\n          if (!results.some(r => r.kind === kind)) {\n            gauge.observe(0, { kind });\n            seen.delete(kind);\n          }\n        });\n      }),\n    registered_locations: meter\n      .createObservableGauge('catalog_registered_locations_count', {\n        description: 'Total amount of registered locations in the catalog',\n      })\n      .addCallback(async gauge => {\n        const total = await knex<DbLocationsRow>('locations').count({\n          count: '*',\n        });\n        gauge.observe(Number(total[0].count));\n      }),\n    relations: meter\n      .createObservableGauge('catalog_relations_count', {\n        description: 'Total amount of relations between entities',\n      })\n      .addCallback(async gauge => {\n        const total = await knex<DbRelationsRow>('relations').count({\n          count: '*',\n        });\n        gauge.observe(Number(total[0].count));\n      }),\n  };\n}\n"],"names":["metrics","createGaugeMetric"],"mappings":";;;;;AAqBO,SAAS,oBAAoB,IAAY,EAAA;AAC9C,EAAM,MAAA,QAAA,uBAAe,GAAY,EAAA;AACjC,EAAM,MAAA,IAAA,uBAAW,GAAY,EAAA;AAC7B,EAAM,MAAA,KAAA,GAAQA,WAAQ,CAAA,QAAA,CAAS,SAAS,CAAA;AACxC,EAAO,OAAA;AAAA,IACL,qBAAqBC,yBAAkB,CAAA;AAAA,MACrC,IAAM,EAAA,wBAAA;AAAA,MACN,IAAM,EAAA,gGAAA;AAAA,MACN,UAAA,EAAY,CAAC,MAAM,CAAA;AAAA,MACnB,MAAM,OAAU,GAAA;AACd,QAAM,MAAA,OAAA,GAAU,MAAM,IAAA,CAAkB,QAAQ,CAAA,CAC7C,MAAM,KAAO,EAAA,GAAA,EAAK,MAAM,CAAA,CACxB,YAAa,CAAA,OAAO,EACpB,MAAO,CAAA,EAAE,IAAM,EAAA,OAAA,EAAS,KAAO,EAAA,IAAA,CAAK,GAAI,CAAA,UAAU,CAAE,EAAC,CACrD,CAAA,OAAA,CAAQ,OAAO,CAAA;AAElB,QAAA,OAAA,CAAQ,OAAQ,CAAA,CAAC,EAAE,IAAA,EAAM,OAAY,KAAA;AACnC,UAAA,QAAA,CAAS,IAAI,IAAI,CAAA;AACjB,UAAA,IAAA,CAAK,IAAI,EAAE,IAAA,EAAQ,EAAA,MAAA,CAAO,KAAK,CAAC,CAAA;AAAA,SACjC,CAAA;AAGD,QAAA,QAAA,CAAS,QAAQ,CAAQ,IAAA,KAAA;AACvB,UAAA,IAAI,CAAC,OAAQ,CAAA,IAAA,CAAK,OAAK,CAAE,CAAA,IAAA,KAAS,IAAI,CAAG,EAAA;AACvC,YAAA,IAAA,CAAK,GAAI,CAAA,EAAE,IAAK,EAAA,EAAG,CAAC,CAAA;AACpB,YAAA,QAAA,CAAS,OAAO,IAAI,CAAA;AAAA;AACtB,SACD,CAAA;AAAA;AACH,KACD,CAAA;AAAA,IACD,2BAA2BA,yBAAkB,CAAA;AAAA,MAC3C,IAAM,EAAA,oCAAA;AAAA,MACN,IAAM,EAAA,4GAAA;AAAA,MACN,MAAM,OAAU,GAAA;AACd,QAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,UAC1D,KAAO,EAAA;AAAA,SACR,CAAA;AACD,QAAA,IAAA,CAAK,IAAI,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA;AACjC,KACD,CAAA;AAAA,IACD,gBAAgBA,yBAAkB,CAAA;AAAA,MAChC,IAAM,EAAA,yBAAA;AAAA,MACN,IAAM,EAAA,mGAAA;AAAA,MACN,MAAM,OAAU,GAAA;AACd,QAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,UAC1D,KAAO,EAAA;AAAA,SACR,CAAA;AACD,QAAA,IAAA,CAAK,IAAI,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA;AACjC,KACD,CAAA;AAAA,IACD,cAAA,EAAgB,KACb,CAAA,qBAAA,CAAsB,wBAA0B,EAAA;AAAA,MAC/C,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAM,MAAA,OAAA,GAAU,MAAM,IAAA,CAAkB,QAAQ,CAAA,CAC7C,MAAM,KAAO,EAAA,GAAA,EAAK,MAAM,CAAA,CACxB,YAAa,CAAA,OAAO,EACpB,MAAO,CAAA,EAAE,IAAM,EAAA,OAAA,EAAS,KAAO,EAAA,IAAA,CAAK,GAAI,CAAA,UAAU,CAAE,EAAC,CACrD,CAAA,OAAA,CAAQ,OAAO,CAAA;AAElB,MAAA,OAAA,CAAQ,OAAQ,CAAA,CAAC,EAAE,IAAA,EAAM,OAAY,KAAA;AACnC,QAAA,IAAA,CAAK,IAAI,IAAI,CAAA;AACb,QAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAK,CAAG,EAAA,EAAE,MAAM,CAAA;AAAA,OACtC,CAAA;AAGD,MAAA,IAAA,CAAK,QAAQ,CAAQ,IAAA,KAAA;AACnB,QAAA,IAAI,CAAC,OAAQ,CAAA,IAAA,CAAK,OAAK,CAAE,CAAA,IAAA,KAAS,IAAI,CAAG,EAAA;AACvC,UAAA,KAAA,CAAM,OAAQ,CAAA,CAAA,EAAG,EAAE,IAAA,EAAM,CAAA;AACzB,UAAA,IAAA,CAAK,OAAO,IAAI,CAAA;AAAA;AAClB,OACD,CAAA;AAAA,KACF,CAAA;AAAA,IACH,oBAAA,EAAsB,KACnB,CAAA,qBAAA,CAAsB,oCAAsC,EAAA;AAAA,MAC3D,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,QAC1D,KAAO,EAAA;AAAA,OACR,CAAA;AACD,MAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA,KACrC,CAAA;AAAA,IACH,SAAA,EAAW,KACR,CAAA,qBAAA,CAAsB,yBAA2B,EAAA;AAAA,MAChD,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,QAC1D,KAAO,EAAA;AAAA,OACR,CAAA;AACD,MAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA,KACrC;AAAA,GACL;AACF;;;;"}
+{"version":3,"file":"metrics.cjs.js","sources":["../../src/database/metrics.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport { createGaugeMetric } from '../util/metrics';\nimport { DbRelationsRow, DbLocationsRow, DbSearchRow } from './tables';\nimport { metrics } from '@opentelemetry/api';\n\nexport function initDatabaseMetrics(knex: Knex) {\n  const seenProm = new Set<string>();\n  const seen = new Set<string>();\n  const meter = metrics.getMeter('default');\n  return {\n    entities_count_prom: createGaugeMetric({\n      name: 'catalog_entities_count',\n      help: 'Total amount of entities in the catalog. DEPRECATED: Please use opentelemetry metrics instead.',\n      labelNames: ['kind'],\n      async collect() {\n        const results = await knex<DbSearchRow>('search')\n          .where('key', '=', 'kind')\n          .whereNotNull('value')\n          .select({ kind: 'value', count: knex.raw('count(*)') })\n          .groupBy('value');\n\n        results.forEach(({ kind, count }) => {\n          seenProm.add(kind);\n          this.set({ kind }, Number(count));\n        });\n\n        // Set all the entities that were not seenProm to 0 and delete them from the seenProm set.\n        seenProm.forEach(kind => {\n          if (!results.some(r => r.kind === kind)) {\n            this.set({ kind }, 0);\n            seenProm.delete(kind);\n          }\n        });\n      },\n    }),\n    registered_locations_prom: createGaugeMetric({\n      name: 'catalog_registered_locations_count',\n      help: 'Total amount of registered locations in the catalog. DEPRECATED: Please use opentelemetry metrics instead.',\n      async collect() {\n        const total = await knex<DbLocationsRow>('locations').count({\n          count: '*',\n        });\n        this.set(Number(total[0].count));\n      },\n    }),\n    relations_prom: createGaugeMetric({\n      name: 'catalog_relations_count',\n      help: 'Total amount of relations between entities. DEPRECATED: Please use opentelemetry metrics instead.',\n      async collect() {\n        const total = await knex<DbRelationsRow>('relations').count({\n          count: '*',\n        });\n        this.set(Number(total[0].count));\n      },\n    }),\n    entities_count: meter\n      .createObservableGauge('catalog_entities_count', {\n        description: 'Total amount of entities in the catalog',\n      })\n      .addCallback(async gauge => {\n        const results = await knex<DbSearchRow>('search')\n          .where('key', '=', 'kind')\n          .whereNotNull('value')\n          .select({ kind: 'value', count: knex.raw('count(*)') })\n          .groupBy('value');\n\n        results.forEach(({ kind, count }) => {\n          seen.add(kind);\n          gauge.observe(Number(count), { kind });\n        });\n\n        // Set all the entities that were not seen to 0 and delete them from the seen set.\n        seen.forEach(kind => {\n          if (!results.some(r => r.kind === kind)) {\n            gauge.observe(0, { kind });\n            seen.delete(kind);\n          }\n        });\n      }),\n    registered_locations: meter\n      .createObservableGauge('catalog_registered_locations_count', {\n        description: 'Total amount of registered locations in the catalog',\n      })\n      .addCallback(async gauge => {\n        if (knex.client.config.client === 'pg') {\n          // https://stackoverflow.com/questions/7943233/fast-way-to-discover-the-row-count-of-a-table-in-postgresql\n          const total = await knex.raw(`\n            SELECT reltuples::bigint AS estimate\n            FROM   pg_class\n            WHERE  oid = 'locations'::regclass;\n          `);\n          gauge.observe(Number(total.rows[0].estimate));\n        } else {\n          const total = await knex<DbLocationsRow>('locations').count({\n            count: '*',\n          });\n          gauge.observe(Number(total[0].count));\n        }\n      }),\n    relations: meter\n      .createObservableGauge('catalog_relations_count', {\n        description: 'Total amount of relations between entities',\n      })\n      .addCallback(async gauge => {\n        if (knex.client.config.client === 'pg') {\n          // https://stackoverflow.com/questions/7943233/fast-way-to-discover-the-row-count-of-a-table-in-postgresql\n          const total = await knex.raw(`\n            SELECT reltuples::bigint AS estimate\n            FROM   pg_class\n            WHERE  oid = 'relations'::regclass;\n          `);\n          gauge.observe(Number(total.rows[0].estimate));\n        } else {\n          const total = await knex<DbRelationsRow>('relations').count({\n            count: '*',\n          });\n          gauge.observe(Number(total[0].count));\n        }\n      }),\n  };\n}\n"],"names":["metrics","createGaugeMetric"],"mappings":";;;;;AAqBO,SAAS,oBAAoB,IAAY,EAAA;AAC9C,EAAM,MAAA,QAAA,uBAAe,GAAY,EAAA;AACjC,EAAM,MAAA,IAAA,uBAAW,GAAY,EAAA;AAC7B,EAAM,MAAA,KAAA,GAAQA,WAAQ,CAAA,QAAA,CAAS,SAAS,CAAA;AACxC,EAAO,OAAA;AAAA,IACL,qBAAqBC,yBAAkB,CAAA;AAAA,MACrC,IAAM,EAAA,wBAAA;AAAA,MACN,IAAM,EAAA,gGAAA;AAAA,MACN,UAAA,EAAY,CAAC,MAAM,CAAA;AAAA,MACnB,MAAM,OAAU,GAAA;AACd,QAAM,MAAA,OAAA,GAAU,MAAM,IAAA,CAAkB,QAAQ,CAAA,CAC7C,MAAM,KAAO,EAAA,GAAA,EAAK,MAAM,CAAA,CACxB,YAAa,CAAA,OAAO,EACpB,MAAO,CAAA,EAAE,IAAM,EAAA,OAAA,EAAS,KAAO,EAAA,IAAA,CAAK,GAAI,CAAA,UAAU,CAAE,EAAC,CACrD,CAAA,OAAA,CAAQ,OAAO,CAAA;AAElB,QAAA,OAAA,CAAQ,OAAQ,CAAA,CAAC,EAAE,IAAA,EAAM,OAAY,KAAA;AACnC,UAAA,QAAA,CAAS,IAAI,IAAI,CAAA;AACjB,UAAA,IAAA,CAAK,IAAI,EAAE,IAAA,EAAQ,EAAA,MAAA,CAAO,KAAK,CAAC,CAAA;AAAA,SACjC,CAAA;AAGD,QAAA,QAAA,CAAS,QAAQ,CAAQ,IAAA,KAAA;AACvB,UAAA,IAAI,CAAC,OAAQ,CAAA,IAAA,CAAK,OAAK,CAAE,CAAA,IAAA,KAAS,IAAI,CAAG,EAAA;AACvC,YAAA,IAAA,CAAK,GAAI,CAAA,EAAE,IAAK,EAAA,EAAG,CAAC,CAAA;AACpB,YAAA,QAAA,CAAS,OAAO,IAAI,CAAA;AAAA;AACtB,SACD,CAAA;AAAA;AACH,KACD,CAAA;AAAA,IACD,2BAA2BA,yBAAkB,CAAA;AAAA,MAC3C,IAAM,EAAA,oCAAA;AAAA,MACN,IAAM,EAAA,4GAAA;AAAA,MACN,MAAM,OAAU,GAAA;AACd,QAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,UAC1D,KAAO,EAAA;AAAA,SACR,CAAA;AACD,QAAA,IAAA,CAAK,IAAI,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA;AACjC,KACD,CAAA;AAAA,IACD,gBAAgBA,yBAAkB,CAAA;AAAA,MAChC,IAAM,EAAA,yBAAA;AAAA,MACN,IAAM,EAAA,mGAAA;AAAA,MACN,MAAM,OAAU,GAAA;AACd,QAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,UAC1D,KAAO,EAAA;AAAA,SACR,CAAA;AACD,QAAA,IAAA,CAAK,IAAI,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA;AACjC,KACD,CAAA;AAAA,IACD,cAAA,EAAgB,KACb,CAAA,qBAAA,CAAsB,wBAA0B,EAAA;AAAA,MAC/C,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAM,MAAA,OAAA,GAAU,MAAM,IAAA,CAAkB,QAAQ,CAAA,CAC7C,MAAM,KAAO,EAAA,GAAA,EAAK,MAAM,CAAA,CACxB,YAAa,CAAA,OAAO,EACpB,MAAO,CAAA,EAAE,IAAM,EAAA,OAAA,EAAS,KAAO,EAAA,IAAA,CAAK,GAAI,CAAA,UAAU,CAAE,EAAC,CACrD,CAAA,OAAA,CAAQ,OAAO,CAAA;AAElB,MAAA,OAAA,CAAQ,OAAQ,CAAA,CAAC,EAAE,IAAA,EAAM,OAAY,KAAA;AACnC,QAAA,IAAA,CAAK,IAAI,IAAI,CAAA;AACb,QAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAK,CAAG,EAAA,EAAE,MAAM,CAAA;AAAA,OACtC,CAAA;AAGD,MAAA,IAAA,CAAK,QAAQ,CAAQ,IAAA,KAAA;AACnB,QAAA,IAAI,CAAC,OAAQ,CAAA,IAAA,CAAK,OAAK,CAAE,CAAA,IAAA,KAAS,IAAI,CAAG,EAAA;AACvC,UAAA,KAAA,CAAM,OAAQ,CAAA,CAAA,EAAG,EAAE,IAAA,EAAM,CAAA;AACzB,UAAA,IAAA,CAAK,OAAO,IAAI,CAAA;AAAA;AAClB,OACD,CAAA;AAAA,KACF,CAAA;AAAA,IACH,oBAAA,EAAsB,KACnB,CAAA,qBAAA,CAAsB,oCAAsC,EAAA;AAAA,MAC3D,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAA,IAAI,IAAK,CAAA,MAAA,CAAO,MAAO,CAAA,MAAA,KAAW,IAAM,EAAA;AAEtC,QAAM,MAAA,KAAA,GAAQ,MAAM,IAAA,CAAK,GAAI,CAAA;AAAA;AAAA;AAAA;AAAA,UAI5B,CAAA,CAAA;AACD,QAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAA,CAAM,KAAK,CAAC,CAAA,CAAE,QAAQ,CAAC,CAAA;AAAA,OACvC,MAAA;AACL,QAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,UAC1D,KAAO,EAAA;AAAA,SACR,CAAA;AACD,QAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA;AACtC,KACD,CAAA;AAAA,IACH,SAAA,EAAW,KACR,CAAA,qBAAA,CAAsB,yBAA2B,EAAA;AAAA,MAChD,WAAa,EAAA;AAAA,KACd,CAAA,CACA,WAAY,CAAA,OAAM,KAAS,KAAA;AAC1B,MAAA,IAAI,IAAK,CAAA,MAAA,CAAO,MAAO,CAAA,MAAA,KAAW,IAAM,EAAA;AAEtC,QAAM,MAAA,KAAA,GAAQ,MAAM,IAAA,CAAK,GAAI,CAAA;AAAA;AAAA;AAAA;AAAA,UAI5B,CAAA,CAAA;AACD,QAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAA,CAAM,KAAK,CAAC,CAAA,CAAE,QAAQ,CAAC,CAAA;AAAA,OACvC,MAAA;AACL,QAAA,MAAM,KAAQ,GAAA,MAAM,IAAqB,CAAA,WAAW,EAAE,KAAM,CAAA;AAAA,UAC1D,KAAO,EAAA;AAAA,SACR,CAAA;AACD,QAAA,KAAA,CAAM,QAAQ,MAAO,CAAA,KAAA,CAAM,CAAC,CAAA,CAAE,KAAK,CAAC,CAAA;AAAA;AACtC,KACD;AAAA,GACL;AACF;;;;"}

package/dist/service/DefaultEntitiesCatalog.cjs.js

@@ -5,12 +5,9 @@ var errors = require('@backstage/errors');
 var lodash = require('lodash');
 var zod = require('zod');
 var util = require('./util.cjs.js');
+var applyEntityFilterToQuery = require('./request/applyEntityFilterToQuery.cjs.js');
 
-const defaultSortField = {
-  field: "metadata.uid",
-  order: "asc"
-};
-const DEFAULT_LIMIT = 20;
+const DEFAULT_LIMIT = 200;
 function parsePagination(input) {
   if (!input) {
     return {};
@@ -46,52 +43,6 @@ function stringifyPagination(input) {
   const base64 = Buffer.from(json, "utf8").toString("base64");
   return base64;
 }
-function addCondition(queryBuilder, db, filter, negate = false, entityIdField = "entity_id") {
-  const key = filter.key.toLowerCase();
-  const values = filter.values?.map((v) => v.toLowerCase());
-  const matchQuery = db("search").select("search.entity_id").where({ key }).andWhere(function keyFilter() {
-    if (values?.length === 1) {
-      this.where({ value: values.at(0) });
-    } else if (values) {
-      this.andWhere("value", "in", values);
-    }
-  });
-  queryBuilder.andWhere(entityIdField, negate ? "not in" : "in", matchQuery);
-}
-function isEntitiesSearchFilter(filter) {
-  return filter.hasOwnProperty("key");
-}
-function isOrEntityFilter(filter) {
-  return filter.hasOwnProperty("anyOf");
-}
-function isNegationEntityFilter(filter) {
-  return filter.hasOwnProperty("not");
-}
-function parseFilter(filter, query, db, negate = false, entityIdField = "entity_id") {
-  if (isNegationEntityFilter(filter)) {
-    return parseFilter(filter.not, query, db, !negate, entityIdField);
-  }
-  if (isEntitiesSearchFilter(filter)) {
-    return query.andWhere(function filterFunction() {
-      addCondition(this, db, filter, negate, entityIdField);
-    });
-  }
-  return query[negate ? "andWhereNot" : "andWhere"](function filterFunction() {
-    if (isOrEntityFilter(filter)) {
-      for (const subFilter of filter.anyOf ?? []) {
-        this.orWhere(
-          (subQuery) => parseFilter(subFilter, subQuery, db, false, entityIdField)
-        );
-      }
-    } else {
-      for (const subFilter of filter.allOf ?? []) {
-        this.andWhere(
-          (subQuery) => parseFilter(subFilter, subQuery, db, false, entityIdField)
-        );
-      }
-    }
-  });
-}
 class DefaultEntitiesCatalog {
   database;
   logger;
@@ -103,6 +54,7 @@ class DefaultEntitiesCatalog {
   }
   async entities(request) {
     const db = this.database;
+    const { limit, offset } = parsePagination(request?.pagination);
     let entitiesQuery = db("final_entities").select("final_entities.*");
     request?.order?.forEach(({ field }, index) => {
       const alias = `order_${index}`;
@@ -115,13 +67,13 @@ class DefaultEntitiesCatalog {
     });
     entitiesQuery = entitiesQuery.whereNotNull("final_entities.final_entity");
     if (request?.filter) {
-      entitiesQuery = parseFilter(
-        request.filter,
-        entitiesQuery,
-        db,
-        false,
-        "final_entities.entity_id"
-      );
+      entitiesQuery = applyEntityFilterToQuery.applyEntityFilterToQuery({
+        filter: request.filter,
+        targetQuery: entitiesQuery,
+        onEntityIdField: "final_entities.entity_id",
+        knex: db,
+        strategy: limit !== void 0 && limit <= 500 ? "join" : "in"
+      });
     }
     request?.order?.forEach(({ order }, index) => {
       if (db.client.config.client === "pg") {
@@ -140,7 +92,6 @@ class DefaultEntitiesCatalog {
     } else {
       entitiesQuery.orderBy("final_entities.entity_id", "asc");
     }
-    const { limit, offset } = parsePagination(request?.pagination);
     if (limit !== void 0) {
       entitiesQuery = entitiesQuery.limit(limit + 1);
     }
@@ -165,17 +116,7 @@ class DefaultEntitiesCatalog {
     if (request?.fields) {
       entities = entities.map((e) => request.fields(e));
     }
-    for (const entity of entities) {
-      if (entity.relations) {
-        for (const relation of entity.relations) {
-          if (!relation.targetRef && relation.target) {
-            relation.targetRef = catalogModel.stringifyEntityRef(relation.target);
-          } else if (!relation.target && relation.targetRef) {
-            relation.target = catalogModel.parseEntityRef(relation.targetRef);
-          }
-        }
-      }
-    }
+    util.expandLegacyCompoundRelationRefsInResponse(entities);
     return {
       entities,
       pageInfo
@@ -189,13 +130,12 @@ class DefaultEntitiesCatalog {
         entity: "final_entities.final_entity"
       }).whereIn("final_entities.entity_ref", chunk);
       if (request?.filter) {
-        query = parseFilter(
-          request.filter,
-          query,
-          this.database,
-          false,
-          "final_entities.entity_id"
-        );
+        query = applyEntityFilterToQuery.applyEntityFilterToQuery({
+          filter: request.filter,
+          targetQuery: query,
+          onEntityIdField: "final_entities.entity_id",
+          knex: this.database
+        });
       }
       for (const row of await query) {
         lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null);
@@ -208,97 +148,137 @@ class DefaultEntitiesCatalog {
     return { items };
   }
   async queryEntities(request) {
-    const db = this.database;
     const limit = request.limit ?? DEFAULT_LIMIT;
     const cursor = {
-      orderFields: [defaultSortField],
+      orderFields: [],
       isPrevious: false,
       ...parseCursorFromRequest(request)
     };
+    const shouldComputeTotalItems = cursor.totalItems === void 0 && !cursor.skipTotalItems;
     const isFetchingBackwards = cursor.isPrevious;
     if (cursor.orderFields.length > 1) {
       this.logger.warn(`Only one sort field is supported, ignoring the rest`);
     }
-    const sortField = {
-      ...defaultSortField,
-      ...cursor.orderFields[0]
-    };
-    const [prevItemOrderFieldValue, prevItemUid] = cursor.orderFieldValues || [];
-    const dbQuery = db("final_entities").leftOuterJoin(
-      "search",
-      (qb) => qb.on("search.entity_id", "final_entities.entity_id").andOnVal("search.key", sortField.field)
-    ).whereNotNull("final_entities.final_entity");
-    if (cursor.filter) {
-      parseFilter(
-        cursor.filter,
-        dbQuery,
-        db,
-        false,
-        "final_entities.entity_id"
-      );
+    const sortField = cursor.orderFields.at(0);
+    const dbQuery = this.database.with(
+      "filtered",
+      ["entity_id", "final_entity", ...sortField ? ["value"] : []],
+      (inner) => {
+        inner.from("final_entities").whereNotNull("final_entity");
+        if (sortField) {
+          inner.leftOuterJoin(
+            "search",
+            (qb) => qb.on("search.entity_id", "final_entities.entity_id").andOnVal("search.key", sortField.field)
+          ).select({
+            entity_id: "final_entities.entity_id",
+            final_entity: "final_entities.final_entity",
+            value: "search.value"
+          });
+        } else {
+          inner.select({
+            entity_id: "final_entities.entity_id",
+            final_entity: "final_entities.final_entity"
+          });
+        }
+        if (cursor.filter) {
+          applyEntityFilterToQuery.applyEntityFilterToQuery({
+            filter: cursor.filter,
+            targetQuery: inner,
+            onEntityIdField: "final_entities.entity_id",
+            knex: this.database,
+            strategy: limit <= 500 ? "join" : "in"
+          });
+        }
+        const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim();
+        const textFilterFields = cursor.fullTextFilter?.fields ?? [
+          sortField?.field || "metadata.uid"
+        ];
+        if (normalizedFullTextFilterTerm) {
+          if (textFilterFields.length === 1 && textFilterFields[0] === sortField?.field) {
+            inner.andWhereRaw(
+              "search.value like ?",
+              `%${normalizedFullTextFilterTerm.toLocaleLowerCase("en-US")}%`
+            );
+          } else {
+            const matchQuery = this.database("search").select("search.entity_id").whereIn(
+              "search.key",
+              textFilterFields.map((field) => field.toLocaleLowerCase("en-US"))
+            ).andWhere(function keyFilter() {
+              this.andWhereRaw(
+                "search.value like ?",
+                `%${normalizedFullTextFilterTerm.toLocaleLowerCase(
+                  "en-US"
+                )}%`
+              );
+            });
+            inner.andWhere("final_entities.entity_id", "in", matchQuery);
+          }
+        }
+      }
+    );
+    if (shouldComputeTotalItems) {
+      dbQuery.with(
+        "filtered_count",
+        ["count"],
+        (inner) => inner.from("filtered").count("*", { as: "count" })
+      ).fromRaw("filtered_count, filtered").select("count", "filtered.*");
+    } else {
+      dbQuery.from("filtered").select("*");
     }
-    const normalizedFullTextFilterTerm = cursor.fullTextFilter?.term?.trim();
-    const textFilterFields = cursor.fullTextFilter?.fields ?? [sortField.field];
-    if (normalizedFullTextFilterTerm) {
-      if (textFilterFields.length === 1 && textFilterFields[0] === sortField.field) {
-        dbQuery.andWhereRaw(
-          "value like ?",
-          `%${normalizedFullTextFilterTerm.toLocaleLowerCase("en-US")}%`
-        );
-      } else {
-        const matchQuery = db("search").select("search.entity_id").whereIn(
-          "key",
-          textFilterFields.map((field) => field.toLocaleLowerCase("en-US"))
-        ).andWhere(function keyFilter() {
-          this.andWhereRaw(
-            "value like ?",
-            `%${normalizedFullTextFilterTerm.toLocaleLowerCase("en-US")}%`
+    const isOrderingDescending = sortField?.order === "desc";
+    if (cursor.orderFieldValues) {
+      if (cursor.orderFieldValues.length === 2) {
+        const [first, second] = cursor.orderFieldValues;
+        dbQuery.andWhere(function nested() {
+          this.where(
+            "filtered.value",
+            isFetchingBackwards !== isOrderingDescending ? "<" : ">",
+            first
+          ).orWhere("filtered.value", "=", first).andWhere(
+            "filtered.entity_id",
+            isFetchingBackwards !== isOrderingDescending ? "<" : ">",
+            second
           );
         });
-        dbQuery.andWhere("final_entities.entity_id", "in", matchQuery);
+      } else if (cursor.orderFieldValues.length === 1) {
+        const [first] = cursor.orderFieldValues;
+        dbQuery.andWhere("entity_id", isFetchingBackwards ? "<" : ">", first);
       }
     }
-    const countQuery = dbQuery.clone();
-    const isOrderingDescending = sortField.order === "desc";
-    if (prevItemOrderFieldValue) {
-      dbQuery.andWhere(function nested() {
-        this.where(
-          "value",
-          isFetchingBackwards !== isOrderingDescending ? "<" : ">",
-          prevItemOrderFieldValue
-        ).orWhere("value", "=", prevItemOrderFieldValue).andWhere(
-          "final_entities.entity_id",
-          isFetchingBackwards !== isOrderingDescending ? "<" : ">",
-          prevItemUid
-        );
-      });
+    let order = sortField?.order ?? "asc";
+    if (isFetchingBackwards) {
+      order = invertOrder(order);
     }
-    if (db.client.config.client === "pg") {
+    if (this.database.client.config.client === "pg") {
       dbQuery.orderBy([
+        ...sortField ? [
+          {
+            column: "filtered.value",
+            order,
+            nulls: "last"
+          }
+        ] : [],
         {
-          column: "search.value",
-          order: isFetchingBackwards ? invertOrder(sortField.order) : sortField.order,
-          nulls: "last"
-        },
-        {
-          column: "final_entities.entity_id",
-          order: isFetchingBackwards ? invertOrder(sortField.order) : sortField.order
+          column: "filtered.entity_id",
+          order
         }
       ]);
     } else {
       dbQuery.orderBy([
+        ...sortField ? [
+          {
+            column: "filtered.value",
+            order: void 0,
+            nulls: "last"
+          },
+          {
+            column: "filtered.value",
+            order
+          }
+        ] : [],
         {
-          column: "search.value",
-          order: void 0,
-          nulls: "last"
-        },
-        {
-          column: "search.value",
-          order: isFetchingBackwards ? invertOrder(sortField.order) : sortField.order
-        },
-        {
-          column: "final_entities.entity_id",
-          order: isFetchingBackwards ? invertOrder(sortField.order) : sortField.order
+          column: "filtered.entity_id",
+          order
         }
       ]);
     }
@@ -306,16 +286,17 @@ class DefaultEntitiesCatalog {
       dbQuery.offset(request.offset);
     }
     dbQuery.limit(isFetchingBackwards ? limit : limit + 1);
-    countQuery.count("final_entities.entity_id", { as: "count" });
-    const [rows, [{ count }]] = await Promise.all([
-      limit > 0 ? dbQuery : [],
-      // for performance reasons we invoke the countQuery
-      // only on the first request.
-      // The result is then embedded into the cursor
-      // for subsequent requests.
-      typeof cursor.totalItems === "undefined" ? countQuery : [{ count: cursor.totalItems }]
-    ]);
-    const totalItems = Number(count);
+    const rows = shouldComputeTotalItems || limit > 0 ? await dbQuery : [];
+    let totalItems;
+    if (cursor.totalItems !== void 0) {
+      totalItems = cursor.totalItems;
+    } else if (cursor.skipTotalItems) {
+      totalItems = 0;
+    } else if (rows.length) {
+      totalItems = Number(rows[0].count);
+    } else {
+      totalItems = 0;
+    }
     if (isFetchingBackwards) {
       rows.reverse();
     }
@@ -326,20 +307,20 @@ class DefaultEntitiesCatalog {
     const isInitialRequest = cursor.firstSortFieldValues === void 0;
     const firstRow = rows[0];
     const lastRow = rows[rows.length - 1];
-    const firstSortFieldValues = cursor.firstSortFieldValues || [
-      firstRow?.value,
-      firstRow?.entity_id
-    ];
+    const firstSortFieldValues = cursor.firstSortFieldValues || sortFieldsFromRow(firstRow, sortField);
     const nextCursor = hasMoreResults ? {
       ...cursor,
-      orderFieldValues: sortFieldsFromRow(lastRow),
+      orderFieldValues: sortFieldsFromRow(lastRow, sortField),
       firstSortFieldValues,
       isPrevious: false,
       totalItems
     } : void 0;
-    const prevCursor = !isInitialRequest && rows.length > 0 && !lodash.isEqual(sortFieldsFromRow(firstRow), cursor.firstSortFieldValues) ? {
+    const prevCursor = !isInitialRequest && rows.length > 0 && !lodash.isEqual(
+      sortFieldsFromRow(firstRow, sortField),
+      cursor.firstSortFieldValues
+    ) ? {
       ...cursor,
-      orderFieldValues: sortFieldsFromRow(firstRow),
+      orderFieldValues: sortFieldsFromRow(firstRow, sortField),
       firstSortFieldValues: cursor.firstSortFieldValues,
       isPrevious: true,
       totalItems
@@ -447,13 +428,12 @@ class DefaultEntitiesCatalog {
       count: this.database.raw("count(*)")
     }).groupBy(["search.key", "search.original_value"]);
     if (request.filter) {
-      parseFilter(
-        request.filter,
-        query,
-        this.database,
-        false,
-        "search.entity_id"
-      );
+      applyEntityFilterToQuery.applyEntityFilterToQuery({
+        filter: request.filter,
+        targetQuery: query,
+        onEntityIdField: "search.entity_id",
+        knex: this.database
+      });
     }
     const rows = await query;
     const facets = {};
@@ -488,21 +468,28 @@ function parseCursorFromRequest(request) {
   if (util.isQueryEntitiesInitialRequest(request)) {
     const {
       filter,
-      orderFields: sortFields = [defaultSortField],
-      fullTextFilter
+      orderFields: sortFields = [],
+      fullTextFilter,
+      skipTotalItems = false
     } = request;
-    return { filter, orderFields: sortFields, fullTextFilter };
+    return { filter, orderFields: sortFields, fullTextFilter, skipTotalItems };
   }
   if (util.isQueryEntitiesCursorRequest(request)) {
-    return request.cursor;
+    return {
+      ...request.cursor,
+      // Doesn't matter here
+      skipTotalItems: false
+    };
   }
-  return {};
+  return {
+    skipTotalItems: false
+  };
 }
 function invertOrder(order) {
   return order === "asc" ? "desc" : "asc";
 }
-function sortFieldsFromRow(row) {
-  return [row.value, row.entity_id];
+function sortFieldsFromRow(row, sortField) {
+  return sortField ? [row?.value, row?.entity_id] : [row?.entity_id];
 }
 
 exports.DefaultEntitiesCatalog = DefaultEntitiesCatalog;