@devrev/meerkat-core 0.0.95 → 0.0.97

package/README.md

@@ -1,11 +1,75 @@
-# meerkat-core
+# @devrev/meerkat-core
 
-This library was generated with [Nx](https://nx.dev).
+`@devrev/meerkat-core` is the foundational library for the Meerkat ecosystem, a TypeScript SDK that seamlessly translates Cube-like queries into DuckDB Abstract Syntax Trees (AST). It provides the core logic for query transformation, designed to be environment-agnostic, running in both Node.js and browser environments.
 
-## Building
+This package focuses exclusively on generating a DuckDB-compatible AST from a JSON-based query object. It does not handle query execution, which is the responsibility of environment-specific packages like `@devrev/meerkat-node` and `@devrev/meerkat-browser`.
 
-Run `nx build meerkat-core` to build the library.
+## Key Features
 
-## Running unit tests
+- **Cube-to-AST Transformation**: Converts Cube-style JSON queries into DuckDB-compatible SQL ASTs.
+- **Environment Agnostic**: Runs in both Node.js and browser environments.
+- **Type-Safe**: Provides strong TypeScript definitions for queries, schemas, and filters.
+- **Advanced Filtering and Joins**: Supports complex filters, logical operators, and multi-table joins.
+- **Extensible by Design**: Leverages DuckDB's native JSON serialization, avoiding the limitations of traditional query builders.
 
-Run `nx test meerkat-core` to execute the unit tests via [Jest](https://jestjs.io).
+## Installation
+
+```bash
+npm install @devrev/meerkat-core
+```
+
+## Core Concepts
+
+`meerkat-core` revolves around two main objects:
+
+1.  **`Query`**: A JSON object that defines your analytics request. It specifies measures, dimensions, filters, and ordering.
+2.  **`TableSchema`**: Defines the structure of your data tables, including columns, measures, dimensions, and joins.
+
+The library uses these objects to generate a DuckDB AST. This AST can then be passed to an execution engine.
+
+## Usage
+
+Here's how to transform a Cube-style query into a DuckDB AST:
+
+```typescript
+import { cubeToDuckdbAST, Query, TableSchema } from '@devrev/meerkat-core';
+
+// 1. Define the schema for your table
+const schema: TableSchema = {
+  name: 'users',
+  sql: 'SELECT * FROM users',
+  columns: [
+    { name: 'id', type: 'INTEGER' },
+    { name: 'name', type: 'VARCHAR' },
+    { name: 'city', type: 'VARCHAR' },
+    { name: 'signed_up_at', type: 'TIMESTAMP' },
+  ],
+};
+
+// 2. Define your query
+const query: Query = {
+  measures: ['users.count'],
+  dimensions: ['users.city'],
+  filters: [
+    {
+      member: 'users.city',
+      operator: 'equals',
+      values: ['New York'],
+    },
+  ],
+  limit: 100,
+};
+
+// 3. Generate the DuckDB AST
+const ast = cubeToDuckdbAST(query, schema);
+
+// The `ast` can now be deserialized into a SQL string for execution.
+console.log(JSON.stringify(ast, null, 2));
+```
+
+## Ecosystem
+
+`meerkat-core` is the foundation for:
+
+- **`@devrev/meerkat-node`**: For server-side analytics in Node.js with `@duckdb/node-api`.
+- **`@devrev/meerkat-browser`**: For client-side analytics in the browser with `@duckdb/duckdb-wasm`.

package/ast-builder/ast-builder.js

@@ -75,7 +75,7 @@ const cubeToDuckdbAST = (query, tableSchema, options)=>{
         ];
     }
     node.modifiers = [];
-    if (query.order) {
+    if (query.order && Object.keys(query.order).length > 0) {
         node.modifiers.push((0, _cubeorderbytransformer.cubeOrderByToAST)(query.order));
     }
     if (query.limit || query.offset) {

package/ast-builder/ast-builder.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../meerkat-core/src/ast-builder/ast-builder.ts"],"sourcesContent":["import { cubeFilterToDuckdbAST } from '../cube-filter-transformer/factory';\nimport { cubeDimensionToGroupByAST } from '../cube-group-by-transformer/cube-group-by-transformer';\nimport { cubeLimitOffsetToAST } from '../cube-limit-offset-transformer/cube-limit-offset-transformer';\nimport { cubeOrderByToAST } from '../cube-order-by-transformer/cube-order-by-transformer';\nimport {\n  QueryFiltersWithInfo,\n  QueryFiltersWithInfoSingular,\n} from '../cube-to-duckdb/cube-filter-to-duckdb';\nimport { traverseAndFilter } from '../filter-params/filter-params-ast';\nimport { memberKeyToSafeKey } from '../member-formatters/member-key-to-safe-key';\nimport {\n  FilterType,\n  MeerkatQueryFilter,\n  Query,\n} from '../types/cube-types/query';\nimport { TableSchema } from '../types/cube-types/table';\nimport { SelectStatement } from '../types/duckdb-serialization-types';\nimport { SelectNode } from '../types/duckdb-serialization-types/serialization/QueryNode';\nimport { getBaseAST } from '../utils/base-ast';\nimport { cubeFiltersEnrichment } from '../utils/cube-filter-enrichment';\nimport { modifyLeafMeerkatFilter } from '../utils/modify-meerkat-filter';\n\nconst formatFilters = (\n  queryFiltersWithInfo: QueryFiltersWithInfo,\n  filterType?: FilterType\n) => {\n  /*\n   * If the type of filter is set to base filter where\n   */\n  return filterType === 'BASE_FILTER'\n    ? queryFiltersWithInfo\n    : (modifyLeafMeerkatFilter(queryFiltersWithInfo, (item) => {\n        return {\n          ...item,\n          member: memberKeyToSafeKey(item.member),\n        };\n      }) as QueryFiltersWithInfo);\n};\n\nconst getFormattedFilters = ({\n  queryFiltersWithInfo,\n  filterType,\n  mapperFn,\n  baseAST,\n}: {\n  queryFiltersWithInfo: QueryFiltersWithInfo;\n  filterType?: FilterType;\n  baseAST: SelectStatement;\n  mapperFn: (val: QueryFiltersWithInfoSingular) => MeerkatQueryFilter | null;\n}) => {\n  const filters = queryFiltersWithInfo\n    .map((item) => mapperFn(item))\n    .filter(Boolean) as QueryFiltersWithInfoSingular[];\n  const formattedFilters = formatFilters(filters, filterType);\n  return cubeFilterToDuckdbAST(formattedFilters, baseAST);\n};\n\nexport const cubeToDuckdbAST = (\n  query: Query,\n  tableSchema: TableSchema,\n  options?: { filterType: FilterType }\n) => {\n  /**\n   * Obviously, if no table schema was found, return null.\n   */\n  if (!tableSchema) {\n    return null;\n  }\n\n  const baseAST = getBaseAST();\n  const node = baseAST.node as SelectNode;\n  if (query.filters && query.filters.length > 0) {\n    /**\n     * Make a copy of the query filters and enrich them with the table schema.\n     */\n    const queryFiltersWithInfo = cubeFiltersEnrichment(\n      JSON.parse(JSON.stringify(query.filters)),\n      tableSchema\n    );\n\n    if (!queryFiltersWithInfo) {\n      return null;\n    }\n\n    const whereClause = getFormattedFilters({\n      baseAST,\n      mapperFn: (item) =>\n        traverseAndFilter(\n          item,\n          (value) => !query.measures.includes(value.member)\n        ),\n      queryFiltersWithInfo,\n      filterType: options?.filterType,\n    });\n\n    const havingClause = getFormattedFilters({\n      baseAST,\n      mapperFn: (item) =>\n        traverseAndFilter(item, (value) =>\n          query.measures.includes(value.member)\n        ),\n      queryFiltersWithInfo,\n      filterType: options?.filterType,\n    });\n\n    // eslint-disable-next-line @typescript-eslint/ban-ts-comment\n    //@ts-ignore\n    node.where_clause = whereClause;\n    node.having = havingClause;\n  }\n  if (\n    query.measures.length &&\n    query.dimensions &&\n    query.dimensions?.length > 0\n  ) {\n    node.group_expressions = cubeDimensionToGroupByAST(query.dimensions);\n    const groupSets = [];\n    /**\n     * We only support one group set for now.\n     */\n    for (let i = 0; i < node.group_expressions.length; i++) {\n      groupSets.push(i);\n    }\n    node.group_sets = [groupSets];\n  }\n  node.modifiers = [];\n  if (query.order) {\n    node.modifiers.push(cubeOrderByToAST(query.order));\n  }\n  if (query.limit || query.offset) {\n    // Type assertion is needed here because the AST is not typed correctly.\n    // eslint-disable-next-line @typescript-eslint/ban-ts-comment\n    //@ts-ignore\n    node.modifiers.push(cubeLimitOffsetToAST(query.limit, query.offset));\n  }\n\n  return baseAST;\n};\n"],"names":["cubeToDuckdbAST","formatFilters","queryFiltersWithInfo","filterType","modifyLeafMeerkatFilter","item","member","memberKeyToSafeKey","getFormattedFilters","mapperFn","baseAST","filters","map","filter","Boolean","formattedFilters","cubeFilterToDuckdbAST","query","tableSchema","options","getBaseAST","node","length","cubeFiltersEnrichment","JSON","parse","stringify","whereClause","traverseAndFilter","value","measures","includes","havingClause","where_clause","having","dimensions","group_expressions","cubeDimensionToGroupByAST","groupSets","i","push","group_sets","modifiers","order","cubeOrderByToAST","limit","offset","cubeLimitOffsetToAST"],"mappings":";+BAyDaA;;;eAAAA;;;;yBAzDyB;wCACI;4CACL;wCACJ;iCAKC;oCACC;yBASR;sCACW;qCACE;AAExC,MAAMC,gBAAgB,CACpBC,sBACAC;IAEA;;GAEC,GACD,OAAOA,eAAe,gBAClBD,uBACCE,IAAAA,4CAAuB,EAACF,sBAAsB,CAACG;QAC9C,OAAO,eACFA;YACHC,QAAQC,IAAAA,sCAAkB,EAACF,KAAKC,MAAM;;IAE1C;AACN;AAEA,MAAME,sBAAsB,CAAC,EAC3BN,oBAAoB,EACpBC,UAAU,EACVM,QAAQ,EACRC,OAAO,EAMR;IACC,MAAMC,UAAUT,qBACbU,GAAG,CAAC,CAACP,OAASI,SAASJ,OACvBQ,MAAM,CAACC;IACV,MAAMC,mBAAmBd,cAAcU,SAASR;IAChD,OAAOa,IAAAA,8BAAqB,EAACD,kBAAkBL;AACjD;AAEO,MAAMV,kBAAkB,CAC7BiB,OACAC,aACAC;QAqDEF;IAnDF;;GAEC,GACD,IAAI,CAACC,aAAa;QAChB,OAAO;IACT;IAEA,MAAMR,UAAUU,IAAAA,mBAAU;IAC1B,MAAMC,OAAOX,QAAQW,IAAI;IACzB,IAAIJ,MAAMN,OAAO,IAAIM,MAAMN,OAAO,CAACW,MAAM,GAAG,GAAG;QAC7C;;KAEC,GACD,MAAMpB,uBAAuBqB,IAAAA,2CAAqB,EAChDC,KAAKC,KAAK,CAACD,KAAKE,SAAS,CAACT,MAAMN,OAAO,IACvCO;QAGF,IAAI,CAAChB,sBAAsB;YACzB,OAAO;QACT;QAEA,MAAMyB,cAAcnB,oBAAoB;YACtCE;YACAD,UAAU,CAACJ,OACTuB,IAAAA,kCAAiB,EACfvB,MACA,CAACwB,QAAU,CAACZ,MAAMa,QAAQ,CAACC,QAAQ,CAACF,MAAMvB,MAAM;YAEpDJ;YACAC,UAAU,EAAEgB,2BAAAA,QAAShB,UAAU;QACjC;QAEA,MAAM6B,eAAexB,oBAAoB;YACvCE;YACAD,UAAU,CAACJ,OACTuB,IAAAA,kCAAiB,EAACvB,MAAM,CAACwB,QACvBZ,MAAMa,QAAQ,CAACC,QAAQ,CAACF,MAAMvB,MAAM;YAExCJ;YACAC,UAAU,EAAEgB,2BAAAA,QAAShB,UAAU;QACjC;QAEA,6DAA6D;QAC7D,YAAY;QACZkB,KAAKY,YAAY,GAAGN;QACpBN,KAAKa,MAAM,GAAGF;IAChB;IACA,IACEf,MAAMa,QAAQ,CAACR,MAAM,IACrBL,MAAMkB,UAAU,IAChBlB,EAAAA,oBAAAA,MAAMkB,UAAU,qBAAhBlB,kBAAkBK,MAAM,IAAG,GAC3B;QACAD,KAAKe,iBAAiB,GAAGC,IAAAA,iDAAyB,EAACpB,MAAMkB,UAAU;QACnE,MAAMG,YAAY,EAAE;QACpB;;KAEC,GACD,IAAK,IAAIC,IAAI,GAAGA,IAAIlB,KAAKe,iBAAiB,CAACd,MAAM,EAAEiB,IAAK;YACtDD,UAAUE,IAAI,CAACD;QACjB;QACAlB,KAAKoB,UAAU,GAAG;YAACH;SAAU;IAC/B;IACAjB,KAAKqB,SAAS,GAAG,EAAE;IACnB,IAAIzB,MAAM0B,KAAK,EAAE;QACftB,KAAKqB,SAAS,CAACF,IAAI,CAACI,IAAAA,wCAAgB,EAAC3B,MAAM0B,KAAK;IAClD;IACA,IAAI1B,MAAM4B,KAAK,IAAI5B,MAAM6B,MAAM,EAAE;QAC/B,wEAAwE;QACxE,6DAA6D;QAC7D,YAAY;QACZzB,KAAKqB,SAAS,CAACF,IAAI,CAACO,IAAAA,gDAAoB,EAAC9B,MAAM4B,KAAK,EAAE5B,MAAM6B,MAAM;IACpE;IAEA,OAAOpC;AACT"}
+{"version":3,"sources":["../../../meerkat-core/src/ast-builder/ast-builder.ts"],"sourcesContent":["import { cubeFilterToDuckdbAST } from '../cube-filter-transformer/factory';\nimport { cubeDimensionToGroupByAST } from '../cube-group-by-transformer/cube-group-by-transformer';\nimport { cubeLimitOffsetToAST } from '../cube-limit-offset-transformer/cube-limit-offset-transformer';\nimport { cubeOrderByToAST } from '../cube-order-by-transformer/cube-order-by-transformer';\nimport {\n  QueryFiltersWithInfo,\n  QueryFiltersWithInfoSingular,\n} from '../cube-to-duckdb/cube-filter-to-duckdb';\nimport { traverseAndFilter } from '../filter-params/filter-params-ast';\nimport { memberKeyToSafeKey } from '../member-formatters/member-key-to-safe-key';\nimport {\n  FilterType,\n  MeerkatQueryFilter,\n  Query,\n} from '../types/cube-types/query';\nimport { TableSchema } from '../types/cube-types/table';\nimport { SelectStatement } from '../types/duckdb-serialization-types';\nimport { SelectNode } from '../types/duckdb-serialization-types/serialization/QueryNode';\nimport { getBaseAST } from '../utils/base-ast';\nimport { cubeFiltersEnrichment } from '../utils/cube-filter-enrichment';\nimport { modifyLeafMeerkatFilter } from '../utils/modify-meerkat-filter';\n\nconst formatFilters = (\n  queryFiltersWithInfo: QueryFiltersWithInfo,\n  filterType?: FilterType\n) => {\n  /*\n   * If the type of filter is set to base filter where\n   */\n  return filterType === 'BASE_FILTER'\n    ? queryFiltersWithInfo\n    : (modifyLeafMeerkatFilter(queryFiltersWithInfo, (item) => {\n        return {\n          ...item,\n          member: memberKeyToSafeKey(item.member),\n        };\n      }) as QueryFiltersWithInfo);\n};\n\nconst getFormattedFilters = ({\n  queryFiltersWithInfo,\n  filterType,\n  mapperFn,\n  baseAST,\n}: {\n  queryFiltersWithInfo: QueryFiltersWithInfo;\n  filterType?: FilterType;\n  baseAST: SelectStatement;\n  mapperFn: (val: QueryFiltersWithInfoSingular) => MeerkatQueryFilter | null;\n}) => {\n  const filters = queryFiltersWithInfo\n    .map((item) => mapperFn(item))\n    .filter(Boolean) as QueryFiltersWithInfoSingular[];\n  const formattedFilters = formatFilters(filters, filterType);\n  return cubeFilterToDuckdbAST(formattedFilters, baseAST);\n};\n\nexport const cubeToDuckdbAST = (\n  query: Query,\n  tableSchema: TableSchema,\n  options?: { filterType: FilterType }\n) => {\n  /**\n   * Obviously, if no table schema was found, return null.\n   */\n  if (!tableSchema) {\n    return null;\n  }\n\n  const baseAST = getBaseAST();\n  const node = baseAST.node as SelectNode;\n  if (query.filters && query.filters.length > 0) {\n    /**\n     * Make a copy of the query filters and enrich them with the table schema.\n     */\n    const queryFiltersWithInfo = cubeFiltersEnrichment(\n      JSON.parse(JSON.stringify(query.filters)),\n      tableSchema\n    );\n\n    if (!queryFiltersWithInfo) {\n      return null;\n    }\n\n    const whereClause = getFormattedFilters({\n      baseAST,\n      mapperFn: (item) =>\n        traverseAndFilter(\n          item,\n          (value) => !query.measures.includes(value.member)\n        ),\n      queryFiltersWithInfo,\n      filterType: options?.filterType,\n    });\n\n    const havingClause = getFormattedFilters({\n      baseAST,\n      mapperFn: (item) =>\n        traverseAndFilter(item, (value) =>\n          query.measures.includes(value.member)\n        ),\n      queryFiltersWithInfo,\n      filterType: options?.filterType,\n    });\n\n    // eslint-disable-next-line @typescript-eslint/ban-ts-comment\n    //@ts-ignore\n    node.where_clause = whereClause;\n    node.having = havingClause;\n  }\n  if (\n    query.measures.length &&\n    query.dimensions &&\n    query.dimensions?.length > 0\n  ) {\n    node.group_expressions = cubeDimensionToGroupByAST(query.dimensions);\n    const groupSets = [];\n    /**\n     * We only support one group set for now.\n     */\n    for (let i = 0; i < node.group_expressions.length; i++) {\n      groupSets.push(i);\n    }\n    node.group_sets = [groupSets];\n  }\n  node.modifiers = [];\n  if (query.order && Object.keys(query.order).length > 0) {\n    node.modifiers.push(cubeOrderByToAST(query.order));\n  }\n  if (query.limit || query.offset) {\n    // Type assertion is needed here because the AST is not typed correctly.\n    // eslint-disable-next-line @typescript-eslint/ban-ts-comment\n    //@ts-ignore\n    node.modifiers.push(cubeLimitOffsetToAST(query.limit, query.offset));\n  }\n\n  return baseAST;\n};\n"],"names":["cubeToDuckdbAST","formatFilters","queryFiltersWithInfo","filterType","modifyLeafMeerkatFilter","item","member","memberKeyToSafeKey","getFormattedFilters","mapperFn","baseAST","filters","map","filter","Boolean","formattedFilters","cubeFilterToDuckdbAST","query","tableSchema","options","getBaseAST","node","length","cubeFiltersEnrichment","JSON","parse","stringify","whereClause","traverseAndFilter","value","measures","includes","havingClause","where_clause","having","dimensions","group_expressions","cubeDimensionToGroupByAST","groupSets","i","push","group_sets","modifiers","order","Object","keys","cubeOrderByToAST","limit","offset","cubeLimitOffsetToAST"],"mappings":";+BAyDaA;;;eAAAA;;;;yBAzDyB;wCACI;4CACL;wCACJ;iCAKC;oCACC;yBASR;sCACW;qCACE;AAExC,MAAMC,gBAAgB,CACpBC,sBACAC;IAEA;;GAEC,GACD,OAAOA,eAAe,gBAClBD,uBACCE,IAAAA,4CAAuB,EAACF,sBAAsB,CAACG;QAC9C,OAAO,eACFA;YACHC,QAAQC,IAAAA,sCAAkB,EAACF,KAAKC,MAAM;;IAE1C;AACN;AAEA,MAAME,sBAAsB,CAAC,EAC3BN,oBAAoB,EACpBC,UAAU,EACVM,QAAQ,EACRC,OAAO,EAMR;IACC,MAAMC,UAAUT,qBACbU,GAAG,CAAC,CAACP,OAASI,SAASJ,OACvBQ,MAAM,CAACC;IACV,MAAMC,mBAAmBd,cAAcU,SAASR;IAChD,OAAOa,IAAAA,8BAAqB,EAACD,kBAAkBL;AACjD;AAEO,MAAMV,kBAAkB,CAC7BiB,OACAC,aACAC;QAqDEF;IAnDF;;GAEC,GACD,IAAI,CAACC,aAAa;QAChB,OAAO;IACT;IAEA,MAAMR,UAAUU,IAAAA,mBAAU;IAC1B,MAAMC,OAAOX,QAAQW,IAAI;IACzB,IAAIJ,MAAMN,OAAO,IAAIM,MAAMN,OAAO,CAACW,MAAM,GAAG,GAAG;QAC7C;;KAEC,GACD,MAAMpB,uBAAuBqB,IAAAA,2CAAqB,EAChDC,KAAKC,KAAK,CAACD,KAAKE,SAAS,CAACT,MAAMN,OAAO,IACvCO;QAGF,IAAI,CAAChB,sBAAsB;YACzB,OAAO;QACT;QAEA,MAAMyB,cAAcnB,oBAAoB;YACtCE;YACAD,UAAU,CAACJ,OACTuB,IAAAA,kCAAiB,EACfvB,MACA,CAACwB,QAAU,CAACZ,MAAMa,QAAQ,CAACC,QAAQ,CAACF,MAAMvB,MAAM;YAEpDJ;YACAC,UAAU,EAAEgB,2BAAAA,QAAShB,UAAU;QACjC;QAEA,MAAM6B,eAAexB,oBAAoB;YACvCE;YACAD,UAAU,CAACJ,OACTuB,IAAAA,kCAAiB,EAACvB,MAAM,CAACwB,QACvBZ,MAAMa,QAAQ,CAACC,QAAQ,CAACF,MAAMvB,MAAM;YAExCJ;YACAC,UAAU,EAAEgB,2BAAAA,QAAShB,UAAU;QACjC;QAEA,6DAA6D;QAC7D,YAAY;QACZkB,KAAKY,YAAY,GAAGN;QACpBN,KAAKa,MAAM,GAAGF;IAChB;IACA,IACEf,MAAMa,QAAQ,CAACR,MAAM,IACrBL,MAAMkB,UAAU,IAChBlB,EAAAA,oBAAAA,MAAMkB,UAAU,qBAAhBlB,kBAAkBK,MAAM,IAAG,GAC3B;QACAD,KAAKe,iBAAiB,GAAGC,IAAAA,iDAAyB,EAACpB,MAAMkB,UAAU;QACnE,MAAMG,YAAY,EAAE;QACpB;;KAEC,GACD,IAAK,IAAIC,IAAI,GAAGA,IAAIlB,KAAKe,iBAAiB,CAACd,MAAM,EAAEiB,IAAK;YACtDD,UAAUE,IAAI,CAACD;QACjB;QACAlB,KAAKoB,UAAU,GAAG;YAACH;SAAU;IAC/B;IACAjB,KAAKqB,SAAS,GAAG,EAAE;IACnB,IAAIzB,MAAM0B,KAAK,IAAIC,OAAOC,IAAI,CAAC5B,MAAM0B,KAAK,EAAErB,MAAM,GAAG,GAAG;QACtDD,KAAKqB,SAAS,CAACF,IAAI,CAACM,IAAAA,wCAAgB,EAAC7B,MAAM0B,KAAK;IAClD;IACA,IAAI1B,MAAM8B,KAAK,IAAI9B,MAAM+B,MAAM,EAAE;QAC/B,wEAAwE;QACxE,6DAA6D;QAC7D,YAAY;QACZ3B,KAAKqB,SAAS,CAACF,IAAI,CAACS,IAAAA,gDAAoB,EAAChC,MAAM8B,KAAK,EAAE9B,MAAM+B,MAAM;IACpE;IAEA,OAAOtC;AACT"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-core",
-  "version": "0.0.95",
+  "version": "0.0.97",
   "dependencies": {
     "@swc/helpers": "~0.5.0"
   },