@malloydata/malloy-profiler 0.0.331 → 0.0.332

package/dist/big_composite.d.ts

@@ -0,0 +1,9 @@
+import type * as Malloy from '@malloydata/malloy-interfaces';
+export declare function generateBigComposite(): {
+    code: string;
+    schema: Malloy.Schema;
+    dimensions: Malloy.FieldInfo[];
+    preaggregates: Malloy.FieldInfo[];
+    partitionField: Malloy.FieldInfoWithDimension;
+    compileQueryRequest: Malloy.CompileQueryRequest;
+};

package/dist/big_composite.js

@@ -0,0 +1,169 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateBigComposite = generateBigComposite;
+const malloy_1 = require("@malloydata/malloy");
+function _getAllCombinations(current, remaining, result) {
+    if (remaining.length === 0) {
+        if (current.length > 0) {
+            result.push(current);
+        }
+        return;
+    }
+    _getAllCombinations([...current, remaining[0]], remaining.slice(1), result);
+    _getAllCombinations(current, remaining.slice(1), result);
+}
+function getAllCombinations(items) {
+    const result = [];
+    _getAllCombinations([], items, result);
+    return result;
+}
+function generateBigComposite() {
+    const dimensionCount = 3;
+    const preaggregateCount = 50;
+    const dimensions = [];
+    for (let i = 0; i < dimensionCount; i++) {
+        dimensions.push({
+            kind: 'dimension',
+            name: `dim_${i}`,
+            type: { kind: 'string_type' },
+        });
+    }
+    const dimensionNames = dimensions.map(f => f.name);
+    const preaggregates = [];
+    for (let i = 0; i < preaggregateCount; i++) {
+        preaggregates.push({
+            kind: 'dimension',
+            name: `preaggregate_${i}`,
+            type: { kind: 'number_type', subtype: 'decimal' },
+        });
+    }
+    const partitionField = {
+        kind: 'dimension',
+        name: 'partition_name',
+        type: { kind: 'string_type' },
+    };
+    const schema = {
+        fields: [...dimensions, ...preaggregates, partitionField],
+    };
+    const tableName = 'cube';
+    const connectionName = 'connection';
+    let code = `
+    ##! experimental { composite_sources access_modifiers }
+  `;
+    const slices = getAllCombinations(dimensionNames);
+    const sliceNames = [];
+    for (const slice of slices) {
+        const partitionName = slice.join('.');
+        const sliceName = `cube:${partitionName}`;
+        sliceNames.push(sliceName);
+        code += `
+source: \`${sliceName}\` is ${connectionName}.table('${tableName}') include {
+  public: ${slice.join(', ')}
+  internal: ${partitionField.name}
+  internal: ${preaggregates.map(a => a.name).join(',')}
+} extend {
+  where: ${partitionField.name} = '${partitionName}'
+}
+
+`;
+    }
+    code += `
+source: cube is compose(
+${sliceNames.map(n => `  \`${n}\``).join(',\n')}
+) include {
+  public: *
+  internal: ${preaggregates.map(a => a.name).join(',')}
+  internal: ${partitionField.name}
+} extend {
+  measure:
+${preaggregates.map(a => `    total_${a.name} is ${a.name}.sum()`).join('\n')}
+}
+
+// Some extensions of the cube to bulk up the size...
+
+source: cube_ext_1 is cube
+source: cube_ext_2 is cube
+source: cube_ext_3 is cube
+source: cube_ext_4 is cube
+source: cube_ext_5 is cube
+`;
+    const fileName = 'file://big_composite.malloy';
+    const compileModelResponse = malloy_1.API.stateless.compileModel({
+        model_url: fileName,
+        compiler_needs: {
+            files: [
+                {
+                    url: fileName,
+                    contents: code,
+                },
+            ],
+            connections: [
+                {
+                    name: connectionName,
+                    dialect: 'duckdb',
+                },
+            ],
+            table_schemas: [
+                {
+                    name: tableName,
+                    connection_name: connectionName,
+                    schema,
+                },
+            ],
+        },
+        exclude_references: true,
+    });
+    const compileQueryRequest = {
+        exclude_references: true,
+        model_url: fileName,
+        query: {
+            definition: {
+                kind: 'arrow',
+                source: {
+                    kind: 'source_reference',
+                    name: 'cube',
+                },
+                view: {
+                    kind: 'segment',
+                    operations: [
+                        {
+                            kind: 'group_by',
+                            field: {
+                                expression: {
+                                    kind: 'field_reference',
+                                    name: dimensions[0].name,
+                                },
+                            },
+                        },
+                        {
+                            kind: 'aggregate',
+                            field: {
+                                expression: {
+                                    kind: 'field_reference',
+                                    name: `total_${preaggregates[0].name}`,
+                                },
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+        compiler_needs: {
+            translations: compileModelResponse.translations,
+        },
+    };
+    return {
+        code,
+        schema,
+        dimensions,
+        preaggregates,
+        partitionField,
+        compileQueryRequest,
+    };
+}
+console.log(JSON.stringify(generateBigComposite().compileQueryRequest));
+//# sourceMappingURL=big_composite.js.map

package/dist/big_composite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"big_composite.js","sourceRoot":"","sources":["../src/big_composite.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAuBH,oDAoJC;AAvKD,+CAAuC;AAEvC,SAAS,mBAAmB,CAAI,OAAY,EAAE,SAAc,EAAE,MAAa;IACzE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QACD,OAAO;IACT,CAAC;IACD,mBAAmB,CAAC,CAAC,GAAG,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC5E,mBAAmB,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,kBAAkB,CAAI,KAAU;IACvC,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,mBAAmB,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACvC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,oBAAoB;IAClC,MAAM,cAAc,GAAG,CAAC,CAAC;IACzB,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAuB,EAAE,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,UAAU,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,WAAW;YACjB,IAAI,EAAE,OAAO,CAAC,EAAE;YAChB,IAAI,EAAE,EAAC,IAAI,EAAE,aAAa,EAAC;SAC5B,CAAC,CAAC;IACL,CAAC;IACD,MAAM,cAAc,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnD,MAAM,aAAa,GAAuB,EAAE,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,aAAa,CAAC,IAAI,CAAC;YACjB,IAAI,EAAE,WAAW;YACjB,IAAI,EAAE,gBAAgB,CAAC,EAAE;YACzB,IAAI,EAAE,EAAC,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,SAAS,EAAC;SAChD,CAAC,CAAC;IACL,CAAC;IACD,MAAM,cAAc,GAAqB;QACvC,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,EAAC,IAAI,EAAE,aAAa,EAAC;KAC5B,CAAC;IACF,MAAM,MAAM,GAAkB;QAC5B,MAAM,EAAE,CAAC,GAAG,UAAU,EAAE,GAAG,aAAa,EAAE,cAAc,CAAC;KAC1D,CAAC;IACF,MAAM,SAAS,GAAG,MAAM,CAAC;IACzB,MAAM,cAAc,GAAG,YAAY,CAAC;IACpC,IAAI,IAAI,GAAG;;GAEV,CAAC;IACF,MAAM,MAAM,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;IAClD,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,SAAS,GAAG,QAAQ,aAAa,EAAE,CAAC;QAC1C,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC3B,IAAI,IAAI;YACA,SAAS,SAAS,cAAc,WAAW,SAAS;YACpD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;cACd,cAAc,CAAC,IAAI;cACnB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;;WAE3C,cAAc,CAAC,IAAI,OAAO,aAAa;;;CAGjD,CAAC;IACA,CAAC;IAED,IAAI,IAAI;;EAER,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;;;cAGjC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;cACxC,cAAc,CAAC,IAAI;;;EAG/B,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;;;CAU5E,CAAC;IAEA,MAAM,QAAQ,GAAG,6BAA6B,CAAC;IAE/C,MAAM,oBAAoB,GAAG,YAAG,CAAC,SAAS,CAAC,YAAY,CAAC;QACtD,SAAS,EAAE,QAAQ;QACnB,cAAc,EAAE;YACd,KAAK,EAAE;gBACL;oBACE,GAAG,EAAE,QAAQ;oBACb,QAAQ,EAAE,IAAI;iBACf;aACF;YACD,WAAW,EAAE;gBACX;oBACE,IAAI,EAAE,cAAc;oBACpB,OAAO,EAAE,QAAQ;iBAClB;aACF;YACD,aAAa,EAAE;gBACb;oBACE,IAAI,EAAE,SAAS;oBACf,eAAe,EAAE,cAAc;oBAC/B,MAAM;iBACP;aACF;SACF;QACD,kBAAkB,EAAE,IAAI;KACzB,CAAC,CAAC;IAEH,MAAM,mBAAmB,GAA+B;QACtD,kBAAkB,EAAE,IAAI;QACxB,SAAS,EAAE,QAAQ;QACnB,KAAK,EAAE;YACL,UAAU,EAAE;gBACV,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE;oBACN,IAAI,EAAE,kBAAkB;oBACxB,IAAI,EAAE,MAAM;iBACb;gBACD,IAAI,EAAE;oBACJ,IAAI,EAAE,SAAS;oBACf,UAAU,EAAE;wBACV;4BACE,IAAI,EAAE,UAAU;4BAChB,KAAK,EAAE;gCACL,UAAU,EAAE;oCACV,IAAI,EAAE,iBAAiB;oCACvB,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI;iCACzB;6BACF;yBACF;wBACD;4BACE,IAAI,EAAE,WAAW;4BACjB,KAAK,EAAE;gCACL,UAAU,EAAE;oCACV,IAAI,EAAE,iBAAiB;oCACvB,IAAI,EAAE,SAAS,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;iCACvC;6BACF;yBACF;qBACF;iBACF;aACF;SACF;QACD,cAAc,EAAE;YACd,YAAY,EAAE,oBAAoB,CAAC,YAAY;SAChD;KACF,CAAC;IAEF,OAAO;QACL,IAAI;QACJ,MAAM;QACN,UAAU;QACV,aAAa;QACb,cAAc;QACd,mBAAmB;KACpB,CAAC;AACJ,CAAC;AAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,oBAAoB,EAAE,CAAC,mBAAmB,CAAC,CAAC,CAAC"}

package/package.json

@@ -21,8 +21,8 @@
     "time": "ts-node src/timer.ts"
   },
   "dependencies": {
-    "@malloydata/malloy": "0.0.331",
-    "@malloydata/malloy-interfaces": "0.0.331"
+    "@malloydata/malloy": "0.0.332",
+    "@malloydata/malloy-interfaces": "0.0.332"
   },
-  "version": "0.0.331"
+  "version": "0.0.332"
 }

package/src/big_composite.ts

@@ -0,0 +1,177 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+/* eslint-disable no-console */
+import type * as Malloy from '@malloydata/malloy-interfaces';
+import {API} from '@malloydata/malloy';
+
+function _getAllCombinations<T>(current: T[], remaining: T[], result: T[][]) {
+  if (remaining.length === 0) {
+    if (current.length > 0) {
+      result.push(current);
+    }
+    return;
+  }
+  _getAllCombinations([...current, remaining[0]], remaining.slice(1), result);
+  _getAllCombinations(current, remaining.slice(1), result);
+}
+
+function getAllCombinations<T>(items: T[]): T[][] {
+  const result: T[][] = [];
+  _getAllCombinations([], items, result);
+  return result;
+}
+
+export function generateBigComposite() {
+  const dimensionCount = 3;
+  const preaggregateCount = 50;
+  const dimensions: Malloy.FieldInfo[] = [];
+  for (let i = 0; i < dimensionCount; i++) {
+    dimensions.push({
+      kind: 'dimension',
+      name: `dim_${i}`,
+      type: {kind: 'string_type'},
+    });
+  }
+  const dimensionNames = dimensions.map(f => f.name);
+  const preaggregates: Malloy.FieldInfo[] = [];
+  for (let i = 0; i < preaggregateCount; i++) {
+    preaggregates.push({
+      kind: 'dimension',
+      name: `preaggregate_${i}`,
+      type: {kind: 'number_type', subtype: 'decimal'},
+    });
+  }
+  const partitionField: Malloy.FieldInfo = {
+    kind: 'dimension',
+    name: 'partition_name',
+    type: {kind: 'string_type'},
+  };
+  const schema: Malloy.Schema = {
+    fields: [...dimensions, ...preaggregates, partitionField],
+  };
+  const tableName = 'cube';
+  const connectionName = 'connection';
+  let code = `
+    ##! experimental { composite_sources access_modifiers }
+  `;
+  const slices = getAllCombinations(dimensionNames);
+  const sliceNames: string[] = [];
+  for (const slice of slices) {
+    const partitionName = slice.join('.');
+    const sliceName = `cube:${partitionName}`;
+    sliceNames.push(sliceName);
+    code += `
+source: \`${sliceName}\` is ${connectionName}.table('${tableName}') include {
+  public: ${slice.join(', ')}
+  internal: ${partitionField.name}
+  internal: ${preaggregates.map(a => a.name).join(',')}
+} extend {
+  where: ${partitionField.name} = '${partitionName}'
+}
+
+`;
+  }
+
+  code += `
+source: cube is compose(
+${sliceNames.map(n => `  \`${n}\``).join(',\n')}
+) include {
+  public: *
+  internal: ${preaggregates.map(a => a.name).join(',')}
+  internal: ${partitionField.name}
+} extend {
+  measure:
+${preaggregates.map(a => `    total_${a.name} is ${a.name}.sum()`).join('\n')}
+}
+
+// Some extensions of the cube to bulk up the size...
+
+source: cube_ext_1 is cube
+source: cube_ext_2 is cube
+source: cube_ext_3 is cube
+source: cube_ext_4 is cube
+source: cube_ext_5 is cube
+`;
+
+  const fileName = 'file://big_composite.malloy';
+
+  const compileModelResponse = API.stateless.compileModel({
+    model_url: fileName,
+    compiler_needs: {
+      files: [
+        {
+          url: fileName,
+          contents: code,
+        },
+      ],
+      connections: [
+        {
+          name: connectionName,
+          dialect: 'duckdb',
+        },
+      ],
+      table_schemas: [
+        {
+          name: tableName,
+          connection_name: connectionName,
+          schema,
+        },
+      ],
+    },
+    exclude_references: true,
+  });
+
+  const compileQueryRequest: Malloy.CompileQueryRequest = {
+    exclude_references: true,
+    model_url: fileName,
+    query: {
+      definition: {
+        kind: 'arrow',
+        source: {
+          kind: 'source_reference',
+          name: 'cube',
+        },
+        view: {
+          kind: 'segment',
+          operations: [
+            {
+              kind: 'group_by',
+              field: {
+                expression: {
+                  kind: 'field_reference',
+                  name: dimensions[0].name,
+                },
+              },
+            },
+            {
+              kind: 'aggregate',
+              field: {
+                expression: {
+                  kind: 'field_reference',
+                  name: `total_${preaggregates[0].name}`,
+                },
+              },
+            },
+          ],
+        },
+      },
+    },
+    compiler_needs: {
+      translations: compileModelResponse.translations,
+    },
+  };
+
+  return {
+    code,
+    schema,
+    dimensions,
+    preaggregates,
+    partitionField,
+    compileQueryRequest,
+  };
+}
+
+console.log(JSON.stringify(generateBigComposite().compileQueryRequest));