@malloydata/malloy-profiler 0.0.300

package/README.md

@@ -0,0 +1,8 @@
+# Malloy Stable Request Profiler
+
+This package enables profiling of the Malloy compiler. Currently it only works with stateless compilation of queries. In the future we can extend it support other kinds of compilation.
+
+Usage:
+
+- Write a Malloy query request to a file.
+- Run `npm run profile path/to/file.json`

package/dist/profile.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/profile.js

@@ -0,0 +1,27 @@
+"use strict";
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable no-console */
+const profiler_1 = require("./profiler");
+async function main() {
+    const fs = require('fs');
+    const filePath = process.argv[2];
+    if (filePath === undefined) {
+        console.log('Missing file path');
+        return;
+    }
+    const fileContent = fs.readFileSync(filePath, 'utf8');
+    console.log(`REQUEST ${fileContent.substring(0, 100)}`);
+    const response = (0, profiler_1.compileQueryStable)(fileContent);
+    if (response.time) {
+        console.log(`TOOK: ${response.time}ms`);
+    }
+    console.log(`RESPONSE ${response.response.substring(0, 100)}`);
+}
+main();
+//# sourceMappingURL=profile.js.map

package/dist/profile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile.js","sourceRoot":"","sources":["../src/profile.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;AAEH,+BAA+B;AAC/B,yCAA8C;AAE9C,KAAK,UAAU,IAAI;IACjB,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IACD,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,WAAW,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAExD,MAAM,QAAQ,GAAG,IAAA,6BAAkB,EAAC,WAAW,CAAC,CAAC;IAEjD,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,SAAS,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,YAAY,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,IAAI,EAAE,CAAC"}

package/dist/profiler.d.ts

@@ -0,0 +1,4 @@
+export declare function compileQueryStable(serializedCompileQueryRequest: string): {
+    response: string;
+    time: number | undefined;
+};

package/dist/profiler.js

@@ -0,0 +1,73 @@
+"use strict";
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.compileQueryStable = compileQueryStable;
+/* eslint-disable no-console */
+const malloy_1 = require("@malloydata/malloy");
+const malloy_interfaces_1 = require("@malloydata/malloy-interfaces");
+function compileQueryStableInternalObjRequest(parsedRequest) {
+    const sessionId = parsedRequest.session_id;
+    delete parsedRequest.session_id;
+    const request = (0, malloy_interfaces_1.convertFromThrift)(parsedRequest, 'CompileQueryRequest');
+    const now = Date.now();
+    const response = malloy_1.API.sessioned.compileQuery(request, {
+        ttl: {
+            seconds: 300,
+        },
+        session_id: sessionId,
+    });
+    const end = Date.now();
+    const took = end - now;
+    const { session_id, ...spreadedResponse } = response;
+    const withNestResponse = (0, malloy_interfaces_1.convertToThrift)(spreadedResponse, 'CompileQueryResponse');
+    // TODO: Remove when session id is part of the response thrift.
+    withNestResponse.session_id =
+        session_id;
+    return { response: withNestResponse, time: took };
+}
+function compileQueryStableInternal(serializedCompileQueryRequest) {
+    const parsedRequest = JSON.parse(serializedCompileQueryRequest);
+    const { response, time } = compileQueryStableInternalObjRequest(parsedRequest);
+    return { response: JSON.stringify(response), time };
+}
+// Compile Query.
+function compileQueryStable(serializedCompileQueryRequest) {
+    var _a;
+    try {
+        return compileQueryStableInternal(serializedCompileQueryRequest);
+    }
+    catch (e) {
+        const response = {
+            logs: [
+                {
+                    message: `Failed to compile Query ${e instanceof Error
+                        ? `${e.message} ${(_a = e.stack) !== null && _a !== void 0 ? _a : ''}`
+                        : JSON.stringify(e)}`,
+                    severity: 'error',
+                    // TODO: location should be optional.
+                    url: '',
+                    range: {
+                        start: {
+                            line: 0,
+                            character: 0,
+                        },
+                        end: {
+                            line: 0,
+                            character: 0,
+                        },
+                    },
+                },
+            ],
+        };
+        return {
+            response: JSON.stringify((0, malloy_interfaces_1.convertToThrift)(response, 'CompileQueryResponse')),
+            time: undefined,
+        };
+    }
+}
+//# sourceMappingURL=profiler.js.map

package/dist/profiler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiler.js","sourceRoot":"","sources":["../src/profiler.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;AA8DH,gDAuCC;AAnGD,+BAA+B;AAC/B,+CAAuC;AAKvC,qEAGuC;AAUvC,SAAS,oCAAoC,CAC3C,aAA+C;IAE/C,MAAM,SAAS,GAAG,aAAa,CAAC,UAAU,CAAC;IAC3C,OAAO,aAAa,CAAC,UAAU,CAAC;IAChC,MAAM,OAAO,GAAG,IAAA,qCAAiB,EAC/B,aAAa,EACb,qBAAqB,CACC,CAAC;IAEzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,QAAQ,GAAG,YAAG,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE;QACnD,GAAG,EAAE;YACH,OAAO,EAAE,GAAG;SACb;QACD,UAAU,EAAE,SAAS;KACtB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,IAAI,GAAG,GAAG,GAAG,GAAG,CAAC;IAEvB,MAAM,EAAC,UAAU,EAAE,GAAG,gBAAgB,EAAC,GAAG,QAAQ,CAAC;IACnD,MAAM,gBAAgB,GAAG,IAAA,mCAAe,EACtC,gBAAgB,EAChB,sBAAsB,CACC,CAAC;IAC1B,+DAA+D;IAC9D,gBAAsD,CAAC,UAAU;QAChE,UAAU,CAAC;IACb,OAAO,EAAC,QAAQ,EAAE,gBAAgB,EAAE,IAAI,EAAE,IAAI,EAAC,CAAC;AAClD,CAAC;AAED,SAAS,0BAA0B,CAAC,6BAAqC;IAIvE,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAChE,MAAM,EAAC,QAAQ,EAAE,IAAI,EAAC,GAAG,oCAAoC,CAAC,aAAa,CAAC,CAAC;IAC7E,OAAO,EAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAC,CAAC;AACpD,CAAC;AAED,iBAAiB;AACjB,SAAgB,kBAAkB,CAAC,6BAAqC;;IAItE,IAAI,CAAC;QACH,OAAO,0BAA0B,CAAC,6BAA6B,CAAC,CAAC;IACnE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,QAAQ,GAAyB;YACrC,IAAI,EAAE;gBACJ;oBACE,OAAO,EAAE,2BACP,CAAC,YAAY,KAAK;wBAChB,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,EAAE;wBACjC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CACtB,EAAE;oBACF,QAAQ,EAAE,OAAO;oBACjB,qCAAqC;oBACrC,GAAG,EAAE,EAAE;oBACP,KAAK,EAAE;wBACL,KAAK,EAAE;4BACL,IAAI,EAAE,CAAC;4BACP,SAAS,EAAE,CAAC;yBACb;wBACD,GAAG,EAAE;4BACH,IAAI,EAAE,CAAC;4BACP,SAAS,EAAE,CAAC;yBACb;qBACF;iBACF;aACF;SACF,CAAC;QAEF,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,SAAS,CACtB,IAAA,mCAAe,EAAC,QAAQ,EAAE,sBAAsB,CAAC,CAClD;YACD,IAAI,EAAE,SAAS;SAChB,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/timer.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/timer.js

@@ -0,0 +1,37 @@
+"use strict";
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable no-console */
+const profiler_1 = require("./profiler");
+async function main() {
+    const fs = require('fs');
+    const filePath = process.argv[2];
+    if (filePath === undefined) {
+        console.log('Missing file path');
+        return;
+    }
+    const fileContent = fs.readFileSync(filePath, 'utf8');
+    console.log(`REQUEST ${fileContent.substring(0, 100)}`);
+    const times = [];
+    for (let i = 0; i < 100; i++) {
+        const response = (0, profiler_1.compileQueryStable)(fileContent);
+        if (i === 0) {
+            console.log(`RESPONSE ${response.response.substring(0, 100)}`);
+        }
+        if (response.time !== undefined) {
+            times.push(response.time);
+        }
+        else {
+            return;
+        }
+    }
+    const avg = times.reduce((a, b) => a + b) / times.length;
+    console.log('AVERAGE TIME = ' + avg + 'ms');
+}
+main();
+//# sourceMappingURL=timer.js.map

package/dist/timer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"timer.js","sourceRoot":"","sources":["../src/timer.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;AAEH,+BAA+B;AAC/B,yCAA8C;AAE9C,KAAK,UAAU,IAAI;IACjB,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IACD,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,WAAW,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACxD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAA,6BAAkB,EAAC,WAAW,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,YAAY,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,OAAO;QACT,CAAC;IACH,CAAC;IACD,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACzD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC;AAC9C,CAAC;AAED,IAAI,EAAE,CAAC"}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@malloydata/malloy-profiler",
+  "license": "MIT",
+  "engines": {
+    "node": ">=20"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "homepage": "https://github.com/malloydata/malloy#readme",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/malloydata/malloy"
+  },
+  "scripts": {
+    "lint": "eslint '**/*.ts{,x}'",
+    "lint-fix": "eslint '**/*.ts{,x}' --fix",
+    "test": "jest --config=../jest.config.js",
+    "build": "tsc --build",
+    "clean": "tsc --build --clean",
+    "profile": "npx clinic flame -- node dist/profile.js",
+    "time": "ts-node src/timer.ts"
+  },
+  "dependencies": {
+    "@malloydata/malloy": "0.0.300",
+    "@malloydata/malloy-interfaces": "0.0.300",
+    "clinic": "^13.0.0"
+  },
+  "devDependencies": {},
+  "version": "0.0.300"
+}

package/src/profile.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/* eslint-disable no-console */
+import {compileQueryStable} from './profiler';
+
+async function main() {
+  const fs = require('fs');
+  const filePath = process.argv[2];
+  if (filePath === undefined) {
+    console.log('Missing file path');
+    return;
+  }
+  const fileContent = fs.readFileSync(filePath, 'utf8');
+  console.log(`REQUEST ${fileContent.substring(0, 100)}`);
+
+  const response = compileQueryStable(fileContent);
+
+  if (response.time) {
+    console.log(`TOOK: ${response.time}ms`);
+  }
+
+  console.log(`RESPONSE ${response.response.substring(0, 100)}`);
+}
+
+main();

package/src/profiler.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/* eslint-disable no-console */
+import {API} from '@malloydata/malloy';
+import type {
+  CompileQueryResponse,
+  CompileQueryRequest,
+} from '@malloydata/malloy-interfaces';
+import {
+  convertFromThrift,
+  convertToThrift,
+} from '@malloydata/malloy-interfaces';
+
+type CompileQueryRequestWithSessionId = CompileQueryRequest & {
+  session_id?: string;
+};
+
+type CompileQueryResponseWithSessionId = CompileQueryResponse & {
+  session_id?: string;
+};
+
+function compileQueryStableInternalObjRequest(
+  parsedRequest: CompileQueryRequestWithSessionId
+): {response: CompileQueryResponse; time: number} {
+  const sessionId = parsedRequest.session_id;
+  delete parsedRequest.session_id;
+  const request = convertFromThrift(
+    parsedRequest,
+    'CompileQueryRequest'
+  ) as CompileQueryRequest;
+
+  const now = Date.now();
+  const response = API.sessioned.compileQuery(request, {
+    ttl: {
+      seconds: 300,
+    },
+    session_id: sessionId,
+  });
+  const end = Date.now();
+  const took = end - now;
+
+  const {session_id, ...spreadedResponse} = response;
+  const withNestResponse = convertToThrift(
+    spreadedResponse,
+    'CompileQueryResponse'
+  ) as CompileQueryResponse;
+  // TODO: Remove when session id is part of the response thrift.
+  (withNestResponse as CompileQueryResponseWithSessionId).session_id =
+    session_id;
+  return {response: withNestResponse, time: took};
+}
+
+function compileQueryStableInternal(serializedCompileQueryRequest: string): {
+  response: string;
+  time: number;
+} {
+  const parsedRequest = JSON.parse(serializedCompileQueryRequest);
+  const {response, time} = compileQueryStableInternalObjRequest(parsedRequest);
+  return {response: JSON.stringify(response), time};
+}
+
+// Compile Query.
+export function compileQueryStable(serializedCompileQueryRequest: string): {
+  response: string;
+  time: number | undefined;
+} {
+  try {
+    return compileQueryStableInternal(serializedCompileQueryRequest);
+  } catch (e) {
+    const response: CompileQueryResponse = {
+      logs: [
+        {
+          message: `Failed to compile Query ${
+            e instanceof Error
+              ? `${e.message} ${e.stack ?? ''}`
+              : JSON.stringify(e)
+          }`,
+          severity: 'error',
+          // TODO: location should be optional.
+          url: '',
+          range: {
+            start: {
+              line: 0,
+              character: 0,
+            },
+            end: {
+              line: 0,
+              character: 0,
+            },
+          },
+        },
+      ],
+    };
+
+    return {
+      response: JSON.stringify(
+        convertToThrift(response, 'CompileQueryResponse')
+      ),
+      time: undefined,
+    };
+  }
+}

package/src/timer.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/* eslint-disable no-console */
+import {compileQueryStable} from './profiler';
+
+async function main() {
+  const fs = require('fs');
+  const filePath = process.argv[2];
+  if (filePath === undefined) {
+    console.log('Missing file path');
+    return;
+  }
+  const fileContent = fs.readFileSync(filePath, 'utf8');
+  console.log(`REQUEST ${fileContent.substring(0, 100)}`);
+  const times: number[] = [];
+  for (let i = 0; i < 100; i++) {
+    const response = compileQueryStable(fileContent);
+    if (i === 0) {
+      console.log(`RESPONSE ${response.response.substring(0, 100)}`);
+    }
+    if (response.time !== undefined) {
+      times.push(response.time);
+    } else {
+      return;
+    }
+  }
+  const avg = times.reduce((a, b) => a + b) / times.length;
+  console.log('AVERAGE TIME = ' + avg + 'ms');
+}
+
+main();

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../tsconfig.packages.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "composite": true,
+    "jsx": "react-jsx",
+    "jsxImportSource": "solid-js"
+  },
+  "include": ["src"],
+  "references": [
+    {
+      "path": "../packages/malloy-interfaces"
+    },
+    {
+      "path": "../packages/malloy"
+    },
+  ]
+}