@malloydata/malloy 0.0.239-dev250226010331 → 0.0.239-dev250227145856

package/dist/api/asynchronous.d.ts

@@ -0,0 +1,13 @@
+import * as Malloy from '@malloydata/malloy-interfaces';
+import { InfoConnection, Connection, LookupConnection } from './connection';
+import { URLReader } from '../runtime_types';
+import { CacheManager } from '../malloy';
+export interface CompilerNeedFetch<T extends InfoConnection> {
+    connections: LookupConnection<T>;
+    urls: URLReader;
+    cacheManager?: CacheManager;
+}
+export declare function compileModel(request: Malloy.CompileModelRequest, fetchers: CompilerNeedFetch<InfoConnection>): Promise<Malloy.CompileModelResponse>;
+export declare function compileSource(request: Malloy.CompileSourceRequest, fetchers: CompilerNeedFetch<InfoConnection>): Promise<Malloy.CompileSourceResponse>;
+export declare function compileQuery(request: Malloy.CompileQueryRequest, fetchers: CompilerNeedFetch<InfoConnection>): Promise<Malloy.CompileQueryResponse>;
+export declare function runQuery(request: Malloy.CompileQueryRequest, fetchers: CompilerNeedFetch<Connection>): Promise<Malloy.CompileQueryResponse>;

package/dist/api/asynchronous.js

@@ -0,0 +1,205 @@
+"use strict";
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runQuery = exports.compileQuery = exports.compileSource = exports.compileModel = void 0;
+const Core = __importStar(require("./core"));
+async function fetchNeeds(needs, fetchers) {
+    var _a, _b, _c, _d, _e, _f, _g;
+    var _h, _j;
+    if (needs === undefined) {
+        throw new Error("Expected compiler to have needs because it didn't return a result");
+    }
+    const result = {};
+    if (needs.connections) {
+        for (const connection of needs.connections) {
+            const info = await fetchers.connections.lookupConnection(connection.name);
+            (_a = result.connections) !== null && _a !== void 0 ? _a : (result.connections = []);
+            result.connections.push({
+                ...connection,
+                dialect: info.dialectName,
+            });
+        }
+    }
+    if (needs.files) {
+        for (const file of needs.files) {
+            // TODO handle checking if the cache has the file...
+            const info = await fetchers.urls.readURL(new URL(file.url));
+            (_b = result.files) !== null && _b !== void 0 ? _b : (result.files = []);
+            if (typeof info === 'string') {
+                result.files.push({
+                    ...file,
+                    contents: info,
+                });
+            }
+            else {
+                result.files.push({
+                    ...file,
+                    contents: info.contents,
+                    invalidation_key: (_c = info.invalidationKey) === null || _c === void 0 ? void 0 : _c.toString(),
+                });
+            }
+        }
+    }
+    if (needs.table_schemas) {
+        const tableSchemasByConnection = {};
+        for (const tableSchema of needs.table_schemas) {
+            (_d = tableSchemasByConnection[_h = tableSchema.connection_name]) !== null && _d !== void 0 ? _d : (tableSchemasByConnection[_h] = []);
+            tableSchemasByConnection[tableSchema.connection_name].push(tableSchema);
+        }
+        for (const connectionName in tableSchemasByConnection) {
+            const connection = await fetchers.connections.lookupConnection(connectionName);
+            const tableNames = tableSchemasByConnection[connectionName].map(t => t.name);
+            const schemas = await Promise.all(tableNames.map(async (tableName) => ({
+                name: tableName,
+                schema: await connection.fetchSchemaForTable(tableName),
+            })));
+            (_e = result.table_schemas) !== null && _e !== void 0 ? _e : (result.table_schemas = []);
+            for (const schema of schemas) {
+                result.table_schemas.push({
+                    connection_name: connectionName,
+                    name: schema.name,
+                    schema: schema.schema,
+                });
+            }
+        }
+    }
+    if (needs.sql_schemas) {
+        const sqlSchemasByConnectionName = {};
+        for (const sqlSchema of needs.sql_schemas) {
+            (_f = sqlSchemasByConnectionName[_j = sqlSchema.connection_name]) !== null && _f !== void 0 ? _f : (sqlSchemasByConnectionName[_j] = []);
+            sqlSchemasByConnectionName[sqlSchema.connection_name].push(sqlSchema);
+        }
+        for (const connectionName in sqlSchemasByConnectionName) {
+            const connection = await fetchers.connections.lookupConnection(connectionName);
+            const sqlQueries = sqlSchemasByConnectionName[connectionName].map(t => t.sql);
+            const schemas = await Promise.all(sqlQueries.map(async (sql) => ({
+                sql,
+                schema: await connection.fetchSchemaForSQLQuery(sql),
+            })));
+            (_g = result.sql_schemas) !== null && _g !== void 0 ? _g : (result.sql_schemas = []);
+            for (const schema of schemas) {
+                result.sql_schemas.push({
+                    connection_name: connectionName,
+                    sql: schema.sql,
+                    schema: schema.schema,
+                });
+            }
+        }
+    }
+    return result;
+}
+async function compileModel(request, fetchers) {
+    const state = Core.newCompileModelState(request);
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const result = Core.statedCompileModel(state);
+        if (result.model) {
+            return result;
+        }
+        const needs = await fetchNeeds(result.compiler_needs, fetchers);
+        Core.updateCompileModelState(state, needs);
+    }
+}
+exports.compileModel = compileModel;
+async function compileSource(request, fetchers) {
+    const state = Core.newCompileSourceState(request);
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const result = Core.statedCompileSource(state, request.name);
+        if (result.source) {
+            return result;
+        }
+        const needs = await fetchNeeds(result.compiler_needs, fetchers);
+        Core.updateCompileModelState(state, needs);
+    }
+}
+exports.compileSource = compileSource;
+async function compileQuery(request, fetchers) {
+    const state = Core.newCompileQueryState(request);
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const result = Core.statedCompileQuery(state);
+        if (result.result) {
+            return result;
+        }
+        const needs = await fetchNeeds(result.compiler_needs, fetchers);
+        Core.updateCompileModelState(state, needs);
+    }
+}
+exports.compileQuery = compileQuery;
+async function runQuery(request, fetchers) {
+    var _a, _b;
+    const compiled = await compileQuery(request, fetchers);
+    if (compiled.result === undefined) {
+        return compiled;
+    }
+    const defaultURL = request.model_url;
+    if (compiled.result.sql === undefined) {
+        return {
+            logs: [
+                ...((_a = compiled.logs) !== null && _a !== void 0 ? _a : []),
+                {
+                    url: defaultURL,
+                    severity: 'error',
+                    message: 'Internal error: Compiler did not generate SQL',
+                    range: Core.DEFAULT_LOG_RANGE,
+                },
+            ],
+        };
+    }
+    try {
+        const connection = await fetchers.connections.lookupConnection(compiled.result.connection_name);
+        const data = await connection.runSQL(compiled.result.sql);
+        return {
+            ...compiled,
+            result: {
+                ...compiled.result,
+                data,
+            },
+        };
+    }
+    catch (error) {
+        return {
+            ...compiled,
+            logs: [
+                ...((_b = compiled.logs) !== null && _b !== void 0 ? _b : []),
+                {
+                    url: defaultURL,
+                    severity: 'error',
+                    message: `Error running SQL: ${error.message}`,
+                    range: Core.DEFAULT_LOG_RANGE,
+                },
+            ],
+        };
+    }
+}
+exports.runQuery = runQuery;
+//# sourceMappingURL=asynchronous.js.map

package/dist/api/asynchronous.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/api/asynchronous.spec.js

@@ -0,0 +1,238 @@
+"use strict";
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const asynchronous_1 = require("./asynchronous");
+describe('api', () => {
+    describe('compile model', () => {
+        test('compile model with table dependency', async () => {
+            const connection = {
+                dialectName: 'duckdb',
+                fetchSchemaForTable: async (_name) => {
+                    return {
+                        fields: [
+                            {
+                                kind: 'dimension',
+                                name: 'carrier',
+                                type: { kind: 'string_type' },
+                            },
+                        ],
+                    };
+                },
+                fetchSchemaForSQLQuery: async (_sql) => {
+                    throw new Error('not implemented');
+                },
+            };
+            const urls = {
+                readURL: async (_url) => {
+                    return "source: flights is connection.table('flights')";
+                },
+            };
+            const connections = {
+                lookupConnection: async (_name) => {
+                    return connection;
+                },
+            };
+            const fetchers = {
+                urls,
+                connections,
+            };
+            const result = await (0, asynchronous_1.compileModel)({
+                model_url: 'file://test.malloy',
+            }, fetchers);
+            const expected = {
+                model: {
+                    entries: [
+                        {
+                            kind: 'source',
+                            name: 'flights',
+                            schema: {
+                                fields: [
+                                    {
+                                        kind: 'dimension',
+                                        name: 'carrier',
+                                        type: { kind: 'string_type' },
+                                    },
+                                ],
+                            },
+                        },
+                    ],
+                    anonymous_queries: [],
+                },
+            };
+            expect(result).toMatchObject(expected);
+        });
+    });
+    describe('compile query', () => {
+        test('compile query with table dependency', async () => {
+            const connection = {
+                dialectName: 'duckdb',
+                fetchSchemaForTable: async (_name) => {
+                    return {
+                        fields: [
+                            {
+                                kind: 'dimension',
+                                name: 'carrier',
+                                type: { kind: 'string_type' },
+                            },
+                        ],
+                    };
+                },
+                fetchSchemaForSQLQuery: async (_sql) => {
+                    throw new Error('not implemented');
+                },
+            };
+            const urls = {
+                readURL: async (_url) => {
+                    return "source: flights is connection.table('flights')";
+                },
+            };
+            const connections = {
+                lookupConnection: async (_name) => {
+                    return connection;
+                },
+            };
+            const fetchers = {
+                urls,
+                connections,
+            };
+            const result = await (0, asynchronous_1.compileQuery)({
+                model_url: 'file://test.malloy',
+                query: {
+                    definition: {
+                        kind: 'arrow',
+                        source_reference: { name: 'flights' },
+                        view: {
+                            kind: 'segment',
+                            operations: [
+                                {
+                                    kind: 'group_by',
+                                    field: {
+                                        expression: { kind: 'field_reference', name: 'carrier' },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                },
+            }, fetchers);
+            const expected = {
+                result: {
+                    connection_name: 'connection',
+                    sql: `SELECT \n\
+   base."carrier" as "carrier"
+FROM flights as base
+GROUP BY 1
+ORDER BY 1 asc NULLS LAST
+`,
+                    schema: {
+                        fields: [
+                            {
+                                kind: 'dimension',
+                                name: 'carrier',
+                                type: { kind: 'string_type' },
+                            },
+                        ],
+                    },
+                },
+            };
+            expect(result).toMatchObject(expected);
+        });
+    });
+    describe('run query', () => {
+        test('run query with table dependency', async () => {
+            const data = {
+                kind: 'table',
+                rows: [
+                    {
+                        cells: [{ kind: 'string_cell', string_value: 'WN' }],
+                    },
+                    {
+                        cells: [{ kind: 'string_cell', string_value: 'AA' }],
+                    },
+                ],
+            };
+            const connection = {
+                dialectName: 'duckdb',
+                fetchSchemaForTable: async (_name) => {
+                    return {
+                        fields: [
+                            {
+                                kind: 'dimension',
+                                name: 'carrier',
+                                type: { kind: 'string_type' },
+                            },
+                        ],
+                    };
+                },
+                fetchSchemaForSQLQuery: async (_sql) => {
+                    throw new Error('not implemented');
+                },
+                runSQL: async (_sql) => {
+                    return data;
+                },
+            };
+            const urls = {
+                readURL: async (_url) => {
+                    return "source: flights is connection.table('flights')";
+                },
+            };
+            const connections = {
+                lookupConnection: async (_name) => {
+                    return connection;
+                },
+            };
+            const fetchers = {
+                urls,
+                connections,
+            };
+            const result = await (0, asynchronous_1.runQuery)({
+                model_url: 'file://test.malloy',
+                query: {
+                    definition: {
+                        kind: 'arrow',
+                        source_reference: { name: 'flights' },
+                        view: {
+                            kind: 'segment',
+                            operations: [
+                                {
+                                    kind: 'group_by',
+                                    field: {
+                                        expression: { kind: 'field_reference', name: 'carrier' },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                },
+            }, fetchers);
+            const expected = {
+                result: {
+                    connection_name: 'connection',
+                    sql: `SELECT \n\
+   base."carrier" as "carrier"
+FROM flights as base
+GROUP BY 1
+ORDER BY 1 asc NULLS LAST
+`,
+                    schema: {
+                        fields: [
+                            {
+                                kind: 'dimension',
+                                name: 'carrier',
+                                type: { kind: 'string_type' },
+                            },
+                        ],
+                    },
+                    data,
+                },
+            };
+            expect(result).toMatchObject(expected);
+        });
+    });
+});
+//# sourceMappingURL=asynchronous.spec.js.map

package/dist/api/connection.d.ts

@@ -0,0 +1,12 @@
+import * as Malloy from '@malloydata/malloy-interfaces';
+export interface InfoConnection {
+    fetchSchemaForTable(name: string): Promise<Malloy.Schema>;
+    fetchSchemaForSQLQuery(sqlQuery: string): Promise<Malloy.Schema>;
+    get dialectName(): string;
+}
+export interface Connection extends InfoConnection {
+    runSQL(sql: string): Promise<Malloy.Data>;
+}
+export interface LookupConnection<T extends InfoConnection> {
+    lookupConnection(connectionName?: string): Promise<T>;
+}

package/dist/api/connection.js

@@ -0,0 +1,9 @@
+"use strict";
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=connection.js.map

package/dist/api/core.d.ts

@@ -0,0 +1,37 @@
+import * as Malloy from '@malloydata/malloy-interfaces';
+import { LogMessage, MalloyTranslator } from '../lang';
+import { ModelDef } from '../model';
+export type CompileResponse = {
+    model: Malloy.ModelInfo;
+    modelDef: ModelDef;
+    compilerNeeds?: undefined;
+    logs?: LogMessage[];
+} | {
+    model?: undefined;
+    modelDef?: undefined;
+    compilerNeeds: Malloy.CompilerNeeds;
+    logs?: LogMessage[];
+} | {
+    model?: undefined;
+    modelDef?: undefined;
+    compilerNeeds?: undefined;
+    logs: LogMessage[];
+};
+export declare function compileQuery(request: Malloy.CompileQueryRequest, state?: CompileModelState): Malloy.CompileQueryResponse;
+export interface CompileModelState {
+    extending?: CompileModelState;
+    translator: MalloyTranslator;
+    done: boolean;
+    hasSource: boolean;
+}
+export declare function updateCompileModelState(state: CompileModelState, needs: Malloy.CompilerNeeds | undefined): void;
+export declare function newCompileModelState(request: Malloy.CompileModelRequest): CompileModelState;
+export declare function newCompileSourceState(request: Malloy.CompileSourceRequest): CompileModelState;
+export declare function statedCompileModel(state: CompileModelState): Malloy.CompileModelResponse;
+export declare function statedCompileSource(state: CompileModelState, name: string): Malloy.CompileSourceResponse;
+export declare function _statedCompileModel(state: CompileModelState): CompileResponse;
+export declare const DEFAULT_LOG_RANGE: Malloy.DocumentRange;
+export declare function compileModel(request: Malloy.CompileModelRequest, state?: CompileModelState): Malloy.CompileModelResponse;
+export declare function compileSource(request: Malloy.CompileSourceRequest): Malloy.CompileSourceResponse;
+export declare function newCompileQueryState(request: Malloy.CompileQueryRequest): CompileModelState;
+export declare function statedCompileQuery(state: CompileModelState): Malloy.CompileQueryResponse;