@malloydata/db-publisher 0.0.315 → 0.0.316

package/src/client/base.ts

@@ -2,7 +2,7 @@
 /* eslint-disable */
 /**
  * Malloy Publisher - Semantic Model Serving API
- * The Malloy Publisher - Semantic Model Serving API serves Malloy packages.  A Malloy package is a directory of Malloy models (.malloy files), Malloy notebooks (.malloynb files), and embedded datbases (.parque files) with a malloy-publisher.json manifest at the package\'s root directory. For example, see the Malloy samples packages (https://github.com/malloydata/malloy-samples) repo.
+ * The Malloy Publisher - Semantic Model Serving API provides comprehensive access to Malloy packages and their associated resources. A Malloy package is a directory containing Malloy models (.malloy files), Malloy notebooks (.malloynb files), and embedded databases (.parquet files) with a malloy-publisher.json manifest at the package\'s root directory.  ## Key Features  - **Project Management**: Create and manage projects with their associated packages and connections - **Package Lifecycle**: Full CRUD operations for Malloy packages and their versions - **Model & Notebook Access**: Retrieve and execute Malloy models and notebooks - **Connection Management**: Secure database connection configuration and testing - **Query Execution**: Execute queries against models and retrieve results - **Watch Mode**: Real-time file watching for development workflows  ## Resource Hierarchy  The API follows a hierarchical resource structure: ``` Projects ├── Connections └── Packages     ├── Models     ├── Notebooks     └── Databases ```  For examples, see the Malloy samples packages (https://github.com/malloydata/malloy-samples) repository. 
  *
  * The version of the OpenAPI document: v0
  * 

package/src/client/common.ts

@@ -2,7 +2,7 @@
 /* eslint-disable */
 /**
  * Malloy Publisher - Semantic Model Serving API
- * The Malloy Publisher - Semantic Model Serving API serves Malloy packages.  A Malloy package is a directory of Malloy models (.malloy files), Malloy notebooks (.malloynb files), and embedded datbases (.parque files) with a malloy-publisher.json manifest at the package\'s root directory. For example, see the Malloy samples packages (https://github.com/malloydata/malloy-samples) repo.
+ * The Malloy Publisher - Semantic Model Serving API provides comprehensive access to Malloy packages and their associated resources. A Malloy package is a directory containing Malloy models (.malloy files), Malloy notebooks (.malloynb files), and embedded databases (.parquet files) with a malloy-publisher.json manifest at the package\'s root directory.  ## Key Features  - **Project Management**: Create and manage projects with their associated packages and connections - **Package Lifecycle**: Full CRUD operations for Malloy packages and their versions - **Model & Notebook Access**: Retrieve and execute Malloy models and notebooks - **Connection Management**: Secure database connection configuration and testing - **Query Execution**: Execute queries against models and retrieve results - **Watch Mode**: Real-time file watching for development workflows  ## Resource Hierarchy  The API follows a hierarchical resource structure: ``` Projects ├── Connections └── Packages     ├── Models     ├── Notebooks     └── Databases ```  For examples, see the Malloy samples packages (https://github.com/malloydata/malloy-samples) repository. 
  *
  * The version of the OpenAPI document: v0
  * 

package/src/client/configuration.ts

@@ -2,7 +2,7 @@
 /* eslint-disable */
 /**
  * Malloy Publisher - Semantic Model Serving API
- * The Malloy Publisher - Semantic Model Serving API serves Malloy packages.  A Malloy package is a directory of Malloy models (.malloy files), Malloy notebooks (.malloynb files), and embedded datbases (.parque files) with a malloy-publisher.json manifest at the package\'s root directory. For example, see the Malloy samples packages (https://github.com/malloydata/malloy-samples) repo.
+ * The Malloy Publisher - Semantic Model Serving API provides comprehensive access to Malloy packages and their associated resources. A Malloy package is a directory containing Malloy models (.malloy files), Malloy notebooks (.malloynb files), and embedded databases (.parquet files) with a malloy-publisher.json manifest at the package\'s root directory.  ## Key Features  - **Project Management**: Create and manage projects with their associated packages and connections - **Package Lifecycle**: Full CRUD operations for Malloy packages and their versions - **Model & Notebook Access**: Retrieve and execute Malloy models and notebooks - **Connection Management**: Secure database connection configuration and testing - **Query Execution**: Execute queries against models and retrieve results - **Watch Mode**: Real-time file watching for development workflows  ## Resource Hierarchy  The API follows a hierarchical resource structure: ``` Projects ├── Connections └── Packages     ├── Models     ├── Notebooks     └── Databases ```  For examples, see the Malloy samples packages (https://github.com/malloydata/malloy-samples) repository. 
  *
  * The version of the OpenAPI document: v0
  * 

package/src/client/index.ts

@@ -2,7 +2,7 @@
 /* eslint-disable */
 /**
  * Malloy Publisher - Semantic Model Serving API
- * The Malloy Publisher - Semantic Model Serving API serves Malloy packages.  A Malloy package is a directory of Malloy models (.malloy files), Malloy notebooks (.malloynb files), and embedded datbases (.parque files) with a malloy-publisher.json manifest at the package\'s root directory. For example, see the Malloy samples packages (https://github.com/malloydata/malloy-samples) repo.
+ * The Malloy Publisher - Semantic Model Serving API provides comprehensive access to Malloy packages and their associated resources. A Malloy package is a directory containing Malloy models (.malloy files), Malloy notebooks (.malloynb files), and embedded databases (.parquet files) with a malloy-publisher.json manifest at the package\'s root directory.  ## Key Features  - **Project Management**: Create and manage projects with their associated packages and connections - **Package Lifecycle**: Full CRUD operations for Malloy packages and their versions - **Model & Notebook Access**: Retrieve and execute Malloy models and notebooks - **Connection Management**: Secure database connection configuration and testing - **Query Execution**: Execute queries against models and retrieve results - **Watch Mode**: Real-time file watching for development workflows  ## Resource Hierarchy  The API follows a hierarchical resource structure: ``` Projects ├── Connections └── Packages     ├── Models     ├── Notebooks     └── Databases ```  For examples, see the Malloy samples packages (https://github.com/malloydata/malloy-samples) repository.
  *
  * The version of the OpenAPI document: v0
  *

package/src/publisher_connection.integration.spec.ts

@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import * as malloy from '@malloydata/malloy';
+import {describeIfDatabaseAvailable} from '@malloydata/malloy/test';
+import {PublisherConnection} from './publisher_connection';
+import {fileURLToPath} from 'url';
+import * as util from 'util';
+import * as fs from 'fs';
+
+const [describe] = describeIfDatabaseAvailable(['publisher']);
+
+describe.skip('db:Publisher Integration Tests', () => {
+  let conn: PublisherConnection | null;
+  let runtime: malloy.Runtime | null;
+
+  beforeEach(async () => {
+    try {
+      conn = await PublisherConnection.create('bq_demo', {
+        connectionUri:
+          'http://localhost:4000/api/v0/projects/malloy-samples/connections/bq_demo',
+      });
+      const files = {
+        readURL: async (url: URL) => {
+          const filePath = fileURLToPath(url);
+          return await util.promisify(fs.readFile)(filePath, 'utf8');
+        },
+      };
+      runtime = new malloy.Runtime({
+        urlReader: files,
+        connection: conn,
+      });
+    } catch (error) {
+      // Skip tests if connection cannot be established
+      conn = null;
+      runtime = null;
+    }
+  });
+
+  afterEach(async () => {
+    if (conn) {
+      await conn.close();
+    }
+  });
+
+  it('tests the connection', async () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    await conn.test();
+  });
+
+  it('correctly identifies the dialect', () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    expect(conn.dialectName).toBe('standardsql');
+  });
+
+  it('correctly identifies the connection as a pooled connection', () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    expect(conn.isPool()).toBe(false);
+  });
+
+  it('correctly identifies the connection as a streaming connection', () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    expect(conn.canStream()).toBe(true);
+  });
+
+  it('correctly identifies the connection as a persistSQLResults connection', () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    expect(conn.canPersist()).toBe(true);
+  });
+
+  it('fetches the table schema', async () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    const schema = await conn.fetchTableSchema(
+      'ecommerce_bq',
+      'ecommerce_bq.users'
+    );
+    expect(schema.type).toBe('table');
+    expect(schema.dialect).toBe('standardsql');
+    expect(schema.tablePath).toBe('ecommerce_bq.users');
+    expect(schema.fields.length).toBe(14);
+    expect(schema.fields[0].name).toBe('id');
+    expect(schema.fields[0].type).toBe('number');
+  });
+
+  it('fetches the sql source schema', async () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    const schema = await conn.fetchSelectSchema({
+      connection: 'bq_demo',
+      selectStr: 'SELECT * FROM ecommerce_bq.users',
+    });
+    expect(schema.type).toBe('sql_select');
+    expect(schema.dialect).toBe('standardsql');
+    expect(schema.fields.length).toBe(14);
+    expect(schema.fields[0].name).toBe('id');
+    expect(schema.fields[0].type).toBe('number');
+  });
+
+  it('runs a SQL query', async () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    const res = await conn.runSQL('SELECT 1 as T');
+    expect(res.rows[0]['T']).toBe(1);
+  });
+
+  it('runs a Malloy query', async () => {
+    if (!conn || !runtime) {
+      pending('Publisher service not available');
+      return;
+    }
+    const sql = await runtime
+      .loadModel("source: users is bq_demo.table('ecommerce_bq.users')")
+      .loadQuery(
+        'run:  users -> { aggregate: cnt is count() group_by: state order_by: cnt desc limit: 10 }'
+      )
+      .getSQL();
+    const res = await conn.runSQL(sql);
+    expect(res.totalRows).toBe(10);
+    let total = 0;
+    for (const row of res.rows) {
+      total += +(row['cnt'] ?? 0);
+    }
+    expect(total).toBeGreaterThan(0);
+  });
+
+  it('runs a Malloy query on an sql source', async () => {
+    if (!conn || !runtime) {
+      pending('Publisher service not available');
+      return;
+    }
+    const sql = await runtime
+      .loadModel(
+        "source: users is bq_demo.sql('SELECT * FROM ecommerce_bq.users')"
+      )
+      .loadQuery(
+        'run:  users -> { aggregate: cnt is count() group_by: state order_by: cnt desc limit: 20 }'
+      )
+      .getSQL();
+    const res = await conn.runSQL(sql);
+    expect(res.totalRows).toBe(20);
+    expect(res.rows[0]['cnt']).toBeGreaterThan(0);
+  });
+
+  it('get temporary table name', async () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    const sql = 'SELECT 1 as T';
+    const tempTableName = await conn.manifestTemporaryTable(sql);
+    expect(tempTableName).toBeDefined();
+    expect(tempTableName.length).toBeGreaterThan(0);
+  });
+
+  it('estimates query cost', async () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    const cost = await conn.estimateQueryCost(
+      'SELECT * FROM ecommerce_bq.users'
+    );
+    expect(cost).toEqual({});
+  });
+
+  it('closes connection', async () => {
+    if (!conn) {
+      pending('Publisher service not available');
+      return;
+    }
+    // close() should not throw and should return void
+    await expect(conn.close()).resolves.toBeUndefined();
+  });
+});

package/src/publisher_connection.ts

@@ -6,7 +6,6 @@
  */
 
 import type {
-  Connection,
   MalloyQueryData,
   PersistSQLResults,
   PooledConnection,
@@ -20,8 +19,14 @@ import type {
   TestableConnection,
 } from '@malloydata/malloy';
 import {BaseConnection} from '@malloydata/malloy/connection';
-import type {ConnectionAttributes, RawAxiosRequestConfig} from './client';
-import {Configuration, ConnectionsApi} from './client';
+import type {
+  Column,
+  Connection,
+  ConnectionAttributes,
+  PostSqlsourceRequest,
+  RawAxiosRequestConfig,
+} from './client';
+import {Configuration, ConnectionsApi, ConnectionsTestApi} from './client';
 
 interface PublisherConnectionOptions {
   connectionUri: string;
@@ -39,7 +44,9 @@ export class PublisherConnection
   public readonly name: string;
   public readonly projectName: string;
   private connectionsApi: ConnectionsApi;
+  private connectionsTestApi: ConnectionsTestApi;
   private connectionAttributes: ConnectionAttributes;
+  private connectionData: Connection;
   private accessToken: string | undefined;
 
   static async create(name: string, options: PublisherConnectionOptions) {
@@ -66,16 +73,25 @@ export class PublisherConnection
       basePath: apiUrl,
     });
     const connectionsApi = new ConnectionsApi(configuration);
+    const connectionsTestApi = new ConnectionsTestApi(configuration);
     const response = await connectionsApi.getConnection(projectName, name, {
       headers: PublisherConnection.getAuthHeaders(options.accessToken),
     });
-    const connectionAttributes = response.data
-      .attributes as ConnectionAttributes;
+    if (!response || !response.data) {
+      throw new Error(
+        `Failed to get connection: ${name} from project: ${projectName}`
+      );
+    }
+    const connectionData = response.data as Connection;
+    const connectionAttributes =
+      connectionData.attributes as ConnectionAttributes;
     const connection = new PublisherConnection(
       name,
       projectName,
       connectionsApi,
+      connectionsTestApi,
       connectionAttributes,
+      connectionData,
       options.accessToken
     );
     await connection.test();
@@ -94,14 +110,18 @@ export class PublisherConnection
     name: string,
     projectName: string,
     connectionsApi: ConnectionsApi,
+    connectionsTestApi: ConnectionsTestApi,
     connectionAttributes: ConnectionAttributes,
+    connectionData: Connection,
     accessToken: string | undefined
   ) {
     super();
     this.name = name;
     this.projectName = projectName;
     this.connectionsApi = connectionsApi;
+    this.connectionsTestApi = connectionsTestApi;
     this.connectionAttributes = connectionAttributes;
+    this.connectionData = connectionData;
     this.accessToken = accessToken;
   }
 
@@ -125,25 +145,41 @@ export class PublisherConnection
     tableKey: string,
     tablePath: string
   ): Promise<TableSourceDef> {
-    const response = await this.connectionsApi.getTablesource(
+    const response = await this.connectionsApi.getTable(
       this.projectName,
       this.name,
-      tableKey,
+      tablePath.split('/')[0],
       tablePath,
       {
         headers: PublisherConnection.getAuthHeaders(this.accessToken),
       }
     );
-    return JSON.parse(response.data.source as string) as TableSourceDef;
+    // Convert the Table response to TableSourceDef format
+    const tableData = response.data;
+    return {
+      type: 'table',
+      name: tableKey,
+      tablePath: tablePath,
+      connection: this.name,
+      dialect: this.dialectName,
+      fields:
+        tableData.columns?.map((col: Column) => ({
+          name: col.name,
+          type: col.type,
+        })) || [],
+    } as TableSourceDef;
   }
 
   public async fetchSelectSchema(
     sqlRef: SQLSourceRequest
   ): Promise<SQLSourceDef> {
-    const response = await this.connectionsApi.getSqlsource(
+    const request: PostSqlsourceRequest = {
+      sqlStatement: sqlRef.selectStr,
+    };
+    const response = await this.connectionsApi.postSqlsource(
       this.projectName,
       this.name,
-      sqlRef.selectStr,
+      request,
       {
         headers: PublisherConnection.getAuthHeaders(this.accessToken),
       }
@@ -162,10 +198,13 @@ export class PublisherConnection
   ): Promise<MalloyQueryData> {
     // TODO: Add support for abortSignal.
     options.abortSignal = undefined;
-    const response = await this.connectionsApi.getQuerydata(
+    const request: PostSqlsourceRequest = {
+      sqlStatement: sql,
+    };
+    const response = await this.connectionsApi.postQuerydata(
       this.projectName,
       this.name,
-      sql,
+      request,
       JSON.stringify(options),
       {
         headers: PublisherConnection.getAuthHeaders(this.accessToken),
@@ -181,10 +220,13 @@ export class PublisherConnection
     // TODO: Add support for abortSignal.
     options.abortSignal = undefined;
     // TODO: Add real streaming support to publisher API.
-    const response = await this.connectionsApi.getQuerydata(
+    const request: PostSqlsourceRequest = {
+      sqlStatement: sqlCommand,
+    };
+    const response = await this.connectionsApi.postQuerydata(
       this.projectName,
       this.name,
-      sqlCommand,
+      request,
       JSON.stringify(options),
       {
         headers: PublisherConnection.getAuthHeaders(this.accessToken),
@@ -199,16 +241,22 @@ export class PublisherConnection
   }
 
   public async test(): Promise<void> {
-    await this.connectionsApi.getTest(this.projectName, this.name, {
-      headers: PublisherConnection.getAuthHeaders(this.accessToken),
-    });
+    await this.connectionsTestApi.testConnectionConfiguration(
+      this.connectionData,
+      {
+        headers: PublisherConnection.getAuthHeaders(this.accessToken),
+      }
+    );
   }
 
   public async manifestTemporaryTable(sqlCommand: string): Promise<string> {
-    const response = await this.connectionsApi.getTemporarytable(
+    const request: PostSqlsourceRequest = {
+      sqlStatement: sqlCommand,
+    };
+    const response = await this.connectionsApi.postTemporarytable(
       this.projectName,
       this.name,
-      sqlCommand,
+      request,
       {
         headers: PublisherConnection.getAuthHeaders(this.accessToken),
       }