@malloydata/db-postgres 0.0.434 → 0.0.435

package/dist/postgres_connection.js

@@ -32,6 +32,23 @@ function addTlsHint(err) {
     }
     return err;
 }
+// Closes an unpooled client without making the caller wait for it. In pg 8.7.3,
+// end() resolves on the connection's next 'end' event and never checks whether
+// one already fired, so after the socket has closed on its own (the server or
+// the network dropped it) an awaited end() never returns.
+function closeClient(client) {
+    client.end().catch(() => { });
+}
+// Fails a row stream when the connection under it drops. pg-query-stream 4.2.3
+// tears a failed stream down by closing its cursor, which waits for a server
+// reply that a dead connection never sends, so without this the stream neither
+// ends nor errors. Returns a function that detaches the listener; call it before
+// the client is closed or released.
+function failStreamOnClientError(client, stream) {
+    const onError = (err) => stream.emit('error', err);
+    client.on('error', onError);
+    return () => client.removeListener('error', onError);
+}
 /**
  * Decode a canonical Postgres dotted-table path into its underlying
  * identifier strings as they appear in `information_schema`. The schema
@@ -134,26 +151,36 @@ class PostgresConnection extends connection_1.BaseConnection {
         }
     }
     async getClient() {
-        return new pg_1.Client(this.buildClientConfig(await this.readConfig()));
+        const client = new pg_1.Client(this.buildClientConfig(await this.readConfig()));
+        // When the socket drops, pg emits 'error' on the client whether or not a
+        // query is running. A running query fails on its own (a row stream through
+        // failStreamOnClientError); without this listener the event itself would
+        // be an uncaught exception.
+        client.on('error', () => { });
+        return client;
     }
     async runPostgresQuery(sqlCommand, _pageSize, _rowIndex, deJSON, values) {
         const client = await this.getClient();
-        await this.withTlsHint(() => client.connect());
-        await this.connectionSetup(client);
-        let result = await client.query(sqlCommand, values);
-        if (Array.isArray(result)) {
-            result = result.pop();
-        }
-        if (deJSON) {
-            for (let i = 0; i < result.rows.length; i++) {
-                result.rows[i] = result.rows[i].row;
+        try {
+            await this.withTlsHint(() => client.connect());
+            await this.connectionSetup(client);
+            let result = await client.query(sqlCommand, values);
+            if (Array.isArray(result)) {
+                result = result.pop();
+            }
+            if (deJSON) {
+                for (let i = 0; i < result.rows.length; i++) {
+                    result.rows[i] = result.rows[i].row;
+                }
             }
+            return {
+                rows: result.rows,
+                totalRows: result.rows.length,
+            };
+        }
+        finally {
+            closeClient(client);
         }
-        await client.end();
-        return {
-            rows: result.rows,
-            totalRows: result.rows.length,
-        };
     }
     async fetchSelectSchema(sqlRef) {
         const structDef = {
@@ -164,23 +191,24 @@ class PostgresConnection extends connection_1.BaseConnection {
             name: (0, malloy_1.sqlKey)(sqlRef.connection, sqlRef.selectStr),
         };
         const client = await this.getClient();
-        await this.withTlsHint(() => client.connect());
-        await this.connectionSetup(client);
-        // 1) Get row-descriptor without fetching data
-        const res = await client.query({
-            text: `SELECT * FROM (${sqlRef.selectStr}) _t LIMIT 0`,
-        });
-        // 2) Resolve every OID we might touch (field, array element, domain base)
-        const neededOids = new Set();
-        res.fields.forEach(f => neededOids.add(f.dataTypeID));
-        // we'll add more OIDs later (typelem / typebasetype) lazily
-        // helper to fetch pg_type rows on demand, with cache
-        const pgTypeCache = new Map();
-        const loadTypes = async (oids) => {
-            if (oids.length === 0)
-                return;
-            const params = oids.map((_, i) => `$${i + 1}`).join(',');
-            const { rows } = await client.query(`
+        try {
+            await this.withTlsHint(() => client.connect());
+            await this.connectionSetup(client);
+            // 1) Get row-descriptor without fetching data
+            const res = await client.query({
+                text: `SELECT * FROM (${sqlRef.selectStr}) _t LIMIT 0`,
+            });
+            // 2) Resolve every OID we might touch (field, array element, domain base)
+            const neededOids = new Set();
+            res.fields.forEach(f => neededOids.add(f.dataTypeID));
+            // we'll add more OIDs later (typelem / typebasetype) lazily
+            // helper to fetch pg_type rows on demand, with cache
+            const pgTypeCache = new Map();
+            const loadTypes = async (oids) => {
+                if (oids.length === 0)
+                    return;
+                const params = oids.map((_, i) => `$${i + 1}`).join(',');
+                const { rows } = await client.query(`
       SELECT
         oid,
         typname,
@@ -192,65 +220,68 @@ class PostgresConnection extends connection_1.BaseConnection {
       FROM pg_type
       WHERE oid IN (${params})
       `, oids);
-            rows.forEach(r => pgTypeCache.set(r.oid, r));
-        };
-        // Prime the cache
-        await loadTypes([...neededOids]);
-        // 3) recursive mapper → info-schema compliant strings
-        const mapDataType = async (oid) => {
-            let t = pgTypeCache.get(oid);
-            if (!t) {
-                await loadTypes([oid]);
-                t = pgTypeCache.get(oid);
+                rows.forEach(r => pgTypeCache.set(r.oid, r));
+            };
+            // Prime the cache
+            await loadTypes([...neededOids]);
+            // 3) recursive mapper → info-schema compliant strings
+            const mapDataType = async (oid) => {
+                let t = pgTypeCache.get(oid);
+                if (!t) {
+                    await loadTypes([oid]);
+                    t = pgTypeCache.get(oid);
+                }
+                // ARRAY?
+                if (t.typcategory === 'A')
+                    return 'ARRAY';
+                // DOMAIN?  recurse to its base type
+                if (t.typtype === 'd')
+                    return mapDataType(t.typbasetype);
+                // ENUM, COMPOSITE, RANGE, MULTIRANGE, PSEUDO  → USER-DEFINED
+                if (['e', 'c', 'r', 'm', 'p'].includes(t.typtype))
+                    return 'USER-DEFINED';
+                // built-in scalar or base type of domain
+                return t.formatted;
+            };
+            // helper to resolve element_type (NULL for scalars)
+            const mapElementType = async (oid) => {
+                let t = pgTypeCache.get(oid);
+                if (!t) {
+                    await loadTypes([oid]);
+                    t = pgTypeCache.get(oid);
+                }
+                if (t.typcategory !== 'A')
+                    return null; // not an array
+                // Ensure element row cached
+                if (!pgTypeCache.has(t.typelem))
+                    await loadTypes([t.typelem]);
+                return mapDataType(t.typelem);
+            };
+            // 4) Build final array in original column order
+            const result = [];
+            for (const field of res.fields) {
+                result.push({
+                    columnName: field.name,
+                    dataType: await mapDataType(field.dataTypeID),
+                    elementType: await mapElementType(field.dataTypeID),
+                });
             }
-            // ARRAY?
-            if (t.typcategory === 'A')
-                return 'ARRAY';
-            // DOMAIN?  recurse to its base type
-            if (t.typtype === 'd')
-                return mapDataType(t.typbasetype);
-            // ENUM, COMPOSITE, RANGE, MULTIRANGE, PSEUDO  → USER-DEFINED
-            if (['e', 'c', 'r', 'm', 'p'].includes(t.typtype))
-                return 'USER-DEFINED';
-            // built-in scalar or base type of domain
-            return t.formatted;
-        };
-        // helper to resolve element_type (NULL for scalars)
-        const mapElementType = async (oid) => {
-            let t = pgTypeCache.get(oid);
-            if (!t) {
-                await loadTypes([oid]);
-                t = pgTypeCache.get(oid);
+            for (const row of result) {
+                const postgresDataType = row.dataType;
+                const name = row.columnName;
+                if (postgresDataType === 'ARRAY') {
+                    const elementType = this.dialect.sqlTypeToMalloyType(row.elementType);
+                    structDef.fields.push((0, malloy_1.mkArrayDef)(elementType, name));
+                }
+                else {
+                    const malloyType = this.dialect.sqlTypeToMalloyType(postgresDataType);
+                    structDef.fields.push({ ...malloyType, name });
+                }
             }
-            if (t.typcategory !== 'A')
-                return null; // not an array
-            // Ensure element row cached
-            if (!pgTypeCache.has(t.typelem))
-                await loadTypes([t.typelem]);
-            return mapDataType(t.typelem);
-        };
-        // 4) Build final array in original column order
-        const result = [];
-        for (const field of res.fields) {
-            result.push({
-                columnName: field.name,
-                dataType: await mapDataType(field.dataTypeID),
-                elementType: await mapElementType(field.dataTypeID),
-            });
         }
-        for (const row of result) {
-            const postgresDataType = row.dataType;
-            const name = row.columnName;
-            if (postgresDataType === 'ARRAY') {
-                const elementType = this.dialect.sqlTypeToMalloyType(row.elementType);
-                structDef.fields.push((0, malloy_1.mkArrayDef)(elementType, name));
-            }
-            else {
-                const malloyType = this.dialect.sqlTypeToMalloyType(postgresDataType);
-                structDef.fields.push({ ...malloyType, name });
-            }
+        finally {
+            closeClient(client);
         }
-        await client.end();
         return structDef;
     }
     async schemaFromQuery(infoQuery, structDef, values) {
@@ -328,20 +359,27 @@ class PostgresConnection extends connection_1.BaseConnection {
         const { rowLimit, abortSignal } = options;
         const query = new pg_query_stream_1.default(this.sqlWithQueryMetadata(sqlCommand, options.queryMetadata));
         const client = await this.getClient();
-        await this.withTlsHint(() => client.connect());
-        await this.connectionSetup(client);
-        const rowStream = client.query(query);
-        let index = 0;
-        for await (const row of rowStream) {
-            yield row.row;
-            index += 1;
-            if ((rowLimit !== undefined && index >= rowLimit) ||
-                (abortSignal === null || abortSignal === void 0 ? void 0 : abortSignal.aborted)) {
-                query.destroy();
-                break;
+        let detach = () => { };
+        try {
+            await this.withTlsHint(() => client.connect());
+            await this.connectionSetup(client);
+            const rowStream = client.query(query);
+            detach = failStreamOnClientError(client, rowStream);
+            let index = 0;
+            for await (const row of rowStream) {
+                yield row.row;
+                index += 1;
+                if ((rowLimit !== undefined && index >= rowLimit) ||
+                    (abortSignal === null || abortSignal === void 0 ? void 0 : abortSignal.aborted)) {
+                    query.destroy();
+                    break;
+                }
             }
         }
-        await client.end();
+        finally {
+            detach();
+            closeClient(client);
+        }
     }
     async estimateQueryCost(_) {
         return {};
@@ -378,6 +416,11 @@ class PooledPostgresConnection extends PostgresConnection {
     async getPool() {
         if (!this._pool) {
             this._pool = new pg_1.Pool(this.buildClientConfig(await this.readConfig()));
+            // An idle client whose socket drops reaches the pool as an 'error' event.
+            // pg-pool has already removed the dead client by then, and there is no
+            // caller to report to, so there is nothing to act on; without a listener
+            // the event would be an uncaught exception.
+            this._pool.on('error', () => { });
             this._pool.on('acquire', client => {
                 client.query("SET TIME ZONE 'UTC'");
                 if (this.setupSQL) {
@@ -418,17 +461,42 @@ class PooledPostgresConnection extends PostgresConnection {
         // `client.query(query)`, which does what it's supposed to.
         const pool = await this.getPool();
         const client = await this.withTlsHint(() => pool.connect());
-        const resultStream = client.query(query);
-        for await (const row of resultStream) {
-            yield row.row;
-            index += 1;
-            if ((rowLimit !== undefined && index >= rowLimit) ||
-                (abortSignal === null || abortSignal === void 0 ? void 0 : abortSignal.aborted)) {
-                query.destroy();
-                break;
+        // pg-pool takes its own 'error' listener off a client while it is checked
+        // out, so this listener is also what keeps a dropped connection from
+        // surfacing as an uncaught exception.
+        const detach = failStreamOnClientError(client, query);
+        let drained = false;
+        try {
+            const resultStream = client.query(query);
+            for await (const row of resultStream) {
+                yield row.row;
+                index += 1;
+                if ((rowLimit !== undefined && index >= rowLimit) ||
+                    (abortSignal === null || abortSignal === void 0 ? void 0 : abortSignal.aborted)) {
+                    // The rows of a result that fits in one batch arrive with its end.
+                    // Once pg-cursor 2.7.3 has that end its state is 'done': no fetch is
+                    // in flight and closing it sends nothing, so the client can be reused.
+                    drained = query.cursor.state === 'done';
+                    query.destroy();
+                    return;
+                }
             }
+            drained = true;
+        }
+        finally {
+            // release(), not end(): this client came from pool.connect(), so it goes
+            // back to the pool rather than closing its session. Nothing else returns
+            // it on a throw or an early consumer exit, and a client that is never
+            // released holds one of the pool's `max` slots for good.
+            //
+            // A stream that stopped before its end can leave a fetch in flight on
+            // the client, so it is released with an error, which makes the pool
+            // discard it instead of handing the next caller a client mid-query.
+            detach();
+            client.release(drained
+                ? undefined
+                : new Error('row stream stopped before reading all its rows'));
         }
-        client.release();
     }
     async close() {
         await this.drain();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-postgres",
-  "version": "0.0.434",
+  "version": "0.0.435",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -15,7 +15,7 @@
   "scripts": {
     "lint": "eslint '**/*.ts{,x}'",
     "lint-fix": "eslint '**/*.ts{,x}' --fix",
-    "test": "jest --config=../../jest.config.js",
+    "test": "cd ../.. && ts-node scripts/check_dialect_connection.ts postgres && jest packages/malloy-db-postgres",
     "build": "tsc --build",
     "dev": "tsc --build",
     "clean": "tsc --build --clean && rm -f tsconfig.tsbuildinfo",
@@ -23,7 +23,7 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "@malloydata/malloy": "0.0.434",
+    "@malloydata/malloy": "0.0.435",
     "@types/pg": "^8.6.1",
     "pg": "8.7.3",
     "pg-query-stream": "4.2.3"