@pgpmjs/core 4.13.0 → 4.13.2

package/esm/export/export-meta.js

@@ -1,5 +1,82 @@
 import { Parser } from 'csv-to-pg';
 import { getPgPool } from 'pg-cache';
+/**
+ * Map PostgreSQL data types to FieldType values.
+ * Uses udt_name from information_schema which gives the base type name.
+ */
+const mapPgTypeToFieldType = (udtName) => {
+    switch (udtName) {
+        case 'uuid':
+            return 'uuid';
+        case '_uuid':
+            return 'uuid[]';
+        case 'text':
+        case 'varchar':
+        case 'bpchar':
+        case 'name':
+            return 'text';
+        case '_text':
+        case '_varchar':
+            return 'text[]';
+        case 'bool':
+            return 'boolean';
+        case 'jsonb':
+        case 'json':
+            return 'jsonb';
+        case 'int4':
+        case 'int8':
+        case 'int2':
+        case 'numeric':
+            return 'int';
+        case 'interval':
+            return 'interval';
+        case 'timestamptz':
+        case 'timestamp':
+            return 'timestamptz';
+        default:
+            return 'text';
+    }
+};
+/**
+ * Query actual columns from information_schema for a given table.
+ * Returns a map of column_name -> udt_name (PostgreSQL type).
+ */
+const getTableColumns = async (pool, schemaName, tableName) => {
+    const result = await pool.query(`
+    SELECT column_name, udt_name
+    FROM information_schema.columns
+    WHERE table_schema = $1 AND table_name = $2
+    ORDER BY ordinal_position
+  `, [schemaName, tableName]);
+    const columns = new Map();
+    for (const row of result.rows) {
+        columns.set(row.column_name, row.udt_name);
+    }
+    return columns;
+};
+/**
+ * Build dynamic fields config by intersecting the hardcoded config with actual database columns.
+ * - Only includes columns that exist in the database
+ * - Preserves special type hints from config (image, upload, url) for columns that exist
+ * - Infers types from PostgreSQL for columns not in config
+ */
+const buildDynamicFields = async (pool, tableConfig) => {
+    const actualColumns = await getTableColumns(pool, tableConfig.schema, tableConfig.table);
+    if (actualColumns.size === 0) {
+        // Table doesn't exist, return empty fields
+        return {};
+    }
+    const dynamicFields = {};
+    // For each column in the hardcoded config, check if it exists in the database
+    for (const [fieldName, fieldType] of Object.entries(tableConfig.fields)) {
+        if (actualColumns.has(fieldName)) {
+            // Column exists - use the config's type hint (preserves special types like 'image', 'upload', 'url')
+            dynamicFields[fieldName] = fieldType;
+        }
+        // If column doesn't exist in database, skip it (this fixes the bug)
+    }
+    return dynamicFields;
+};
 const config = {
     // =============================================================================
     // metaschema_public tables
@@ -771,15 +848,41 @@ export const exportMeta = async ({ opts, dbname, database_id }) => {
         database: dbname
     });
     const sql = {};
-    const parsers = Object.entries(config).reduce((m, [name, config]) => {
-        m[name] = new Parser(config);
-        return m;
-    }, {});
+    // Cache for dynamically built parsers
+    const parsers = {};
+    // Build parser dynamically by querying actual columns from the database
+    const getParser = async (key) => {
+        if (parsers[key]) {
+            return parsers[key];
+        }
+        const tableConfig = config[key];
+        if (!tableConfig) {
+            return null;
+        }
+        // Build fields dynamically based on actual database columns
+        const dynamicFields = await buildDynamicFields(pool, tableConfig);
+        if (Object.keys(dynamicFields).length === 0) {
+            // No columns found (table doesn't exist or no matching columns)
+            return null;
+        }
+        const parser = new Parser({
+            schema: tableConfig.schema,
+            table: tableConfig.table,
+            conflictDoNothing: tableConfig.conflictDoNothing,
+            fields: dynamicFields
+        });
+        parsers[key] = parser;
+        return parser;
+    };
     const queryAndParse = async (key, query) => {
         try {
+            const parser = await getParser(key);
+            if (!parser) {
+                return;
+            }
             const result = await pool.query(query, [database_id]);
             if (result.rows.length) {
-                const parsed = await parsers[key].parse(result.rows);
+                const parsed = await parser.parse(result.rows);
                 if (parsed) {
                     sql[key] = parsed;
                 }

package/export/export-meta.js

@@ -3,6 +3,83 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.exportMeta = void 0;
 const csv_to_pg_1 = require("csv-to-pg");
 const pg_cache_1 = require("pg-cache");
+/**
+ * Map PostgreSQL data types to FieldType values.
+ * Uses udt_name from information_schema which gives the base type name.
+ */
+const mapPgTypeToFieldType = (udtName) => {
+    switch (udtName) {
+        case 'uuid':
+            return 'uuid';
+        case '_uuid':
+            return 'uuid[]';
+        case 'text':
+        case 'varchar':
+        case 'bpchar':
+        case 'name':
+            return 'text';
+        case '_text':
+        case '_varchar':
+            return 'text[]';
+        case 'bool':
+            return 'boolean';
+        case 'jsonb':
+        case 'json':
+            return 'jsonb';
+        case 'int4':
+        case 'int8':
+        case 'int2':
+        case 'numeric':
+            return 'int';
+        case 'interval':
+            return 'interval';
+        case 'timestamptz':
+        case 'timestamp':
+            return 'timestamptz';
+        default:
+            return 'text';
+    }
+};
+/**
+ * Query actual columns from information_schema for a given table.
+ * Returns a map of column_name -> udt_name (PostgreSQL type).
+ */
+const getTableColumns = async (pool, schemaName, tableName) => {
+    const result = await pool.query(`
+    SELECT column_name, udt_name
+    FROM information_schema.columns
+    WHERE table_schema = $1 AND table_name = $2
+    ORDER BY ordinal_position
+  `, [schemaName, tableName]);
+    const columns = new Map();
+    for (const row of result.rows) {
+        columns.set(row.column_name, row.udt_name);
+    }
+    return columns;
+};
+/**
+ * Build dynamic fields config by intersecting the hardcoded config with actual database columns.
+ * - Only includes columns that exist in the database
+ * - Preserves special type hints from config (image, upload, url) for columns that exist
+ * - Infers types from PostgreSQL for columns not in config
+ */
+const buildDynamicFields = async (pool, tableConfig) => {
+    const actualColumns = await getTableColumns(pool, tableConfig.schema, tableConfig.table);
+    if (actualColumns.size === 0) {
+        // Table doesn't exist, return empty fields
+        return {};
+    }
+    const dynamicFields = {};
+    // For each column in the hardcoded config, check if it exists in the database
+    for (const [fieldName, fieldType] of Object.entries(tableConfig.fields)) {
+        if (actualColumns.has(fieldName)) {
+            // Column exists - use the config's type hint (preserves special types like 'image', 'upload', 'url')
+            dynamicFields[fieldName] = fieldType;
+        }
+        // If column doesn't exist in database, skip it (this fixes the bug)
+    }
+    return dynamicFields;
+};
 const config = {
     // =============================================================================
     // metaschema_public tables
@@ -774,15 +851,41 @@ const exportMeta = async ({ opts, dbname, database_id }) => {
         database: dbname
     });
     const sql = {};
-    const parsers = Object.entries(config).reduce((m, [name, config]) => {
-        m[name] = new csv_to_pg_1.Parser(config);
-        return m;
-    }, {});
+    // Cache for dynamically built parsers
+    const parsers = {};
+    // Build parser dynamically by querying actual columns from the database
+    const getParser = async (key) => {
+        if (parsers[key]) {
+            return parsers[key];
+        }
+        const tableConfig = config[key];
+        if (!tableConfig) {
+            return null;
+        }
+        // Build fields dynamically based on actual database columns
+        const dynamicFields = await buildDynamicFields(pool, tableConfig);
+        if (Object.keys(dynamicFields).length === 0) {
+            // No columns found (table doesn't exist or no matching columns)
+            return null;
+        }
+        const parser = new csv_to_pg_1.Parser({
+            schema: tableConfig.schema,
+            table: tableConfig.table,
+            conflictDoNothing: tableConfig.conflictDoNothing,
+            fields: dynamicFields
+        });
+        parsers[key] = parser;
+        return parser;
+    };
     const queryAndParse = async (key, query) => {
         try {
+            const parser = await getParser(key);
+            if (!parser) {
+                return;
+            }
             const result = await pool.query(query, [database_id]);
             if (result.rows.length) {
-                const parsed = await parsers[key].parse(result.rows);
+                const parsed = await parser.parse(result.rows);
                 if (parsed) {
                     sql[key] = parsed;
                 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pgpmjs/core",
-  "version": "4.13.0",
+  "version": "4.13.2",
   "author": "Constructive <developers@constructive.io>",
   "description": "PGPM Package and Migration Tools",
   "main": "index.js",
@@ -52,7 +52,7 @@
     "@pgpmjs/logger": "^1.3.7",
     "@pgpmjs/server-utils": "^2.8.14",
     "@pgpmjs/types": "^2.14.0",
-    "csv-to-pg": "^3.4.0",
+    "csv-to-pg": "^3.4.1",
     "genomic": "^5.2.3",
     "glob": "^13.0.0",
     "komoji": "^0.7.14",
@@ -64,5 +64,5 @@
     "pgsql-parser": "^17.9.11",
     "yanse": "^0.1.11"
   },
-  "gitHead": "eff12e839462a1d5435f04318db1f44fc141e235"
+  "gitHead": "6a66f8849db582924e8a24e938a6a38ae3122467"
 }