@pgpmjs/export 0.20.1 → 0.20.2

package/esm/export-graphql-meta.js

@@ -6,8 +6,70 @@
  * generate SQL INSERT statements.
  */
 import { Parser } from 'csv-to-pg';
+import { toSnakeCase } from 'inflekt';
 import { META_TABLE_CONFIG } from './export-utils';
-import { buildFieldsFragment, getGraphQLQueryName, graphqlRowToPostgresRow, intervalToPostgres } from './graphql-naming';
+import { buildFieldsFragment, getGraphQLQueryName, getGraphQLTypeName, graphqlRowToPostgresRow, intervalToPostgres, mapGraphQLTypeToFieldType } from './graphql-naming';
+import { lookupByGqlType } from './type-map';
+/**
+ * Discover fields dynamically from a GraphQL type via introspection.
+ * Queries `__type` to enumerate fields and infer their `FieldType` via
+ * `mapGraphQLTypeToFieldType`. Tracks ENUM fields separately so callers
+ * can normalize their CONSTANT_CASE values back to lowercase.
+ *
+ * `typeOverrides` from the config are applied on top for special types
+ * (image, upload, url) that cannot be inferred from the GraphQL type alone.
+ */
+const buildDynamicFieldsFromGraphQL = async (client, tableConfig) => {
+    const emptyResult = { fields: {}, enumFields: new Set() };
+    const typeName = tableConfig.gqlTypeName || getGraphQLTypeName(tableConfig.table);
+    try {
+        const introspectedFields = await client.introspectType(typeName);
+        const dynamicFields = {};
+        const enumFields = new Set();
+        for (const [camelName, typeInfo] of introspectedFields) {
+            const snakeName = toSnakeCase(camelName);
+            // Skip internal GraphQL fields
+            if (camelName.startsWith('__'))
+                continue;
+            // Track enum fields for lowercase normalization (custom inflector uppercases them)
+            if (typeInfo.kind === 'ENUM') {
+                enumFields.add(snakeName);
+            }
+            // Skip non-scalar fields (relations/computed columns like "database" of type Database)
+            // Only SCALAR and ENUM kinds can be selected without sub-field selections
+            // EXCEPTION: types registered as non-SCALAR in PG_TYPE_MAP (e.g. Interval=OBJECT)
+            // are handled via buildFieldsFragment sub-selections and intervalToPostgres conversion
+            if (typeInfo.kind !== 'SCALAR' && typeInfo.kind !== 'ENUM') {
+                const mapEntry = lookupByGqlType(typeInfo.typeName);
+                if (mapEntry && mapEntry.gqlKind !== 'SCALAR') {
+                    dynamicFields[snakeName] = mapEntry.fieldType;
+                    continue;
+                }
+                continue;
+            }
+            dynamicFields[snakeName] = mapGraphQLTypeToFieldType(typeInfo.typeName, typeInfo.list);
+        }
+        // Apply type overrides (e.g., image, upload, url)
+        if (tableConfig.typeOverrides) {
+            for (const [fieldName, fieldType] of Object.entries(tableConfig.typeOverrides)) {
+                if (dynamicFields[fieldName]) {
+                    dynamicFields[fieldName] = fieldType;
+                }
+            }
+        }
+        return { fields: dynamicFields, enumFields };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        if (message.includes('Cannot query field') ||
+            message.includes('is not defined by type') ||
+            message.includes('Unknown field')) {
+            // Type not available in the GraphQL schema — return empty
+            return emptyResult;
+        }
+        throw err;
+    }
+};
 /**
  * Fetch metadata via GraphQL and generate SQL INSERT statements.
  * This is the GraphQL equivalent of exportMeta() in export-meta.ts.
@@ -18,8 +80,12 @@ export const exportGraphQLMeta = async ({ client, database_id }) => {
         const tableConfig = META_TABLE_CONFIG[key];
         if (!tableConfig)
             return;
-        const pgFieldNames = Object.keys(tableConfig.fields);
-        const graphqlFieldsFragment = buildFieldsFragment(pgFieldNames, tableConfig.fields);
+        // Build fields dynamically: either from hardcoded config or via introspection
+        const { fields: configFields, enumFields } = await buildDynamicFieldsFromGraphQL(client, tableConfig);
+        if (Object.keys(configFields).length === 0)
+            return;
+        const pgFieldNames = Object.keys(configFields);
+        const graphqlFieldsFragment = buildFieldsFragment(pgFieldNames, configFields);
         const graphqlQueryName = getGraphQLQueryName(tableConfig.table);
         // The 'database' table is fetched by id, not by database_id
         const condition = key === 'database'
@@ -30,13 +96,30 @@ export const exportGraphQLMeta = async ({ client, database_id }) => {
             if (rows.length > 0) {
                 // Convert camelCase GraphQL keys back to snake_case for the Parser
                 // Also convert interval objects back to Postgres interval strings
+                // and normalize enum values from CONSTANT_CASE back to lowercase
                 const pgRows = rows.map(row => {
                     const pgRow = graphqlRowToPostgresRow(row);
-                    // Convert any interval fields from {seconds, minutes, ...} objects to strings
-                    for (const [fieldName, fieldType] of Object.entries(tableConfig.fields)) {
+                    for (const [fieldName, fieldType] of Object.entries(configFields)) {
+                        // Convert interval fields from {seconds, minutes, ...} objects to strings
                         if (fieldType === 'interval' && pgRow[fieldName] && typeof pgRow[fieldName] === 'object') {
                             pgRow[fieldName] = intervalToPostgres(pgRow[fieldName]);
                         }
+                        // Truncate timestamptz to second precision for parity with SQL flow
+                        // PostGraphile's Datetime scalar preserves full millisecond precision,
+                        // but the pg driver + our SQL flow truncates to .000Z via Date rounding
+                        if (fieldType === 'timestamptz' && typeof pgRow[fieldName] === 'string') {
+                            const d = new Date(pgRow[fieldName]);
+                            if (!isNaN(d.getTime())) {
+                                pgRow[fieldName] = new Date(Math.floor(d.getTime() / 1000) * 1000).toISOString();
+                            }
+                        }
+                    }
+                    // Normalize enum values: custom inflector uppercases them to CONSTANT_CASE,
+                    // but PostgreSQL stores them in lowercase — convert back for parity with SQL flow
+                    for (const fieldName of enumFields) {
+                        if (typeof pgRow[fieldName] === 'string') {
+                            pgRow[fieldName] = pgRow[fieldName].toLowerCase();
+                        }
                     }
                     return pgRow;
                 });
@@ -49,7 +132,7 @@ export const exportGraphQLMeta = async ({ client, database_id }) => {
                     }
                 }
                 const dynamicFields = {};
-                for (const [fieldName, fieldType] of Object.entries(tableConfig.fields)) {
+                for (const [fieldName, fieldType] of Object.entries(configFields)) {
                     if (returnedKeys.has(fieldName)) {
                         dynamicFields[fieldName] = fieldType;
                     }
@@ -133,6 +216,7 @@ export const exportGraphQLMeta = async ({ client, database_id }) => {
         queryAndParse('permissions_module'),
         queryAndParse('limits_module'),
         queryAndParse('levels_module'),
+        queryAndParse('events_module'),
         queryAndParse('users_module'),
         queryAndParse('hierarchy_module'),
         queryAndParse('membership_types_module'),
@@ -142,13 +226,14 @@ export const exportGraphQLMeta = async ({ client, database_id }) => {
         queryAndParse('user_state_module'),
         queryAndParse('profiles_module'),
         queryAndParse('config_secrets_user_module'),
+        queryAndParse('user_credentials_module'),
+        queryAndParse('user_settings_module'),
         queryAndParse('connected_accounts_module'),
         queryAndParse('phone_numbers_module'),
         queryAndParse('crypto_addresses_module'),
         queryAndParse('crypto_auth_module'),
         queryAndParse('field_module'),
         queryAndParse('table_module'),
-        queryAndParse('table_template_module'),
         queryAndParse('secure_table_provision'),
         queryAndParse('uuid_module'),
         queryAndParse('default_ids_module'),
@@ -166,8 +251,21 @@ export const exportGraphQLMeta = async ({ client, database_id }) => {
         queryAndParse('realtime_module'),
         queryAndParse('session_secrets_module'),
         queryAndParse('config_secrets_org_module'),
+        queryAndParse('config_secrets_module'),
+        queryAndParse('i18n_module'),
+        queryAndParse('agent_module'),
+        queryAndParse('function_module'),
+        queryAndParse('namespace_module'),
+        queryAndParse('merkle_store_module'),
+        queryAndParse('graph_module'),
+        queryAndParse('compute_log_module'),
+        queryAndParse('db_usage_module'),
+        queryAndParse('storage_log_module'),
+        queryAndParse('transfer_log_module'),
         queryAndParse('webauthn_auth_module'),
-        queryAndParse('webauthn_credentials_module')
+        queryAndParse('webauthn_credentials_module'),
+        queryAndParse('inference_log_module'),
+        queryAndParse('rate_limit_meters_module')
     ]);
     return sql;
 };

package/esm/export-meta.js

@@ -1,6 +1,6 @@
 import { Parser } from 'csv-to-pg';
 import { getPgPool } from 'pg-cache';
-import { META_TABLE_CONFIG } from './export-utils';
+import { META_TABLE_CONFIG, mapPgTypeToFieldType } from './export-utils';
 /**
  * Query actual columns from information_schema for a given table.
  * Returns a map of column_name -> udt_name (PostgreSQL type).
@@ -19,10 +19,10 @@ const getTableColumns = async (pool, schemaName, tableName) => {
     return columns;
 };
 /**
- * Build dynamic fields config by intersecting the hardcoded config with actual database columns.
- * - Only includes columns that exist in the database
- * - Preserves special type hints from config (image, upload, url) for columns that exist
- * - Infers types from PostgreSQL for columns not in config
+ * Build dynamic fields config from the database via information_schema.
+ * - All fields are derived from `information_schema.columns` + `mapPgTypeToFieldType`.
+ * - `typeOverrides` from the config are applied on top for special types
+ *   (image, upload, url) that cannot be inferred from PG types alone.
  */
 const buildDynamicFields = async (pool, tableConfig) => {
     const actualColumns = await getTableColumns(pool, tableConfig.schema, tableConfig.table);
@@ -31,13 +31,17 @@ const buildDynamicFields = async (pool, tableConfig) => {
         return {};
     }
     const dynamicFields = {};
-    // For each column in the hardcoded config, check if it exists in the database
-    for (const [fieldName, fieldType] of Object.entries(tableConfig.fields)) {
-        if (actualColumns.has(fieldName)) {
-            // Column exists - use the config's type hint (preserves special types like 'image', 'upload', 'url')
-            dynamicFields[fieldName] = fieldType;
+    // Derive all fields from information_schema
+    for (const [columnName, udtName] of actualColumns) {
+        dynamicFields[columnName] = mapPgTypeToFieldType(udtName);
+    }
+    // Apply type overrides (image, upload, url)
+    if (tableConfig.typeOverrides) {
+        for (const [fieldName, fieldType] of Object.entries(tableConfig.typeOverrides)) {
+            if (dynamicFields[fieldName]) {
+                dynamicFields[fieldName] = fieldType;
+            }
         }
-        // If column doesn't exist in database, skip it (this fixes the bug)
     }
     return dynamicFields;
 };
@@ -47,8 +51,9 @@ export const exportMeta = async ({ opts, dbname, database_id }) => {
         database: dbname
     });
     const sql = {};
-    // Cache for dynamically built parsers
+    // Cache for dynamically built parsers and their field configs
     const parsers = {};
+    const parserFields = {};
     // Build parser dynamically by querying actual columns from the database
     const getParser = async (key) => {
         if (parsers[key]) {
@@ -64,6 +69,7 @@ export const exportMeta = async ({ opts, dbname, database_id }) => {
             // No columns found (table doesn't exist or no matching columns)
             return null;
         }
+        parserFields[key] = dynamicFields;
         const parser = new Parser({
             schema: tableConfig.schema,
             table: tableConfig.table,
@@ -81,6 +87,23 @@ export const exportMeta = async ({ opts, dbname, database_id }) => {
             }
             const result = await pool.query(query, [database_id]);
             if (result.rows.length) {
+                // Truncate timestamptz to second precision to match PostGraphile's Datetime scalar
+                // which truncates milliseconds in the GraphQL flow
+                const fields = parserFields[key];
+                if (fields) {
+                    for (const row of result.rows) {
+                        for (const [fieldName, fieldType] of Object.entries(fields)) {
+                            if (fieldType === 'timestamptz') {
+                                const val = row[fieldName];
+                                if (val instanceof Date) {
+                                    // Truncate to second precision and convert to ISO string
+                                    // so both SQL and GraphQL flows pass the same value type to the Parser
+                                    row[fieldName] = new Date(Math.floor(val.getTime() / 1000) * 1000).toISOString();
+                                }
+                            }
+                        }
+                    }
+                }
                 const parsed = await parser.parse(result.rows);
                 if (parsed) {
                     sql[key] = parsed;
@@ -145,6 +168,7 @@ export const exportMeta = async ({ opts, dbname, database_id }) => {
     await queryAndParse('permissions_module', `SELECT * FROM metaschema_modules_public.permissions_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('limits_module', `SELECT * FROM metaschema_modules_public.limits_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('levels_module', `SELECT * FROM metaschema_modules_public.levels_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('events_module', `SELECT * FROM metaschema_modules_public.events_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('users_module', `SELECT * FROM metaschema_modules_public.users_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('hierarchy_module', `SELECT * FROM metaschema_modules_public.hierarchy_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('membership_types_module', `SELECT * FROM metaschema_modules_public.membership_types_module WHERE database_id = $1 ORDER BY id`);
@@ -154,13 +178,14 @@ export const exportMeta = async ({ opts, dbname, database_id }) => {
     await queryAndParse('user_state_module', `SELECT * FROM metaschema_modules_public.user_state_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('profiles_module', `SELECT * FROM metaschema_modules_public.profiles_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('config_secrets_user_module', `SELECT * FROM metaschema_modules_public.config_secrets_user_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('user_credentials_module', `SELECT * FROM metaschema_modules_public.user_credentials_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('user_settings_module', `SELECT * FROM metaschema_modules_public.user_settings_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('connected_accounts_module', `SELECT * FROM metaschema_modules_public.connected_accounts_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('phone_numbers_module', `SELECT * FROM metaschema_modules_public.phone_numbers_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('crypto_addresses_module', `SELECT * FROM metaschema_modules_public.crypto_addresses_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('crypto_auth_module', `SELECT * FROM metaschema_modules_public.crypto_auth_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('field_module', `SELECT * FROM metaschema_modules_public.field_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('table_module', `SELECT * FROM metaschema_modules_public.table_module WHERE database_id = $1 ORDER BY id`);
-    await queryAndParse('table_template_module', `SELECT * FROM metaschema_modules_public.table_template_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('secure_table_provision', `SELECT * FROM metaschema_modules_public.secure_table_provision WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('uuid_module', `SELECT * FROM metaschema_modules_public.uuid_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('default_ids_module', `SELECT * FROM metaschema_modules_public.default_ids_module WHERE database_id = $1 ORDER BY id`);
@@ -178,7 +203,20 @@ export const exportMeta = async ({ opts, dbname, database_id }) => {
     await queryAndParse('realtime_module', `SELECT * FROM metaschema_modules_public.realtime_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('session_secrets_module', `SELECT * FROM metaschema_modules_public.session_secrets_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('config_secrets_org_module', `SELECT * FROM metaschema_modules_public.config_secrets_org_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('config_secrets_module', `SELECT * FROM metaschema_modules_public.config_secrets_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('i18n_module', `SELECT * FROM metaschema_modules_public.i18n_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('agent_module', `SELECT * FROM metaschema_modules_public.agent_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('function_module', `SELECT * FROM metaschema_modules_public.function_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('namespace_module', `SELECT * FROM metaschema_modules_public.namespace_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('merkle_store_module', `SELECT * FROM metaschema_modules_public.merkle_store_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('graph_module', `SELECT * FROM metaschema_modules_public.graph_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('compute_log_module', `SELECT * FROM metaschema_modules_public.compute_log_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('db_usage_module', `SELECT * FROM metaschema_modules_public.db_usage_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('storage_log_module', `SELECT * FROM metaschema_modules_public.storage_log_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('transfer_log_module', `SELECT * FROM metaschema_modules_public.transfer_log_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('webauthn_auth_module', `SELECT * FROM metaschema_modules_public.webauthn_auth_module WHERE database_id = $1 ORDER BY id`);
     await queryAndParse('webauthn_credentials_module', `SELECT * FROM metaschema_modules_public.webauthn_credentials_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('inference_log_module', `SELECT * FROM metaschema_modules_public.inference_log_module WHERE database_id = $1 ORDER BY id`);
+    await queryAndParse('rate_limit_meters_module', `SELECT * FROM metaschema_modules_public.rate_limit_meters_module WHERE database_id = $1 ORDER BY id`);
     return sql;
 };