@genesislcap/pbc-reporting-ui 14.353.3 → 14.353.4

package/src/new/utils/transformers.test.ts

@@ -0,0 +1,487 @@
+import { assert, sinon, suite } from '@genesislcap/foundation-testing';
+import type { Genesis } from '../types';
+import { buildDatasourceName, transformFromServerPayload } from './transformers';
+
+const TransformFromServerPayload = suite('transformFromServerPayload');
+
+TransformFromServerPayload.before.each(() => {
+  sinon.restore();
+});
+
+TransformFromServerPayload('cleans up orphaned fields from INCLUDE_COLUMNS', async () => {
+  const mockGetSchema = sinon.stub();
+  const validSchema: Genesis.JSONSchema7 = {
+    properties: {
+      COUNTERPARTY_NAME: { genesisType: 'STRING' },
+      AMOUNT: { genesisType: 'DOUBLE' },
+    },
+  };
+  mockGetSchema.resolves(validSchema);
+
+  const payload: Genesis.ServerReportConfig = {
+    ID: 'test-id',
+    NAME: 'Test Report',
+    SCHEDULES: [],
+    DATA_SOURCES: [
+      {
+        KEY: 'COUNTERPARTY',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'COUNTERPARTY',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: {
+          INCLUDE_COLUMNS: ['COUNTERPARTY_NAME', 'AMOUNT', 'NAME'], // NAME is orphaned
+        },
+      },
+    ],
+    DESTINATION_IDS: [],
+    OUTPUT_DIRECTORY: '/reports',
+  };
+
+  const result = await transformFromServerPayload(payload, mockGetSchema);
+
+  const datasourceKey = buildDatasourceName('COUNTERPARTY', 'REQ_REP');
+  const cleanedConfig = result.datasourceConfig[datasourceKey].TRANSFORMER_CONFIGURATION;
+
+  assert.equal(cleanedConfig.INCLUDE_COLUMNS?.length, 2);
+  assert.ok(cleanedConfig.INCLUDE_COLUMNS?.includes('COUNTERPARTY_NAME'));
+  assert.ok(cleanedConfig.INCLUDE_COLUMNS?.includes('AMOUNT'));
+  assert.ok(!cleanedConfig.INCLUDE_COLUMNS?.includes('NAME'));
+});
+
+TransformFromServerPayload('cleans up orphaned fields from COLUMN_RENAMES', async () => {
+  const mockGetSchema = sinon.stub();
+  const validSchema: Genesis.JSONSchema7 = {
+    properties: {
+      COUNTERPARTY_NAME: { genesisType: 'STRING' },
+      AMOUNT: { genesisType: 'DOUBLE' },
+    },
+  };
+  mockGetSchema.resolves(validSchema);
+
+  const payload: Genesis.ServerReportConfig = {
+    ID: 'test-id',
+    NAME: 'Test Report',
+    SCHEDULES: [],
+    DATA_SOURCES: [
+      {
+        KEY: 'COUNTERPARTY',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'COUNTERPARTY',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: {
+          COLUMN_RENAMES: {
+            COUNTERPARTY_NAME: 'Counterparty Name',
+            AMOUNT: 'Amount',
+            NAME: 'Old Name', // orphaned
+          },
+        },
+      },
+    ],
+    DESTINATION_IDS: [],
+    OUTPUT_DIRECTORY: '/reports',
+  };
+
+  const result = await transformFromServerPayload(payload, mockGetSchema);
+
+  const datasourceKey = buildDatasourceName('COUNTERPARTY', 'REQ_REP');
+  const cleanedConfig = result.datasourceConfig[datasourceKey].TRANSFORMER_CONFIGURATION;
+
+  assert.equal(Object.keys(cleanedConfig.COLUMN_RENAMES || {}).length, 2);
+  assert.equal(cleanedConfig.COLUMN_RENAMES?.['COUNTERPARTY_NAME'], 'Counterparty Name');
+  assert.equal(cleanedConfig.COLUMN_RENAMES?.['AMOUNT'], 'Amount');
+  assert.ok(!cleanedConfig.COLUMN_RENAMES?.['NAME']);
+});
+
+TransformFromServerPayload('cleans up orphaned fields from COLUMN_FORMATS', async () => {
+  const mockGetSchema = sinon.stub();
+  const validSchema: Genesis.JSONSchema7 = {
+    properties: {
+      COUNTERPARTY_NAME: { genesisType: 'STRING' },
+      AMOUNT: { genesisType: 'DOUBLE' },
+    },
+  };
+  mockGetSchema.resolves(validSchema);
+
+  const payload: Genesis.ServerReportConfig = {
+    ID: 'test-id',
+    NAME: 'Test Report',
+    SCHEDULES: [],
+    DATA_SOURCES: [
+      {
+        KEY: 'COUNTERPARTY',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'COUNTERPARTY',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: {
+          COLUMN_FORMATS: {
+            COUNTERPARTY_NAME: 'text',
+            AMOUNT: '#,##0.00',
+            NAME: 'text', // orphaned
+          },
+        },
+      },
+    ],
+    DESTINATION_IDS: [],
+    OUTPUT_DIRECTORY: '/reports',
+  };
+
+  const result = await transformFromServerPayload(payload, mockGetSchema);
+
+  const datasourceKey = buildDatasourceName('COUNTERPARTY', 'REQ_REP');
+  const cleanedConfig = result.datasourceConfig[datasourceKey].TRANSFORMER_CONFIGURATION;
+
+  assert.equal(Object.keys(cleanedConfig.COLUMN_FORMATS || {}).length, 2);
+  assert.equal(cleanedConfig.COLUMN_FORMATS?.['COUNTERPARTY_NAME'], 'text');
+  assert.equal(cleanedConfig.COLUMN_FORMATS?.['AMOUNT'], '#,##0.00');
+  assert.ok(!cleanedConfig.COLUMN_FORMATS?.['NAME']);
+});
+
+TransformFromServerPayload('cleans up orphaned fields from COLUMN_TRANSFORMS', async () => {
+  const mockGetSchema = sinon.stub();
+  const validSchema: Genesis.JSONSchema7 = {
+    properties: {
+      COUNTERPARTY_NAME: { genesisType: 'STRING' },
+      AMOUNT: { genesisType: 'DOUBLE' },
+    },
+  };
+  mockGetSchema.resolves(validSchema);
+
+  const payload: Genesis.ServerReportConfig = {
+    ID: 'test-id',
+    NAME: 'Test Report',
+    SCHEDULES: [],
+    DATA_SOURCES: [
+      {
+        KEY: 'COUNTERPARTY',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'COUNTERPARTY',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: {
+          COLUMN_TRANSFORMS: {
+            COUNTERPARTY_NAME: { TYPE: 'BINARY_EXPRESSION', LEFT: {}, OPERATION: 'AND', RIGHT: {} },
+            AMOUNT: { TYPE: 'METHOD_EXPRESSION', METHOD: 'TRIM', PARAMETERS: [] },
+            NAME: { TYPE: 'BINARY_EXPRESSION', LEFT: {}, OPERATION: 'AND', RIGHT: {} }, // orphaned
+          } as any,
+        },
+      },
+    ],
+    DESTINATION_IDS: [],
+    OUTPUT_DIRECTORY: '/reports',
+  };
+
+  const result = await transformFromServerPayload(payload, mockGetSchema);
+
+  const datasourceKey = buildDatasourceName('COUNTERPARTY', 'REQ_REP');
+  const cleanedConfig = result.datasourceConfig[datasourceKey].TRANSFORMER_CONFIGURATION;
+
+  assert.equal(Object.keys(cleanedConfig.COLUMN_TRANSFORMS || {}).length, 2);
+  assert.ok(cleanedConfig.COLUMN_TRANSFORMS?.['COUNTERPARTY_NAME']);
+  assert.ok(cleanedConfig.COLUMN_TRANSFORMS?.['AMOUNT']);
+  assert.ok(!cleanedConfig.COLUMN_TRANSFORMS?.['NAME']);
+});
+
+TransformFromServerPayload('handles schema fetch failure gracefully', async () => {
+  const mockGetSchema = sinon.stub();
+  const consoleWarnStub = sinon.stub(console, 'warn');
+  mockGetSchema.rejects(new Error('Schema fetch failed'));
+
+  const originalConfig = {
+    INCLUDE_COLUMNS: ['NAME', 'AMOUNT'],
+    COLUMN_RENAMES: { NAME: 'Name' },
+  };
+
+  const payload: Genesis.ServerReportConfig = {
+    ID: 'test-id',
+    NAME: 'Test Report',
+    SCHEDULES: [],
+    DATA_SOURCES: [
+      {
+        KEY: 'COUNTERPARTY',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'COUNTERPARTY',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: originalConfig,
+      },
+    ],
+    DESTINATION_IDS: [],
+    OUTPUT_DIRECTORY: '/reports',
+  };
+
+  const result = await transformFromServerPayload(payload, mockGetSchema);
+
+  const datasourceKey = buildDatasourceName('COUNTERPARTY', 'REQ_REP');
+  const cleanedConfig = result.datasourceConfig[datasourceKey].TRANSFORMER_CONFIGURATION;
+
+  // Should keep original config when schema fetch fails
+  assert.equal(JSON.stringify(cleanedConfig), JSON.stringify(originalConfig));
+  assert.ok(consoleWarnStub.calledOnce);
+  assert.ok(consoleWarnStub.calledWithMatch(sinon.match.string, sinon.match.instanceOf(Error)));
+
+  consoleWarnStub.restore();
+});
+
+TransformFromServerPayload('handles multiple datasources independently', async () => {
+  const mockGetSchema = sinon.stub();
+  mockGetSchema
+    .onFirstCall()
+    .resolves({
+      properties: {
+        COUNTERPARTY_NAME: { genesisType: 'STRING' },
+      },
+    } as Genesis.JSONSchema7)
+    .onSecondCall()
+    .resolves({
+      properties: {
+        TRADE_ID: { genesisType: 'STRING' },
+      },
+    } as Genesis.JSONSchema7);
+
+  const payload: Genesis.ServerReportConfig = {
+    ID: 'test-id',
+    NAME: 'Test Report',
+    SCHEDULES: [],
+    DATA_SOURCES: [
+      {
+        KEY: 'COUNTERPARTY',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'COUNTERPARTY',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: {
+          INCLUDE_COLUMNS: ['COUNTERPARTY_NAME', 'NAME'], // NAME is orphaned
+        },
+      },
+      {
+        KEY: 'TRADE',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'TRADE',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: {
+          INCLUDE_COLUMNS: ['TRADE_ID', 'OLD_FIELD'], // OLD_FIELD is orphaned
+        },
+      },
+    ],
+    DESTINATION_IDS: [],
+    OUTPUT_DIRECTORY: '/reports',
+  };
+
+  const result = await transformFromServerPayload(payload, mockGetSchema);
+
+  const counterpartyKey = buildDatasourceName('COUNTERPARTY', 'REQ_REP');
+  const tradeKey = buildDatasourceName('TRADE', 'REQ_REP');
+
+  assert.equal(
+    result.datasourceConfig[counterpartyKey].TRANSFORMER_CONFIGURATION.INCLUDE_COLUMNS?.length,
+    1,
+  );
+  assert.ok(
+    result.datasourceConfig[counterpartyKey].TRANSFORMER_CONFIGURATION.INCLUDE_COLUMNS?.includes(
+      'COUNTERPARTY_NAME',
+    ),
+  );
+
+  assert.equal(
+    result.datasourceConfig[tradeKey].TRANSFORMER_CONFIGURATION.INCLUDE_COLUMNS?.length,
+    1,
+  );
+  assert.ok(
+    result.datasourceConfig[tradeKey].TRANSFORMER_CONFIGURATION.INCLUDE_COLUMNS?.includes(
+      'TRADE_ID',
+    ),
+  );
+});
+
+TransformFromServerPayload('handles empty transformer configuration', async () => {
+  const mockGetSchema = sinon.stub();
+  const validSchema: Genesis.JSONSchema7 = {
+    properties: {
+      COUNTERPARTY_NAME: { genesisType: 'STRING' },
+    },
+  };
+  mockGetSchema.resolves(validSchema);
+
+  const payload: Genesis.ServerReportConfig = {
+    ID: 'test-id',
+    NAME: 'Test Report',
+    SCHEDULES: [],
+    DATA_SOURCES: [
+      {
+        KEY: 'COUNTERPARTY',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'COUNTERPARTY',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: {},
+      },
+    ],
+    DESTINATION_IDS: [],
+    OUTPUT_DIRECTORY: '/reports',
+  };
+
+  const result = await transformFromServerPayload(payload, mockGetSchema);
+
+  const datasourceKey = buildDatasourceName('COUNTERPARTY', 'REQ_REP');
+  const cleanedConfig = result.datasourceConfig[datasourceKey].TRANSFORMER_CONFIGURATION;
+
+  assert.equal(JSON.stringify(cleanedConfig), JSON.stringify({}));
+});
+
+TransformFromServerPayload('handles missing schema properties', async () => {
+  const mockGetSchema = sinon.stub();
+  const schemaWithoutProperties: Genesis.JSONSchema7 = {};
+  mockGetSchema.resolves(schemaWithoutProperties);
+
+  const payload: Genesis.ServerReportConfig = {
+    ID: 'test-id',
+    NAME: 'Test Report',
+    SCHEDULES: [],
+    DATA_SOURCES: [
+      {
+        KEY: 'COUNTERPARTY',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'COUNTERPARTY',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: {
+          INCLUDE_COLUMNS: ['NAME', 'AMOUNT'],
+        },
+      },
+    ],
+    DESTINATION_IDS: [],
+    OUTPUT_DIRECTORY: '/reports',
+  };
+
+  const result = await transformFromServerPayload(payload, mockGetSchema);
+
+  const datasourceKey = buildDatasourceName('COUNTERPARTY', 'REQ_REP');
+  const cleanedConfig = result.datasourceConfig[datasourceKey].TRANSFORMER_CONFIGURATION;
+
+  // All fields should be removed when schema has no properties
+  assert.equal(cleanedConfig.INCLUDE_COLUMNS?.length, 0);
+});
+
+TransformFromServerPayload('preserves base config correctly', async () => {
+  const mockGetSchema = sinon.stub();
+  const validSchema: Genesis.JSONSchema7 = {
+    properties: {
+      COUNTERPARTY_NAME: { genesisType: 'STRING' },
+    },
+  };
+  mockGetSchema.resolves(validSchema);
+
+  const payload: Genesis.ServerReportConfig = {
+    ID: 'test-id',
+    NAME: 'Test Report',
+    DESCRIPTION: 'Test Description',
+    OUTPUT_FORMAT: 'CSV',
+    FILE_NAME: 'test.csv',
+    AUTO_DISPATCH: true,
+    DESTINATION_IDS: ['dest1', 'dest2'],
+    OUTPUT_DIRECTORY: '/reports',
+    DOCUMENT_TEMPLATE_ID: 'template-123',
+    SCHEDULES: [
+      {
+        CRON_EXPRESSION: '0 0 0 * * ?',
+        TIME_ZONE: 'UTC',
+      },
+    ],
+    DATA_SOURCES: [
+      {
+        KEY: 'COUNTERPARTY',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'COUNTERPARTY',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: {},
+      },
+    ],
+  };
+
+  const result = await transformFromServerPayload(payload, mockGetSchema);
+
+  assert.equal(result.baseConfig.ID, 'test-id');
+  assert.equal(result.baseConfig.NAME, 'Test Report');
+  assert.equal(result.baseConfig.DESCRIPTION, 'Test Description');
+  assert.equal(result.baseConfig.OUTPUT_FORMAT, 'CSV');
+  assert.equal(result.baseConfig.FILE_NAME, 'test.csv');
+  assert.equal(result.baseConfig.AUTO_DISPATCH, true);
+  assert.equal(
+    JSON.stringify(result.baseConfig.DESTINATION_IDS),
+    JSON.stringify(['dest1', 'dest2']),
+  );
+  assert.equal(result.baseConfig.OUTPUT_DIRECTORY, '/reports');
+  assert.equal(result.baseConfig.DOCUMENT_TEMPLATE_ID, 'template-123');
+  assert.ok(result.baseConfig.SCHEDULE);
+  assert.equal(result.baseConfig.SCHEDULE?.CRON_EXPRESSION, '0 0 0 * * ?');
+  assert.equal(result.baseConfig.SCHEDULE?.TIME_ZONE, 'UTC');
+});
+
+TransformFromServerPayload('handles all transformer config types together', async () => {
+  const mockGetSchema = sinon.stub();
+  const validSchema: Genesis.JSONSchema7 = {
+    properties: {
+      COUNTERPARTY_NAME: { genesisType: 'STRING' },
+      AMOUNT: { genesisType: 'DOUBLE' },
+    },
+  };
+  mockGetSchema.resolves(validSchema);
+
+  const payload: Genesis.ServerReportConfig = {
+    ID: 'test-id',
+    NAME: 'Test Report',
+    SCHEDULES: [],
+    DATA_SOURCES: [
+      {
+        KEY: 'COUNTERPARTY',
+        INPUT_TYPE: 'REQ_REP',
+        NAME: 'COUNTERPARTY',
+        OUTPUT_TYPE: 'TABLE',
+        TRANSFORMER_CONFIGURATION: {
+          INCLUDE_COLUMNS: ['COUNTERPARTY_NAME', 'AMOUNT', 'NAME'], // NAME orphaned
+          COLUMN_RENAMES: {
+            COUNTERPARTY_NAME: 'Counterparty',
+            NAME: 'Old Name', // orphaned
+          },
+          COLUMN_FORMATS: {
+            AMOUNT: '#,##0.00',
+            NAME: 'text', // orphaned
+          },
+          COLUMN_TRANSFORMS: {
+            COUNTERPARTY_NAME: { TYPE: 'BINARY_EXPRESSION', LEFT: {}, OPERATION: 'AND', RIGHT: {} },
+            NAME: { TYPE: 'METHOD_EXPRESSION', METHOD: 'TRIM', PARAMETERS: [] }, // orphaned
+          } as any,
+          ROW_FILTERS: {
+            TYPE: 'PREDICATE_EXPRESSION' as const,
+            OPERATION: 'AND' as const,
+            EXPRESSIONS: [],
+          },
+        },
+      },
+    ],
+    DESTINATION_IDS: [],
+    OUTPUT_DIRECTORY: '/reports',
+  };
+
+  const result = await transformFromServerPayload(payload, mockGetSchema);
+
+  const datasourceKey = buildDatasourceName('COUNTERPARTY', 'REQ_REP');
+  const cleanedConfig = result.datasourceConfig[datasourceKey].TRANSFORMER_CONFIGURATION;
+
+  // INCLUDE_COLUMNS should only have valid fields
+  assert.equal(cleanedConfig.INCLUDE_COLUMNS?.length, 2);
+  assert.ok(cleanedConfig.INCLUDE_COLUMNS?.includes('COUNTERPARTY_NAME'));
+  assert.ok(cleanedConfig.INCLUDE_COLUMNS?.includes('AMOUNT'));
+
+  // COLUMN_RENAMES should only have valid fields
+  assert.equal(Object.keys(cleanedConfig.COLUMN_RENAMES || {}).length, 1);
+  assert.equal(cleanedConfig.COLUMN_RENAMES?.['COUNTERPARTY_NAME'], 'Counterparty');
+
+  // COLUMN_FORMATS should only have valid fields
+  assert.equal(Object.keys(cleanedConfig.COLUMN_FORMATS || {}).length, 1);
+  assert.equal(cleanedConfig.COLUMN_FORMATS?.['AMOUNT'], '#,##0.00');
+
+  // COLUMN_TRANSFORMS should only have valid fields
+  assert.equal(Object.keys(cleanedConfig.COLUMN_TRANSFORMS || {}).length, 1);
+  assert.ok(cleanedConfig.COLUMN_TRANSFORMS?.['COUNTERPARTY_NAME']);
+
+  // ROW_FILTERS should be preserved
+  assert.ok(cleanedConfig.ROW_FILTERS);
+});
+
+TransformFromServerPayload.run();

package/src/new/utils/transformers.ts

@@ -1,4 +1,6 @@
+import type { JSONSchema7 } from 'json-schema';
 import type { BaseConfig, DatasourceInputTypes, DatasourceName, ReportingConfig } from '../store';
+import type { TransformerConfig } from '../store/slices/types';
 import { Display, Genesis } from '../types';
 
 export function transformToServerPayload(state: ReportingConfig): Genesis.ServerReportConfig {
@@ -20,9 +22,41 @@ export function transformToServerPayload(state: ReportingConfig): Genesis.Server
   };
 }
 
+/**
+ * Cleans up orphaned field references from transformer configuration
+ * by validating against current schema
+ */
+function cleanupTransformerConfig(
+  config: TransformerConfig,
+  validFields: Set<string>,
+): TransformerConfig {
+  return {
+    INCLUDE_COLUMNS: config.INCLUDE_COLUMNS?.filter((field) => validFields.has(field)),
+    COLUMN_RENAMES: config.COLUMN_RENAMES
+      ? Object.fromEntries(
+          Object.entries(config.COLUMN_RENAMES).filter(([key]) => validFields.has(key)),
+        )
+      : undefined,
+    COLUMN_FORMATS: config.COLUMN_FORMATS
+      ? Object.fromEntries(
+          Object.entries(config.COLUMN_FORMATS).filter(([key]) => validFields.has(key)),
+        )
+      : undefined,
+    COLUMN_TRANSFORMS: config.COLUMN_TRANSFORMS
+      ? Object.fromEntries(
+          Object.entries(config.COLUMN_TRANSFORMS).filter(([key]) => validFields.has(key)),
+        )
+      : undefined,
+    ROW_FILTERS: config.ROW_FILTERS,
+  };
+}
+
 // TODO: we'll suppress the bigints on the server so won't need to do this
 // horrible destructing
-export function transformFromServerPaylaod(payload: Genesis.ServerReportConfig): ReportingConfig {
+export async function transformFromServerPayload(
+  payload: Genesis.ServerReportConfig,
+  getSchema: (name: string) => Promise<JSONSchema7>,
+): Promise<ReportingConfig> {
   const {
     DATA_SOURCES,
     ID,
@@ -48,18 +82,44 @@ export function transformFromServerPaylaod(payload: Genesis.ServerReportConfig):
     DOCUMENT_TEMPLATE_ID,
     ...(SCHEDULES.length > 0 && { SCHEDULE: SCHEDULES[0] }),
   };
+
+  const cleanedDatasources = await Promise.all(
+    DATA_SOURCES.map(
+      async ({ KEY, INPUT_TYPE, NAME: DS_NAME, OUTPUT_TYPE, TRANSFORMER_CONFIGURATION }) => {
+        try {
+          const schema = await getSchema(KEY);
+          const validFields = new Set(Object.keys(schema.properties || {}));
+
+          const cleanedConfig = cleanupTransformerConfig(
+            TRANSFORMER_CONFIGURATION || {},
+            validFields,
+          );
+
+          return {
+            KEY,
+            INPUT_TYPE,
+            NAME: DS_NAME,
+            OUTPUT_TYPE,
+            TRANSFORMER_CONFIGURATION: cleanedConfig,
+          };
+        } catch (error) {
+          // If schema fetch fails, log warning but keep original config
+          console.warn(`Failed to fetch schema for ${KEY}, skipping cleanup:`, error);
+          return {
+            KEY,
+            INPUT_TYPE,
+            NAME: DS_NAME,
+            OUTPUT_TYPE,
+            TRANSFORMER_CONFIGURATION: TRANSFORMER_CONFIGURATION || {},
+          };
+        }
+      },
+    ),
+  );
+
   return {
     baseConfig,
-
-    datasourceConfig: DATA_SOURCES.map(
-      ({ KEY, INPUT_TYPE, NAME: DS_NAME, OUTPUT_TYPE, TRANSFORMER_CONFIGURATION }) => ({
-        KEY,
-        INPUT_TYPE,
-        NAME: DS_NAME,
-        OUTPUT_TYPE,
-        TRANSFORMER_CONFIGURATION,
-      }),
-    ).reduce(
+    datasourceConfig: cleanedDatasources.reduce(
       (acum, ds) => ({
         ...acum,
         [buildDatasourceName(ds.NAME, ds.INPUT_TYPE)]: ds,