@cdc/editor 4.26.2 → 4.26.4

package/src/components/DataImport/components/DataImport.tsx

@@ -1,6 +1,7 @@
 import React, { useState, useContext, useEffect } from 'react'
 import { useDropzone } from 'react-dropzone'
 import axios from 'axios'
+import { csvFormat } from 'd3'
 
 import { DataTransform } from '@cdc/core/helpers/DataTransform'
 
@@ -19,12 +20,13 @@ import CloseIcon from '@cdc/core/assets/icon-close.svg'
 import DataDesigner from '@cdc/core/components/managers/DataDesigner'
 import Tooltip from '@cdc/core/components/ui/Tooltip'
 import Icon from '@cdc/core/components/ui/Icon'
+import Button from '@cdc/core/components/elements/Button'
 import { isSolrCsv, isSolrJson } from '@cdc/core/helpers/isSolr'
 import { type Visualization } from '@cdc/core/types/Visualization'
 import { type DataSet } from '@cdc/core/types/DataSet'
 
 import './data-import.scss'
-import '@cdc/core/styles/v2/components/data-designer.scss'
+import '@cdc/core/components/managers/data-designer.scss'
 
 import { errorMessages, maxFileSize } from '../../../helpers/errorMessages'
 import { displaySize } from '../helpers/displaySize'
@@ -32,6 +34,7 @@ import { supportedDataTypes } from '../helpers/supportedDataTypes'
 import { getFileExtension } from '../helpers/getFileExtension'
 import { parseTextByMimeType } from '../helpers/parseTextByMimeType'
 import { getMimeType } from '../helpers/getMimeType'
+import { applyAutoDetectedDateParseFormat } from '../helpers/applyAutoDetectedDateParseFormat'
 import {
   extractCoveData,
   getSampleVegaJson,
@@ -39,6 +42,7 @@ import {
   parseVegaConfig,
   updateVegaData
 } from '@cdc/core/helpers/vegaConfig'
+import { extractDataAndMetadata } from '@cdc/core/helpers/extractDataAndMetadata'
 
 const DataImport = () => {
   const { config, errors, tempConfig, sharepath } = useContext(ConfigContext)
@@ -179,12 +183,13 @@ const DataImport = () => {
     const filereader = new FileReader()
 
     filereader.onload = function () {
-      const handleSetConfig = (newData: Object[], useTempConfig = false) => {
+      const handleSetConfig = (newData: Object[], useTempConfig = false, dataMetadata = {}) => {
         const setDataURL = keepURL && fileSourceType === 'url'
         if (config.type === 'dashboard') {
           const dataFileFormat = mimeType.split('/')[1].toUpperCase()
           const dataset = {
             data: newData,
+            dataMetadata,
             dataFileSize: fileSize,
             dataFileName: fileSource, // new file source
             dataFileSourceType: fileSourceType, // new file source type
@@ -206,13 +211,21 @@ const DataImport = () => {
             payload: { datasetKey: newDatasetName || fileSource, dataset, oldDatasetKey }
           })
         } else {
+          const configWithAutoDetectedDateFormat = applyAutoDetectedDateParseFormat(
+            {
+              ...config,
+              ...tempConfig
+            },
+            newData as Record<string, unknown>[]
+          )
+
           let newConfig = {
-            ...config,
-            ...tempConfig,
+            ...configWithAutoDetectedDateFormat,
             data: newData,
+            dataMetadata,
             dataFileName: fileSource, // new file source
             dataFileSourceType: fileSourceType, // new file source type
-            formattedData: transform.developerStandardize(newData, config.dataDescription)
+            formattedData: transform.developerStandardize(newData, configWithAutoDetectedDateFormat.dataDescription)
           }
           if (setDataURL) {
             newConfig.dataUrl = fileSource
@@ -227,14 +240,16 @@ const DataImport = () => {
         if (config.vegaConfig) {
           return updateDataFromVegaData(result, fileSource, fileSourceType)
         }
-        const text = transform.autoStandardize(result)
+        const { data: extractedData, dataMetadata } = extractDataAndMetadata(result)
+        const text = transform.autoStandardize(extractedData)
         if (config.data && config.series) {
           if (dataExists(text, config.series, config?.xAxis.dataKey)) {
-            handleSetConfig(text, true)
+            handleSetConfig(text, true, dataMetadata)
           } else {
             resetEditor(
               {
                 data: text,
+                dataMetadata,
                 dataFileName: fileSource,
                 dataFileSourceType: fileSourceType
               } as Visualization,
@@ -242,7 +257,7 @@ const DataImport = () => {
             )
           }
         } else {
-          handleSetConfig(text)
+          handleSetConfig(text, false, dataMetadata)
         }
 
         if (editingDataset) {
@@ -354,7 +369,7 @@ const DataImport = () => {
           Always load from URL (normally will only pull once)
         </label>
         <div className='d-flex justify-content-end mt-2 mb-3'>
-          <button
+          <Button
             className='btn btn-primary px-4'
             type='submit'
             id='load-data'
@@ -362,7 +377,7 @@ const DataImport = () => {
             onClick={() => loadData(null, externalURL, editingDataset)}
           >
             Save & Load
-          </button>
+          </Button>
         </div>
       </>
     )
@@ -383,7 +398,7 @@ const DataImport = () => {
     return (
       //todo convert to modal
       <>
-        <button
+        <Button
           className='btn danger'
           onClick={() =>
             resetEditor(
@@ -394,7 +409,7 @@ const DataImport = () => {
         >
           Clear
           <CloseIcon />
-        </button>
+        </Button>
         {/* DEV-851 link to replace file should pop file dialog */}
         {config.dataFileSourceType === 'file' && (
           <div className='link link-replace' {...getRootProps2()}>
@@ -430,6 +445,49 @@ const DataImport = () => {
     dispatch({ type: 'DELETE_DASHBOARD_DATASET', payload: { datasetKey } })
   }
 
+  const downloadCSV = (data: Array<Record<string, unknown>>, filename: string) => {
+    // Normalize filename: strip query/hash, remove path, replace extension with .csv
+    let baseName = filename || ''
+
+    const queryIndex = baseName.indexOf('?')
+    const hashIndex = baseName.indexOf('#')
+    const cutoffIndex = queryIndex === -1 ? hashIndex : hashIndex === -1 ? queryIndex : Math.min(queryIndex, hashIndex)
+    if (cutoffIndex !== -1) {
+      baseName = baseName.substring(0, cutoffIndex)
+    }
+
+    const lastSlash = Math.max(baseName.lastIndexOf('/'), baseName.lastIndexOf('\\'))
+    if (lastSlash !== -1) {
+      baseName = baseName.substring(lastSlash + 1)
+    }
+
+    const lastDot = baseName.lastIndexOf('.')
+    if (lastDot > 0) {
+      baseName = baseName.substring(0, lastDot)
+    }
+
+    if (!baseName) {
+      baseName = 'data'
+    }
+
+    const name = `${baseName}.csv`
+    const blob = new Blob(['\uFEFF' + csvFormat(data)], { type: 'text/csv;charset=utf-8;' })
+    // @ts-ignore
+    if (typeof window.navigator.msSaveBlob === 'function') {
+      // @ts-ignore
+      navigator.msSaveBlob(blob, name)
+    } else {
+      const url = URL.createObjectURL(blob)
+      const link = document.createElement('a')
+      link.href = url
+      link.download = name
+      document.body.appendChild(link)
+      link.click()
+      document.body.removeChild(link)
+      URL.revokeObjectURL(url)
+    }
+  }
+
   const updateDataFromVegaConfig = pastedConfig => {
     const vegaConfig = parseVegaConfig(JSON.parse(pastedConfig))
     const newData = extractCoveData(vegaConfig)
@@ -604,7 +662,7 @@ const DataImport = () => {
             </fieldset>
           )
         )}
-      <button
+      <Button
         className='btn full-width btn-primary'
         onClick={() => {
           setConfig({
@@ -616,7 +674,7 @@ const DataImport = () => {
         }}
       >
         Add New URL Filter
-      </button>
+      </Button>
     </>
   )
 
@@ -634,7 +692,7 @@ const DataImport = () => {
                   <th>Name</th>
                   <th>Size</th>
                   <th>Type</th>
-                  <th colSpan={3}>Actions</th>
+                  <th colSpan={4}>Actions</th>
                 </tr>
               </thead>
               <tbody>
@@ -646,17 +704,33 @@ const DataImport = () => {
                         <td className='p-1'>{displaySize(config.datasets[datasetKey].dataFileSize)}</td>
                         <td className='p-1'>{config.datasets[datasetKey].dataFileFormat}</td>
                         <td>
-                          <button
-                            className='btn btn-link p-1'
+                          <Button
+                            variant='link'
+                            className='p-1'
                             onClick={() => setGlobalDatasetProp(datasetKey, 'preview', true)}
                           >
                             Preview
+                          </Button>
+                        </td>
+                        <td>
+                          <button
+                            className='btn btn-link p-1'
+                            disabled={!config.datasets[datasetKey].data?.length}
+                            onClick={() =>
+                              downloadCSV(
+                                config.datasets[datasetKey].data,
+                                config.datasets[datasetKey].dataFileName || datasetKey
+                              )
+                            }
+                          >
+                            Download
                           </button>
                         </td>
                         <td>
                           {config.datasets[datasetKey].dataFileSourceType === 'url' && (
-                            <button
-                              className='btn btn-link p-1'
+                            <Button
+                              variant='link'
+                              className='p-1'
                               onClick={() => {
                                 if (editingDataset === datasetKey) {
                                   setEditingDataset(undefined)
@@ -672,13 +746,13 @@ const DataImport = () => {
                               }}
                             >
                               Edit
-                            </button>
+                            </Button>
                           )}
                         </td>
                         <td>
-                          <button className='btn btn-danger' onClick={() => removeDataset(datasetKey)}>
+                          <Button variant='danger' onClick={() => removeDataset(datasetKey)}>
                             X
-                          </button>
+                          </Button>
                         </td>
                       </tr>
                     )
@@ -714,6 +788,14 @@ const DataImport = () => {
                         )}
                       </div>
                       <div>{resetButton()}</div>
+                      {config.data?.length > 0 && (
+                        <button
+                          className='btn btn-link p-1'
+                          onClick={() => downloadCSV(config.data, config.dataFileName || 'data')}
+                        >
+                          Download CSV
+                        </button>
+                      )}
                     </div>
                   )}
 
@@ -722,6 +804,14 @@ const DataImport = () => {
                       <div className='url-source-options'>
                         <div>{loadDataFromUrl()}</div>
                         <div>{resetButton()}</div>
+                        {config.data?.length > 0 && (
+                          <button
+                            className='btn btn-link p-1'
+                            onClick={() => downloadCSV(config.data, config.dataFileName || 'data')}
+                          >
+                            Download CSV
+                          </button>
+                        )}
                       </div>
                       {config.dataUrl && (config.type === 'chart' || config.type === 'map') && urlFilters}
                     </>
@@ -745,15 +835,16 @@ const DataImport = () => {
                         placeholder='{ }'
                       />
                       <div className='mb-3 d-flex justify-content-end'>
-                        <button
-                          className='btn btn-primary px-4'
+                        <Button
+                          variant='primary'
+                          className='px-4'
                           type='submit'
                           id='load-data'
                           disabled={!pastedConfig}
                           onClick={() => updateDataFromVegaConfig(pastedConfig)}
                         >
                           Save & Load
-                        </button>
+                        </Button>
                       </div>
                     </div>
                   </TabPane>
@@ -807,15 +898,16 @@ const DataImport = () => {
                       />
                     </label>
                     <div className='d-flex justify-content-end mt-2 mb-3'>
-                      <button
-                        className='btn btn-primary px-4'
+                      <Button
+                        variant='primary'
+                        className='px-4'
                         type='submit'
                         id='load-data'
                         disabled={!newDatasetName || !externalURL}
                         onClick={() => loadData(null, externalURL, editingDataset)}
                       >
                         Save & Load
-                      </button>
+                      </Button>
                     </div>
                   </TabPane>
                 </Tabs>
@@ -890,20 +982,17 @@ const DataImport = () => {
 
         {config.type === 'dashboard' && !addingDataset && (
           <div className='mt-2'>
-            <button className='btn btn-primary' onClick={() => setAddingDataset(true)}>
+            <Button variant='primary' onClick={() => setAddingDataset(true)}>
               + Add More Files
-            </button>
+            </Button>
           </div>
         )}
 
         {readyToConfigure && (
           <div className='mt-2'>
-            <button
-              className='btn btn-primary'
-              onClick={() => dispatch({ type: 'EDITOR_SET_GLOBALACTIVE', payload: 2 })}
-            >
+            <Button variant='primary' onClick={() => dispatch({ type: 'EDITOR_SET_GLOBALACTIVE', payload: 2 })}>
               Configure your visualization
-            </button>
+            </Button>
           </div>
         )}
 

package/src/components/DataImport/helpers/applyAutoDetectedDateParseFormat.ts

@@ -0,0 +1,35 @@
+import { getAutoDetectedDateParseFormat, isDateScale } from '@cdc/core/helpers/cove/date'
+import { type Visualization } from '@cdc/core/types/Visualization'
+
+const isChartWithDateAxis = (config: Visualization) =>
+  config?.type === 'chart' && !!config?.xAxis && isDateScale(config.xAxis)
+
+export const applyAutoDetectedDateParseFormat = (
+  config: Visualization,
+  importedData: Record<string, unknown>[]
+): Visualization => {
+  if (!isChartWithDateAxis(config) || !config?.xAxis?.dataKey || !importedData?.length) {
+    return config
+  }
+
+  const existingDateParseFormat = config.xAxis.dateParseFormat
+
+  if (typeof existingDateParseFormat === 'string' && existingDateParseFormat.trim() !== '') {
+    return config
+  }
+  const xAxisKey = config.xAxis.dataKey
+
+  const autoDetectedDateParseFormat = getAutoDetectedDateParseFormat(importedData, xAxisKey)
+
+  if (!autoDetectedDateParseFormat) {
+    return config
+  }
+
+  return {
+    ...config,
+    xAxis: {
+      ...config.xAxis,
+      dateParseFormat: autoDetectedDateParseFormat
+    }
+  }
+}

package/src/components/DataImport/tests/applyAutoDetectedDateParseFormat.test.ts

@@ -0,0 +1,128 @@
+import { describe, expect, it } from 'vitest'
+import { applyAutoDetectedDateParseFormat } from '../helpers/applyAutoDetectedDateParseFormat'
+
+describe('applyAutoDetectedDateParseFormat', () => {
+  const baseChartConfig: any = {
+    type: 'chart',
+    xAxis: {
+      type: 'date',
+      dataKey: 'date',
+      dateParseFormat: ''
+    },
+    tooltips: {
+      dateDisplayFormat: '%B %-d, %Y'
+    },
+    table: {
+      dateDisplayFormat: '%b %-d %Y'
+    }
+  }
+
+  const reliableSlashDateRows = [
+    { date: '2024/03/15', value: 10 },
+    { date: '2025/11/09', value: 12 }
+  ]
+
+  const ambiguousUsOrEuDateRows = [
+    { date: '01/02/2024', value: 10 },
+    { date: '02/03/2024', value: 12 }
+  ]
+
+  const missingXAxisDateRows = [{ otherDate: '2024/03/15', value: 10 }]
+  const sparseFirstRowReliableDateRows = [
+    { otherDate: 'ignore-me', value: 8 },
+    { date: '2024/03/15', value: 10 },
+    { date: '2025/11/09', value: 12 }
+  ]
+
+  it('auto-fills the chart xAxis date parse format when detection is reliable', () => {
+    const result: any = applyAutoDetectedDateParseFormat(baseChartConfig, reliableSlashDateRows)
+
+    expect(result.xAxis.dateParseFormat).toBe('%Y/%m/%d')
+    expect(result.tooltips.dateDisplayFormat).toBe('%B %-d, %Y')
+    expect(result.table.dateDisplayFormat).toBe('%b %-d %Y')
+  })
+
+  it('auto-fills the chart xAxis date parse format for date-time axes when detection is reliable', () => {
+    const result: any = applyAutoDetectedDateParseFormat(
+      {
+        ...baseChartConfig,
+        xAxis: {
+          ...baseChartConfig.xAxis,
+          type: 'date-time'
+        }
+      },
+      reliableSlashDateRows
+    )
+
+    expect(result.xAxis.dateParseFormat).toBe('%Y/%m/%d')
+    expect(result.tooltips.dateDisplayFormat).toBe('%B %-d, %Y')
+    expect(result.table.dateDisplayFormat).toBe('%b %-d %Y')
+  })
+
+  it('leaves the config unchanged when the sample is ambiguous', () => {
+    const result: any = applyAutoDetectedDateParseFormat(baseChartConfig, ambiguousUsOrEuDateRows)
+
+    expect(result).toEqual(baseChartConfig)
+  })
+
+  it('leaves the config unchanged when xAxis.dateParseFormat is already configured', () => {
+    const preconfiguredChartConfig: any = {
+      ...baseChartConfig,
+      xAxis: {
+        ...baseChartConfig.xAxis,
+        dateParseFormat: '%d/%m/%Y'
+      }
+    }
+
+    const result: any = applyAutoDetectedDateParseFormat(preconfiguredChartConfig, reliableSlashDateRows)
+
+    expect(result).toEqual(preconfiguredChartConfig)
+  })
+  it('does nothing for non-date chart axes', () => {
+    const result: any = applyAutoDetectedDateParseFormat(
+      {
+        ...baseChartConfig,
+        xAxis: {
+          ...baseChartConfig.xAxis,
+          type: 'categorical'
+        }
+      },
+      [{ date: '2024/03/15', value: 10 }]
+    )
+
+    expect(result).toEqual({
+      ...baseChartConfig,
+      xAxis: {
+        ...baseChartConfig.xAxis,
+        type: 'categorical'
+      }
+    })
+  })
+
+  it('does nothing for non-chart visualizations', () => {
+    const result: any = applyAutoDetectedDateParseFormat(
+      {
+        ...baseChartConfig,
+        type: 'map'
+      },
+      [{ date: '2024/03/15', value: 10 }]
+    )
+
+    expect(result).toEqual({
+      ...baseChartConfig,
+      type: 'map'
+    })
+  })
+
+  it('does nothing when the configured x-axis column is missing', () => {
+    const result: any = applyAutoDetectedDateParseFormat(baseChartConfig, missingXAxisDateRows)
+
+    expect(result).toEqual(baseChartConfig)
+  })
+
+  it('still detects the date parse format when the first row is missing the x-axis key', () => {
+    const result: any = applyAutoDetectedDateParseFormat(baseChartConfig, sparseFirstRowReliableDateRows)
+
+    expect(result.xAxis.dateParseFormat).toBe('%Y/%m/%d')
+  })
+})