@graphenedata/cli 0.0.1

package/dist/ui/component-utilities/formatTitle.js

@@ -0,0 +1,24 @@
+import {applyTitleTagReplacement} from './formatting'
+
+export default function formatTitle (column, columnFormat) {
+  let result = applyTitleTagReplacement(column, columnFormat)
+  // Allow some acronyms to remain fully capitalized in titles:
+  let acronyms = ['id', 'gdp']
+  // Allow some joining words to remain fully lowercased in title:
+  let lowercase = ['of', 'the', 'and', 'in', 'on']
+  // Set name to proper casing:
+  function toTitleCase (str) {
+    return str.replace(/\S*/g, function (txt) {
+      if (!acronyms.includes(txt) && !lowercase.includes(txt)) {
+        return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase()
+      } else if (acronyms.includes(txt)) {
+        return txt.toUpperCase()
+      } else {
+        return txt.toLowerCase()
+      }
+    })
+  }
+  // Remove all underscores & double quotes before passing to title case function:
+  result = toTitleCase(column.replace(/"/g, '').replace(/_/g, ' '))
+  return result
+}

package/dist/ui/component-utilities/formatting.js

@@ -0,0 +1,258 @@
+import ssf from 'ssf'
+import {getContext} from 'svelte'
+import {CUSTOM_FORMATTING_SETTINGS_CONTEXT_KEY} from './globalContexts'
+import {findImplicitAutoFormat, autoFormat, fallbackFormat, isAutoFormat} from './autoFormatting'
+import {BUILT_IN_FORMATS} from './builtInFormats'
+import {standardizeDateString} from './dateParsing'
+
+const AXIS_FORMATTING_CONTEXT = 'axis'
+const VALUE_FORMATTING_CONTEXT = 'value'
+
+export const getCustomFormats = () => {
+  try {
+    return getContext(CUSTOM_FORMATTING_SETTINGS_CONTEXT_KEY)?.getCustomFormats() || []
+  } catch {
+    return []
+  }
+}
+
+/**
+ * @param {*} columnName the name of the column
+ * @returns a format object (built-in or custom) based on the column name if it matches the pattern column_${formatTag}, otherwise returns undefined
+ */
+export const lookupColumnFormat = (columnName, columnEvidenceType, columnUnitSummary) => {
+  let potentialFormatTag = maybeExtractFormatTag(columnName)
+
+  if (columnEvidenceType.evidenceType === 'string') {
+    return undefined
+  }
+
+  if (potentialFormatTag) {
+    let customFormats = getCustomFormats()
+    let matchingFormat = [...BUILT_IN_FORMATS, ...customFormats].find(
+      (format) => format.formatTag?.toLowerCase() === potentialFormatTag?.toLowerCase?.(),
+    )
+    if (matchingFormat) {
+      return matchingFormat
+    }
+  }
+
+  let matchingImplicitAutoFormat = findImplicitAutoFormat(
+    columnName,
+    columnEvidenceType,
+    columnUnitSummary,
+  )
+  if (matchingImplicitAutoFormat) {
+    return matchingImplicitAutoFormat
+  }
+
+  return undefined
+}
+
+/**
+ * Returns an Evidence format object to be used in the applyFormatting function
+ * @param {string} formatString string containing an Excel-style format code, or a format name matching a built-in or custom format
+ * @param {'number' | 'date' | 'boolean' | 'string'} [valueType] optional - a string representing the data type within the column that will be formatted ('number', 'date', 'boolean', or 'string)
+ * @returns a format object based on the formatString matching a built-in or custom format name, or a new custom format object containing an Excel-style format code
+ */
+export function getFormatObjectFromString (formatString, valueType = undefined) {
+  let potentialFormatTag = formatString
+  let customFormats = getCustomFormats()
+  let matchingFormat = [...BUILT_IN_FORMATS, ...customFormats].find(
+    (format) => format.formatTag?.toLowerCase() === potentialFormatTag?.toLowerCase?.(),
+  )
+  let newFormat = {}
+  if (matchingFormat) {
+    return matchingFormat
+  } else {
+    newFormat = {
+      formatTag: 'custom',
+      formatCode: potentialFormatTag,
+    }
+    if (valueType) {
+      newFormat.valueType = valueType
+    }
+    return newFormat
+  }
+}
+
+export const formatValue = (value, columnFormat = undefined, columnUnitSummary = undefined) => {
+  try {
+    return applyFormatting(value, columnFormat, columnUnitSummary, VALUE_FORMATTING_CONTEXT)
+  } catch (error) {
+    //fallback to default
+    console.warn(
+      `Unexpected error calling applyFormatting(${value}, ${columnFormat}, ${VALUE_FORMATTING_CONTEXT}, ${columnUnitSummary}). Error=${error}`,
+    )
+    return value
+  }
+}
+
+export const formatAxisValue = (value, columnFormat = undefined, columnUnitSummary = undefined) => {
+  try {
+    return applyFormatting(value, columnFormat, columnUnitSummary, AXIS_FORMATTING_CONTEXT)
+  } catch {
+    //fallback to default
+  }
+  return value
+}
+
+export const applyTitleTagReplacement = (columnName, columnFormatSettings) => {
+  let result = columnName
+  if (columnName && columnFormatSettings?.formatTag) {
+    let lastIndexOfTag = columnName
+      .toLowerCase()
+      .lastIndexOf(`_${columnFormatSettings.formatTag.toLowerCase()}`)
+    let titleTagReplacement = ''
+    if (lastIndexOfTag > 0) {
+      //explicitly ignore columns starting with _, hence >0 instead of => 0
+      if (typeof columnFormatSettings?.titleTagReplacement === 'string') {
+        titleTagReplacement = columnFormatSettings.titleTagReplacement
+      }
+      result = columnName.substring(0, lastIndexOfTag) + titleTagReplacement
+    }
+  }
+  return result
+}
+
+export const defaultExample = (valueType) => {
+  switch (valueType) {
+    case 'number':
+      return 1234
+    case 'date':
+      return '2022-01-03'
+    default:
+      return undefined
+  }
+}
+
+export const formatExample = (format) => {
+  let normalizedUserInput = format.userInput?.trim()
+  let preFormattedValue =
+    normalizedUserInput || format.exampleInput || defaultExample(format.valueType)
+
+  if (preFormattedValue) {
+    try {
+      let columnUnitSummary = undefined
+      if (format.valueType === 'number') {
+        let numericValue = Number(preFormattedValue)
+        columnUnitSummary = {
+          min: numericValue,
+          max: numericValue,
+          median: numericValue,
+          maxDecimals: numericValue.toString().split('.')[1]?.length || 0,
+          unitType: 'number',
+        }
+      }
+      return applyFormatting(
+        preFormattedValue,
+        format,
+        columnUnitSummary,
+        VALUE_FORMATTING_CONTEXT,
+      )
+    } catch {
+      //return default value
+    }
+  }
+  return ''
+}
+
+function applyFormatting (
+  value,
+  columnFormat = undefined,
+  columnUnitSummary = undefined,
+  formattingContext = VALUE_FORMATTING_CONTEXT,
+) {
+  if (value === undefined || value === null) {
+    return '-'
+  }
+
+  let result = undefined
+  if (columnFormat) {
+    try {
+      let formattingCode = getEffectiveFormattingCode(columnFormat, formattingContext)
+      let typedValue
+      try {
+        if (columnFormat.valueType === 'date' && typeof value === 'string') {
+          typedValue = new Date(standardizeDateString(value))
+        } else if (value instanceof Date) {
+          // "2023-09-06T22:40:43.000Z" minus the Z is interpreted
+          // as local time
+          // similar in behavior to standardizeDateString
+          typedValue = new Date(value.toISOString().slice(0, -1))
+        } else if (
+          columnFormat.valueType === 'number' &&
+          typeof value !== 'number' &&
+          !Number.isNaN(value)
+        ) {
+          typedValue = Number(value)
+        } else {
+          typedValue = value
+        }
+      } catch {
+        typedValue = value
+      }
+      if (isAutoFormat(columnFormat, formattingCode)) {
+        try {
+          result = autoFormat(typedValue, columnFormat, columnUnitSummary)
+        } catch (error) {
+          console.warn(`Unexpected error applying auto formatting. Error=${error}`)
+        }
+      } else {
+        result = ssf.format(formattingCode, typedValue)
+      }
+    } catch (error) {
+      console.warn(`Unexpected error applying formatting ${error}`)
+    }
+  }
+  if (result === undefined) {
+    result = fallbackFormat(value, columnUnitSummary)
+  }
+  return result
+}
+function getEffectiveFormattingCode (columnFormat, formattingContext = VALUE_FORMATTING_CONTEXT) {
+  if (typeof columnFormat === 'string') {
+    // This should only be used by end users, not by components.
+    return columnFormat
+  } else {
+    if (formattingContext === AXIS_FORMATTING_CONTEXT && columnFormat?.axisFormatCode) {
+      return columnFormat.axisFormatCode
+    }
+    return columnFormat?.formatCode
+  }
+}
+
+/**
+ * Extracts a possible format tag from a column name based on the column name pattern
+ * @returns "column_${formatTag}" will return ${formatTag} or undefined if the columnName doesn't match the pattern
+ */
+function maybeExtractFormatTag (columnName) {
+  let normalizedColName = columnName.toLowerCase()
+  let lastUnderScoreIndex = normalizedColName.lastIndexOf('_')
+
+  if (lastUnderScoreIndex > 0) {
+    return normalizedColName.substr(lastUnderScoreIndex).replace('_', '')
+  } else {
+    return undefined
+  }
+}
+
+/**
+ * Formats a value to whichever format is passed in
+ * @param {*} value the value to be formatted
+ * @param {string} format string containing an Excel-style format code, or a format name matching a built-in or custom format
+ * @returns a formatted value
+ */
+export function fmt (value, format) {
+  let formatObj = getFormatObjectFromString(format)
+  let valueType = valueTypeFromSample(value)
+  formatObj.valueType = valueType
+  return formatValue(value, formatObj)
+}
+
+function valueTypeFromSample (value) {
+  if (typeof value === 'number') return 'number'
+  if (typeof value === 'boolean') return 'boolean'
+  if (value instanceof Date) return 'date'
+  return 'string'
+}

package/dist/ui/component-utilities/getColumnExtents.js

@@ -0,0 +1,79 @@
+import {tidy, summarize, min, max, median, mean, n, nDistinct, sum} from '@tidyjs/tidy'
+
+/**
+ *
+ * @param {Record<string, unknown>[]} data
+ * @param {string} columnName
+ * @param {boolean} [isNumeric]
+ * @returns {{ min?: number, max?: number, median?: number, mean?: number, count?: number, countDistinct?: number, sum?: number, maxDecimals: number, unitType: string }}
+ */
+export function getColumnUnitSummary (data, columnName, isNumeric = true) {
+  let seriesExtents = tidy(
+    data,
+    isNumeric
+      ? summarize({
+        count: n(columnName),
+        countDistinct: nDistinct(columnName),
+        min: min(columnName),
+        max: max(columnName),
+        median: median(columnName),
+        mean: mean(columnName),
+        sum: sum(columnName),
+      })
+      : summarize({count: n(columnName), countDistinct: nDistinct(columnName)}),
+  )[0]
+
+  // TODO try to use summarize spec in tidy
+  let {maxDecimals, unitType} = summarizeUnits(data.map((row) => row[columnName]))
+
+  return {
+    min: seriesExtents.min,
+    max: seriesExtents.max,
+    median: seriesExtents.median,
+    mean: seriesExtents.mean,
+    count: seriesExtents.count,
+    countDistinct: seriesExtents.countDistinct,
+    sum: seriesExtents.sum,
+    maxDecimals: maxDecimals,
+    unitType: unitType,
+  }
+}
+
+/**
+ *
+ * @param {Record<string, unknown>[]} data
+ * @param {string} column
+ * @returns {[number?, number?]}
+ */
+export function getColumnExtentsLegacy (data, column) {
+  let domainData = tidy(data, summarize({min: min(column), max: max(column)}))[0]
+  return [domainData.min, domainData.max]
+}
+
+/**
+ *
+ * @param {number[]} series
+ * @returns {{ maxDecimals: number, unitType: string }}
+ */
+function summarizeUnits (series) {
+  if (series === undefined || series === null || series.length === 0) {
+    return {
+      maxDecimals: 0,
+      unitType: 'unknown',
+    }
+  } else {
+    let maxDecimals = 0
+
+    for (let element of series) {
+      let decimal_places = element?.toString().split('.')[1]?.length
+      if (decimal_places > maxDecimals) {
+        maxDecimals = decimal_places
+      }
+    }
+
+    return {
+      maxDecimals: maxDecimals,
+      unitType: 'number',
+    }
+  }
+}

package/dist/ui/component-utilities/getColumnSummary.js

@@ -0,0 +1,67 @@
+import {getColumnUnitSummary} from './getColumnExtents.js'
+import {lookupColumnFormat} from './formatting.js'
+import formatTitle from './formatTitle.js'
+
+const EvidenceType = {
+  BOOLEAN: 'boolean',
+  NUMBER: 'number',
+  STRING: 'string',
+  DATE: 'date',
+}
+
+/**
+ * @typedef {Object} ColumnSummary
+ * @property {string} title
+ * @property {string} type
+ * @property {Object} evidenceColumnType
+ * @property {ReturnType<typeof lookupColumnFormat>} format
+ * @property {ReturnType<typeof getColumnUnitSummary>} columnUnitSummary
+ */
+
+/**
+ * @function
+ * @template T
+ * @param {Record<string, unknown>[]} data
+ * @param {T} returnType
+ * @returns {T extends 'object' ? Record<string, ColumnSummary> : (ColumnSummary & { id: string })[]}
+ */
+export default function getColumnSummary (data, returnType = 'object') {
+  /** @type {Record<string, ColumnSummary>} */
+  let columnSummary = {}
+
+  let types = Array.isArray(data?._evidenceColumnTypes) ? data._evidenceColumnTypes : []
+
+  for (let colName of Object.keys(data[0])) {
+    let evidenceColumnType = types.find(
+      (item) => item.name?.toLowerCase() === colName?.toLowerCase(),
+    ) ?? {
+      name: colName,
+      evidenceType: EvidenceType.STRING,
+    }
+    let type = evidenceColumnType.evidenceType
+    let columnUnitSummary =
+      evidenceColumnType.evidenceType === EvidenceType.NUMBER
+        ? getColumnUnitSummary(data, colName, true)
+        : getColumnUnitSummary(data, colName, false)
+
+    if (evidenceColumnType.evidenceType !== EvidenceType.NUMBER) {
+      columnUnitSummary.maxDecimals = 0
+      columnUnitSummary.unitType = evidenceColumnType.evidenceType
+    }
+    let format = lookupColumnFormat(colName, evidenceColumnType, columnUnitSummary)
+
+    columnSummary[colName] = {
+      title: formatTitle(colName, format),
+      type,
+      evidenceColumnType,
+      format,
+      columnUnitSummary,
+    }
+  }
+
+  if (returnType !== 'object') {
+    return Object.entries(columnSummary).map(([key, value]) => ({id: key, ...value}))
+  }
+
+  return columnSummary
+}

package/dist/ui/component-utilities/getCompletedData.js

@@ -0,0 +1,114 @@
+import {tidy, complete} from '@tidyjs/tidy'
+import getDistinctValues from './getDistinctValues'
+import {findInterval, vectorSeq} from './helpers/getCompletedData.helpers.js'
+
+/**
+ * This function fills missing data points in the given data array for a specific series.
+ *
+ * @param {Record<string, unknown>[]} _data - The data as an array of objects.
+ * @param {string} x - The property used as x-axis.
+ * @param {string} y - The property used as y-axis.
+ * @param {string} series - The specific series in the data to be filled.
+ * @param {boolean} [nullsZero=false] - A flag indicating whether nulls should be replaced with zero.
+ * @param {boolean} [fillX=false] - A flag indicating whether the x-axis values should be filled (based on the found interval distance).
+ * @return {Record<string, unknown>[]} An array containing the filled data objects.
+ */
+export default function getCompletedData (_data, x, y, series, nullsZero = false, fillX = false) {
+  let xIsDate = false
+  let data = _data
+    .map((d) =>
+      Object.assign({}, d, {
+        [x]: d[x] instanceof Date ? ((xIsDate = true), d[x].toISOString()) : d[x],
+      }),
+    )
+    .filter((d) => d[x] !== undefined && d[x] !== null)
+  let groups = Array.from(data).reduce((a, v) => {
+    if (v[x] instanceof Date) {
+      v[x] = v[x].toISOString()
+      xIsDate = true
+    }
+    if (series) {
+      if (!a[v[series] ?? 'null']) a[v[series] ?? 'null'] = []
+      a[v[series] ?? 'null'].push(v)
+    } else {
+      if (!a.default) a.default = []
+      a.default.push(v)
+    }
+    return a
+  }, {})
+
+  // Ensures that all permutations of this map exist in the output
+  // e.g. can include series and x values to ensure that all series have all x values
+  let expandKeys = {}
+
+  /** @type {Array<number | string>} */
+  let xDistinct
+
+  let exampleX =
+    data.find((item) => item && item[x] !== null && item[x] !== undefined)?.[x] ?? null
+  // const exampleX = data[0]?.[x];
+  switch (typeof exampleX) {
+    case 'object':
+      if (exampleX === null) {
+        throw new Error(
+          `Column '${x}' is entirely null. Column must contain at least one non-null value.`,
+        )
+      } else {
+        throw new Error('Unexpected object property, expected string, date, or number')
+      }
+    case 'number':
+      // Numbers are the most straightforward
+      xDistinct = getDistinctValues(data, x)
+      if (fillX) {
+        // Attempt to derive the interval between X values and interpolate missing values in that set (within the bounds of min/max)
+        let interval = findInterval(xDistinct)
+        expandKeys[x] = vectorSeq(xDistinct, interval)
+      }
+      break
+    case 'string':
+      xDistinct = getDistinctValues(data, x)
+      expandKeys[x] = xDistinct
+      break
+  }
+
+  let output = []
+
+  for (let value of Object.values(groups)) {
+    let nullySpec = series ? {[series]: null} : {}
+    if (nullsZero) {
+      if (y instanceof Array) {
+        for (let i = 0; i < y.length; i++) {
+          nullySpec[y[i]] = 0
+        }
+      } else {
+        nullySpec[y] = 0
+      }
+    } else {
+      // Ensure null for consistency
+      if (y instanceof Array) {
+        for (let i = 0; i < y.length; i++) {
+          nullySpec[y[i]] = null
+        }
+      } else {
+        nullySpec[y] = null
+      }
+    }
+
+    if (series) {
+      expandKeys[series] = series
+    }
+
+    let tidyFuncs = []
+    if (Object.keys(expandKeys).length === 0) {
+      // empty object, no special configuration
+      tidyFuncs.push(complete([x], nullySpec))
+    } else {
+      tidyFuncs.push(complete(expandKeys, nullySpec))
+    }
+
+    output.push(tidy(value, ...tidyFuncs))
+  }
+  if (xIsDate) return output.flat().map((r) => ({...r, [x]: new Date(r[x])}))
+
+  return output.flat()
+}

package/dist/ui/component-utilities/getDistinctCount.js

@@ -0,0 +1,7 @@
+import {tidy, summarize, nDistinct} from '@tidyjs/tidy'
+
+export default function getDistinctCount (data, column) {
+  let distinctCount = tidy(data, summarize({count: nDistinct(column)}))[0].count
+
+  return distinctCount
+}

package/dist/ui/component-utilities/getDistinctValues.js

@@ -0,0 +1,15 @@
+/**
+ * Extracts an array of distinct values from a specified column in a dataset.
+ *
+ * This function iterates over the dataset, collecting values from the specified
+ * column into a Set to ensure uniqueness. It then converts the Set into an array
+ * of distinct values and returns this array.
+ *
+ * @param {Object[]} data - The dataset to process, represented as an array of objects.
+ * @param {string} column - The name of the column from which to extract distinct values.
+ * @returns {any[]} An array containing distinct values from the specified column of the dataset.
+ */
+export default function getDistinctValues (data, column) {
+  let set = new Set(data.map((val) => val[column]))
+  return Array.from(set)
+}