@rokkit/data 1.0.0-next.100

package/src/list.js

@@ -0,0 +1,232 @@
+import { writable, get } from 'svelte/store'
+import { ascending } from 'd3-array'
+import { nest } from 'd3-collection'
+import { v4 as uuid } from '@lukeed/uuid'
+
+/**
+ * Compares two items for sorting. When grouped, uses group comparison before item comparison
+ *
+ * @param {*} a first item
+ * @param {*} b second item
+ * @param {*} data
+ * @returns
+ */
+export function compare(a, b, data) {
+	let result = 0
+
+	if (data.groupKey) {
+		result = ascending(data.lookup[a[data.groupKey]], data.lookup[b[data.groupKey]])
+	}
+	if (result === 0) {
+		result = ascending(a[data.sortKey], b[data.sortKey])
+	}
+	return result
+}
+
+/**
+ * Search the data for existence of string in the data
+ *
+ * @param {List} data
+ * @param {string} value
+ * @returns
+ */
+export function quickSearch(data, value) {
+	return data.data.filter((d) => d[data.filterKey].toLowerCase().includes(value.toLowerCase()))
+}
+
+/**
+ * Implements a list containing data and functionality for sorting, filtering, and grouping.
+ */
+export class List {
+	/**
+	 * Creates a List instance with initial data and default settings for keys and functions.
+	 * @param {Array<object>} data - The initial array of objects to populate the list.
+	 */
+	constructor(data) {
+		this.data = data
+
+		this.primaryKey = 'id'
+		this.filterKey = undefined
+		this.sortKey = undefined
+		this.groupKey = undefined
+		this.lookup = {}
+		this.searchText = ''
+
+		this.filterUsing = undefined
+		this.sortUsing = undefined
+		this.filtered = writable(data)
+	}
+
+	/**
+	 * Sets a primary key attribute for list items and assigns a unique identifier if not present.
+	 *
+	 * @param {string} value - The property name to be set as the primary key.
+	 * @returns {List} This list instance for method chaining.
+	 */
+	key(value) {
+		this.primaryKey = value
+		this.data = this.data.map((item) => (value in item ? item : { ...item, [value]: uuid() }))
+
+		return this
+	}
+
+	/**
+	 * Configures sorting by a given key and sorting function.
+	 *
+	 * @param {string} key - The property name to sort by.
+	 * @param {Function} [using=compare] - The function used to sort the list items.
+	 * @returns {List} This list instance for method chaining.
+	 */
+	sortBy(key, using = compare) {
+		this.sortKey = key
+		this.sortUsing = using
+
+		return this.sort().refresh()
+	}
+
+	/**
+	 * Groups list items by a given key and maps group keys for display using an optional lookup object.
+	 *
+	 * @param {string} key - The property name to group by.
+	 * @param {object|null} [lookup=null] - An object mapping group keys to display names.
+	 * @returns {List} This list instance for method chaining.
+	 */
+	groupBy(key, lookup = null) {
+		this.groupKey = key
+
+		if (lookup) this.lookup = lookup
+		else
+			this.lookup = this.data
+				.map((item) => item[key])
+				.reduce((obj, value) => ({ ...obj, [value]: value }), {})
+
+		return this.sort().refresh()
+	}
+
+	/**
+	 * Sets the key and function for filtering list items.
+	 *
+	 * @param {string} key - The property name to filter by.
+	 * @param {Function} [using=quickSearch] - The function used to filter the list items.
+	 * @returns {List} This list instance for method chaining.
+	 */
+	filterBy(key, using = quickSearch) {
+		this.filterKey = key
+		this.filterUsing = using
+		return this
+	}
+
+	/**
+	 * Sorts the list based on the configured sorting function.
+	 *
+	 * @returns {List} This list instance for method chaining.
+	 */
+	sort() {
+		if (this.sortUsing) this.data.sort((a, b) => this.sortUsing(a, b, this))
+		return this
+	}
+
+	/**
+	 * Adds a new item to the list, assigning a primary key if necessary, and then sorts and refreshes the list.
+	 *
+	 * @param {object} item - The item to add to the list.
+	 * @returns {List} This list instance for method chaining.
+	 */
+	add(item) {
+		const index = this.data.findIndex((d) => d[this.primaryKey] === item[this.primaryKey])
+		if (index === -1) {
+			if (!(this.primaryKey in item)) {
+				item[this.primaryKey] = uuid()
+			}
+			this.data.push(item)
+			this.sort().refresh()
+		}
+		return this
+	}
+
+	/**
+	 * Removes an item from the list by primary key and refreshes the list.
+	 *
+	 * @param {object} item - The item to remove from the list.
+	 * @returns {List} This list instance for method chaining.
+	 */
+	remove(item) {
+		const index = this.data.findIndex((d) => d[this.primaryKey] === item[this.primaryKey])
+		if (index > -1) {
+			this.data = [...this.data.slice(0, index), ...this.data.slice(index + 1)]
+			this.refresh()
+		}
+		return this
+	}
+
+	/**
+	 * Modifies an existing item in the list, identified by primary key, and then sorts and refreshes the list.
+	 *
+	 * @param {object} item - The item to modify in the list.
+	 * @returns {List} This list instance for method chaining.
+	 */
+	modify(item) {
+		const index = this.data.findIndex((d) => d[this.primaryKey] === item[this.primaryKey])
+		if (index > -1) {
+			this.data[index] = { ...item }
+			this.sort().refresh()
+		}
+		return this
+	}
+
+	/**
+	 * Sets the text to search for in the list's items and then refreshes the list.
+	 *
+	 * @param {string} text - The text to search for.
+	 * @returns {List} This list instance for method chaining.
+	 */
+	search(text) {
+		this.searchText = text
+		return this.refresh()
+	}
+
+	/**
+	 * Refreshes the list based on current sorting, grouping, and filtering configuration.
+	 *
+	 * @returns {List} This list instance for method chaining.
+	 */
+	refresh() {
+		let data = [...this.data]
+
+		if (this.filterKey && this.searchText) {
+			data = this.filterUsing(this, this.searchText)
+		}
+
+		if (this.groupKey) {
+			data = nest()
+				.key((d) => d[this.groupKey])
+				.entries(data)
+				.map(({ key, values }) => ({
+					key,
+					name: this.lookup[key],
+					data: values
+				}))
+		}
+
+		this.filtered.set(data)
+		return this
+	}
+	/**
+	 * Gets the current state of the filtered list data.
+	 *
+	 * @returns {Array<object>} The current filtered data of the list.
+	 */
+	current() {
+		return get(this.filtered)
+	}
+}
+
+/**
+ * Create a new list
+ *
+ * @param {Array<object>} data
+ * @returns
+ */
+export function list(data) {
+	return new List(data)
+}

package/src/metadata.js

@@ -0,0 +1,106 @@
+import { defaultPathOptions } from './constants'
+import { getDeepScanSample } from './infer'
+import { typeOf } from './utils'
+
+/**
+ * Merges currency-related attributes into the corresponding base columns.
+ *
+ * @param {Array<Object>} input - An array of column definitions.
+ * @param {string} currencySuffix - The suffix used to identify currency columns.
+ * @returns {Array<Object>} A new array of merged column definitions.
+ */
+function mergeCurrencyAttributes(input, currencySuffix) {
+	let currencyColumns = input.filter((col) => col.name.endsWith(currencySuffix))
+	const columns = input.filter((col) => !col.name.endsWith(currencySuffix))
+
+	currencyColumns = currencyColumns
+		.map((col) => {
+			const matched = columns.find((column) => column.name === col.name.replace(currencySuffix, ''))
+			if (matched) {
+				matched.fields.currency = col.name
+				matched.type = 'currency'
+				matched.digits = 2
+				return null
+			}
+			return col
+		})
+		.filter((col) => col !== null)
+
+	return [...columns, ...currencyColumns]
+}
+
+/**
+ * Adds a path modifier to column definitions that match a specific path.
+ *
+ * @param {Array<Object>} columns - The original array of column definitions.
+ * @param {string|null} path - The path used to identify columns that require path modifiers.
+ * @param {string} separator - The separator used in conjunction with the path.
+ * @returns {Array<Object>} The array of column definitions with path modifiers added as needed.
+ */
+export function addPathModifier(columns, path, separator) {
+	if (!path) return columns
+
+	const index = columns.findIndex((col) => col.name === path)
+	if (index > -1) {
+		return [
+			{ ...columns[index], path: true, separator },
+			...columns.slice(0, index),
+			...columns.slice(index + 1)
+		]
+	}
+	return columns
+}
+
+/**
+ * Derives column properties for a given sample and options.
+ *
+ * @param {Object} sample - The sample object used to derive column properties.
+ * @param {Object} options - The options used to modify column properties.
+ *        It can specify path, separator, and currencySuffix for column customization.
+ * @returns {Array<Object>} An array of derived column definitions.
+ */
+export function deriveColumnProperties(sample, options) {
+	const { path, separator, currencySuffix } = { ...defaultPathOptions, ...options }
+
+	let columns = Object.keys(sample).map((key) => ({
+		name: key,
+		type: typeOf(sample[key]),
+		sortable: true,
+		filterable: true,
+		sorted: 'none',
+		fields: { text: key }
+	}))
+
+	columns = addPathModifier(columns, path, separator)
+	return mergeCurrencyAttributes(columns, currencySuffix)
+}
+
+/**
+ * Derives the column metadata from the provided data and options.
+ *
+ * @param {Array} data                       - The data to derive the column metadata from.
+ * @param {Object} [options]                 - Optional parameters to control the metadata derivation.
+ * @param {Array} [options.metadata]         - The metadata to use instead of deriving it from the data.
+ * @param {boolean} [options.deepScan=false] - Determines if the metadata derivation should perform a deep scan of the data.
+ *
+ * @returns {import('./types').Metadata} The derived column metadata.
+ */
+export function deriveColumnMetadata(data, options = {}) {
+	const { metadata, deepScan = false } = options
+	if (Array.isArray(metadata) && metadata.length > 0) return metadata
+	if (data.length === 0) return []
+	const sample = deepScan ? getDeepScanSample(data) : data[0]
+	return Object.entries(sample).map(([name, value]) => ({ name, type: typeOf(value) }))
+}
+
+/**
+ * Derive column index from metadata.
+ * @param {Array} metadata - The metadata array.
+ * @returns {Object} - The column index object.
+ */
+// export function deriveColumnIndex(metadata) {
+// 	return metadata.reduce((acc, col, index) => {
+// 		acc[col.name] = index
+// 		return acc
+// 	}, {})
+// }

package/src/model.js

@@ -0,0 +1,100 @@
+import { identity, clone as copy } from 'ramda'
+import { getDeepScanSample } from './infer'
+import { typeOf } from './utils'
+
+/**
+ * A model is a representation of a dataset
+ * @returns an object with methods to manipulate the model
+ */
+export function model() {
+	let data = []
+
+	const actions = {
+		get: () => data,
+		/*
+		 * Creates a clone from another model
+		 * @param {Object} other - the other model object to clone from
+		 */
+		clone: (other) => {
+			data = copy(other.get())
+			return actions
+		},
+		/**
+		 * Renames the reference names using a renamer
+		 * @param {Function} rename - the renamer function
+		 */
+		renameUsing: (rename = identity) => {
+			if (rename !== identity) {
+				data = data.map((x) => ({ ...x, name: rename(x.name) }))
+			}
+			return actions
+		},
+		/**
+		 * Analyzes the input data and derives a model from it
+		 *
+		 * @param {Array|Object} value      - the data to derive the model from
+		 * @param {boolean}      sparseData - indicates that rows may have missing attributes
+		 * @returns {Object}     this
+		 */
+		from: (value, sparseData) => {
+			data = deriveModel(value, sparseData)
+			return actions
+		},
+		/**
+		 * Merges the model with another model
+		 *
+		 * @param {Array}   other    - the model to merge with
+		 * @param {Boolean} override - whether to override the type of the first model
+		 * @returns {Object} this
+		 */
+		merge: (other, override = false) => {
+			data = mergeModels(data, other.get(), override)
+			return actions
+		}
+	}
+	return actions
+}
+
+/**
+ * Derives a model from the input data
+ *
+ * @param {Array|Object} data  - the data to derive the model from
+ * @param {Boolean}      sparse - whether the data set contains sparse data
+ * @returns {Array}             - the derived model
+ */
+function deriveModel(data, sparse = false) {
+	const result = []
+	let item = data
+
+	if (Array.isArray(data)) item = sparse ? getDeepScanSample(data) : data[0]
+
+	const kv = Object.entries(item)
+	kv.forEach(([key, value]) => {
+		result.push({
+			name: key,
+			type: typeOf(value)
+		})
+	})
+	return result
+}
+
+/**
+ * Merges two models, returning a new model
+ *
+ * @param {Array} first      - the first model
+ * @param {Array} second     - the second model
+ * @param {Boolean} override - whether to override the type of the first model
+ * @returns {Array}      - the merged model
+ */
+function mergeModels(first, second, override = false) {
+	second.forEach(({ name, type }) => {
+		const existing = first.find((x) => x.name === name)
+		if (existing) {
+			if (existing.type !== type) existing.mixedTypes = true
+			if (override) existing.type = type
+		} else {
+			first.push({ name, type })
+		}
+	})
+	return first
+}

package/src/parser.js

@@ -0,0 +1,111 @@
+/**
+ * Constructs a regular expression pattern for matching search filter strings.
+ * The pattern captures "column", "operator", and "value" groups for filter expressions.
+ * Supported operators include common comparison and wildcard operators.
+ *
+ * Examples of valid expressions this regex can match:
+ * - "username:john_doe"
+ * - "age>30"
+ * - "status!=active"
+ * - "title~\"Product Manager\""
+ * - "completed!~*yes"
+ *
+ * The "column" group matches one or more word characters.
+ * The "operator" group matches one among the specified comparison or wildcard operators:
+ * :, >, <, >=, <=, =<, =>, =, !=, ~, ~*, !~, !~*.
+ * The "value" group matches either a double-quoted string or a single unquoted word
+ * that doesn't include whitespace or any of the operator characters.
+ *
+ * @returns {RegExp} - The regular expression pattern to identify search filter elements.
+ */
+export function getRegex() {
+	const column = '[\\w]+'
+	const operator = ':|>|<|>=|<=|=<|=>|=|!=|~|~\\*|!~|!~\\*'
+	const value = '("[^"]+"|[^\\s=:<>!~*]+)'
+
+	const pattern = `(?<group>((?<column>${column})\\s?(?<operator>${operator})\\s?)(?<value>${value}))`
+
+	return new RegExp(pattern, 'gm')
+}
+
+/**
+ * Parses a search string and extracts filters based on a predefined regular expression pattern.
+ * Each filter captures column, operator, and value components. Any remaining text that
+ * does not fit the pattern is considered a general search term.
+ *
+ * @param {string} string - The search string to parse for filters.
+ * @returns {Object[]} - An array of filter objects each containing the column, operator, and value,
+ *                       plus an additional object for general search if there is remaining text.
+ */
+export function parseFilters(string) {
+	const results = []
+	const regex = getRegex()
+
+	// Split the string into an array of tokens
+	const tokens = string.matchAll(regex)
+	let search = string
+
+	// Iterate over the tokens
+	for (const token of tokens) {
+		// Extract the named groups from the token
+		const { group, column } = token.groups
+		let { operator, value } = token.groups
+		search = search.replace(group, '').trim()
+
+		operator = replaceOperators(operator)
+		value = processValue(value, operator)
+
+		if (column && value) results.push({ column, operator, value })
+	}
+
+	if (search.length > 0) {
+		results.push({
+			operator: '~*',
+			value: new RegExp(removeQuotes(search), 'i')
+		})
+	}
+
+	return results
+}
+
+/**
+ * Replaces various shorthand operators with their standardized equivalent for filtering.
+ *
+ * @param {string} operator - The operator to be replaced.
+ * @returns {string} - The replaced operator string.
+ */
+function replaceOperators(operator) {
+	return operator.replace(':', '~*').replace('=>', '>=').replace('=<', '<=')
+}
+
+/**
+ * Processes the filter value provided, converting it into a format suitable for filtering.
+ * If the operator includes a tilde (~), the value is converted to a RegExp.
+ *
+ * @param {string|number} value - The value to be processed. It could be a string needing quote removal or a number that needs parsing.
+ * @param {string} operator - The operator that determines the type of processing to be applied to the value.
+ * @returns {string|number|RegExp} - The processed value, which may be a RegExp if the operator involves pattern matching.
+ */
+function processValue(value, operator) {
+	// If the value can be parsed as an integer, do so. Otherwise, strip quotes if present.
+
+	value = !isNaN(parseInt(value)) ? parseInt(value) : removeQuotes(value)
+	// If the operator includes a tilde (~), treat the value as a regular expression,
+	// case-insensitive if '*' is present, sensitive otherwise.
+	if (operator.includes('~')) {
+		value = operator.includes('*') ? new RegExp(value, 'i') : new RegExp(value)
+	}
+
+	return value
+}
+
+/**
+ * Removes double quotes from the start and end of a string.
+ *
+ * @param {string} str - The input string from which the surrounding quotes should be removed.
+ * @returns {string} - The unquoted string.
+ */
+function removeQuotes(str) {
+	const quoteMatch = str.match(/^"([^"]+)"$/)
+	return quoteMatch ? quoteMatch[1] : str
+}

package/src/renamer.js

@@ -0,0 +1,98 @@
+import { omit, identity } from 'ramda'
+
+/**
+ *
+ *
+ * @typedef {Object} OptionsToRenameKeys
+ * @property {string} [prefix]        - The prefix to add to the key.
+ * @property {string} [suffix]        - The suffix to add to the key.
+ * @property {string} [separator='_'] - The separator to use between the key and the prefix or suffix.
+ * @property {Array}  [keys]          - The keys to rename.
+ */
+
+/**
+ * A generator function that produces a function to rename keys.
+ *
+ * @param {OptionsToRenameKeys} options - Options to rename keys
+ * @returns {Object}                    - A renamer object.
+ *
+ * Example:
+ *
+ * renamer({ prefix: 'a', keys: ['a', 'b'] }).get()
+ * renamer({ prefix: 'a', separator: '-' }).get()
+ * renamer().setPrefix('a')
+ *          .setSeparator('-')
+ *          .setKeys(['a', 'b'])
+ *          .get()
+ * renamer().setPrefix('a').get()
+ * renamer().setSuffix('b').get()
+ * renamer().setKeys(['a', 'b']).get()
+ */
+export function renamer(options = {}) {
+	const { prefix, suffix, separator = '_', keys = [] } = options
+	const rename = getRenamer(prefix, suffix, separator)
+	const renameObject = getObjectRenamer(rename, keys)
+
+	return {
+		get: () => ({ rename, renameObject }),
+		setPrefix: (value) => renamer({ ...omit(['suffix'], options), prefix: value }),
+		setSuffix: (value) => renamer({ ...omit(['prefix'], options), suffix: value }),
+		setKeys: (value) => renamer({ ...options, keys: value }),
+		setSeparator: (value) => renamer({ ...options, separator: value })
+	}
+}
+
+/**
+ * Generates a renamer function which adds a prefix or suffix to a string.
+ *
+ * @param {string} prefix    - The prefix to add to the key.
+ * @param {string} suffix    - The suffix to add to the key.
+ * @param {string} separator - The separator to use between the key and the prefix or suffix.
+ * @returns {Function} - A function that takes a string and adds a prefix or suffix.
+ */
+function getRenamer(prefix, suffix, separator) {
+	let rename = identity
+
+	if (prefix) {
+		rename = (x) => [prefix, x].join(separator)
+	} else if (suffix) {
+		rename = (x) => [x, suffix].join(separator)
+	}
+	return rename
+}
+
+/**
+ * Returns a function that renames the keys of an object.
+ *
+ * @param {Function} rename - The function to rename the keys.
+ * @param {Array}    keys   - The keys to rename.
+ * @returns {Function}      - A function that takes an object and renames the keys.
+ */
+function getObjectRenamer(rename, keys) {
+	if (rename === identity) return identity
+	if (keys.length > 0) {
+		const lookup = keys.reduce((acc, key) => ({ ...acc, [key]: rename(key) }), {})
+		return usingLookup(lookup)
+	}
+	return forObject(rename)
+}
+
+/**
+ * Renames the keys of an object using a lookup table.
+ *
+ * @param {Object} lookup - The lookup table.
+ * @returns {Function} - A function that takes an object and renames the keys using the lookup table.
+ */
+function usingLookup(lookup) {
+	return (row) => Object.keys(row).reduce((acc, key) => ({ ...acc, [lookup[key]]: row[key] }), {})
+}
+
+/**
+ * Renames the keys of an object using a function.
+ *
+ * @param {Function} rename - The function to rename the keys.
+ * @returns {Function} - A function that takes an object and renames the keys using the function.
+ */
+function forObject(rename) {
+	return (row) => Object.keys(row).reduce((acc, key) => ({ ...acc, [rename(key)]: row[key] }), {})
+}