@rokkit/data 1.0.0-next.98 → 1.0.1

package/src/dataset.svelte.js

@@ -0,0 +1,181 @@
+/**
+ * ReactiveDataSet — Svelte 5 rune-based reactive wrapper for dataset operations.
+ *
+ * Use this in Svelte 5 components when you need the pipeline to auto-recompute
+ * when source data changes (e.g. live search, CRUD lists).
+ *
+ * Unlike `DataSet`, which is imperatively pipeline-driven, `ReactiveDataSet` holds
+ * reactive configuration ($state) and exposes a `rows` getter backed by $derived —
+ * so templates can bind to `ds.rows` directly without calling `.select()`.
+ *
+ * File must remain `.svelte.js` for Svelte 5 rune compilation.
+ */
+
+import { executeFused } from './pipeline.js'
+import { deriveColumnDefs } from './infer.js'
+
+export class ReactiveDataSet {
+	/** @type {any[]} */
+	#source = $state([])
+
+	/** @type {Function|null} */
+	#filter = $state(null)
+
+	/** @type {Array<{fn: Function}>} */
+	#transforms = $state([])
+
+	/** @type {string[]} */
+	#sortFields = $state([])
+
+	/** @type {Array<Object>} Per-column overrides merged into auto-derived column defs. */
+	#columnEnhancements = $state([])
+
+	/** @type {Object} Options forwarded to deriveColumnDefs (language, scanMode, etc.) */
+	#columnOptions = $state({})
+
+	/**
+	 * @param {any[]} data - Initial source rows.
+	 */
+	constructor(data = []) {
+		this.#source = data
+	}
+
+	/**
+	 * Reactive computed result of filter + transforms + sort.
+	 * Bind to this in templates instead of calling `.select()`.
+	 */
+	get rows() {
+		const ops = []
+		if (this.#filter) ops.push({ type: 'filter', fn: this.#filter })
+		for (const t of this.#transforms) ops.push({ type: 'map', fn: t.fn })
+		if (this.#sortFields.length > 0) ops.push({ type: 'sort', fields: this.#sortFields })
+		return executeFused(this.#source, ops)
+	}
+
+	/**
+	 * Reactive derived column definitions: name, type, scale (discrete|continuous),
+	 * sortable, filterable, fields, formatter — merged with any `withColumns` overrides.
+	 *
+	 * Re-computes when source data or column enhancements change.
+	 * Bind to this in templates for Table / TreeTable column specs.
+	 */
+	get columns() {
+		return deriveColumnDefs(this.#source, {
+			...this.#columnOptions,
+			enhancements: this.#columnEnhancements
+		})
+	}
+
+	// ── Configuration (chainable) ────────────────────────────────────────────
+
+	/**
+	 * Set the filter predicate. Replaces any previous filter.
+	 * @param {Function|null} fn
+	 */
+	where(fn) {
+		this.#filter = fn
+		return this
+	}
+
+	/**
+	 * Append a row transform. Each call to `apply` adds a new stage.
+	 * @param {Function} fn
+	 */
+	apply(fn) {
+		this.#transforms = [...this.#transforms, { fn }]
+		return this
+	}
+
+	/**
+	 * Set sort fields. Replaces any previous sort.
+	 * @param {...string} fields
+	 */
+	sortBy(...fields) {
+		this.#sortFields = fields
+		return this
+	}
+
+	/**
+	 * Set per-column enhancements / overrides for the reactive `columns` property.
+	 *
+	 * Each entry is matched by `name` and merged into the auto-derived column def.
+	 * Unknown names are appended as synthetic columns (useful for action columns).
+	 *
+	 * @param {Array<Object>} enhancements - e.g. [{ name: 'revenue', label: 'Revenue ($)', scale: 'continuous' }]
+	 * @param {Object} [options] - Options forwarded to deriveColumnDefs (language, scanMode).
+	 */
+	withColumns(enhancements, options = {}) {
+		this.#columnEnhancements = enhancements
+		this.#columnOptions = options
+		return this
+	}
+
+	/** Clear the current filter. */
+	clearFilter() {
+		this.#filter = null
+		return this
+	}
+
+	/** Clear all transforms. */
+	clearTransforms() {
+		this.#transforms = []
+		return this
+	}
+
+	// ── Data mutation ────────────────────────────────────────────────────────
+
+	/**
+	 * Replace the entire source dataset.
+	 * @param {any[]} rows
+	 */
+	setData(rows) {
+		this.#source = rows
+		return this
+	}
+
+	/**
+	 * Append a single row to the source.
+	 * @param {any} row
+	 */
+	push(row) {
+		this.#source = [...this.#source, row]
+		return this
+	}
+
+	/**
+	 * Remove rows matching the predicate from the source.
+	 * @param {Function} predicate
+	 */
+	remove(predicate) {
+		this.#source = this.#source.filter((r) => !predicate(r))
+		return this
+	}
+
+	/**
+	 * Update rows matching the predicate with patch (object or function).
+	 * @param {Function} predicate
+	 * @param {Object|Function} patch
+	 */
+	update(predicate, patch) {
+		this.#source = this.#source.map((row) => {
+			if (!predicate(row)) return row
+			return typeof patch === 'function' ? patch(row) : { ...row, ...patch }
+		})
+		return this
+	}
+
+	/** Return a plain snapshot of the current reactive result (non-reactive copy). */
+	snapshot() {
+		return [...this.rows]
+	}
+}
+
+/**
+ * Factory function for `ReactiveDataSet`.
+ *
+ * @param {any[]} [data=[]] - Initial source rows.
+ * @returns {ReactiveDataSet}
+ */
+export function reactiveDataset(data = []) {
+	return new ReactiveDataSet(data)
+}

package/src/filter.js

@@ -1,5 +1,22 @@
 import { filterOperations } from './constants'
 
+/**
+ * Checks if a row matches the given value based on the specified operator.
+ * Supports both standard and negated operators by checking the operator string
+ * for a leading '!' character, which indicates negation.
+ *
+ * @param {Object} row - The row object in which to search for the value match.
+ * @param {any} value - The value to match against the row's properties.
+ * @param {string} operator - The operator used to compare the row's properties to the value.
+ *                            When operator starts with '!', it negates the result of the comparison.
+ * @returns {boolean} - Returns true if the row matches the condition; otherwise, returns false.
+ */
+function findMatchingRow(row, value, operator) {
+	const op = operator.startsWith('!') ? operator.slice(1) : operator
+	const matched = Object.keys(row).some((key) => filterOperations[op](row[key], value))
+
+	return operator.startsWith('!') ? !matched : matched
+}
 /**
  * Filters an array of objects based on a specified condition defined in the options.
  * If a column is provided, it filters based on the value of that column. Otherwise, it filters
@@ -22,24 +39,6 @@ export function filterObjectArray(data, options) {
 	return data.filter((row) => findMatchingRow(row, value, operator))
 }
 
-/**
- * Checks if a row matches the given value based on the specified operator.
- * Supports both standard and negated operators by checking the operator string
- * for a leading '!' character, which indicates negation.
- *
- * @param {Object} row - The row object in which to search for the value match.
- * @param {any} value - The value to match against the row's properties.
- * @param {string} operator - The operator used to compare the row's properties to the value.
- *                            When operator starts with '!', it negates the result of the comparison.
- * @returns {boolean} - Returns true if the row matches the condition; otherwise, returns false.
- */
-function findMatchingRow(row, value, operator) {
-	const op = operator.startsWith('!') ? operator.slice(1) : operator
-	const matched = Object.keys(row).some((key) => filterOperations[op](row[key], value))
-
-	return operator.startsWith('!') ? !matched : matched
-}
-
 /**
  * Applies multiple filtering conditions to an array of objects. This function iterates through each filter option,
  * successively narrowing down the data.

package/src/formatter.js

@@ -1,20 +1,4 @@
-import { identity } from 'ramda'
-
-const formatterMap = {
-	currency: getCurrencyFormatter,
-	integer: (language) => (val) => val.toLocaleString(language, { maximumFractionDigits: 0 }),
-	number: (language, decimalPlaces) => (val) =>
-		val.toLocaleString(language, {
-			minimumFractionDigits: decimalPlaces,
-			maximumFractionDigits: decimalPlaces
-		}),
-	date: (language) => (val) => val.toLocaleDateString(language),
-	time: (language) => (val) => val.toLocaleTimeString(language),
-	object: () => JSON.stringify,
-	array: () => JSON.stringify, // Combined formatter for 'object' and 'array'
-	ellipsis: () => () => '...' // Static formatter, no need for parameters
-}
-
+const identity = (x) => x
 /**
  * Returns a currency formatter function that formats a value as a currency string.
  * @param {string} language       - The IETF language tag used for locale-specific formatting.
@@ -39,6 +23,21 @@ function getCurrencyFormatter(language, decimalPlaces) {
 	return formatter
 }
 
+const formatterMap = {
+	currency: getCurrencyFormatter,
+	integer: (language) => (val) => val.toLocaleString(language, { maximumFractionDigits: 0 }),
+	number: (language, decimalPlaces) => (val) =>
+		val.toLocaleString(language, {
+			minimumFractionDigits: decimalPlaces,
+			maximumFractionDigits: decimalPlaces
+		}),
+	date: (language) => (val) => val.toLocaleDateString(language),
+	time: (language) => (val) => val.toLocaleTimeString(language),
+	object: () => JSON.stringify,
+	array: () => JSON.stringify, // Combined formatter for 'object' and 'array'
+	ellipsis: () => () => '...' // Static formatter, no need for parameters
+}
+
 /**
  * Creates a formatter function that formats a value according to the specified type and optional locale settings.
  * Supported types include 'default' (no formatting), 'integer', 'number', 'date', 'time',, 'object', 'array', 'currency' & 'ellipsis''

package/src/index.js

@@ -3,5 +3,17 @@ export * from './types'
 export { typeOf } from './utils'
 export { renamer } from './renamer'
 export { model } from './model'
+export { innerJoin, leftJoin, rightJoin, fullJoin, crossJoin, antiJoin, semiJoin } from './join'
 export { dataset } from './dataset'
 export { dataview } from './view'
+export {
+	deriveColumns,
+	deriveMetadata,
+	deriveSortableColumn,
+	inferScale,
+	addScales,
+	deriveColumnDefs,
+	mergeColumnEnhancements
+} from './infer'
+export { parseFilters } from './parser'
+export { filterData, filterObjectArray } from './filter'

package/src/infer.js

@@ -1,33 +1,35 @@
 import { ascending, descending } from 'd3-array'
 import { compact } from '@rokkit/core'
-import { omit } from 'ramda'
 import { createFormatter } from './formatter'
 import { defaultViewOptions, defaultActionOrder } from './constants'
 import { typeOf } from './utils'
 import { deriveColumnProperties } from './metadata'
 
 /**
- * Derives the sorting information for a column based on the type of the input value.
+ * Validates an array to ensure it can be used to create a sorting configuration.
  *
- * @param {import('./types').SortableColumn} value - The value that determines the sorting behavior of the column.
- * @returns {import('./types').ColumnSorter}         An object containing the 'name' of the column and a 'sorter' function.
- * @throws {Error} Throws an error if the value type is not supported or the value format is invalid.
+ * @param {Array} value - The array to validate.
+ * @throws {Error} Throws an error if the array does not correspond to a [string, boolean] pair.
  */
-export function deriveSortableColumn(value) {
-	const type = typeOf(value)
-
-	const configGenerators = {
-		array: arrayConfigGenerator,
-		object: objectConfigGenerator,
-		string: stringConfigGenerator
+function arrayValidator(value) {
+	if (value.length !== 2 || typeof value[0] !== 'string' || typeof value[1] !== 'boolean') {
+		throw new Error('Array should be a pair of [string, boolean]')
 	}
+}
 
-	const configGenerator = configGenerators[type]
-	if (!configGenerator) {
-		throw new Error(`Invalid value type: ${type}`)
+/**
+ * Validates an object to ensure it has the necessary properties for creating a sorting configuration.
+ *
+ * @param {Object} value - The object to validate.
+ * @throws {Error} Throws an error if the object does not have a 'name' property or has an invalid 'sorter'.
+ */
+function objectValidator(value) {
+	if (typeof value.name !== 'string') {
+		throw new Error('The property "name" is required and must be a string')
+	}
+	if (value.sorter !== undefined && typeof value.sorter !== 'function') {
+		throw new Error('Sorter should be a function')
 	}
-
-	return configGenerator(value)
 }
 
 /**
@@ -63,32 +65,28 @@ function stringConfigGenerator(value) {
 }
 
 /**
- * Validates an array to ensure it can be used to create a sorting configuration.
+ * Derives the sorting information for a column based on the type of the input value.
  *
- * @param {Array} value - The array to validate.
- * @throws {Error} Throws an error if the array does not correspond to a [string, boolean] pair.
+ * @param {import('./types').SortableColumn} value - The value that determines the sorting behavior of the column.
+ * @returns {import('./types').ColumnSorter}         An object containing the 'name' of the column and a 'sorter' function.
+ * @throws {Error} Throws an error if the value type is not supported or the value format is invalid.
  */
-function arrayValidator(value) {
-	if (value.length !== 2 || typeof value[0] !== 'string' || typeof value[1] !== 'boolean') {
-		throw new Error('Array should be a pair of [string, boolean]')
-	}
-}
+export function deriveSortableColumn(value) {
+	const type = typeOf(value)
 
-/**
- * Validates an object to ensure it has the necessary properties for creating a sorting configuration.
- *
- * @param {Object} value - The object to validate.
- * @throws {Error} Throws an error if the object does not have a 'name' property or has an invalid 'sorter'.
- */
-function objectValidator(value) {
-	if (typeof value.name !== 'string') {
-		throw new Error('The property "name" is required and must be a string')
+	const configGenerators = {
+		array: arrayConfigGenerator,
+		object: objectConfigGenerator,
+		string: stringConfigGenerator
 	}
-	if (value.sorter !== undefined && typeof value.sorter !== 'function') {
-		throw new Error('Sorter should be a function')
+
+	const configGenerator = configGenerators[type]
+	if (!configGenerator) {
+		throw new Error(`Invalid value type: ${type}`)
 	}
-}
 
+	return configGenerator(value)
+}
 /**
  * Creates a deep scan sample object that contains a union of all keys from all
  * objects in the array, pulling the values from the last item that contains the key.
@@ -96,11 +94,13 @@ function objectValidator(value) {
  * @param {Array<Object>} data - An array of objects to deep scan and collect sample values.
  * @returns {Object} A sample object consolidating all unique keys with their latest non-null values.
  */
-export function getDeepScanSample(data) {
+export function getSample(data, deepScan = true) {
+	if (data.length === 0) return {}
+	if (!deepScan) return data[0]
 	return data.reduce(
 		(acc, cur) => ({
 			...acc,
-			...compact(omit(Object.keys(acc), cur))
+			...compact(Object.fromEntries(Object.entries(cur).filter(([k]) => !(k in acc))))
 		}),
 		{}
 	)
@@ -117,7 +117,7 @@ export function deriveColumns(data, options) {
 	if (data.length === 0) return []
 
 	const { scanMode } = { ...defaultViewOptions, ...options }
-	const sample = scanMode === 'fast' ? data[0] : getDeepScanSample(data)
+	const sample = scanMode === 'fast' ? data[0] : getSample(data)
 
 	return deriveColumnProperties(sample, options)
 }
@@ -178,31 +178,6 @@ export function inferDataType(values) {
 	return type
 }
 
-/**
- * Derives column metadata from the data to be used in a tabular component.
- *
- * @param {Array} data - The data to derive column metadata from.
- * @returns {Array<import('./types').ColumnMetadata>} - The derived column metadata.
- */
-export function deriveMetadata(dataArray, options = {}) {
-	let { columns = [] } = options
-	if (dataArray.length === 0) return columns
-	const { actions, language } = { ...defaultViewOptions, ...options }
-
-	if (columns.length === 0) {
-		columns = deriveColumns(dataArray, options)
-	} else {
-		columns.forEach((col) => {
-			// if (col.sortable && !col.order) col.order = 'none'
-			if (!col.fields) col.fields = { text: col.name }
-		})
-	}
-	columns = addFormatters(columns, language)
-	if (actions.length > 0) columns = deriveActions(columns, actions)
-
-	return columns
-}
-
 /**
  * Adds a formatter function to each column in the array based on the column type. If a formatter
  * function is already present, it will not be overwritten. The formatter is created based on the
@@ -261,3 +236,142 @@ export function deriveActions(columns, input) {
 		}))
 	return [...actionColumns, ...columns]
 }
+
+/**
+ * Derives column metadata from the data to be used in a tabular component.
+ * @param {Array<import('./types').ColumnMetadata>} columns
+ * @param {Array} dataArray - The data to derive column metadata from.
+ * @param {Object} [options] - The options to use when deriving column metadata.
+ * @returns {Array<import('./types').ColumnMetadata>} - The derived column metadata.
+ */
+function deriveColumnMetadata(columns, dataArray, options) {
+	if (columns.length === 0) {
+		columns = deriveColumns(dataArray, options)
+	} else {
+		columns.forEach((col) => {
+			// if (col.sortable && !col.order) col.order = 'none'
+			if (!col.fields) col.fields = { text: col.name }
+		})
+	}
+	return columns
+}
+/**
+ * Derives column metadata from the data to be used in a tabular component.
+ *
+ * @param {Array} data - The data to derive column metadata from.
+ * @param {Object} [options] - The options to use when deriving column metadata.
+ * @returns {Array<import('./types').ColumnMetadata>} - The derived column metadata.
+ */
+export function deriveMetadata(dataArray, options = {}) {
+	let { columns = [] } = options
+	if (dataArray.length === 0) return columns
+	const { actions, language } = { ...defaultViewOptions, ...options }
+
+	columns = deriveColumnMetadata(columns, dataArray, options)
+	columns = addFormatters(columns, language)
+	if (actions.length > 0) columns = deriveActions(columns, actions)
+
+	return columns
+}
+
+// ── Scale inference ──────────────────────────────────────────────────────────
+
+/**
+ * Infers whether a column's values are discrete (categorical) or continuous (quantitative).
+ *
+ * Rules:
+ *  - string, boolean           → always discrete
+ *  - date, number (float)      → always continuous
+ *  - integer                   → discrete when ≤20 unique values OR unique ratio < 5 %,
+ *                                continuous otherwise (e.g. large ID column)
+ *  - anything else             → discrete (safe default for unknown types)
+ *
+ * @param {any[]} values - All values for the column (may include nulls).
+ * @returns {'discrete'|'continuous'}
+ */
+function inferIntegerScale(nonNull) {
+	const unique = new Set(nonNull).size
+	return unique <= 20 || unique / nonNull.length < 0.05 ? 'discrete' : 'continuous'
+}
+
+const SCALE_BY_TYPE = {
+	string: () => 'discrete',
+	boolean: () => 'discrete',
+	date: () => 'continuous',
+	number: () => 'continuous',
+	integer: (nonNull) => inferIntegerScale(nonNull)
+}
+
+export function inferScale(values) {
+	const nonNull = values.filter((v) => v !== null && v !== undefined)
+	if (nonNull.length === 0) return 'discrete'
+	const type = typeOf(nonNull[0])
+	return (SCALE_BY_TYPE[type] ?? (() => 'discrete'))(nonNull)
+}
+
+/**
+ * Adds a `scale` property to each column definition.
+ *
+ * @param {Array<Object>} columns - Column definitions (from deriveColumns / deriveMetadata).
+ * @param {any[]} data - Source rows.
+ * @returns {Array<Object>} Column definitions with `scale` added.
+ */
+export function addScales(columns, data) {
+	return columns.map((col) => ({
+		...col,
+		scale: inferScale(data.map((r) => r[col.name]))
+	}))
+}
+
+/**
+ * Merges user-provided enhancements into auto-derived column definitions.
+ * Enhancements are matched by `name` and merged (user values win).
+ * Enhancements for unknown column names are appended as synthetic columns.
+ *
+ * @param {Array<Object>} derived - Auto-derived column definitions.
+ * @param {Array<Object>} enhancements - User-provided overrides/additions.
+ * @returns {Array<Object>} Merged column definitions.
+ */
+export function mergeColumnEnhancements(derived, enhancements) {
+	const enhMap = new Map(enhancements.map((e) => [e.name, e]))
+	const result = derived.map((col) => {
+		const enh = enhMap.get(col.name)
+		return enh ? { ...col, ...enh } : col
+	})
+	const derivedNames = new Set(derived.map((c) => c.name))
+	for (const e of enhancements) {
+		if (!derivedNames.has(e.name)) result.push(e)
+	}
+	return result
+}
+
+/**
+ * Full column definition pipeline: derive → format → scale → merge enhancements.
+ *
+ * Produces column descriptors suitable for use in Table, TreeTable, and chart
+ * components. Each column gets: name, type, scale, sortable, filterable, fields,
+ * formatter — plus any user-supplied properties from `enhancements`.
+ *
+ * @param {any[]} data - Source rows.
+ * @param {Object} [options] - Options forwarded to deriveColumns / addFormatters.
+ * @param {Array<Object>} [options.enhancements=[]] - Per-column overrides keyed by name.
+ *        Use to override scale, add a label, attach a custom formatter, etc.
+ * @param {string} [options.language='en-US'] - Locale for formatters.
+ * @param {string} [options.scanMode='fast'] - 'fast' (first row) or 'deep' (all rows).
+ * @returns {Array<Object>} Column definitions with scale and formatters.
+ */
+export function deriveColumnDefs(data, options = {}) {
+	const { enhancements = [], language, scanMode } = { ...defaultViewOptions, ...options }
+
+	if (data.length === 0) return enhancements
+
+	let columns = deriveColumns(data, { scanMode })
+	columns = addFormatters(columns, language)
+	columns = addScales(columns, data)
+
+	if (enhancements.length > 0) {
+		columns = mergeColumnEnhancements(columns, enhancements)
+	}
+
+	return columns
+}