@rokkit/data 1.0.0-next.148 → 1.0.0-next.150

package/src/dataset.js

@@ -1,7 +1,7 @@
-import { equals, identity, pick, omit, clone } from 'ramda'
 import { descending } from 'd3-array'
 import { pickAllowedConfig, defaultConfig } from './constants'
-import { deriveSortableColumn } from './infer'
+import { deriveColumnDefs } from './infer'
+import { executeFused, sortDataByFields } from './pipeline.js'
 import { groupDataByKeys, fillAlignedData, getAlignGenerator, aggregateData } from './rollup'
 import {
 	leftJoin,
@@ -14,19 +14,54 @@ import {
 	nestedJoin
 } from './join'
 
+// ── Native helpers (replace Ramda) ─────────────────────────────────────────
+
+function pickKeys(keys) {
+	return (row) => {
+		const out = {}
+		for (const k of keys) if (k in row) out[k] = row[k]
+		return out
+	}
+}
+
+function omitKeys(keys) {
+	const skip = new Set(keys)
+	return (row) => {
+		const out = {}
+		for (const k of Object.keys(row)) if (!skip.has(k)) out[k] = row[k]
+		return out
+	}
+}
+
+function deepEqual(a, b) {
+	return JSON.stringify(a) === JSON.stringify(b)
+}
+
+function cloneConfig(config) {
+	return {
+		...config,
+		group_by: [...config.group_by],
+		align_by: [...config.align_by],
+		summaries: [...config.summaries],
+		template: { ...config.template }
+	}
+}
+
+// ───────────────────────────────────────────────────────────────────────────
+
 /**
  * Adds a summary field to the dataset.
- *
- * @param {Object} config                        - The configuration object to be updated.
- * @param {string|Array<string>|Function}  from  - The field or function to fetch data for summary
- * @param {string|Object<string:Function>} using - The target field & formula to use for summarizing.
- * @returns {Object}                             - The updated configuration.
  */
 function summarize(config, from, using) {
-	const mapper = typeof from === 'function' ? from : Array.isArray(from) ? pick(from) : pick([from])
+	const mapper =
+		typeof from === 'function'
+			? from
+			: Array.isArray(from)
+				? pickKeys(from)
+				: pickKeys([from])
 	const reducers = []
 
-	if (typeof using === 'string') reducers.push({ field: using, formula: identity })
+	if (typeof using === 'string') reducers.push({ field: using, formula: (x) => x })
 	if (typeof using === 'object')
 		Object.entries(using).forEach(([field, formula]) => reducers.push({ field, formula }))
 
@@ -37,11 +72,6 @@ function summarize(config, from, using) {
 
 /**
  * Updates rows in the data based on the condition
- *
- * @param {Array<Object>}   data      - The data to update rows in.
- * @param {Function}        filter    - The condition to update rows on.
- * @param {Object|Function} value     - The values to update the rows with.
- * @returns {Array<Object>}           - The updated data with rows updated based on the condition.
  */
 function updateRows(data, filter, value) {
 	if (typeof value !== 'function' && typeof value !== 'object') {
@@ -54,17 +84,8 @@ function updateRows(data, filter, value) {
 
 /**
  * Fills missing values in the data based on the provided value.
- *
- * @param {Array<Object>} data  - The data to fill missing values in.
- * @param {Object}        value - The value to fill missing values with.
- * @returns {Array<Object>}     - The updated data with missing values filled.
  */
 function fillNA(data, value) {
-	/**
-	 * Fills missing values in a single row based on the provided value.
-	 *
-	 * @param {Object} row - The row to fill missing values in.
-	 */
 	const fill = (row) => {
 		const filled = {}
 		Object.entries(value).forEach(([k, v]) => {
@@ -79,9 +100,6 @@ function fillNA(data, value) {
 
 /**
  * Returns a function that renames the keys of a single row based on the provided mapping.
- *
- * @param {Object} lookup - The mapping of old keys to new keys.
- * @returns {Function}    - The function that renames the keys of a single row.
  */
 function renameKeysUsingMap(lookup) {
 	return (row) =>
@@ -91,12 +109,7 @@ function renameKeysUsingMap(lookup) {
 }
 
 /**
- * Renames the keys of the data based on the provided mapping, or function
- * that returns the new key. If the new key already exists, the value is overwritten.
- *
- * @param {Array<Object>}   data - The data to rename the keys of.
- * @param {Object|Function} how  - The mapping of old keys to new keys, or a function that performs renaming.
- * returns Array<Object>         - The updated data with renamed keys.
+ * Renames the keys of the data based on the provided mapping, or function.
  */
 function renameKeys(data, how) {
 	const rename = typeof how === 'function' ? how : renameKeysUsingMap(how)
@@ -104,47 +117,17 @@ function renameKeys(data, how) {
 }
 
 /**
- * Sorts data by the specified columns.
- *
- * @param {Array<Object>}                    data   - The data to sort.
- * @param {import('./types').SortableColumn} fields - The columns to sort by.
- * @returns {Array<Object>}                           The sorted data
- */
-function sortDataBy(data, ...fields) {
-	const sorters = fields.map(deriveSortableColumn)
-
-	data.sort((a, b) => {
-		let result = 0
-		for (let i = 0; i < sorters.length && result === 0; i++) {
-			const { name, sorter } = sorters[i]
-			result = sorter(a[name], b[name])
-		}
-		return result
-	})
-	return data
-}
-
-/**
- * Returns the default aggregator which rolls up the columns other than the group_by columns into a single object.
- *
- * @param {import('./types').Metadata} metadata - The metadata for the columns to be aggregated.
- * @param {Object} config                       - The configuration used to build the aggregator.
- *
- * @returns {Object} An object containing the default aggregator for the specified metadata and configuration.
+ * Returns the default aggregator which rolls up non-group_by columns into a children array.
  */
 export function defaultAggregator(config) {
 	return {
-		mapper: omit(config.group_by),
-		reducers: [{ field: config.children, formula: identity }]
+		mapper: omitKeys(config.group_by),
+		reducers: [{ field: config.children, formula: (x) => x }]
 	}
 }
 
 /**
  * Summarizes the dataset by the specified columns.
- * @param {import('./types').dataset} df - The dataset object to summarize.
- * @param {Array} summaries                  - The columns to summarize.
- *
- * @returns {import('./types').dataset}  The summarized dataset object.
  */
 function rollup(data, config) {
 	if (config.group_by.length === 0 && config.summaries.length === 0) {
@@ -153,7 +136,7 @@ function rollup(data, config) {
 		)
 	}
 
-	const summaries = clone(config.summaries)
+	const summaries = [...config.summaries]
 	const hasAlignBy = config.align_by.length > 0
 	if (summaries.length === 0) summaries.push(defaultAggregator(config))
 
@@ -163,19 +146,18 @@ function rollup(data, config) {
 		const fillRows = getAlignGenerator(data, config)
 		alignedData = fillAlignedData(alignedData, config, fillRows)
 		alignedData.forEach((group) => {
-			group[config.children] = sortDataBy(group[config.children], ...config.align_by)
+			group[config.children] = sortDataByFields(group[config.children], ...config.align_by)
 		})
 	}
 
-	const aggregatedData = aggregateData(alignedData, summaries)
-	// const newMetadata = buildMetadata(aggregatedData, df.metadata, df.config.group_by, summaries)
-
-	return aggregatedData
+	return aggregateData(alignedData, summaries)
 }
 
 export class DataSet {
-	#config = clone(defaultConfig)
+	#config = cloneConfig(defaultConfig)
 	#data = []
+	/** @type {Array<{type:string, fn?:Function, fields?:string[]}>} */
+	#ops = []
 
 	constructor(data) {
 		this.#data = data
@@ -198,110 +180,157 @@ export class DataSet {
 		return this
 	}
 
+	// ── Lazy linear ops (fused in select) ──────────────────────────────────
+
 	where(condition) {
-		this.#config.filter = condition
-		return this
-	}
-	groupBy(...fields) {
-		this.#config.group_by = fields
-		return this
-	}
-	alignBy(...fields) {
-		this.#config.align_by = fields
+		this.#ops.push({ type: 'filter', fn: condition })
 		return this
 	}
-	usingTemplate(template) {
-		this.#config.template = template
+	apply(callback) {
+		this.#ops.push({ type: 'map', fn: callback })
 		return this
 	}
-	summarize(from, fields) {
-		this.config = summarize(this.#config, from, fields)
+	sortBy(...fields) {
+		this.#ops.push({ type: 'sort', fields })
 		return this
 	}
 
+	// ── Eager ops (commit to this.#data immediately) ────────────────────────
+
 	rename(how) {
-		this.data = renameKeys(this.data, how)
+		this.#data = renameKeys(this.#data, how)
 		return this
 	}
 	drop(...fields) {
-		const newDataSet = new DataSet(this.data.map((row) => omit(fields, row)))
-		newDataSet.config = this.config
+		const omit = omitKeys(fields)
+		const newDataSet = new DataSet(this.#data.map(omit))
+		newDataSet.config = this.#config
 		return newDataSet
 	}
-	// alter rows
-	sortBy(...fields) {
-		this.data = sortDataBy(this.data, ...fields)
+	fillNA(value) {
+		this.#data = fillNA(this.#data, value)
 		return this
 	}
+
+	// ── Mutation ops (flush lazy ops first, then mutate) ───────────────────
+
 	remove() {
-		if (this.#config.filter) {
-			this.#data = this.data.filter((row) => !this.#config.filter(row))
+		// Flush pending ops so the filter context is respected
+		const pending = this.#ops.splice(0)
+		if (pending.length > 0) {
+			// The last filter op is the one to use for removal
+			const filterOp = [...pending].reverse().find((o) => o.type === 'filter')
+			if (filterOp) {
+				this.#data = this.#data.filter((row) => !filterOp.fn(row))
+			} else {
+				this.#data = []
+			}
 		} else {
 			this.#data = []
 		}
-		this.#config.filter = undefined
-		// this.data = deleteRows(this.data, this.#config.filter || includeAll)
 		return this
 	}
 	update(value) {
-		this.data = updateRows(this.data, this.#config.filter || identity, value)
-		this.#config.filter = undefined
+		// Flush pending ops to get the filter
+		const pending = this.#ops.splice(0)
+		const filterOp = [...pending].reverse().find((o) => o.type === 'filter')
+		const filter = filterOp ? filterOp.fn : () => true
+		this.#data = updateRows(this.#data, filter, value)
 		return this
 	}
-	fillNA(value) {
-		this.data = fillNA(this.#data, value)
+
+	// ── Aggregation ─────────────────────────────────────────────────────────
+
+	groupBy(...fields) {
+		this.#config.group_by = fields
 		return this
 	}
-	// transform data
-	apply(callback) {
-		this.data = this.data.map(callback)
+	alignBy(...fields) {
+		this.#config.align_by = fields
+		return this
+	}
+	usingTemplate(template) {
+		this.#config.template = template
+		return this
+	}
+	summarize(from, fields) {
+		this.config = summarize(this.#config, from, fields)
 		return this
 	}
 	rollup() {
-		return new DataSet(rollup(this.data, this.#config))
+		const data = executeFused(this.#data, this.#ops)
+		this.#ops = []
+		return new DataSet(rollup(data, this.#config))
 	}
+
+	// ── Terminal ────────────────────────────────────────────────────────────
+
 	select(...cols) {
-		const result = this.#config.filter ? this.data.filter(this.#config.filter) : this.data
-		if (cols.length > 0) return result.map(pick(cols))
+		const result = executeFused(this.#data, this.#ops)
+		this.#ops = []
+		if (cols.length > 0) {
+			const pick = pickKeys(cols)
+			return result.map(pick)
+		}
 		return result
 	}
-	// set operations
+
+	/**
+	 * Derive column definitions from the current data.
+	 *
+	 * Returns `{ name, type, scale, sortable, filterable, fields, formatter }` for each
+	 * column, plus any user-supplied properties from `options.enhancements`.
+	 *
+	 * @param {Object} [options]
+	 * @param {Array<Object>} [options.enhancements=[]] - Per-column overrides keyed by name.
+	 * @param {string} [options.language='en-US'] - Locale for formatters.
+	 * @param {string} [options.scanMode='fast'] - 'fast' or 'deep'.
+	 * @returns {Array<Object>}
+	 */
+	columnDefs(options = {}) {
+		return deriveColumnDefs(this.#data, options)
+	}
+
+	// ── Set operations ──────────────────────────────────────────────────────
+
 	union(other) {
 		if (Array.isArray(other)) {
-			return new DataSet(this.data.concat(other))
+			return new DataSet(this.#data.concat(other))
 		}
-		return new DataSet(this.data.concat(other.data))
+		return new DataSet(this.#data.concat(other.data))
 	}
 	minus(other) {
-		return new DataSet(this.data.filter((d) => !other.find((x) => equals(x, d))))
+		return new DataSet(this.#data.filter((d) => !other.find((x) => deepEqual(x, d))))
 	}
 	intersect(other) {
-		return new DataSet(this.data.filter((d) => other.find((x) => equals(x, d))))
+		return new DataSet(this.#data.filter((d) => other.find((x) => deepEqual(x, d))))
 	}
-	// joins
+
+	// ── Joins ───────────────────────────────────────────────────────────────
+
 	innerJoin(other, condition) {
-		return new DataSet(innerJoin(this.data, other.data, condition))
+		return new DataSet(innerJoin(this.#data, other.data, condition))
 	}
 	leftJoin(other, condition) {
-		return new DataSet(leftJoin(this.data, other.data, condition))
+		return new DataSet(leftJoin(this.#data, other.data, condition))
 	}
 	rightJoin(other, condition) {
-		return new DataSet(rightJoin(this.data, other.data, condition))
+		return new DataSet(rightJoin(this.#data, other.data, condition))
 	}
 	fullJoin(other, condition) {
-		return new DataSet(fullJoin(this.data, other.data, condition))
+		return new DataSet(fullJoin(this.#data, other.data, condition))
 	}
 	crossJoin(other) {
-		return new DataSet(crossJoin(this.data, other.data))
+		return new DataSet(crossJoin(this.#data, other.data))
 	}
 	semiJoin(other, condition) {
-		return new DataSet(semiJoin(this.data, other.data, condition))
+		return new DataSet(semiJoin(this.#data, other.data, condition))
 	}
 	antiJoin(other, condition) {
-		return new DataSet(antiJoin(this.data, other.data, condition))
+		return new DataSet(antiJoin(this.#data, other.data, condition))
 	}
 	nestedJoin(other, condition) {
-		return new DataSet(nestedJoin(this.data, other.data, condition, this.#config.children))
+		return new DataSet(nestedJoin(this.#data, other.data, condition, this.#config.children))
 	}
 }
 

package/src/dataset.svelte.js

@@ -0,0 +1,181 @@
+/**
+ * ReactiveDataSet — Svelte 5 rune-based reactive wrapper for dataset operations.
+ *
+ * Use this in Svelte 5 components when you need the pipeline to auto-recompute
+ * when source data changes (e.g. live search, CRUD lists).
+ *
+ * Unlike `DataSet`, which is imperatively pipeline-driven, `ReactiveDataSet` holds
+ * reactive configuration ($state) and exposes a `rows` getter backed by $derived —
+ * so templates can bind to `ds.rows` directly without calling `.select()`.
+ *
+ * File must remain `.svelte.js` for Svelte 5 rune compilation.
+ */
+
+import { executeFused } from './pipeline.js'
+import { deriveColumnDefs } from './infer.js'
+
+export class ReactiveDataSet {
+	/** @type {any[]} */
+	#source = $state([])
+
+	/** @type {Function|null} */
+	#filter = $state(null)
+
+	/** @type {Array<{fn: Function}>} */
+	#transforms = $state([])
+
+	/** @type {string[]} */
+	#sortFields = $state([])
+
+	/** @type {Array<Object>} Per-column overrides merged into auto-derived column defs. */
+	#columnEnhancements = $state([])
+
+	/** @type {Object} Options forwarded to deriveColumnDefs (language, scanMode, etc.) */
+	#columnOptions = $state({})
+
+	/**
+	 * @param {any[]} data - Initial source rows.
+	 */
+	constructor(data = []) {
+		this.#source = data
+	}
+
+	/**
+	 * Reactive computed result of filter + transforms + sort.
+	 * Bind to this in templates instead of calling `.select()`.
+	 */
+	get rows() {
+		const ops = []
+		if (this.#filter) ops.push({ type: 'filter', fn: this.#filter })
+		for (const t of this.#transforms) ops.push({ type: 'map', fn: t.fn })
+		if (this.#sortFields.length > 0) ops.push({ type: 'sort', fields: this.#sortFields })
+		return executeFused(this.#source, ops)
+	}
+
+	/**
+	 * Reactive derived column definitions: name, type, scale (discrete|continuous),
+	 * sortable, filterable, fields, formatter — merged with any `withColumns` overrides.
+	 *
+	 * Re-computes when source data or column enhancements change.
+	 * Bind to this in templates for Table / TreeTable column specs.
+	 */
+	get columns() {
+		return deriveColumnDefs(this.#source, {
+			...this.#columnOptions,
+			enhancements: this.#columnEnhancements
+		})
+	}
+
+	// ── Configuration (chainable) ────────────────────────────────────────────
+
+	/**
+	 * Set the filter predicate. Replaces any previous filter.
+	 * @param {Function|null} fn
+	 */
+	where(fn) {
+		this.#filter = fn
+		return this
+	}
+
+	/**
+	 * Append a row transform. Each call to `apply` adds a new stage.
+	 * @param {Function} fn
+	 */
+	apply(fn) {
+		this.#transforms = [...this.#transforms, { fn }]
+		return this
+	}
+
+	/**
+	 * Set sort fields. Replaces any previous sort.
+	 * @param {...string} fields
+	 */
+	sortBy(...fields) {
+		this.#sortFields = fields
+		return this
+	}
+
+	/**
+	 * Set per-column enhancements / overrides for the reactive `columns` property.
+	 *
+	 * Each entry is matched by `name` and merged into the auto-derived column def.
+	 * Unknown names are appended as synthetic columns (useful for action columns).
+	 *
+	 * @param {Array<Object>} enhancements - e.g. [{ name: 'revenue', label: 'Revenue ($)', scale: 'continuous' }]
+	 * @param {Object} [options] - Options forwarded to deriveColumnDefs (language, scanMode).
+	 */
+	withColumns(enhancements, options = {}) {
+		this.#columnEnhancements = enhancements
+		this.#columnOptions = options
+		return this
+	}
+
+	/** Clear the current filter. */
+	clearFilter() {
+		this.#filter = null
+		return this
+	}
+
+	/** Clear all transforms. */
+	clearTransforms() {
+		this.#transforms = []
+		return this
+	}
+
+	// ── Data mutation ────────────────────────────────────────────────────────
+
+	/**
+	 * Replace the entire source dataset.
+	 * @param {any[]} rows
+	 */
+	setData(rows) {
+		this.#source = rows
+		return this
+	}
+
+	/**
+	 * Append a single row to the source.
+	 * @param {any} row
+	 */
+	push(row) {
+		this.#source = [...this.#source, row]
+		return this
+	}
+
+	/**
+	 * Remove rows matching the predicate from the source.
+	 * @param {Function} predicate
+	 */
+	remove(predicate) {
+		this.#source = this.#source.filter((r) => !predicate(r))
+		return this
+	}
+
+	/**
+	 * Update rows matching the predicate with patch (object or function).
+	 * @param {Function} predicate
+	 * @param {Object|Function} patch
+	 */
+	update(predicate, patch) {
+		this.#source = this.#source.map((row) => {
+			if (!predicate(row)) return row
+			return typeof patch === 'function' ? patch(row) : { ...row, ...patch }
+		})
+		return this
+	}
+
+	/** Return a plain snapshot of the current reactive result (non-reactive copy). */
+	snapshot() {
+		return [...this.rows]
+	}
+}
+
+/**
+ * Factory function for `ReactiveDataSet`.
+ *
+ * @param {any[]} [data=[]] - Initial source rows.
+ * @returns {ReactiveDataSet}
+ */
+export function reactiveDataset(data = []) {
+	return new ReactiveDataSet(data)
+}

package/src/formatter.js

@@ -1,4 +1,4 @@
-import { identity } from 'ramda'
+const identity = (x) => x
 /**
  * Returns a currency formatter function that formats a value as a currency string.
  * @param {string} language       - The IETF language tag used for locale-specific formatting.

package/src/index.js

@@ -6,6 +6,14 @@ export { model } from './model'
 export { innerJoin, leftJoin, rightJoin, fullJoin, crossJoin, antiJoin, semiJoin } from './join'
 export { dataset } from './dataset'
 export { dataview } from './view'
-export { deriveColumns, deriveMetadata, deriveSortableColumn } from './infer'
+export {
+	deriveColumns,
+	deriveMetadata,
+	deriveSortableColumn,
+	inferScale,
+	addScales,
+	deriveColumnDefs,
+	mergeColumnEnhancements
+} from './infer'
 export { parseFilters } from './parser'
 export { filterData, filterObjectArray } from './filter'