@rokkit/data 1.0.0-next.147 → 1.0.0-next.149

package/src/infer.js

@@ -1,6 +1,5 @@
 import { ascending, descending } from 'd3-array'
 import { compact } from '@rokkit/core'
-import { omit } from 'ramda'
 import { createFormatter } from './formatter'
 import { defaultViewOptions, defaultActionOrder } from './constants'
 import { typeOf } from './utils'
@@ -101,7 +100,7 @@ export function getSample(data, deepScan = true) {
 	return data.reduce(
 		(acc, cur) => ({
 			...acc,
-			...compact(omit(Object.keys(acc), cur))
+			...compact(Object.fromEntries(Object.entries(cur).filter(([k]) => !(k in acc))))
 		}),
 		{}
 	)
@@ -274,3 +273,102 @@ export function deriveMetadata(dataArray, options = {}) {
 
 	return columns
 }
+
+// ── Scale inference ──────────────────────────────────────────────────────────
+
+/**
+ * Infers whether a column's values are discrete (categorical) or continuous (quantitative).
+ *
+ * Rules:
+ *  - string, boolean           → always discrete
+ *  - date, number (float)      → always continuous
+ *  - integer                   → discrete when ≤20 unique values OR unique ratio < 5 %,
+ *                                continuous otherwise (e.g. large ID column)
+ *  - anything else             → discrete (safe default for unknown types)
+ *
+ * @param {any[]} values - All values for the column (may include nulls).
+ * @returns {'discrete'|'continuous'}
+ */
+export function inferScale(values) {
+	const nonNull = values.filter((v) => v !== null && v !== undefined)
+	if (nonNull.length === 0) return 'discrete'
+
+	const type = typeOf(nonNull[0])
+
+	if (type === 'string' || type === 'boolean') return 'discrete'
+	if (type === 'date' || type === 'number') return 'continuous'
+
+	if (type === 'integer') {
+		const unique = new Set(nonNull).size
+		return unique <= 20 || unique / nonNull.length < 0.05 ? 'discrete' : 'continuous'
+	}
+
+	return 'discrete'
+}
+
+/**
+ * Adds a `scale` property to each column definition.
+ *
+ * @param {Array<Object>} columns - Column definitions (from deriveColumns / deriveMetadata).
+ * @param {any[]} data - Source rows.
+ * @returns {Array<Object>} Column definitions with `scale` added.
+ */
+export function addScales(columns, data) {
+	return columns.map((col) => ({
+		...col,
+		scale: inferScale(data.map((r) => r[col.name]))
+	}))
+}
+
+/**
+ * Merges user-provided enhancements into auto-derived column definitions.
+ * Enhancements are matched by `name` and merged (user values win).
+ * Enhancements for unknown column names are appended as synthetic columns.
+ *
+ * @param {Array<Object>} derived - Auto-derived column definitions.
+ * @param {Array<Object>} enhancements - User-provided overrides/additions.
+ * @returns {Array<Object>} Merged column definitions.
+ */
+export function mergeColumnEnhancements(derived, enhancements) {
+	const enhMap = new Map(enhancements.map((e) => [e.name, e]))
+	const result = derived.map((col) => {
+		const enh = enhMap.get(col.name)
+		return enh ? { ...col, ...enh } : col
+	})
+	const derivedNames = new Set(derived.map((c) => c.name))
+	for (const e of enhancements) {
+		if (!derivedNames.has(e.name)) result.push(e)
+	}
+	return result
+}
+
+/**
+ * Full column definition pipeline: derive → format → scale → merge enhancements.
+ *
+ * Produces column descriptors suitable for use in Table, TreeTable, and chart
+ * components. Each column gets: name, type, scale, sortable, filterable, fields,
+ * formatter — plus any user-supplied properties from `enhancements`.
+ *
+ * @param {any[]} data - Source rows.
+ * @param {Object} [options] - Options forwarded to deriveColumns / addFormatters.
+ * @param {Array<Object>} [options.enhancements=[]] - Per-column overrides keyed by name.
+ *        Use to override scale, add a label, attach a custom formatter, etc.
+ * @param {string} [options.language='en-US'] - Locale for formatters.
+ * @param {string} [options.scanMode='fast'] - 'fast' (first row) or 'deep' (all rows).
+ * @returns {Array<Object>} Column definitions with scale and formatters.
+ */
+export function deriveColumnDefs(data, options = {}) {
+	const { enhancements = [], language, scanMode } = { ...defaultViewOptions, ...options }
+
+	if (data.length === 0) return enhancements
+
+	let columns = deriveColumns(data, { scanMode })
+	columns = addFormatters(columns, language)
+	columns = addScales(columns, data)
+
+	if (enhancements.length > 0) {
+		columns = mergeColumnEnhancements(columns, enhancements)
+	}
+
+	return columns
+}

package/src/join.js

@@ -13,14 +13,19 @@
  * @returns {Array<Object>}            - An object with the inner and outer join result.
  */
 function joinData(first, second, condition) {
-	let inner = []
+	const inner = []
 	const outer = []
 
-	first.forEach((f) => {
-		const matches = second.filter((s) => condition(f, s)).map((m) => ({ ...m, ...f }))
-		inner = inner.concat(matches)
-		if (matches.length === 0) outer.push(f)
-	})
+	for (const f of first) {
+		let matched = false
+		for (const s of second) {
+			if (condition(f, s)) {
+				inner.push({ ...s, ...f })
+				matched = true
+			}
+		}
+		if (!matched) outer.push(f)
+	}
 	return { inner, outer }
 }
 

package/src/model.js

@@ -1,4 +1,4 @@
-import { identity, clone as copy } from 'ramda'
+const copy = (val) => structuredClone(val)
 import { getSample } from './infer'
 import { typeOf } from './utils'
 
@@ -67,8 +67,8 @@ export function model() {
 		 * Renames the reference names using a renamer
 		 * @param {Function} rename - the renamer function
 		 */
-		renameUsing: (rename = identity) => {
-			if (rename !== identity) {
+		renameUsing: (rename) => {
+			if (rename) {
 				data = data.map((x) => ({ ...x, name: rename(x.name) }))
 			}
 			return actions

package/src/pipeline.js

@@ -0,0 +1,91 @@
+/**
+ * Shared pipeline execution logic used by both DataSet and ReactiveDataSet.
+ *
+ * Kept in a plain `.js` file so it can be imported from both regular modules
+ * and `.svelte.js` files without triggering rune compilation errors.
+ */
+
+import { deriveSortableColumn } from './infer.js'
+
+/**
+ * Sort an array in-place by one or more field specs.
+ * @param {any[]} data - Array to sort (mutated in place).
+ * @param {...string|Object|Array} fields - Sort field specs (same as DataSet.sortBy).
+ * @returns {any[]}
+ */
+export function sortDataByFields(data, ...fields) {
+	const sorters = fields.map(deriveSortableColumn)
+	data.sort((a, b) => {
+		let result = 0
+		for (let i = 0; i < sorters.length && result === 0; i++) {
+			const { name, sorter } = sorters[i]
+			result = sorter(a[name], b[name])
+		}
+		return result
+	})
+	return data
+}
+
+/**
+ * Execute a list of pipeline ops in a single fused pass over `data`.
+ *
+ * Ops:
+ *   { type: 'filter', fn }  — keep rows where fn(row) is truthy
+ *   { type: 'map',    fn }  — transform row via fn(row)
+ *   { type: 'sort', fields } — sort the result by fields
+ *
+ * Consecutive filter/map ops are merged into one loop.
+ * Sort ops are applied after the linear pass.
+ *
+ * @param {any[]} data
+ * @param {Array<{type:string, fn?:Function, fields?:string[]}>} ops
+ * @returns {any[]}
+ */
+export function executeFused(data, ops) {
+	if (ops.length === 0) return data
+
+	let result = data
+	let i = 0
+
+	while (i < ops.length) {
+		// Collect a run of linear (filter/map) ops
+		const linearOps = []
+		while (i < ops.length && ops[i].type !== 'sort') {
+			linearOps.push(ops[i])
+			i++
+		}
+
+		if (linearOps.length > 0) {
+			const out = []
+			for (const row of result) {
+				let cur = row
+				let keep = true
+				for (const op of linearOps) {
+					if (op.type === 'filter') {
+						if (!op.fn(cur)) {
+							keep = false
+							break
+						}
+					} else {
+						cur = op.fn(cur)
+					}
+				}
+				if (keep) out.push(cur)
+			}
+			result = out
+		}
+
+		// Collect a run of sort ops and apply them as a single sort
+		const sortFields = []
+		while (i < ops.length && ops[i].type === 'sort') {
+			sortFields.push(...ops[i].fields)
+			i++
+		}
+		if (sortFields.length > 0) {
+			result = [...result]
+			sortDataByFields(result, ...sortFields)
+		}
+	}
+
+	return result
+}

package/src/renamer.js

@@ -1,4 +1,11 @@
-import { omit, identity } from 'ramda'
+const identity = (x) => x
+
+function omit(keys, obj) {
+	const skip = new Set(keys)
+	const out = {}
+	for (const k of Object.keys(obj)) if (!skip.has(k)) out[k] = obj[k]
+	return out
+}
 
 /**
  *

package/src/rollup.js

@@ -1,5 +1,3 @@
-import { omit, pick, mergeLeft, map, difference, uniq, pipe } from 'ramda'
-
 /**
  * Sets initial values for summaries.
  *
@@ -28,7 +26,7 @@ export function addToSummaries(group, row, summaries) {
 }
 
 /**
- * Groups data by specified keys.
+ * Groups data by specified keys using a composite string key (Phase 3: replaces JSON.stringify).
  *
  * @param {Array} data - An array of data points.
  * @param {Array} groupByKeys - Keys to group the data by.
@@ -36,17 +34,22 @@ export function addToSummaries(group, row, summaries) {
  * @returns {Object} Grouped data object with aggregation placeholders.
  */
 export function groupDataByKeys(data, groupByKeys, summaries) {
-	const keysExtractor = pick(groupByKeys)
-	const groupedData = data.reduce((grouped, row) => {
-		const key = JSON.stringify(keysExtractor(row))
-		if (!grouped[key]) {
-			grouped[key] = { ...keysExtractor(row), ...initialValues(summaries) }
+	const groupedData = new Map()
+
+	for (const row of data) {
+		// Composite key: avoids JSON.stringify overhead
+		const key = groupByKeys.map((k) => row[k]).join('\x00')
+
+		if (!groupedData.has(key)) {
+			const group = {}
+			for (const k of groupByKeys) group[k] = row[k]
+			Object.assign(group, initialValues(summaries))
+			groupedData.set(key, group)
 		}
-		addToSummaries(grouped[key], row, summaries)
-		return grouped
-	}, {})
+		addToSummaries(groupedData.get(key), row, summaries)
+	}
 
-	return Object.values(groupedData)
+	return [...groupedData.values()]
 }
 
 /**
@@ -57,9 +60,8 @@ export function groupDataByKeys(data, groupByKeys, summaries) {
  * @returns {Array} Aligned data array.
  */
 export function fillAlignedData(groupedData, config, fillRowsFunc) {
-	// const fillRowsFunc = getAlignGenerator(df.data, config)
 	const { actual_flag, children } = config
-	return Object.values(groupedData).map((row) => ({
+	return groupedData.map((row) => ({
 		...row,
 		[children]: [
 			...row[children].map((value) => ({ ...value, [actual_flag]: 1 })),
@@ -68,6 +70,49 @@ export function fillAlignedData(groupedData, config, fillRowsFunc) {
 	}))
 }
 
+/**
+ * Creates a generator that produces missing rows in a dataset based on specified columns.
+ *
+ * @param {Array<Object>} data - The array of objects representing the dataset.
+ * @param {Object} config - The configuration object.
+ * @returns {Function} A generator function that when called, produces the missing rows.
+ */
+export function getAlignGenerator(data, config) {
+	const { align_by, group_by, actual_flag } = config
+
+	// Build template: omit align_by and group_by keys, add actual_flag=0
+	const omitKeys = new Set([...align_by, ...group_by])
+	const template = { [actual_flag]: 0 }
+	for (const [k, v] of Object.entries(config.template)) {
+		if (!omitKeys.has(k)) template[k] = v
+	}
+
+	// Collect all unique combinations of align_by fields from full dataset
+	const seen = new Map()
+	for (const row of data) {
+		const key = align_by.map((k) => row[k]).join('\x00')
+		if (!seen.has(key)) {
+			const combo = {}
+			for (const k of align_by) combo[k] = row[k]
+			seen.set(key, combo)
+		}
+	}
+	const allCombos = [...seen.values()]
+
+	// Return a function that, given a group's children, yields the missing combos
+	return (children) => {
+		const childKeys = new Set(children.map((c) => align_by.map((k) => c[k]).join('\x00')))
+		const missing = []
+		for (const combo of allCombos) {
+			const key = align_by.map((k) => combo[k]).join('\x00')
+			if (!childKeys.has(key)) {
+				missing.push({ ...combo, ...template })
+			}
+		}
+		return missing
+	}
+}
+
 /**
  * Performs aggregation operations on grouped data.
  *
@@ -88,51 +133,3 @@ export function aggregateData(dataArray, summaries) {
 		)
 	}))
 }
-
-/**
- * Creates a generator that produces missing rows in a dataset based on specified columns.
- * The function determines all unique combinations of the specified columns in config.align_by
- * and returns a generator function to create the missing combinations, potentially with default
- * values for other columns.
- * @param {Array<Object>} data - The array of objects representing the dataset.
- * @param {Object} config - The configuration object.
- *
- * @returns {Function} A generator function that when called, produces the missing rows.
- */
-export function getAlignGenerator(data, config) {
-	const { align_by, group_by, actual_flag } = config
-	const template = { ...omit([...align_by, ...group_by], config.template), [actual_flag]: 0 }
-	const subset = pipe(map(pick(align_by)), uniq)(data)
-
-	return pipe(map(pick(align_by)), uniq, difference(subset), map(mergeLeft(template)))
-}
-
-/**
- * Returns an aggregator object with a mapper and reducer function.
- *
- * @param {Array<string>} from  - The key or keys to aggregate.
- * @param {Function}      using - The aggregation function.
- * @returns {import('./types').SummaryConfig}  - An object with a mapper and reducer function.
- */
-// export function getAggregator(from, into, using = identity) {
-// 	// const mapper = pick(Array.isArray ? from : [from])
-// 	return {
-// 		mapper: pick(from),
-// 		reducers: [{ field: into, formula: using }]
-// 	}
-// }
-
-/**
- * Returns the default aggregator which rolls up the columns other than the group_by columns into a single object.
- *
- * @param {import('./types').Metadata} metadata - The metadata for the columns to be aggregated.
- * @param {Object} config                       - The configuration used to build the aggregator.
- *
- * @returns {Object} An object containing the default aggregator for the specified metadata and configuration.
- */
-// export function defaultAggregator(metadata, config) {
-// 	const child = metadata.filter((col) => !config.group_by.includes(col.name))
-// 	const keys = child.map((col) => col.name)
-
-// 	return { ...getAggregator(keys, config.children), metadata: child }
-// }

package/src/view.js

@@ -1,4 +1,3 @@
-import { omit } from 'ramda'
 import { get, writable } from 'svelte/store'
 import { deriveMetadata, deriveSortableColumn } from './infer'
 
@@ -162,7 +161,7 @@ export function dataview(data, options) {
 	let sortGroup = []
 
 	return {
-		...omit(['set', 'update'], store),
+		...Object.fromEntries(Object.entries(store).filter(([k]) => k !== 'set' && k !== 'update')),
 		// filter: noop,
 		/**
 		 * Clears the applied sort order from the hierarchy.