@rokkit/data 1.0.0-next.98 → 1.0.1

package/src/join.js

@@ -0,0 +1,157 @@
+/**
+ * Joins two datasets together based on a condition. Result includes all rows from the first
+ * dataset and matching rows from the second dataset. In case of multiple matches, all
+ * combinations are returned. When combining the rows, the columns from the first dataset take
+ * precedence.
+ *
+ * inner: only the rows that have a match in both datasets.
+ * outer: all rows from the first dataset and matching rows from the second dataset.
+ *
+ * @param {Array<Object>}    first     - The first dataset to join.
+ * @param {Array<Object>}    second    - The second dataset to join.
+ * @param {Function}         condition - The condition to join the datasets on.
+ * @returns {Array<Object>}            - An object with the inner and outer join result.
+ */
+function joinData(first, second, condition) {
+	const inner = []
+	const outer = []
+
+	for (const f of first) {
+		let matched = false
+		for (const s of second) {
+			if (condition(f, s)) {
+				inner.push({ ...s, ...f })
+				matched = true
+			}
+		}
+		if (!matched) outer.push(f)
+	}
+	return { inner, outer }
+}
+
+/**
+ * Filters the results of the first dataset based on the condition using the second dataset.
+ *
+ * @param {Array<Object>}    first     - The first dataset to filter.
+ * @param {Array<Object>}    second    - The second dataset to filter by.
+ * @param {Function}        condition - The condition to filter the first dataset by.
+ * @returns {Array<Object>}            - The filtered dataset.
+ */
+export function antiJoin(first, second, condition) {
+	return first.filter((f) => !second.find((s) => condition(f, s)))
+}
+
+/**
+ * Joins two datasets together based on a condition. Result includes all rows from the first
+ * dataset and matching rows from the second dataset. In case of multiple matches, all
+ * combinations are returned. When combining the rows, the columns from the first dataset take
+ * precedence.
+ *
+ * @param {Array<Object>}    first     - The first dataset to join.
+ * @param {Array<Object>}    second    - The second dataset to join.
+ * @param {Function}      condition - The condition to join the datasets on.
+ * @returns {Array<Object>}            - The joined dataset.
+ */
+export function innerJoin(first, second, condition) {
+	const { inner } = joinData(first, second, condition)
+	return inner
+}
+
+/**
+ * Performs a left join on two datasets based on a condition.
+ *
+ * - Result includes all rows from the first dataset and matching rows from the second dataset.
+ * - In case of multiple matches, all combinations are returned.
+ * - When combining the rows, the columns from the first dataset take precedence.
+ * - If there is no match in the second dataset, only the row from the first dataset is returned.
+ *
+ * This can be used to perform a right join by swapping the datasets.
+ *
+ * @param {Array<Object>}    first     - The first dataset to join.
+ * @param {Array<Object>}    second    - The second dataset to join.
+ * @param {Function}         condition - The condition to join the datasets on.
+ * @returns {Array<Object>}            - The joined dataset.
+ */
+export function leftJoin(first, second, condition) {
+	const { inner, outer } = joinData(first, second, condition)
+	return inner.concat(outer)
+}
+
+/**
+ * Performs a right join on two datasets based on a condition.
+ *
+ * - Result includes all rows from both datasets.
+ * - In case of multiple matches, all combinations are returned.
+ * - When combining the rows, the columns from the second dataset take precedence.
+ * - If there is no match in the first dataset, only the row from the second dataset is returned.
+ *
+ * @param {Array<Object>}    first     - The first dataset to join.
+ * @param {Array<Object>}    second    - The second dataset to join.
+ * @param {Function}   condition - The condition to join the datasets on.
+ * @returns {Array<Object>}            - The joined dataset.
+ */
+export function rightJoin(first, second, condition) {
+	return leftJoin(second, first, condition)
+}
+
+/**
+ * Performs a full join on two datasets based on a condition.
+ *
+ * - Result includes all rows from both datasets.
+ * - In case of multiple matches, all combinations are returned.
+ * - When combining the rows, the columns from the second dataset take precedence.
+ * - If there is no match in the first dataset, only the row from the second dataset is returned.
+ *
+ * @param {Array<Object>}    first     - The first dataset to join.
+ * @param {Array<Object>}    second    - The second dataset to join.
+ * @param {Function}         condition - The condition to join the datasets on.
+ * @returns {Array<Object>}            - The joined dataset.
+ */
+export function fullJoin(first, second, condition) {
+	const { inner, outer } = joinData(first, second, condition)
+	const rightOuter = antiJoin(second, first, condition)
+	return inner.concat(outer).concat(rightOuter)
+}
+
+/**
+ * Performs a cross join on two datasets. Result includes all combinations of rows from both datasets.
+ *
+ * @param {Array<Object>} first  - The first dataset to join.
+ * @param {Array<Object>} second - The second dataset to join.
+ * @returns {Array<Object>}      - The joined dataset.
+ */
+export function crossJoin(first, second) {
+	return first.map((f) => second.map((s) => ({ ...f, ...s }))).flat()
+}
+
+/**
+ * Performs a semi join on two datasets based on a condition. Result includes all rows from the first
+ * dataset that have a match in the second dataset.
+ *
+ * @param {Array<Object>}    first     - The first dataset to join.
+ * @param {Array<Object>}    second    - The second dataset to join.
+ * @param {Function}     condition - The condition to join the datasets on.
+ * @returns {Array<Object>}            - The joined dataset.
+ */
+export function semiJoin(first, second, condition) {
+	return first.filter((f) => second.find((s) => condition(f, s)))
+}
+
+/**
+ * Performs a nested join on two datasets based on a condition. Result includes all rows from the first
+ * dataset that have a match in the second dataset. The result is nested with the matching rows from the
+ * second dataset.
+ *
+ * @param {Array<Object>}    first            - The first dataset to join.
+ * @param {Array<Object>}    second           - The second dataset to join.
+ * @param {Function}         condition        - The condition to join the datasets on.
+ * @param {String}   [key='children'] - The key to nest the matching rows under.
+ * @returns {Array<Object>}            - The joined dataset.
+ */
+export function nestedJoin(first, second, condition, key = 'children') {
+	const result = first.map((f) => ({
+		...f,
+		[key]: second.filter((s) => condition(f, s))
+	}))
+	return result
+}

package/src/metadata.js

@@ -1,5 +1,5 @@
 import { defaultPathOptions } from './constants'
-import { getDeepScanSample } from './infer'
+import { getSample } from './infer'
 import { typeOf } from './utils'
 
 /**
@@ -88,8 +88,8 @@ export function deriveColumnProperties(sample, options) {
 export function deriveColumnMetadata(data, options = {}) {
 	const { metadata, deepScan = false } = options
 	if (Array.isArray(metadata) && metadata.length > 0) return metadata
-	if (data.length === 0) return []
-	const sample = deepScan ? getDeepScanSample(data) : data[0]
+	// if (data.length === 0) return []
+	const sample = getSample(data, deepScan)
 	return Object.entries(sample).map(([name, value]) => ({ name, type: typeOf(value) }))
 }
 

package/src/model.js

@@ -1,7 +1,51 @@
-import { identity, clone as copy } from 'ramda'
-import { getDeepScanSample } from './infer'
+const copy = (val) => structuredClone(val)
+import { getSample } from './infer'
 import { typeOf } from './utils'
 
+/**
+ * Derives a model from the input data
+ *
+ * @param {Array|Object} data  - the data to derive the model from
+ * @param {Boolean}      sparse - whether the data set contains sparse data
+ * @returns {Array}             - the derived model
+ */
+function deriveModel(data, sparse = false) {
+	const result = []
+	let item = data
+
+	if (Array.isArray(data)) item = sparse ? getSample(data) : data[0]
+
+	const kv = Object.entries(item)
+	kv.forEach(([key, value]) => {
+		result.push({
+			name: key,
+			type: typeOf(value)
+		})
+	})
+	return result
+}
+
+/**
+ * Merges two models, returning a new model
+ *
+ * @param {Array} first      - the first model
+ * @param {Array} second     - the second model
+ * @param {Boolean} override - whether to override the type of the first model
+ * @returns {Array}      - the merged model
+ */
+function mergeModels(first, second, override = false) {
+	second.forEach(({ name, type }) => {
+		const existing = first.find((x) => x.name === name)
+		if (existing) {
+			if (existing.type !== type) existing.mixedTypes = true
+			if (override) existing.type = type
+		} else {
+			first.push({ name, type })
+		}
+	})
+	return first
+}
+
 /**
  * A model is a representation of a dataset
  * @returns an object with methods to manipulate the model
@@ -23,8 +67,8 @@ export function model() {
 		 * Renames the reference names using a renamer
 		 * @param {Function} rename - the renamer function
 		 */
-		renameUsing: (rename = identity) => {
-			if (rename !== identity) {
+		renameUsing: (rename) => {
+			if (rename) {
 				data = data.map((x) => ({ ...x, name: rename(x.name) }))
 			}
 			return actions
@@ -54,47 +98,3 @@ export function model() {
 	}
 	return actions
 }
-
-/**
- * Derives a model from the input data
- *
- * @param {Array|Object} data  - the data to derive the model from
- * @param {Boolean}      sparse - whether the data set contains sparse data
- * @returns {Array}             - the derived model
- */
-function deriveModel(data, sparse = false) {
-	const result = []
-	let item = data
-
-	if (Array.isArray(data)) item = sparse ? getDeepScanSample(data) : data[0]
-
-	const kv = Object.entries(item)
-	kv.forEach(([key, value]) => {
-		result.push({
-			name: key,
-			type: typeOf(value)
-		})
-	})
-	return result
-}
-
-/**
- * Merges two models, returning a new model
- *
- * @param {Array} first      - the first model
- * @param {Array} second     - the second model
- * @param {Boolean} override - whether to override the type of the first model
- * @returns {Array}      - the merged model
- */
-function mergeModels(first, second, override = false) {
-	second.forEach(({ name, type }) => {
-		const existing = first.find((x) => x.name === name)
-		if (existing) {
-			if (existing.type !== type) existing.mixedTypes = true
-			if (override) existing.type = type
-		} else {
-			first.push({ name, type })
-		}
-	})
-	return first
-}

package/src/parser.js

@@ -28,6 +28,48 @@ export function getRegex() {
 	return new RegExp(pattern, 'gm')
 }
 
+/**
+ * Replaces various shorthand operators with their standardized equivalent for filtering.
+ *
+ * @param {string} operator - The operator to be replaced.
+ * @returns {string} - The replaced operator string.
+ */
+function replaceOperators(operator) {
+	return operator.replace(':', '~*').replace('=>', '>=').replace('=<', '<=')
+}
+
+/**
+ * Removes double quotes from the start and end of a string.
+ *
+ * @param {string} str - The input string from which the surrounding quotes should be removed.
+ * @returns {string} - The unquoted string.
+ */
+function removeQuotes(str) {
+	const quoteMatch = str.match(/^"([^"]+)"$/)
+	return quoteMatch ? quoteMatch[1] : str
+}
+
+/**
+ * Processes the filter value provided, converting it into a format suitable for filtering.
+ * If the operator includes a tilde (~), the value is converted to a RegExp.
+ *
+ * @param {string|number} value - The value to be processed. It could be a string needing quote removal or a number that needs parsing.
+ * @param {string} operator - The operator that determines the type of processing to be applied to the value.
+ * @returns {string|number|RegExp} - The processed value, which may be a RegExp if the operator involves pattern matching.
+ */
+function processValue(value, operator) {
+	// If the value can be parsed as an integer, do so. Otherwise, strip quotes if present.
+
+	value = !isNaN(parseInt(value)) ? parseInt(value) : removeQuotes(value)
+	// If the operator includes a tilde (~), treat the value as a regular expression,
+	// case-insensitive if '*' is present, sensitive otherwise.
+	if (operator.includes('~')) {
+		value = operator.includes('*') ? new RegExp(value, 'i') : new RegExp(value)
+	}
+
+	return value
+}
+
 /**
  * Parses a search string and extracts filters based on a predefined regular expression pattern.
  * Each filter captures column, operator, and value components. Any remaining text that
@@ -67,45 +109,3 @@ export function parseFilters(string) {
 
 	return results
 }
-
-/**
- * Replaces various shorthand operators with their standardized equivalent for filtering.
- *
- * @param {string} operator - The operator to be replaced.
- * @returns {string} - The replaced operator string.
- */
-function replaceOperators(operator) {
-	return operator.replace(':', '~*').replace('=>', '>=').replace('=<', '<=')
-}
-
-/**
- * Processes the filter value provided, converting it into a format suitable for filtering.
- * If the operator includes a tilde (~), the value is converted to a RegExp.
- *
- * @param {string|number} value - The value to be processed. It could be a string needing quote removal or a number that needs parsing.
- * @param {string} operator - The operator that determines the type of processing to be applied to the value.
- * @returns {string|number|RegExp} - The processed value, which may be a RegExp if the operator involves pattern matching.
- */
-function processValue(value, operator) {
-	// If the value can be parsed as an integer, do so. Otherwise, strip quotes if present.
-
-	value = !isNaN(parseInt(value)) ? parseInt(value) : removeQuotes(value)
-	// If the operator includes a tilde (~), treat the value as a regular expression,
-	// case-insensitive if '*' is present, sensitive otherwise.
-	if (operator.includes('~')) {
-		value = operator.includes('*') ? new RegExp(value, 'i') : new RegExp(value)
-	}
-
-	return value
-}
-
-/**
- * Removes double quotes from the start and end of a string.
- *
- * @param {string} str - The input string from which the surrounding quotes should be removed.
- * @returns {string} - The unquoted string.
- */
-function removeQuotes(str) {
-	const quoteMatch = str.match(/^"([^"]+)"$/)
-	return quoteMatch ? quoteMatch[1] : str
-}

package/src/pipeline.js

@@ -0,0 +1,97 @@
+/**
+ * Shared pipeline execution logic used by both DataSet and ReactiveDataSet.
+ *
+ * Kept in a plain `.js` file so it can be imported from both regular modules
+ * and `.svelte.js` files without triggering rune compilation errors.
+ */
+
+import { deriveSortableColumn } from './infer.js'
+
+/**
+ * Sort an array in-place by one or more field specs.
+ * @param {any[]} data - Array to sort (mutated in place).
+ * @param {...string|Object|Array} fields - Sort field specs (same as DataSet.sortBy).
+ * @returns {any[]}
+ */
+export function sortDataByFields(data, ...fields) {
+	const sorters = fields.map(deriveSortableColumn)
+	data.sort((a, b) => {
+		let result = 0
+		for (let i = 0; i < sorters.length && result === 0; i++) {
+			const { name, sorter } = sorters[i]
+			result = sorter(a[name], b[name])
+		}
+		return result
+	})
+	return data
+}
+
+/**
+ * Execute a list of pipeline ops in a single fused pass over `data`.
+ *
+ * Ops:
+ *   { type: 'filter', fn }  — keep rows where fn(row) is truthy
+ *   { type: 'map',    fn }  — transform row via fn(row)
+ *   { type: 'sort', fields } — sort the result by fields
+ *
+ * Consecutive filter/map ops are merged into one loop.
+ * Sort ops are applied after the linear pass.
+ *
+ * @param {any[]} data
+ * @param {Array<{type:string, fn?:Function, fields?:string[]}>} ops
+ * @returns {any[]}
+ */
+function applyOpsToRow(row, linearOps) {
+	let cur = row
+	for (const op of linearOps) {
+		if (op.type === 'filter') {
+			if (!op.fn(cur)) return null
+		} else {
+			cur = op.fn(cur)
+		}
+	}
+	return cur
+}
+
+function applyLinearOps(data, linearOps) {
+	const out = []
+	for (const row of data) {
+		const result = applyOpsToRow(row, linearOps)
+		if (result !== null) out.push(result)
+	}
+	return out
+}
+
+function collectLinearOps(ops, i) {
+	const linearOps = []
+	while (i < ops.length && ops[i].type !== 'sort') linearOps.push(ops[i++])
+	return { linearOps, i }
+}
+
+function collectSortFields(ops, i) {
+	const sortFields = []
+	while (i < ops.length && ops[i].type === 'sort') sortFields.push(...ops[i++].fields)
+	return { sortFields, i }
+}
+
+export function executeFused(data, ops) {
+	if (ops.length === 0) return data
+
+	let result = data
+	let i = 0
+
+	while (i < ops.length) {
+		const linear = collectLinearOps(ops, i)
+		i = linear.i
+		if (linear.linearOps.length > 0) result = applyLinearOps(result, linear.linearOps)
+
+		const sorted = collectSortFields(ops, i)
+		i = sorted.i
+		if (sorted.sortFields.length > 0) {
+			result = [...result]
+			sortDataByFields(result, ...sorted.sortFields)
+		}
+	}
+
+	return result
+}

package/src/renamer.js

@@ -1,4 +1,11 @@
-import { omit, identity } from 'ramda'
+const identity = (x) => x
+
+function omit(keys, obj) {
+	const skip = new Set(keys)
+	const out = {}
+	for (const k of Object.keys(obj)) if (!skip.has(k)) out[k] = obj[k]
+	return out
+}
 
 /**
  *
@@ -9,39 +16,25 @@ import { omit, identity } from 'ramda'
  * @property {string} [separator='_'] - The separator to use between the key and the prefix or suffix.
  * @property {Array}  [keys]          - The keys to rename.
  */
-
 /**
- * A generator function that produces a function to rename keys.
- *
- * @param {OptionsToRenameKeys} options - Options to rename keys
- * @returns {Object}                    - A renamer object.
- *
- * Example:
+ * Renames the keys of an object using a lookup table.
  *
- * renamer({ prefix: 'a', keys: ['a', 'b'] }).get()
- * renamer({ prefix: 'a', separator: '-' }).get()
- * renamer().setPrefix('a')
- *          .setSeparator('-')
- *          .setKeys(['a', 'b'])
- *          .get()
- * renamer().setPrefix('a').get()
- * renamer().setSuffix('b').get()
- * renamer().setKeys(['a', 'b']).get()
+ * @param {Object} lookup - The lookup table.
+ * @returns {Function} - A function that takes an object and renames the keys using the lookup table.
  */
-export function renamer(options = {}) {
-	const { prefix, suffix, separator = '_', keys = [] } = options
-	const rename = getRenamer(prefix, suffix, separator)
-	const renameObject = getObjectRenamer(rename, keys)
-
-	return {
-		get: () => ({ rename, renameObject }),
-		setPrefix: (value) => renamer({ ...omit(['suffix'], options), prefix: value }),
-		setSuffix: (value) => renamer({ ...omit(['prefix'], options), suffix: value }),
-		setKeys: (value) => renamer({ ...options, keys: value }),
-		setSeparator: (value) => renamer({ ...options, separator: value })
-	}
+function usingLookup(lookup) {
+	return (row) => Object.keys(row).reduce((acc, key) => ({ ...acc, [lookup[key]]: row[key] }), {})
 }
 
+/**
+ * Renames the keys of an object using a function.
+ *
+ * @param {Function} rename - The function to rename the keys.
+ * @returns {Function} - A function that takes an object and renames the keys using the function.
+ */
+function forObject(rename) {
+	return (row) => Object.keys(row).reduce((acc, key) => ({ ...acc, [rename(key)]: row[key] }), {})
+}
 /**
  * Generates a renamer function which adds a prefix or suffix to a string.
  *
@@ -78,21 +71,33 @@ function getObjectRenamer(rename, keys) {
 }
 
 /**
- * Renames the keys of an object using a lookup table.
+ * A generator function that produces a function to rename keys.
  *
- * @param {Object} lookup - The lookup table.
- * @returns {Function} - A function that takes an object and renames the keys using the lookup table.
- */
-function usingLookup(lookup) {
-	return (row) => Object.keys(row).reduce((acc, key) => ({ ...acc, [lookup[key]]: row[key] }), {})
-}
-
-/**
- * Renames the keys of an object using a function.
+ * @param {OptionsToRenameKeys} options - Options to rename keys
+ * @returns {Object}                    - A renamer object.
  *
- * @param {Function} rename - The function to rename the keys.
- * @returns {Function} - A function that takes an object and renames the keys using the function.
+ * Example:
+ *
+ * renamer({ prefix: 'a', keys: ['a', 'b'] }).get()
+ * renamer({ prefix: 'a', separator: '-' }).get()
+ * renamer().setPrefix('a')
+ *          .setSeparator('-')
+ *          .setKeys(['a', 'b'])
+ *          .get()
+ * renamer().setPrefix('a').get()
+ * renamer().setSuffix('b').get()
+ * renamer().setKeys(['a', 'b']).get()
  */
-function forObject(rename) {
-	return (row) => Object.keys(row).reduce((acc, key) => ({ ...acc, [rename(key)]: row[key] }), {})
+export function renamer(options = {}) {
+	const { prefix, suffix, separator = '_', keys = [] } = options
+	const rename = getRenamer(prefix, suffix, separator)
+	const renameObject = getObjectRenamer(rename, keys)
+
+	return {
+		get: () => ({ rename, renameObject }),
+		setPrefix: (value) => renamer({ ...omit(['suffix'], options), prefix: value }),
+		setSuffix: (value) => renamer({ ...omit(['prefix'], options), suffix: value }),
+		setKeys: (value) => renamer({ ...options, keys: value }),
+		setSeparator: (value) => renamer({ ...options, separator: value })
+	}
 }