@rokkit/data 1.0.0-next.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +50 -0
- package/package.json +56 -0
- package/src/aggregators.js +24 -0
- package/src/constants.js +72 -0
- package/src/convert.js +36 -0
- package/src/dataset.js +428 -0
- package/src/filter.js +57 -0
- package/src/formatter.js +56 -0
- package/src/hierarchy.js +136 -0
- package/src/index.js +7 -0
- package/src/infer.js +263 -0
- package/src/list.js +232 -0
- package/src/metadata.js +106 -0
- package/src/model.js +100 -0
- package/src/parser.js +111 -0
- package/src/renamer.js +98 -0
- package/src/rollup.js +138 -0
- package/src/types.js +173 -0
- package/src/utils.js +36 -0
- package/src/view.js +181 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Jerry Thomas
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# @rokkit/data
|
|
2
|
+
|
|
3
|
+
This package provides a set of tools for managing and manipulating data in user interfaces. It is framework agnostic and can be used with any ui framework.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Datasets:** Manage and manipulate datasets with ease
|
|
8
|
+
- filtering, sorting, and aggregation.
|
|
9
|
+
- **Joins:** Combine datasets effortlessly with support for inner, outer, left, and right joins.
|
|
10
|
+
- **Rollup:** Summarize data across multiple dimensions, calculating aggregate statistics like sum, average, min, and max.
|
|
11
|
+
- **Modeling:** Create and manage data models, including merging, renaming, and resolving conflicts.
|
|
12
|
+
- **Renaming:** Simplify key renaming in JavaScript objects, with options for prefixes, suffixes, and custom separators.
|
|
13
|
+
- **Dataviews:** Create interactive views for exploring and analyzing datasets, including sortable and filterable columns.
|
|
14
|
+
|
|
15
|
+
## Dataset
|
|
16
|
+
|
|
17
|
+
- The `dataset` function initializes a dataset from an array of objects.
|
|
18
|
+
- This dataset comprises data rows that are sortable, filterable, and groupable.
|
|
19
|
+
- Leverage dataset for SQL-like operations such as selecting, updating, and deleting rows.
|
|
20
|
+
- Employ aggregate functions like sum, average, and count for summarizing multiple attributes of the data.
|
|
21
|
+
- Utilize the dataset for various data operations like joining and merging data from disparate sources.
|
|
22
|
+
- Establish connected data views with joins to visualize data relationships, particularly for tracking parent-child connections.
|
|
23
|
+
|
|
24
|
+
This makes it easier to build a tweenable data structure for animations. Here is an example of how to use the `dataset` to generate a tweenable data:
|
|
25
|
+
|
|
26
|
+
```js
|
|
27
|
+
function tweenable(data, by, group, template) {
|
|
28
|
+
return dataset(data)
|
|
29
|
+
.groupBy(by)
|
|
30
|
+
.using(template)
|
|
31
|
+
.alignBy(...group)
|
|
32
|
+
.rollup()
|
|
33
|
+
.sortBy(by)
|
|
34
|
+
.select()
|
|
35
|
+
}
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Refer to [dataset](docs/dataset.md) documentation for more details.
|
|
39
|
+
|
|
40
|
+
## DataView
|
|
41
|
+
|
|
42
|
+
- Utilize the `dataview` function to create a data view, enabling dynamic data manipulation and observation.
|
|
43
|
+
- This view facilitates sorting, filtering, and grouping data while providing insight into selected or hidden rows.
|
|
44
|
+
- The original dataset remains untouched while the view offers flexible data presentation options.
|
|
45
|
+
- Associate actions such as select, edit, and delete with data rows using action columns within the view.
|
|
46
|
+
- Manage data effectively, including parent-child links, even within a flat array structure.
|
|
47
|
+
|
|
48
|
+
Refer to [dataview](docs/dataview.md) documentation for more details.
|
|
49
|
+
|
|
50
|
+
The `dataview` function initializes a manageable data state with methods that allow for manipulation and observation of data changes, integrating smoothly with UI updates.
|
package/package.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@rokkit/data",
|
|
3
|
+
"version": "1.0.0-next.100",
|
|
4
|
+
"description": "Contains generic data manipulation functions that can be used in various components.",
|
|
5
|
+
"author": "Jerry Thomas <me@jerrythomas.name>",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"main": "index.js",
|
|
8
|
+
"module": "src/index.js",
|
|
9
|
+
"types": "dist/index.d.ts",
|
|
10
|
+
"type": "module",
|
|
11
|
+
"publishConfig": {
|
|
12
|
+
"access": "public"
|
|
13
|
+
},
|
|
14
|
+
"devDependencies": {
|
|
15
|
+
"@sveltejs/vite-plugin-svelte": "^3.1.2",
|
|
16
|
+
"@testing-library/svelte": "^5.2.1",
|
|
17
|
+
"@types/ramda": "^0.30.2",
|
|
18
|
+
"@vitest/coverage-v8": "^2.1.1",
|
|
19
|
+
"@vitest/ui": "~2.1.1",
|
|
20
|
+
"typescript": "^5.6.2",
|
|
21
|
+
"vite": "^5.4.6",
|
|
22
|
+
"vitest": "~2.1.1",
|
|
23
|
+
"shared-config": "1.0.0-next.100"
|
|
24
|
+
},
|
|
25
|
+
"files": [
|
|
26
|
+
"src/**/*.js",
|
|
27
|
+
"src/**/*.svelte"
|
|
28
|
+
],
|
|
29
|
+
"exports": {
|
|
30
|
+
"./src": "./src",
|
|
31
|
+
"./package.json": "./package.json",
|
|
32
|
+
".": {
|
|
33
|
+
"types": "./dist/index.d.ts",
|
|
34
|
+
"import": "./src/index.js",
|
|
35
|
+
"svelte": "./src/index.js"
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
"dependencies": {
|
|
39
|
+
"@lukeed/uuid": "^2.0.1",
|
|
40
|
+
"d3-array": "^3.2.4",
|
|
41
|
+
"d3-collection": "^1.0.7",
|
|
42
|
+
"ramda": "^0.30.1",
|
|
43
|
+
"svelte": "^4.2.19",
|
|
44
|
+
"@rokkit/core": "1.0.0-next.100"
|
|
45
|
+
},
|
|
46
|
+
"scripts": {
|
|
47
|
+
"format": "prettier --write .",
|
|
48
|
+
"lint": "eslint --fix .",
|
|
49
|
+
"test:ci": "vitest run",
|
|
50
|
+
"test:ui": "vitest --ui",
|
|
51
|
+
"test": "vitest",
|
|
52
|
+
"coverage": "vitest run --coverage",
|
|
53
|
+
"latest": "pnpm upgrade --latest && pnpm test:ci",
|
|
54
|
+
"release": "pnpm publish --access public"
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Counts the number of values in an array.
|
|
3
|
+
*
|
|
4
|
+
* @param {Array} values - An array of values to count.
|
|
5
|
+
* @returns {number} The count of values in the array.
|
|
6
|
+
*/
|
|
7
|
+
export const counter = (values) => values.length
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Calculates the quartiles and interquartile range (IQR) of a numeric data array.
|
|
11
|
+
* Assumes that the `quantile` function is defined elsewhere and can calculate a
|
|
12
|
+
* quantile for a given array of numbers and a probability.
|
|
13
|
+
*
|
|
14
|
+
* @param {number[]} values - An array of numeric values to calculate the quartiles for.
|
|
15
|
+
* @returns {Object} An object containing the first quartile (q1), third quartile (q3),
|
|
16
|
+
* interquartile range (iqr), and the minimum and maximum range values (qr_min, qr_max)
|
|
17
|
+
* after applying the IQR rule for identifying potential outliers.
|
|
18
|
+
*/
|
|
19
|
+
export const violin = (values) => {
|
|
20
|
+
const { q1, q3 } = values
|
|
21
|
+
const iqr = q3 - q1
|
|
22
|
+
|
|
23
|
+
return { ...values, q1, q3, iqr, qr_min: q1 - 1.5 * iqr, qr_max: q1 + 1.5 * iqr }
|
|
24
|
+
}
|
package/src/constants.js
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { identity, pick } from 'ramda'
|
|
2
|
+
|
|
3
|
+
export const pickAllowedConfig = pick(['children', 'actual_flag'])
|
|
4
|
+
|
|
5
|
+
/** @type {import('./types').DataFrameConfig */
|
|
6
|
+
export const defaultConfig = {
|
|
7
|
+
children: 'children',
|
|
8
|
+
actual_flag: 'actual_flag',
|
|
9
|
+
inplace: false,
|
|
10
|
+
group_by: [],
|
|
11
|
+
align_by: [],
|
|
12
|
+
template: {},
|
|
13
|
+
summaries: []
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export const defaultPathOptions = {
|
|
17
|
+
path: null,
|
|
18
|
+
separator: '/',
|
|
19
|
+
currencySuffix: '_currency'
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export const defaultViewOptions = {
|
|
23
|
+
...defaultPathOptions,
|
|
24
|
+
expanded: false,
|
|
25
|
+
actions: [],
|
|
26
|
+
language: 'en-US',
|
|
27
|
+
scanMode: 'fast'
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export const defaultActionOrder = {
|
|
31
|
+
select: 0,
|
|
32
|
+
edit: 1,
|
|
33
|
+
delete: 2
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Returns the default properties for a column
|
|
38
|
+
* @param {string} key - the key to use for the column
|
|
39
|
+
* @returns {Object} the default properties for a column
|
|
40
|
+
*/
|
|
41
|
+
// export const defaultProps = (key) => ({
|
|
42
|
+
// sortable: true,
|
|
43
|
+
// filterable: true,
|
|
44
|
+
// sorted: 'none',
|
|
45
|
+
// fields: { text: key }
|
|
46
|
+
// })
|
|
47
|
+
|
|
48
|
+
export const filterOperations = {
|
|
49
|
+
'=': (value, pattern) => value === pattern,
|
|
50
|
+
'<': (value, pattern) => value < pattern,
|
|
51
|
+
'>': (value, pattern) => value > pattern,
|
|
52
|
+
'<=': (value, pattern) => value <= pattern,
|
|
53
|
+
'>=': (value, pattern) => value >= pattern,
|
|
54
|
+
'~*': (value, pattern) => pattern.test(value),
|
|
55
|
+
'~': (value, pattern) => pattern.test(value),
|
|
56
|
+
'!=': (value, pattern) => value !== pattern,
|
|
57
|
+
'!~*': (value, pattern) => !pattern.test(value),
|
|
58
|
+
'!~': (value, pattern) => !pattern.test(value)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export const typeConverters = {
|
|
62
|
+
string: (value) => String(value),
|
|
63
|
+
number: (value) => Number(value),
|
|
64
|
+
boolean: (value) => value === 'true' || value === true,
|
|
65
|
+
date: (value) => new Date(value),
|
|
66
|
+
mixed: identity
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* A filter function that includes all rows.
|
|
71
|
+
*/
|
|
72
|
+
export const includeAll = () => true
|
package/src/convert.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { inferDataType } from './infer'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Converts an array of objects into a columnar structure, returning an object with two attributes:
|
|
5
|
+
*
|
|
6
|
+
* - columns, which contains an array of column names
|
|
7
|
+
* - data, which is an object with column names as keys and arrays as values.
|
|
8
|
+
*
|
|
9
|
+
* @param {Array<Object>} input
|
|
10
|
+
* @returns {import('./types').Dataframe}
|
|
11
|
+
*/
|
|
12
|
+
export function fromArray(input) {
|
|
13
|
+
const data = {}
|
|
14
|
+
let columns = []
|
|
15
|
+
|
|
16
|
+
input.forEach((items, index) => {
|
|
17
|
+
const keys = Object.keys(items)
|
|
18
|
+
const missing = columns.filter((x) => !keys.includes(x))
|
|
19
|
+
|
|
20
|
+
columns = [...columns, ...keys.filter((x) => !columns.includes(x))]
|
|
21
|
+
|
|
22
|
+
missing.forEach((key) => {
|
|
23
|
+
data[key] = [...data[key], null]
|
|
24
|
+
})
|
|
25
|
+
keys.forEach((key) => {
|
|
26
|
+
if (key in data) {
|
|
27
|
+
data[key] = [...data[key], items[key]]
|
|
28
|
+
} else {
|
|
29
|
+
data[key] = [...Array(index).fill(null), items[key]]
|
|
30
|
+
}
|
|
31
|
+
})
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
const types = columns.reduce((acc, x) => ({ ...acc, [x]: inferDataType(data[x]) }), {})
|
|
35
|
+
return { data, columns, types }
|
|
36
|
+
}
|
package/src/dataset.js
ADDED
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
import { equals, identity, pick, omit, clone } from 'ramda'
|
|
2
|
+
import { descending } from 'd3-array'
|
|
3
|
+
import { pickAllowedConfig, defaultConfig, includeAll } from './constants'
|
|
4
|
+
import { deriveSortableColumn } from './infer'
|
|
5
|
+
import { groupDataByKeys, fillAlignedData, getAlignGenerator, aggregateData } from './rollup'
|
|
6
|
+
/**
|
|
7
|
+
* Dataset is a collection of data with a set of operations that can be performed on it.
|
|
8
|
+
* @param {Array} data - The data to be stored in the dataset.
|
|
9
|
+
* @param {Object} options - The configuration options for the dataset.
|
|
10
|
+
* @returns {Object} - An object with a set of operations that can be performed on the dataset.
|
|
11
|
+
*/
|
|
12
|
+
// eslint-disable-next-line max-lines-per-function
|
|
13
|
+
export function dataset(data, options = {}) {
|
|
14
|
+
const config = { ...clone(defaultConfig), ...options }
|
|
15
|
+
|
|
16
|
+
const actions = {
|
|
17
|
+
// configuration
|
|
18
|
+
override: (props) => dataset(data, { ...config, ...pickAllowedConfig(props) }),
|
|
19
|
+
where: (condition) => dataset(data, where(config, condition)),
|
|
20
|
+
groupBy: (...fields) => dataset(data, groupBy(config, ...fields)),
|
|
21
|
+
alignBy: (...fields) => dataset(data, alignBy(config, ...fields)),
|
|
22
|
+
using: (template) => dataset(data, usingTemplate(config, template)),
|
|
23
|
+
summarize: (from, fields) => dataset(data, summarize(config, from, fields)),
|
|
24
|
+
// alter keys
|
|
25
|
+
rename: (how) => dataset(renameKeys(data, how)),
|
|
26
|
+
drop: (...fields) => dataset(dropKeys(data, ...fields)),
|
|
27
|
+
// alter rows
|
|
28
|
+
sortBy: (...fields) => dataset(sortDataBy(data, ...fields)),
|
|
29
|
+
delete: () => dataset(deleteRows(data, config.filter || includeAll)),
|
|
30
|
+
update: (value) => dataset(updateRows(data, config.filter || identity, value)),
|
|
31
|
+
fillNA: (value) => dataset(fillNA(data, value)),
|
|
32
|
+
// transform data
|
|
33
|
+
apply: (callback) => dataset(data.map(callback)),
|
|
34
|
+
rollup: () => dataset(rollup(data, config)),
|
|
35
|
+
select: (...cols) => selectKeys(data, config, ...cols),
|
|
36
|
+
// set operations
|
|
37
|
+
union: (other) => dataset(data.concat(other)),
|
|
38
|
+
minus: (other) => dataset(data.filter((d) => !other.find((x) => equals(x, d)))),
|
|
39
|
+
intersect: (other) => dataset(data.filter((d) => other.find((x) => equals(x, d)))),
|
|
40
|
+
// joins
|
|
41
|
+
innerJoin: (other, condition) => dataset(innerJoin(data, other.select(), condition)),
|
|
42
|
+
leftJoin: (other, condition) => dataset(leftJoin(data, other.select(), condition)),
|
|
43
|
+
rightJoin: (other, condition) => dataset(leftJoin(other.select(), data, condition)),
|
|
44
|
+
fullJoin: (other, condition) => dataset(fullJoin(data, other.select(), condition)),
|
|
45
|
+
crossJoin: (other) => dataset(crossJoin(data, other.select())),
|
|
46
|
+
semiJoin: (other, condition) => dataset(semiJoin(data, other.select(), condition)),
|
|
47
|
+
antiJoin: (other, condition) => dataset(antiJoin(data, other.select(), condition)),
|
|
48
|
+
nestedJoin: (other, condition) =>
|
|
49
|
+
dataset(nestedJoin(data, other.select(), condition, config.children))
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return actions
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Adds a filter to the dataset using the provided condition. This filter is applied
|
|
57
|
+
* in subsequent operations like select, delete, and update.
|
|
58
|
+
*
|
|
59
|
+
* @param {Object} config - The configuration object to be updated.
|
|
60
|
+
* @param {Function} condition - The condition function to apply.
|
|
61
|
+
* @returns {Object} - The updated configuration
|
|
62
|
+
*/
|
|
63
|
+
function where(config, condition) {
|
|
64
|
+
config.filter = condition
|
|
65
|
+
return config
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Groups the dataset by the specified columns.
|
|
69
|
+
* @param {Object} config - The configuration object to be updated.
|
|
70
|
+
* @param {Object} actions - The actions object for method chaining.
|
|
71
|
+
* @param {...string} fields - The columns to group by.
|
|
72
|
+
* @returns {Object} - The updated configurations.
|
|
73
|
+
*/
|
|
74
|
+
function groupBy(config, ...fields) {
|
|
75
|
+
config.group_by = fields
|
|
76
|
+
return config
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Aligns the columns of the dataset using the provided fields.
|
|
81
|
+
*
|
|
82
|
+
* @param {Object} config - The configuration object to be updated.
|
|
83
|
+
* @param {...string} fields - The fields to align.
|
|
84
|
+
* @returns {Object} - The updated configuration.
|
|
85
|
+
*/
|
|
86
|
+
function alignBy(config, ...fields) {
|
|
87
|
+
config.align_by = fields
|
|
88
|
+
return config
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Sets the template for adding empty rows in the dataset.
|
|
92
|
+
* @param {Object} config - The configuration object to be updated.
|
|
93
|
+
* @param {Object} template - The template to use for adding empty rows.
|
|
94
|
+
* @returns {Object} - The updated configuration.
|
|
95
|
+
*/
|
|
96
|
+
function usingTemplate(config, template) {
|
|
97
|
+
config.template = template
|
|
98
|
+
return config
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Adds a summary field to the dataset.
|
|
103
|
+
*
|
|
104
|
+
* @param {Object} config - The configuration object to be updated.
|
|
105
|
+
* @param {string|Array<string>|Function} from - The field or function to fetch data for summary
|
|
106
|
+
* @param {string|Object<string:Function>} using - The target field & formula to use for summarizing.
|
|
107
|
+
* @returns {Object} - The updated configuration.
|
|
108
|
+
*/
|
|
109
|
+
function summarize(config, from, using) {
|
|
110
|
+
const mapper = typeof from === 'function' ? from : Array.isArray(from) ? pick(from) : pick([from])
|
|
111
|
+
const reducers = []
|
|
112
|
+
|
|
113
|
+
if (typeof using === 'string') reducers.push({ field: using, formula: identity })
|
|
114
|
+
if (typeof using === 'object')
|
|
115
|
+
Object.entries(using).forEach(([field, formula]) => reducers.push({ field, formula }))
|
|
116
|
+
|
|
117
|
+
config.summaries.push({ mapper, reducers })
|
|
118
|
+
|
|
119
|
+
return config
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Joins two datasets together based on a condition. Result includes all rows from the first
|
|
124
|
+
* dataset and matching rows from the second dataset. In case of multiple matches, all
|
|
125
|
+
* combinations are returned. When combining the rows, the columns from the first dataset take
|
|
126
|
+
* precedence.
|
|
127
|
+
*
|
|
128
|
+
* inner: only the rows that have a match in both datasets.
|
|
129
|
+
* outer: all rows from the first dataset and matching rows from the second dataset.
|
|
130
|
+
*
|
|
131
|
+
* @param {Array} first - The first dataset to join.
|
|
132
|
+
* @param {Array} second - The second dataset to join.
|
|
133
|
+
* @param {Function} condition - The condition to join the datasets on.
|
|
134
|
+
* @returns {Object} - An object with the inner and outer join result.
|
|
135
|
+
*/
|
|
136
|
+
function joinData(first, second, condition) {
|
|
137
|
+
let inner = []
|
|
138
|
+
const outer = []
|
|
139
|
+
|
|
140
|
+
first.forEach((f) => {
|
|
141
|
+
const matches = second.filter((s) => condition(f, s)).map((m) => ({ ...m, ...f }))
|
|
142
|
+
inner = inner.concat(matches)
|
|
143
|
+
if (matches.length === 0) outer.push(f)
|
|
144
|
+
})
|
|
145
|
+
return { inner, outer }
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Filters the results of the first dataset based on the condition using the second dataset.
|
|
150
|
+
*
|
|
151
|
+
* @param {Array} first - The first dataset to filter.
|
|
152
|
+
* @param {Array} second - The second dataset to filter by.
|
|
153
|
+
* @param {Function} condition - The condition to filter the first dataset by.
|
|
154
|
+
* @returns {Array} - The filtered dataset.
|
|
155
|
+
*/
|
|
156
|
+
function antiJoin(first, second, condition) {
|
|
157
|
+
return first.filter((f) => !second.find((s) => condition(f, s)))
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Joins two datasets together based on a condition. Result includes all rows from the first
|
|
162
|
+
* dataset and matching rows from the second dataset. In case of multiple matches, all
|
|
163
|
+
* combinations are returned. When combining the rows, the columns from the first dataset take
|
|
164
|
+
* precedence.
|
|
165
|
+
*
|
|
166
|
+
* @param {Array} first - The first dataset to join.
|
|
167
|
+
* @param {Array} second - The second dataset to join.
|
|
168
|
+
* @param {Function} condition - The condition to join the datasets on.
|
|
169
|
+
* @returns {Array} - The joined dataset.
|
|
170
|
+
*/
|
|
171
|
+
function innerJoin(first, second, condition) {
|
|
172
|
+
const { inner } = joinData(first, second, condition)
|
|
173
|
+
return inner
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Performs a left join on two datasets based on a condition. Result includes all rows from the first
|
|
178
|
+
* dataset and matching rows from the second dataset. In case of multiple matches, all combinations
|
|
179
|
+
* are returned. When combining the rows, the columns from the first dataset take precedence.
|
|
180
|
+
* If there is no match in the second dataset, only the row from the first dataset is returned.
|
|
181
|
+
*
|
|
182
|
+
* This can be used to perform a right join by swapping the datasets.
|
|
183
|
+
*
|
|
184
|
+
* @param {Array} first - The first dataset to join.
|
|
185
|
+
* @param {Array} second - The second dataset to join.
|
|
186
|
+
* @param {Function} condition - The condition to join the datasets on.
|
|
187
|
+
* @returns {Array} - The joined dataset.
|
|
188
|
+
*/
|
|
189
|
+
function leftJoin(first, second, condition) {
|
|
190
|
+
const { inner, outer } = joinData(first, second, condition)
|
|
191
|
+
return inner.concat(outer)
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Performs a full join on two datasets based on a condition. Result includes all rows from both
|
|
196
|
+
* datasets. In case of multiple matches, all combinations are returned. When combining the rows,
|
|
197
|
+
* the columns from the first dataset take precedence. If there is no match in the second dataset,
|
|
198
|
+
* only the row from the first dataset is returned. If there is no match in the first dataset, only
|
|
199
|
+
* the row from the second dataset is returned.
|
|
200
|
+
*
|
|
201
|
+
* @param {Array} first - The first dataset to join.
|
|
202
|
+
* @param {Array} second - The second dataset to join.
|
|
203
|
+
* @param {Function} condition - The condition to join the datasets on.
|
|
204
|
+
* @returns {Array} - The joined dataset.
|
|
205
|
+
*/
|
|
206
|
+
function fullJoin(first, second, condition) {
|
|
207
|
+
const { inner, outer } = joinData(first, second, condition)
|
|
208
|
+
const rightOuter = antiJoin(second, first, condition)
|
|
209
|
+
return inner.concat(outer).concat(rightOuter)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Performs a cross join on two datasets. Result includes all combinations of rows from both datasets.
|
|
214
|
+
*
|
|
215
|
+
* @param {Array} first - The first dataset to join.
|
|
216
|
+
* @param {Array} second - The second dataset to join.
|
|
217
|
+
* @returns {Array} - The joined dataset.
|
|
218
|
+
*/
|
|
219
|
+
function crossJoin(first, second) {
|
|
220
|
+
return first.map((f) => second.map((s) => ({ ...f, ...s }))).flat()
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Performs a semi join on two datasets based on a condition. Result includes all rows from the first
|
|
225
|
+
* dataset that have a match in the second dataset.
|
|
226
|
+
*
|
|
227
|
+
* @param {Array} first - The first dataset to join.
|
|
228
|
+
* @param {Array} second - The second dataset to join.
|
|
229
|
+
* @param {Function} condition - The condition to join the datasets on.
|
|
230
|
+
* @returns {Array} - The joined dataset.
|
|
231
|
+
*/
|
|
232
|
+
function semiJoin(first, second, condition) {
|
|
233
|
+
return first.filter((f) => second.find((s) => condition(f, s)))
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Performs a nested join on two datasets based on a condition. Result includes all rows from the first
|
|
238
|
+
* dataset that have a match in the second dataset. The result is nested with the matching rows from the
|
|
239
|
+
* second dataset.
|
|
240
|
+
*
|
|
241
|
+
* @param {Array} first - The first dataset to join.
|
|
242
|
+
* @param {Array} second - The second dataset to join.
|
|
243
|
+
* @param {Function} condition - The condition to join the datasets on.
|
|
244
|
+
* @param {String} [key='children'] - The key to nest the matching rows under.
|
|
245
|
+
* @returns {Array} - The joined dataset.
|
|
246
|
+
*/
|
|
247
|
+
function nestedJoin(first, second, condition, key = 'children') {
|
|
248
|
+
const result = first.map((f) => ({
|
|
249
|
+
...f,
|
|
250
|
+
[key]: second.filter((s) => condition(f, s))
|
|
251
|
+
}))
|
|
252
|
+
return result
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Renames the keys of the data based on the provided mapping, or function
|
|
257
|
+
* that returns the new key. If the new key already exists, the value is overwritten.
|
|
258
|
+
*
|
|
259
|
+
* @param {Array<Object>} data - The data to rename the keys of.
|
|
260
|
+
* @param {Object|Function} how - The mapping of old keys to new keys, or a function that performs renaming.
|
|
261
|
+
* returns Array<Object> - The updated data with renamed keys.
|
|
262
|
+
*/
|
|
263
|
+
function renameKeys(data, how) {
|
|
264
|
+
const rename = typeof how === 'function' ? how : renameKeysUsingMap(how)
|
|
265
|
+
return data.map(rename)
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Drops the specified keys from the data.
|
|
270
|
+
*
|
|
271
|
+
* @param {Array<Object>} data - The data to drop keys from.
|
|
272
|
+
* @param {Array<String>} keys - The keys to drop.
|
|
273
|
+
* @returns {Array<Object>} - The updated data with the specified keys dropped.
|
|
274
|
+
*/
|
|
275
|
+
function dropKeys(data, ...keys) {
|
|
276
|
+
return data.map((row) => omit(keys, row))
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Deletes rows from the data based on the condition
|
|
281
|
+
*
|
|
282
|
+
* @param {Array<Object>} data - The data to delete rows from.
|
|
283
|
+
* @param {Function} condition - The condition to delete rows on.
|
|
284
|
+
* @returns {Array<Object>} - The updated data with rows deleted based on the condition.
|
|
285
|
+
*/
|
|
286
|
+
function deleteRows(data, condition) {
|
|
287
|
+
return data.filter((row) => !condition(row))
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Updates rows in the data based on the condition
|
|
292
|
+
*
|
|
293
|
+
* @param {Array<Object>} data - The data to update rows in.
|
|
294
|
+
* @param {Function} filter - The condition to update rows on.
|
|
295
|
+
* @param {Object|Function} value - The values to update the rows with.
|
|
296
|
+
* @returns {Array<Object>} - The updated data with rows updated based on the condition.
|
|
297
|
+
*/
|
|
298
|
+
function updateRows(data, filter, value) {
|
|
299
|
+
if (typeof value !== 'function' && typeof value !== 'object') {
|
|
300
|
+
throw new Error('Value must be an object or function')
|
|
301
|
+
}
|
|
302
|
+
return data.map((row) =>
|
|
303
|
+
filter(row) ? (typeof value === 'function' ? value(row) : { ...row, ...value }) : row
|
|
304
|
+
)
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Fills missing values in the data based on the provided value.
|
|
309
|
+
*
|
|
310
|
+
* @param {Array<Object>} data - The data to fill missing values in.
|
|
311
|
+
* @param {Object} value - The value to fill missing values with.
|
|
312
|
+
* @returns {Array<Object>} - The updated data with missing values filled.
|
|
313
|
+
*/
|
|
314
|
+
function fillNA(data, value) {
|
|
315
|
+
/**
|
|
316
|
+
* Fills missing values in a single row based on the provided value.
|
|
317
|
+
*
|
|
318
|
+
* @param {Object} row - The row to fill missing values in.
|
|
319
|
+
*/
|
|
320
|
+
const fill = (row) => {
|
|
321
|
+
const filled = {}
|
|
322
|
+
Object.entries(value).forEach(([k, v]) => {
|
|
323
|
+
if (row[k] === undefined || row[k] === null) {
|
|
324
|
+
filled[k] = v
|
|
325
|
+
}
|
|
326
|
+
})
|
|
327
|
+
return { ...row, ...filled }
|
|
328
|
+
}
|
|
329
|
+
return data.map(fill)
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Returns a function that renames the keys of a single row based on the provided mapping.
|
|
334
|
+
*
|
|
335
|
+
* @param {Object} lookup - The mapping of old keys to new keys.
|
|
336
|
+
* @returns {Function} - The function that renames the keys of a single row.
|
|
337
|
+
*/
|
|
338
|
+
function renameKeysUsingMap(lookup) {
|
|
339
|
+
return (row) =>
|
|
340
|
+
Object.entries(row)
|
|
341
|
+
.sort((a, b) => descending(a[0], b[0]))
|
|
342
|
+
.reduce((acc, [k, v]) => ({ ...acc, [lookup[k] || k]: v }), {})
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* Sorts data by the specified columns.
|
|
347
|
+
*
|
|
348
|
+
* @param {Array<Object>} data - The data to sort.
|
|
349
|
+
* @param {import('./types').SortableColumn} fields - The columns to sort by.
|
|
350
|
+
* @returns {Array<Object>} The sorted data
|
|
351
|
+
*/
|
|
352
|
+
function sortDataBy(data, ...fields) {
|
|
353
|
+
const sorters = fields.map(deriveSortableColumn)
|
|
354
|
+
|
|
355
|
+
data.sort((a, b) => {
|
|
356
|
+
let result = 0
|
|
357
|
+
for (let i = 0; i < sorters.length && result === 0; i++) {
|
|
358
|
+
const { name, sorter } = sorters[i]
|
|
359
|
+
result = sorter(a[name], b[name])
|
|
360
|
+
}
|
|
361
|
+
return result
|
|
362
|
+
})
|
|
363
|
+
return data
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Selects the specified columns from the dataset.
|
|
368
|
+
*
|
|
369
|
+
* @param {Array<Object>} data - The data to select from.
|
|
370
|
+
* @param {Object} config - The configuration object.
|
|
371
|
+
* @param {Function} config.filter - The filter function to apply to the data.
|
|
372
|
+
* @param {Array<String>} cols - The columns to select.
|
|
373
|
+
* @returns {Array<Object>} The selected data.
|
|
374
|
+
*/
|
|
375
|
+
function selectKeys(data, config, ...cols) {
|
|
376
|
+
const result = config.filter ? data.filter(config.filter) : data
|
|
377
|
+
if (cols.length > 0) return result.map(pick(cols))
|
|
378
|
+
return result
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Summarizes the dataset by the specified columns.
|
|
383
|
+
* @param {import('./types').dataset} df - The dataset object to summarize.
|
|
384
|
+
* @param {Array} summaries - The columns to summarize.
|
|
385
|
+
*
|
|
386
|
+
* @returns {import('./types').dataset} The summarized dataset object.
|
|
387
|
+
*/
|
|
388
|
+
function rollup(data, config) {
|
|
389
|
+
if (config.group_by.length === 0 && config.summaries.length === 0) {
|
|
390
|
+
throw new Error(
|
|
391
|
+
'Use groupBy to specify the columns to group by or use summarize to add aggregators.'
|
|
392
|
+
)
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const summaries = clone(config.summaries)
|
|
396
|
+
const hasAlignBy = config.align_by.length > 0
|
|
397
|
+
if (summaries.length === 0) summaries.push(defaultAggregator(config))
|
|
398
|
+
|
|
399
|
+
let alignedData = groupDataByKeys(data, config.group_by, summaries)
|
|
400
|
+
|
|
401
|
+
if (hasAlignBy) {
|
|
402
|
+
const fillRows = getAlignGenerator(data, config)
|
|
403
|
+
alignedData = fillAlignedData(alignedData, config, fillRows)
|
|
404
|
+
alignedData.forEach((group) => {
|
|
405
|
+
group[config.children] = sortDataBy(group[config.children], ...config.align_by)
|
|
406
|
+
})
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
const aggregatedData = aggregateData(alignedData, summaries)
|
|
410
|
+
// const newMetadata = buildMetadata(aggregatedData, df.metadata, df.config.group_by, summaries)
|
|
411
|
+
|
|
412
|
+
return aggregatedData
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Returns the default aggregator which rolls up the columns other than the group_by columns into a single object.
|
|
417
|
+
*
|
|
418
|
+
* @param {import('./types').Metadata} metadata - The metadata for the columns to be aggregated.
|
|
419
|
+
* @param {Object} config - The configuration used to build the aggregator.
|
|
420
|
+
*
|
|
421
|
+
* @returns {Object} An object containing the default aggregator for the specified metadata and configuration.
|
|
422
|
+
*/
|
|
423
|
+
export function defaultAggregator(config) {
|
|
424
|
+
return {
|
|
425
|
+
mapper: omit(config.group_by),
|
|
426
|
+
reducers: [{ field: config.children, formula: identity }]
|
|
427
|
+
}
|
|
428
|
+
}
|