df-script 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +125 -9
- package/dist/api.d.ts +3 -2
- package/dist/assets/index-DBhGK6Tp.css +1 -0
- package/dist/assets/index-DEJEV_tU.js +195 -0
- package/dist/columnExpressions/ColumnExpr.d.ts +14 -7
- package/dist/columnExpressions/ExprBase.d.ts +26 -12
- package/dist/columnExpressions/constants.d.ts +1 -0
- package/dist/columnExpressions/functions/all.d.ts +23 -0
- package/dist/columnExpressions/functions/coalesce.d.ts +31 -2
- package/dist/columnExpressions/functions/element.d.ts +26 -1
- package/dist/columnExpressions/functions/exclude.d.ts +24 -0
- package/dist/columnExpressions/functions/implode.d.ts +28 -0
- package/dist/columnExpressions/functions/lit.d.ts +27 -0
- package/dist/columnExpressions/functions/seq_range.d.ts +36 -0
- package/dist/columnExpressions/functions/struct.d.ts +31 -0
- package/dist/columnExpressions/functions/when.d.ts +38 -8
- package/dist/columnExpressions/index.d.ts +2 -1
- package/dist/columnExpressions/mixins/AggregationExpr.d.ts +336 -2
- package/dist/columnExpressions/mixins/ArithmeticExpr.d.ts +620 -10
- package/dist/columnExpressions/mixins/ArrayExpr.d.ts +567 -0
- package/dist/columnExpressions/mixins/ComparisonExpr.d.ts +353 -0
- package/dist/columnExpressions/mixins/LogicalExpr.d.ts +82 -0
- package/dist/columnExpressions/mixins/ManipulationExpr.d.ts +40 -0
- package/dist/columnExpressions/mixins/StringExpr.d.ts +658 -5
- package/dist/columnExpressions/mixins/StructExpr.d.ts +70 -0
- package/dist/columnExpressions/mixins/TemporalExpr.d.ts +530 -4
- package/dist/columnExpressions/mixins/WindowExpr.d.ts +313 -2
- package/dist/columnExpressions/types.d.ts +2 -0
- package/dist/constants.d.ts +11 -0
- package/dist/dataframe/dataframe.d.ts +933 -3
- package/dist/dataframe/grouped/grouped.d.ts +41 -6
- package/dist/dataframe/types.d.ts +1 -0
- package/dist/dataframe/utils.d.ts +1 -0
- package/dist/datatypes/index.d.ts +3 -2
- package/dist/datatypes/types.d.ts +52 -7
- package/dist/exceptions/index.d.ts +23 -0
- package/dist/functions/concat.d.ts +50 -0
- package/dist/functions/read_csv.d.ts +9 -0
- package/dist/functions/read_json.d.ts +7 -2
- package/dist/index.html +17 -0
- package/dist/index.js +6 -6
- package/dist/index.mjs +6 -0
- package/dist/types.d.ts +51 -16
- package/dist/utils/{list.d.ts → array.d.ts} +68 -15
- package/dist/utils/binary.d.ts +3 -0
- package/dist/utils/csv.d.ts +1 -0
- package/dist/utils/date.d.ts +23 -54
- package/dist/utils/index.d.ts +3 -2
- package/dist/utils/json.d.ts +1 -0
- package/dist/utils/number.d.ts +15 -29
- package/dist/utils/object.d.ts +27 -0
- package/dist/utils/string.d.ts +15 -5
- package/package.json +18 -4
- package/dist/api.js +0 -26
- package/dist/bundle.min.js +0 -1
- package/dist/columnExpressions/ColumnExpr.js +0 -219
- package/dist/columnExpressions/ExprBase.js +0 -92
- package/dist/columnExpressions/constants.js +0 -7
- package/dist/columnExpressions/functions/all.js +0 -11
- package/dist/columnExpressions/functions/coalesce.js +0 -44
- package/dist/columnExpressions/functions/element.js +0 -11
- package/dist/columnExpressions/functions/exclude.js +0 -13
- package/dist/columnExpressions/functions/implode.js +0 -7
- package/dist/columnExpressions/functions/lit.js +0 -17
- package/dist/columnExpressions/functions/repeat.d.ts +0 -47
- package/dist/columnExpressions/functions/repeat.js +0 -131
- package/dist/columnExpressions/functions/seq_range.js +0 -91
- package/dist/columnExpressions/functions/when.js +0 -85
- package/dist/columnExpressions/index.js +0 -38
- package/dist/columnExpressions/mixins/AggregationExpr.js +0 -76
- package/dist/columnExpressions/mixins/ArithmeticExpr.js +0 -131
- package/dist/columnExpressions/mixins/BooleanExpr.d.ts +0 -40
- package/dist/columnExpressions/mixins/BooleanExpr.js +0 -67
- package/dist/columnExpressions/mixins/ComparisonExpr.js +0 -250
- package/dist/columnExpressions/mixins/ListExpr.d.ts +0 -39
- package/dist/columnExpressions/mixins/ListExpr.js +0 -224
- package/dist/columnExpressions/mixins/LogicalExpr.js +0 -53
- package/dist/columnExpressions/mixins/ManipulationExpr.js +0 -71
- package/dist/columnExpressions/mixins/NumericExpr.d.ts +0 -92
- package/dist/columnExpressions/mixins/NumericExpr.js +0 -259
- package/dist/columnExpressions/mixins/StringExpr.js +0 -237
- package/dist/columnExpressions/mixins/StructExpr.js +0 -139
- package/dist/columnExpressions/mixins/TemporalExpr.js +0 -126
- package/dist/columnExpressions/mixins/WindowExpr.js +0 -151
- package/dist/columnExpressions/types.js +0 -2
- package/dist/columnExpressions/utils.js +0 -45
- package/dist/dataframe/constants.js +0 -7
- package/dist/dataframe/dataframe.js +0 -990
- package/dist/dataframe/grouped/grouped.js +0 -114
- package/dist/dataframe/index.js +0 -20
- package/dist/dataframe/types.js +0 -2
- package/dist/dataframe/utils.js +0 -307
- package/dist/datatypes/DataType.js +0 -76
- package/dist/datatypes/index.js +0 -46
- package/dist/datatypes/types/Binary.d.ts +0 -9
- package/dist/datatypes/types/Binary.js +0 -18
- package/dist/datatypes/types/Boolean.d.ts +0 -9
- package/dist/datatypes/types/Boolean.js +0 -19
- package/dist/datatypes/types/Date.d.ts +0 -8
- package/dist/datatypes/types/Date.js +0 -21
- package/dist/datatypes/types/Datetime.d.ts +0 -8
- package/dist/datatypes/types/Datetime.js +0 -17
- package/dist/datatypes/types/Decimal.d.ts +0 -10
- package/dist/datatypes/types/Decimal.js +0 -28
- package/dist/datatypes/types/Duration.d.ts +0 -8
- package/dist/datatypes/types/Duration.js +0 -17
- package/dist/datatypes/types/Float32.d.ts +0 -8
- package/dist/datatypes/types/Float32.js +0 -17
- package/dist/datatypes/types/Float64.d.ts +0 -8
- package/dist/datatypes/types/Float64.js +0 -17
- package/dist/datatypes/types/Int16.d.ts +0 -8
- package/dist/datatypes/types/Int16.js +0 -17
- package/dist/datatypes/types/Int32.d.ts +0 -8
- package/dist/datatypes/types/Int32.js +0 -17
- package/dist/datatypes/types/Int64.d.ts +0 -8
- package/dist/datatypes/types/Int64.js +0 -17
- package/dist/datatypes/types/Int8.d.ts +0 -8
- package/dist/datatypes/types/Int8.js +0 -17
- package/dist/datatypes/types/List.d.ts +0 -10
- package/dist/datatypes/types/List.js +0 -31
- package/dist/datatypes/types/Null.d.ts +0 -9
- package/dist/datatypes/types/Null.js +0 -17
- package/dist/datatypes/types/Object.d.ts +0 -9
- package/dist/datatypes/types/Object.js +0 -17
- package/dist/datatypes/types/Struct.d.ts +0 -14
- package/dist/datatypes/types/Struct.js +0 -39
- package/dist/datatypes/types/Time.d.ts +0 -8
- package/dist/datatypes/types/Time.js +0 -29
- package/dist/datatypes/types/UInt16.d.ts +0 -8
- package/dist/datatypes/types/UInt16.js +0 -17
- package/dist/datatypes/types/UInt32.d.ts +0 -8
- package/dist/datatypes/types/UInt32.js +0 -17
- package/dist/datatypes/types/UInt64.d.ts +0 -8
- package/dist/datatypes/types/UInt64.js +0 -17
- package/dist/datatypes/types/UInt8.d.ts +0 -8
- package/dist/datatypes/types/UInt8.js +0 -17
- package/dist/datatypes/types/Utf8.d.ts +0 -10
- package/dist/datatypes/types/Utf8.js +0 -20
- package/dist/datatypes/types.js +0 -290
- package/dist/exceptions/index.js +0 -46
- package/dist/exceptions/utils.js +0 -28
- package/dist/functions/concat.js +0 -260
- package/dist/functions/index.js +0 -9
- package/dist/functions/read_csv.js +0 -53
- package/dist/functions/read_json.js +0 -21
- package/dist/functions/transpose.d.ts +0 -7
- package/dist/functions/transpose.js +0 -84
- package/dist/types.js +0 -2
- package/dist/utils/boolean.d.ts +0 -1
- package/dist/utils/boolean.js +0 -18
- package/dist/utils/csv.js +0 -313
- package/dist/utils/date.js +0 -348
- package/dist/utils/guards.d.ts +0 -12
- package/dist/utils/guards.js +0 -112
- package/dist/utils/index.js +0 -23
- package/dist/utils/json.js +0 -217
- package/dist/utils/list.js +0 -361
- package/dist/utils/number.js +0 -401
- package/dist/utils/string.js +0 -209
|
@@ -1,114 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.GroupedData = void 0;
|
|
4
|
-
const dataframe_1 = require("../dataframe");
|
|
5
|
-
const utils_1 = require("../utils");
|
|
6
|
-
const columnExpressions_1 = require("../../columnExpressions");
|
|
7
|
-
class GroupedData {
|
|
8
|
-
groups;
|
|
9
|
-
keys;
|
|
10
|
-
allKeys;
|
|
11
|
-
parentColumns;
|
|
12
|
-
parentHeight;
|
|
13
|
-
parentSchema;
|
|
14
|
-
constructor(groups, keys, allKeys, parentColumns, parentHeight, parentSchema) {
|
|
15
|
-
this.groups = groups;
|
|
16
|
-
this.keys = keys;
|
|
17
|
-
this.allKeys = allKeys;
|
|
18
|
-
this.parentColumns = parentColumns;
|
|
19
|
-
this.parentHeight = parentHeight;
|
|
20
|
-
this.parentSchema = parentSchema;
|
|
21
|
-
}
|
|
22
|
-
to_dataframe() {
|
|
23
|
-
const keysLen = this.keys.length;
|
|
24
|
-
const keysStr = new Array(keysLen);
|
|
25
|
-
for (let i = 0; i < keysLen; i++) {
|
|
26
|
-
keysStr[i] = String(this.keys[i]);
|
|
27
|
-
}
|
|
28
|
-
const numGroups = this.groups.size;
|
|
29
|
-
const newColumns = {};
|
|
30
|
-
for (let i = 0; i < keysStr.length; i++) {
|
|
31
|
-
newColumns[keysStr[i]] = new Array(numGroups);
|
|
32
|
-
}
|
|
33
|
-
let groupIdx = 0;
|
|
34
|
-
for (const indices of this.groups.values()) {
|
|
35
|
-
if (indices.length === 0)
|
|
36
|
-
continue;
|
|
37
|
-
const firstIdx = indices[0];
|
|
38
|
-
for (let i = 0; i < keysStr.length; i++) {
|
|
39
|
-
const k = keysStr[i];
|
|
40
|
-
const val = this.parentColumns[k][firstIdx];
|
|
41
|
-
newColumns[k][groupIdx] = val === undefined ? null : val;
|
|
42
|
-
}
|
|
43
|
-
groupIdx++;
|
|
44
|
-
}
|
|
45
|
-
const outSchema = {};
|
|
46
|
-
for (const k of keysStr) {
|
|
47
|
-
outSchema[k] = this.parentSchema[k];
|
|
48
|
-
}
|
|
49
|
-
return dataframe_1.DataFrame._createDirect(newColumns, outSchema, groupIdx);
|
|
50
|
-
}
|
|
51
|
-
agg(...exprs) {
|
|
52
|
-
const allKeysLen = this.allKeys.length;
|
|
53
|
-
const allKeysStr = new Array(allKeysLen);
|
|
54
|
-
for (let i = 0; i < allKeysLen; i++) {
|
|
55
|
-
allKeysStr[i] = String(this.allKeys[i]);
|
|
56
|
-
}
|
|
57
|
-
const keysLen = this.keys.length;
|
|
58
|
-
const keysStr = new Array(keysLen);
|
|
59
|
-
for (let i = 0; i < keysLen; i++) {
|
|
60
|
-
keysStr[i] = String(this.keys[i]);
|
|
61
|
-
}
|
|
62
|
-
const expandedExprs = (0, columnExpressions_1.resolveColumnSelectors)(exprs.flat(), allKeysStr, keysStr, this.parentSchema, this.parentColumns);
|
|
63
|
-
const numGroups = this.groups.size;
|
|
64
|
-
const newColumns = {};
|
|
65
|
-
for (let i = 0; i < keysStr.length; i++) {
|
|
66
|
-
newColumns[keysStr[i]] = new Array(numGroups);
|
|
67
|
-
}
|
|
68
|
-
let groupIdx = 0;
|
|
69
|
-
for (const indices of this.groups.values()) {
|
|
70
|
-
if (indices.length === 0)
|
|
71
|
-
continue;
|
|
72
|
-
const firstIdx = indices[0];
|
|
73
|
-
for (let i = 0; i < keysStr.length; i++) {
|
|
74
|
-
const k = keysStr[i];
|
|
75
|
-
const val = this.parentColumns[k][firstIdx];
|
|
76
|
-
newColumns[k][groupIdx] = val === undefined ? null : val;
|
|
77
|
-
}
|
|
78
|
-
groupIdx++;
|
|
79
|
-
}
|
|
80
|
-
for (let i = 0; i < expandedExprs.length; i++) {
|
|
81
|
-
const e = expandedExprs[i];
|
|
82
|
-
const targetKey = e.outputName || e.colName || columnExpressions_1.ALL_COLUMNS_MARKER;
|
|
83
|
-
if (!e.aggFn) {
|
|
84
|
-
newColumns[targetKey] = e.evaluate(newColumns, numGroups);
|
|
85
|
-
}
|
|
86
|
-
else {
|
|
87
|
-
const preGroupedCol = e.evaluatePre(e.groupingOpsIndex, this.parentColumns, this.parentHeight);
|
|
88
|
-
const aggregatedGroupValues = new Array(numGroups);
|
|
89
|
-
let gIdx = 0;
|
|
90
|
-
for (const indices of this.groups.values()) {
|
|
91
|
-
if (indices.length === 0)
|
|
92
|
-
continue;
|
|
93
|
-
const groupValues = new Array(indices.length);
|
|
94
|
-
for (let k = 0; k < indices.length; k++) {
|
|
95
|
-
groupValues[k] = preGroupedCol[indices[k]];
|
|
96
|
-
}
|
|
97
|
-
aggregatedGroupValues[gIdx] = e.aggFn(groupValues);
|
|
98
|
-
gIdx++;
|
|
99
|
-
}
|
|
100
|
-
newColumns[targetKey] = e.evaluatePost(e.groupingOpsIndex, aggregatedGroupValues, newColumns);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
const outSchema = {};
|
|
104
|
-
for (const k of keysStr) {
|
|
105
|
-
outSchema[k] = this.parentSchema[k];
|
|
106
|
-
}
|
|
107
|
-
for (const e of expandedExprs) {
|
|
108
|
-
const targetKey = e.outputName || e.colName || columnExpressions_1.ALL_COLUMNS_MARKER;
|
|
109
|
-
outSchema[targetKey] = (0, utils_1.inferColumnType)(newColumns[targetKey]);
|
|
110
|
-
}
|
|
111
|
-
return dataframe_1.DataFrame._createDirect(newColumns, outSchema, groupIdx);
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
exports.GroupedData = GroupedData;
|
package/dist/dataframe/index.js
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./dataframe"), exports);
|
|
18
|
-
__exportStar(require("./utils"), exports);
|
|
19
|
-
__exportStar(require("./types"), exports);
|
|
20
|
-
__exportStar(require("./grouped/grouped"), exports);
|
package/dist/dataframe/types.js
DELETED
package/dist/dataframe/utils.js
DELETED
|
@@ -1,307 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.resolveWindowExpr = resolveWindowExpr;
|
|
4
|
-
exports.rowsToColumns = rowsToColumns;
|
|
5
|
-
exports.columnsToRows = columnsToRows;
|
|
6
|
-
exports.getRowFromColumns = getRowFromColumns;
|
|
7
|
-
exports.inferColumnType = inferColumnType;
|
|
8
|
-
exports.gatherColumnsByIndices = gatherColumnsByIndices;
|
|
9
|
-
exports.computeRowHash = computeRowHash;
|
|
10
|
-
exports.coerceColumn = coerceColumn;
|
|
11
|
-
exports.writeStringToFileOrStream = writeStringToFileOrStream;
|
|
12
|
-
const datatypes_1 = require("../datatypes");
|
|
13
|
-
const constants_1 = require("./constants");
|
|
14
|
-
const utils_1 = require("../utils");
|
|
15
|
-
const exceptions_1 = require("../exceptions");
|
|
16
|
-
function partition_by_columns(columns, height, partitionKeys) {
|
|
17
|
-
const partitionMap = new Map();
|
|
18
|
-
const pKeysLen = partitionKeys.length;
|
|
19
|
-
const keyColumns = new Array(pKeysLen);
|
|
20
|
-
for (let i = 0; i < pKeysLen; i++) {
|
|
21
|
-
const pKey = partitionKeys[i];
|
|
22
|
-
if (typeof pKey === "string") {
|
|
23
|
-
(0, exceptions_1.assertColumnExists)(pKey, columns, "Partition key", " in the DataFrame.");
|
|
24
|
-
keyColumns[i] = columns[pKey];
|
|
25
|
-
}
|
|
26
|
-
else {
|
|
27
|
-
keyColumns[i] = pKey.evaluate(columns, height);
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
if (pKeysLen === 1) {
|
|
31
|
-
const keyCol = keyColumns[0];
|
|
32
|
-
for (let i = 0; i < height; i++) {
|
|
33
|
-
const val = keyCol[i];
|
|
34
|
-
const hash = val == null ? "" : (0, utils_1.toCanonicalString)(val);
|
|
35
|
-
let group = partitionMap.get(hash);
|
|
36
|
-
if (group === undefined) {
|
|
37
|
-
group = [];
|
|
38
|
-
partitionMap.set(hash, group);
|
|
39
|
-
}
|
|
40
|
-
group.push(i);
|
|
41
|
-
}
|
|
42
|
-
return partitionMap;
|
|
43
|
-
}
|
|
44
|
-
for (let i = 0; i < height; i++) {
|
|
45
|
-
const keyValues = new Array(pKeysLen);
|
|
46
|
-
for (let j = 0; j < pKeysLen; j++) {
|
|
47
|
-
const val = keyColumns[j][i];
|
|
48
|
-
keyValues[j] = val == null ? "" : (0, utils_1.toCanonicalString)(val);
|
|
49
|
-
}
|
|
50
|
-
const hash = keyValues.join(constants_1.KEY_SEPARATOR);
|
|
51
|
-
let group = partitionMap.get(hash);
|
|
52
|
-
if (group === undefined) {
|
|
53
|
-
group = [];
|
|
54
|
-
partitionMap.set(hash, group);
|
|
55
|
-
}
|
|
56
|
-
group.push(i);
|
|
57
|
-
}
|
|
58
|
-
return partitionMap;
|
|
59
|
-
}
|
|
60
|
-
function resolveWindowExpr(expr, columns, height) {
|
|
61
|
-
const results = new Array(height);
|
|
62
|
-
if (height === 0)
|
|
63
|
-
return results;
|
|
64
|
-
const partitionKeys = expr.partitionBy || [];
|
|
65
|
-
const partitionGroups = partition_by_columns(columns, height, partitionKeys);
|
|
66
|
-
const prePartitionArray = expr.evaluatePre(expr.partitionOpsIndex, columns, height);
|
|
67
|
-
for (const indices of partitionGroups.values()) {
|
|
68
|
-
const groupLen = indices.length;
|
|
69
|
-
const groupPreValues = new Array(groupLen);
|
|
70
|
-
for (let k = 0; k < groupLen; k++) {
|
|
71
|
-
groupPreValues[k] = prePartitionArray[indices[k]];
|
|
72
|
-
}
|
|
73
|
-
if (expr.evaluateWindow) {
|
|
74
|
-
for (let k = 0; k < groupLen; k++) {
|
|
75
|
-
results[indices[k]] = expr.evaluateWindow(groupPreValues, indices, k);
|
|
76
|
-
}
|
|
77
|
-
continue;
|
|
78
|
-
}
|
|
79
|
-
if (expr.aggFn) {
|
|
80
|
-
const aggregatedVal = expr.aggFn(groupPreValues);
|
|
81
|
-
for (let k = 0; k < groupLen; k++) {
|
|
82
|
-
results[indices[k]] = aggregatedVal;
|
|
83
|
-
}
|
|
84
|
-
continue;
|
|
85
|
-
}
|
|
86
|
-
for (let k = 0; k < groupLen; k++) {
|
|
87
|
-
results[indices[k]] = prePartitionArray[indices[k]];
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
return expr.evaluatePost(expr.partitionOpsIndex, results, columns);
|
|
91
|
-
}
|
|
92
|
-
function rowsToColumns(rows) {
|
|
93
|
-
if (!Array.isArray(rows) || rows.length === 0) {
|
|
94
|
-
return { columns: {}, height: 0 };
|
|
95
|
-
}
|
|
96
|
-
const height = rows.length;
|
|
97
|
-
const keysSet = new Set();
|
|
98
|
-
for (let r = 0; r < height; r++) {
|
|
99
|
-
const row = rows[r];
|
|
100
|
-
if ((0, utils_1.isObj)(row)) {
|
|
101
|
-
const rowKeys = Object.keys(row);
|
|
102
|
-
for (let i = 0; i < rowKeys.length; i++) {
|
|
103
|
-
keysSet.add(rowKeys[i]);
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
const keys = Array.from(keysSet);
|
|
108
|
-
const columns = {};
|
|
109
|
-
for (let i = 0; i < keys.length; i++) {
|
|
110
|
-
columns[keys[i]] = new Array(height);
|
|
111
|
-
}
|
|
112
|
-
for (let r = 0; r < height; r++) {
|
|
113
|
-
const row = rows[r] || {};
|
|
114
|
-
for (let i = 0; i < keys.length; i++) {
|
|
115
|
-
const k = keys[i];
|
|
116
|
-
const val = row[k];
|
|
117
|
-
columns[k][r] = val === undefined ? null : val;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
return { columns, height };
|
|
121
|
-
}
|
|
122
|
-
function columnsToRows(columns, height) {
|
|
123
|
-
const keys = Object.keys(columns);
|
|
124
|
-
const rows = new Array(height);
|
|
125
|
-
for (let r = 0; r < height; r++) {
|
|
126
|
-
const row = {};
|
|
127
|
-
for (let i = 0; i < keys.length; i++) {
|
|
128
|
-
const k = keys[i];
|
|
129
|
-
const val = columns[k][r];
|
|
130
|
-
row[k] = val === undefined ? null : val;
|
|
131
|
-
}
|
|
132
|
-
rows[r] = row;
|
|
133
|
-
}
|
|
134
|
-
return rows;
|
|
135
|
-
}
|
|
136
|
-
function getRowFromColumns(columns, idx, keys) {
|
|
137
|
-
const row = {};
|
|
138
|
-
for (let i = 0; i < keys.length; i++) {
|
|
139
|
-
const k = keys[i];
|
|
140
|
-
const val = columns[k][idx];
|
|
141
|
-
row[k] = val === undefined ? null : val;
|
|
142
|
-
}
|
|
143
|
-
return row;
|
|
144
|
-
}
|
|
145
|
-
function inferColumnType(col) {
|
|
146
|
-
if (col.length === 0)
|
|
147
|
-
return datatypes_1.DataTypeRegistry.Utf8;
|
|
148
|
-
let isBoolean = true;
|
|
149
|
-
let isInteger = true;
|
|
150
|
-
let isNumeric = true;
|
|
151
|
-
let isBigInt = true;
|
|
152
|
-
let isDate = true;
|
|
153
|
-
let isList = true;
|
|
154
|
-
let isBinary = true;
|
|
155
|
-
let isObject = true;
|
|
156
|
-
let hasDateObj = false;
|
|
157
|
-
let hasNonNull = false;
|
|
158
|
-
const allListElements = [];
|
|
159
|
-
for (let i = 0; i < col.length; i++) {
|
|
160
|
-
const val = col[i];
|
|
161
|
-
if (val == null)
|
|
162
|
-
continue;
|
|
163
|
-
hasNonNull = true;
|
|
164
|
-
if (!(val instanceof Uint8Array)) {
|
|
165
|
-
isBinary = false;
|
|
166
|
-
}
|
|
167
|
-
if (!(0, utils_1.isArrayOrTypedArray)(val) || val instanceof Uint8Array) {
|
|
168
|
-
isList = false;
|
|
169
|
-
}
|
|
170
|
-
else {
|
|
171
|
-
const valArr = val;
|
|
172
|
-
const subLen = valArr.length;
|
|
173
|
-
for (let j = 0; j < subLen; j++) {
|
|
174
|
-
allListElements.push(valArr[j]);
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
if (val instanceof Date)
|
|
178
|
-
hasDateObj = true;
|
|
179
|
-
if (typeof val !== "boolean")
|
|
180
|
-
isBoolean = false;
|
|
181
|
-
if (typeof val !== "bigint")
|
|
182
|
-
isBigInt = false;
|
|
183
|
-
if (typeof val !== "number") {
|
|
184
|
-
isNumeric = false;
|
|
185
|
-
isInteger = false;
|
|
186
|
-
}
|
|
187
|
-
else {
|
|
188
|
-
if (!Number.isInteger(val))
|
|
189
|
-
isInteger = false;
|
|
190
|
-
}
|
|
191
|
-
if (!(val instanceof Date) && (typeof val !== "string" || isNaN(Date.parse(val)))) {
|
|
192
|
-
isDate = false;
|
|
193
|
-
}
|
|
194
|
-
if (!(0, utils_1.isObj)(val) || val instanceof Date || val instanceof Uint8Array || Array.isArray(val)) {
|
|
195
|
-
isObject = false;
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
if (!hasNonNull)
|
|
199
|
-
return datatypes_1.DataTypeRegistry.Utf8;
|
|
200
|
-
if (isBinary)
|
|
201
|
-
return datatypes_1.DataTypeRegistry.Binary;
|
|
202
|
-
if (isList) {
|
|
203
|
-
const innerType = inferColumnType(allListElements);
|
|
204
|
-
return datatypes_1.DataTypeRegistry.List(innerType);
|
|
205
|
-
}
|
|
206
|
-
if (isBoolean)
|
|
207
|
-
return datatypes_1.DataTypeRegistry.Boolean;
|
|
208
|
-
if (isBigInt)
|
|
209
|
-
return datatypes_1.DataTypeRegistry.Int64;
|
|
210
|
-
if (isNumeric && !isInteger)
|
|
211
|
-
return datatypes_1.DataTypeRegistry.Float64;
|
|
212
|
-
if (isNumeric && isInteger) {
|
|
213
|
-
let fitsInInt32 = true;
|
|
214
|
-
for (let i = 0; i < col.length; i++) {
|
|
215
|
-
const val = col[i];
|
|
216
|
-
if (val == null)
|
|
217
|
-
continue;
|
|
218
|
-
if (val < -2147483648 || val > 2147483647) {
|
|
219
|
-
fitsInInt32 = false;
|
|
220
|
-
break;
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
return fitsInInt32 ? datatypes_1.DataTypeRegistry.Int32 : datatypes_1.DataTypeRegistry.Float64;
|
|
224
|
-
}
|
|
225
|
-
if (isDate && hasDateObj)
|
|
226
|
-
return datatypes_1.DataTypeRegistry.Datetime;
|
|
227
|
-
if (isObject)
|
|
228
|
-
return datatypes_1.DataTypeRegistry.Object;
|
|
229
|
-
return datatypes_1.DataTypeRegistry.Utf8;
|
|
230
|
-
}
|
|
231
|
-
function gatherColumnsByIndices(columns, indices) {
|
|
232
|
-
const keys = Object.keys(columns);
|
|
233
|
-
const numKeys = keys.length;
|
|
234
|
-
const newHeight = indices.length;
|
|
235
|
-
const res = {};
|
|
236
|
-
for (let j = 0; j < numKeys; j++) {
|
|
237
|
-
const k = keys[j];
|
|
238
|
-
const oldCol = columns[k];
|
|
239
|
-
const newCol = (0, utils_1.isTypedArray)(oldCol)
|
|
240
|
-
? new oldCol.constructor(newHeight)
|
|
241
|
-
: new Array(newHeight);
|
|
242
|
-
for (let idx = 0; idx < newHeight; idx++) {
|
|
243
|
-
newCol[idx] = oldCol[indices[idx]];
|
|
244
|
-
}
|
|
245
|
-
res[k] = newCol;
|
|
246
|
-
}
|
|
247
|
-
return res;
|
|
248
|
-
}
|
|
249
|
-
/**
|
|
250
|
-
* Computes a hash string for a row at the given index, using one or more column keys.
|
|
251
|
-
* Includes a single-key fast path to avoid array allocation and join overhead.
|
|
252
|
-
*/
|
|
253
|
-
function computeRowHash(columns, keys, rowIndex) {
|
|
254
|
-
const len = keys.length;
|
|
255
|
-
if (len === 1) {
|
|
256
|
-
const val = columns[keys[0]][rowIndex];
|
|
257
|
-
return val == null ? "" : (0, utils_1.toCanonicalString)(val);
|
|
258
|
-
}
|
|
259
|
-
const vals = new Array(len);
|
|
260
|
-
for (let i = 0; i < len; i++) {
|
|
261
|
-
const val = columns[keys[i]][rowIndex];
|
|
262
|
-
vals[i] = val == null ? "" : (0, utils_1.toCanonicalString)(val);
|
|
263
|
-
}
|
|
264
|
-
return vals.join(constants_1.KEY_SEPARATOR);
|
|
265
|
-
}
|
|
266
|
-
function coerceColumn(col, type, height) {
|
|
267
|
-
let newCol = type.allocate ? type.allocate(height) : new Array(height);
|
|
268
|
-
const isTyped = (0, utils_1.isTypedArray)(newCol);
|
|
269
|
-
if (isTyped) {
|
|
270
|
-
const typedCol = newCol;
|
|
271
|
-
for (let i = 0; i < height; i++) {
|
|
272
|
-
const coerced = type.coerce(col[i]);
|
|
273
|
-
if (coerced == null) {
|
|
274
|
-
const fallback = new Array(height);
|
|
275
|
-
for (let j = 0; j < i; j++) {
|
|
276
|
-
fallback[j] = typedCol[j];
|
|
277
|
-
}
|
|
278
|
-
fallback[i] = null;
|
|
279
|
-
for (let j = i + 1; j < height; j++) {
|
|
280
|
-
fallback[j] = type.coerce(col[j]);
|
|
281
|
-
}
|
|
282
|
-
return fallback;
|
|
283
|
-
}
|
|
284
|
-
typedCol[i] = coerced;
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
else {
|
|
288
|
-
for (let i = 0; i < height; i++) {
|
|
289
|
-
newCol[i] = type.coerce(col[i]);
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
return newCol;
|
|
293
|
-
}
|
|
294
|
-
function writeStringToFileOrStream(file, content) {
|
|
295
|
-
if (!file)
|
|
296
|
-
return;
|
|
297
|
-
if (typeof file === "string") {
|
|
298
|
-
const fs = require("fs");
|
|
299
|
-
fs.writeFileSync(file, content, "utf8");
|
|
300
|
-
}
|
|
301
|
-
else if (typeof file === "object" && typeof file.write === "function") {
|
|
302
|
-
file.write(content);
|
|
303
|
-
}
|
|
304
|
-
else {
|
|
305
|
-
throw new TypeError("Invalid file argument. Expected a file path string or a writable stream/object with a write method.");
|
|
306
|
-
}
|
|
307
|
-
}
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.NestedDataType = exports.TemporalDataType = exports.FloatDataType = exports.UnsignedIntegerType = exports.SignedIntegerType = exports.IntegerDataType = exports.NumericDataType = exports.DataType = void 0;
|
|
4
|
-
class DataType {
|
|
5
|
-
matches(selector) {
|
|
6
|
-
if (selector == null)
|
|
7
|
-
return false;
|
|
8
|
-
if (selector instanceof DataType) {
|
|
9
|
-
if (this.equals(selector))
|
|
10
|
-
return true;
|
|
11
|
-
if (this.name.startsWith("Decimal") && selector.name.startsWith("Decimal")) {
|
|
12
|
-
if (selector.precision === undefined && selector.scale === undefined) {
|
|
13
|
-
return true;
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
return false;
|
|
17
|
-
}
|
|
18
|
-
if (typeof selector === "function") {
|
|
19
|
-
if (selector.prototype instanceof DataType) {
|
|
20
|
-
return this instanceof selector;
|
|
21
|
-
}
|
|
22
|
-
try {
|
|
23
|
-
const dummy = selector();
|
|
24
|
-
if (dummy instanceof DataType) {
|
|
25
|
-
return this.constructor === dummy.constructor;
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
catch {
|
|
29
|
-
// Ignore errors
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
return false;
|
|
33
|
-
}
|
|
34
|
-
get isNumeric() { return false; }
|
|
35
|
-
get isInteger() { return false; }
|
|
36
|
-
get isFloat() { return false; }
|
|
37
|
-
get isSigned() { return false; }
|
|
38
|
-
get isUnsigned() { return false; }
|
|
39
|
-
get isTemporal() { return false; }
|
|
40
|
-
get isNested() { return false; }
|
|
41
|
-
get isBoolean() { return false; }
|
|
42
|
-
get isString() { return false; }
|
|
43
|
-
get isUtf8() { return false; }
|
|
44
|
-
get isObject() { return false; }
|
|
45
|
-
get isNull() { return false; }
|
|
46
|
-
get isBinary() { return false; }
|
|
47
|
-
}
|
|
48
|
-
exports.DataType = DataType;
|
|
49
|
-
class NumericDataType extends DataType {
|
|
50
|
-
get isNumeric() { return true; }
|
|
51
|
-
}
|
|
52
|
-
exports.NumericDataType = NumericDataType;
|
|
53
|
-
class IntegerDataType extends NumericDataType {
|
|
54
|
-
get isInteger() { return true; }
|
|
55
|
-
}
|
|
56
|
-
exports.IntegerDataType = IntegerDataType;
|
|
57
|
-
class SignedIntegerType extends IntegerDataType {
|
|
58
|
-
get isSigned() { return true; }
|
|
59
|
-
}
|
|
60
|
-
exports.SignedIntegerType = SignedIntegerType;
|
|
61
|
-
class UnsignedIntegerType extends IntegerDataType {
|
|
62
|
-
get isUnsigned() { return true; }
|
|
63
|
-
}
|
|
64
|
-
exports.UnsignedIntegerType = UnsignedIntegerType;
|
|
65
|
-
class FloatDataType extends NumericDataType {
|
|
66
|
-
get isFloat() { return true; }
|
|
67
|
-
}
|
|
68
|
-
exports.FloatDataType = FloatDataType;
|
|
69
|
-
class TemporalDataType extends DataType {
|
|
70
|
-
get isTemporal() { return true; }
|
|
71
|
-
}
|
|
72
|
-
exports.TemporalDataType = TemporalDataType;
|
|
73
|
-
class NestedDataType extends DataType {
|
|
74
|
-
get isNested() { return true; }
|
|
75
|
-
}
|
|
76
|
-
exports.NestedDataType = NestedDataType;
|
package/dist/datatypes/index.js
DELETED
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.DataTypeRegistry = exports.DataType = void 0;
|
|
18
|
-
const DataType_1 = require("./DataType");
|
|
19
|
-
Object.defineProperty(exports, "DataType", { enumerable: true, get: function () { return DataType_1.DataType; } });
|
|
20
|
-
const types_1 = require("./types");
|
|
21
|
-
exports.DataTypeRegistry = {
|
|
22
|
-
Int8: types_1.Int8,
|
|
23
|
-
Int16: types_1.Int16,
|
|
24
|
-
Int32: types_1.Int32,
|
|
25
|
-
Int64: types_1.Int64,
|
|
26
|
-
UInt8: types_1.UInt8,
|
|
27
|
-
UInt16: types_1.UInt16,
|
|
28
|
-
UInt32: types_1.UInt32,
|
|
29
|
-
UInt64: types_1.UInt64,
|
|
30
|
-
Float32: types_1.Float32,
|
|
31
|
-
Float64: types_1.Float64,
|
|
32
|
-
Decimal: (precision, scale) => new types_1.DecimalType(precision, scale),
|
|
33
|
-
Boolean: types_1.Boolean,
|
|
34
|
-
Utf8: types_1.Utf8,
|
|
35
|
-
Binary: types_1.Binary,
|
|
36
|
-
Date: types_1.Date,
|
|
37
|
-
Datetime: types_1.Datetime,
|
|
38
|
-
Time: types_1.Time,
|
|
39
|
-
Duration: types_1.Duration,
|
|
40
|
-
Object: types_1.Object,
|
|
41
|
-
Null: types_1.Null,
|
|
42
|
-
List: types_1.List,
|
|
43
|
-
Struct: types_1.Struct
|
|
44
|
-
};
|
|
45
|
-
__exportStar(require("./types"), exports);
|
|
46
|
-
__exportStar(require("./DataType"), exports);
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { DataType } from "../DataType";
|
|
2
|
-
export declare class BinaryType extends DataType {
|
|
3
|
-
readonly name = "Binary";
|
|
4
|
-
get isBinary(): boolean;
|
|
5
|
-
coerce(val: any): Uint8Array | null;
|
|
6
|
-
equals(other: DataType): boolean;
|
|
7
|
-
allocate(size: number): (Uint8Array | null)[];
|
|
8
|
-
}
|
|
9
|
-
export declare const Binary: BinaryType;
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Binary = exports.BinaryType = void 0;
|
|
4
|
-
const DataType_1 = require("../DataType");
|
|
5
|
-
const utils_1 = require("../../utils");
|
|
6
|
-
class BinaryType extends DataType_1.DataType {
|
|
7
|
-
name = "Binary";
|
|
8
|
-
get isBinary() { return true; }
|
|
9
|
-
coerce(val) {
|
|
10
|
-
return (0, utils_1.toValidBinary)(val);
|
|
11
|
-
}
|
|
12
|
-
equals(other) {
|
|
13
|
-
return other.name === "Binary";
|
|
14
|
-
}
|
|
15
|
-
allocate(size) { return new Array(size).fill(null); }
|
|
16
|
-
}
|
|
17
|
-
exports.BinaryType = BinaryType;
|
|
18
|
-
exports.Binary = new BinaryType();
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { DataType } from "../DataType";
|
|
2
|
-
export declare class BooleanType extends DataType {
|
|
3
|
-
readonly name = "Boolean";
|
|
4
|
-
get isBoolean(): boolean;
|
|
5
|
-
coerce(val: any): boolean | null;
|
|
6
|
-
equals(other: DataType): boolean;
|
|
7
|
-
allocate(size: number): (boolean | null)[];
|
|
8
|
-
}
|
|
9
|
-
export declare const Boolean: BooleanType;
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Boolean = exports.BooleanType = void 0;
|
|
4
|
-
const DataType_1 = require("../DataType");
|
|
5
|
-
class BooleanType extends DataType_1.DataType {
|
|
6
|
-
name = "Boolean";
|
|
7
|
-
get isBoolean() { return true; }
|
|
8
|
-
coerce(val) {
|
|
9
|
-
if (val == null)
|
|
10
|
-
return null;
|
|
11
|
-
return !!val;
|
|
12
|
-
}
|
|
13
|
-
equals(other) {
|
|
14
|
-
return other.name === "Boolean";
|
|
15
|
-
}
|
|
16
|
-
allocate(size) { return new Array(size).fill(null); }
|
|
17
|
-
}
|
|
18
|
-
exports.BooleanType = BooleanType;
|
|
19
|
-
exports.Boolean = new BooleanType();
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { TemporalDataType, DataType } from "../DataType";
|
|
2
|
-
export declare class DateType extends TemporalDataType {
|
|
3
|
-
readonly name = "Date";
|
|
4
|
-
coerce(val: any): Date | null;
|
|
5
|
-
equals(other: DataType): boolean;
|
|
6
|
-
allocate(size: number): (Date | null)[];
|
|
7
|
-
}
|
|
8
|
-
export declare const Date: DateType;
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Date = exports.DateType = void 0;
|
|
4
|
-
const DataType_1 = require("../DataType");
|
|
5
|
-
const utils_1 = require("../../utils");
|
|
6
|
-
class DateType extends DataType_1.TemporalDataType {
|
|
7
|
-
name = "Date";
|
|
8
|
-
coerce(val) {
|
|
9
|
-
const d = (0, utils_1.toValidDate)(val);
|
|
10
|
-
if (!d)
|
|
11
|
-
return null;
|
|
12
|
-
d.setUTCHours(0, 0, 0, 0);
|
|
13
|
-
return d;
|
|
14
|
-
}
|
|
15
|
-
equals(other) {
|
|
16
|
-
return other.name === "Date";
|
|
17
|
-
}
|
|
18
|
-
allocate(size) { return new Array(size).fill(null); }
|
|
19
|
-
}
|
|
20
|
-
exports.DateType = DateType;
|
|
21
|
-
exports.Date = new DateType();
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import { TemporalDataType, DataType } from "../DataType";
|
|
2
|
-
export declare class DatetimeType extends TemporalDataType {
|
|
3
|
-
readonly name = "Datetime";
|
|
4
|
-
coerce(val: any): Date | null;
|
|
5
|
-
equals(other: DataType): boolean;
|
|
6
|
-
allocate(size: number): (Date | null)[];
|
|
7
|
-
}
|
|
8
|
-
export declare const Datetime: DatetimeType;
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Datetime = exports.DatetimeType = void 0;
|
|
4
|
-
const DataType_1 = require("../DataType");
|
|
5
|
-
const utils_1 = require("../../utils");
|
|
6
|
-
class DatetimeType extends DataType_1.TemporalDataType {
|
|
7
|
-
name = "Datetime";
|
|
8
|
-
coerce(val) {
|
|
9
|
-
return (0, utils_1.toValidDate)(val);
|
|
10
|
-
}
|
|
11
|
-
equals(other) {
|
|
12
|
-
return other.name === "Datetime";
|
|
13
|
-
}
|
|
14
|
-
allocate(size) { return new Array(size).fill(null); }
|
|
15
|
-
}
|
|
16
|
-
exports.DatetimeType = DatetimeType;
|
|
17
|
-
exports.Datetime = new DatetimeType();
|