@vizabi/reader-ddfcsv 4.2.0 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.nyc_output/728a89bae7b247a7244820c1b6596c8b.json +1 -0
- package/.nyc_output/{b4e1a7f86ae858c85e0528842263588d.json → e875174b7b217be31b58605962f05304.json} +0 -0
- package/.travis.yml +0 -0
- package/CHANGELOG.md +0 -5
- package/LICENSE +0 -0
- package/README.md +0 -0
- package/coverage/base.css +0 -0
- package/coverage/block-navigation.js +0 -0
- package/coverage/index.html +39 -39
- package/coverage/prettify.css +0 -0
- package/coverage/prettify.js +0 -0
- package/coverage/sort-arrow-sprite.png +0 -0
- package/coverage/sorter.js +0 -0
- package/coverage/src/ddf-csv.ts.html +377 -377
- package/coverage/src/ddfcsv-error.ts.html +6 -6
- package/coverage/src/ddfcsv-reader.ts.html +52 -106
- package/coverage/src/file-readers/backend-file-reader.ts.html +15 -27
- package/coverage/src/file-readers/github-file-reader.ts.html +168 -0
- package/coverage/src/file-readers/index.html +26 -26
- package/coverage/src/index.html +29 -29
- package/coverage/src/index.ts.html +19 -4
- package/coverage/src/resource-selection-optimizer/in-clause-under-conjunction.ts.html +236 -236
- package/coverage/src/resource-selection-optimizer/index.html +20 -20
- package/coverage/src/resource-selection-optimizer/index.ts.html +3 -3
- package/coverage/src/test-cases/concepts.ts.html +1 -1
- package/coverage/src/test-cases/entities.ts.html +1 -1
- package/coverage/src/test-cases/index.html +1 -1
- package/deploy.js +0 -0
- package/dist/reader-ddfcsv-polyfill.js +1 -1
- package/dist/reader-ddfcsv-polyfill.js.map +1 -1
- package/dist/reader-ddfcsv.js +1 -1
- package/dist/reader-ddfcsv.js.map +1 -1
- package/icon.png +0 -0
- package/lib/src/ddf-csv.d.ts +0 -0
- package/lib/src/ddf-csv.js +0 -0
- package/lib/src/ddf-csv.js.map +0 -0
- package/lib/src/ddfcsv-error.d.ts +0 -0
- package/lib/src/ddfcsv-error.js +0 -0
- package/lib/src/ddfcsv-error.js.map +0 -0
- package/lib/src/ddfcsv-reader.d.ts +0 -0
- package/lib/src/ddfcsv-reader.js +1 -1
- package/lib/src/ddfcsv-reader.js.map +0 -0
- package/lib/src/file-readers/backend-file-reader.d.ts +0 -0
- package/lib/src/file-readers/backend-file-reader.js +0 -0
- package/lib/src/file-readers/backend-file-reader.js.map +0 -0
- package/lib/src/file-readers/github-path-adapter.d.ts +0 -0
- package/lib/src/file-readers/github-path-adapter.js +0 -0
- package/lib/src/file-readers/github-path-adapter.js.map +0 -0
- package/lib/src/index.d.ts +0 -0
- package/lib/src/index.js +0 -0
- package/lib/src/index.js.map +0 -0
- package/lib/src/interfaces.d.ts +0 -0
- package/lib/src/interfaces.js +0 -0
- package/lib/src/interfaces.js.map +0 -0
- package/lib/src/resource-selection-optimizer/in-clause-under-conjunction.d.ts +0 -0
- package/lib/src/resource-selection-optimizer/in-clause-under-conjunction.js +0 -0
- package/lib/src/resource-selection-optimizer/in-clause-under-conjunction.js.map +0 -0
- package/lib/src/resource-selection-optimizer/index.d.ts +0 -0
- package/lib/src/resource-selection-optimizer/index.js +0 -0
- package/lib/src/resource-selection-optimizer/index.js.map +0 -0
- package/lib-web/src/ddf-csv.d.ts +0 -0
- package/lib-web/src/ddf-csv.js +262 -376
- package/lib-web/src/ddf-csv.js.map +1 -1
- package/lib-web/src/ddfcsv-error.d.ts +0 -0
- package/lib-web/src/ddfcsv-error.js +8 -12
- package/lib-web/src/ddfcsv-error.js.map +1 -1
- package/lib-web/src/ddfcsv-reader.d.ts +0 -0
- package/lib-web/src/ddfcsv-reader.js +81 -123
- package/lib-web/src/ddfcsv-reader.js.map +1 -1
- package/lib-web/src/file-readers/frontend-file-reader.d.ts +0 -0
- package/lib-web/src/file-readers/frontend-file-reader.js +13 -16
- package/lib-web/src/file-readers/frontend-file-reader.js.map +1 -1
- package/lib-web/src/file-readers/github-path-adapter.d.ts +0 -0
- package/lib-web/src/file-readers/github-path-adapter.js +5 -5
- package/lib-web/src/file-readers/github-path-adapter.js.map +1 -1
- package/lib-web/src/index-web.d.ts +0 -0
- package/lib-web/src/index-web.js +2 -2
- package/lib-web/src/index-web.js.map +1 -1
- package/lib-web/src/interfaces.d.ts +0 -0
- package/lib-web/src/interfaces.js +0 -0
- package/lib-web/src/interfaces.js.map +0 -0
- package/lib-web/src/resource-selection-optimizer/in-clause-under-conjunction.d.ts +0 -0
- package/lib-web/src/resource-selection-optimizer/in-clause-under-conjunction.js +169 -348
- package/lib-web/src/resource-selection-optimizer/in-clause-under-conjunction.js.map +1 -1
- package/lib-web/src/resource-selection-optimizer/index.d.ts +0 -0
- package/lib-web/src/resource-selection-optimizer/index.js +4 -4
- package/lib-web/src/resource-selection-optimizer/index.js.map +1 -1
- package/package.json +2 -1
- package/scripts/set-own-version.js +0 -0
- package/src/ddf-csv.ts +0 -0
- package/src/ddfcsv-error.ts +0 -0
- package/src/ddfcsv-reader.ts +0 -0
- package/src/file-readers/backend-file-reader.ts +0 -0
- package/src/file-readers/frontend-file-reader.ts +0 -0
- package/src/file-readers/github-path-adapter.ts +0 -0
- package/src/index-web.ts +0 -0
- package/src/index.ts +0 -0
- package/src/interfaces.ts +0 -0
- package/src/resource-selection-optimizer/in-clause-under-conjunction.ts +0 -0
- package/src/resource-selection-optimizer/index.ts +0 -0
- package/src/test-cases/concepts.ts +0 -0
- package/src/test-cases/entities.ts +0 -0
- package/test/assets-fixtures/world-50m.json +0 -0
- package/test/assets.spec.ts +0 -0
- package/test/common.ts +0 -0
- package/test/definition/concepts-definition.spec.ts +0 -0
- package/test/definition/datapoints-definition.spec.ts +0 -0
- package/test/definition/entities-definition.spec.ts +0 -0
- package/test/definition/schema-definition.spec.ts +0 -0
- package/test/diagnostics.spec.ts +0 -0
- package/test/features-service.spec.ts +0 -0
- package/test/high-load.spec.ts +0 -0
- package/test/main.spec.ts +0 -0
- package/test/mocha.opts +0 -0
- package/test/multi-instances.spec.ts +0 -0
- package/test/result-fixtures/datapoints-assets.json +0 -0
- package/test/result-fixtures/in-clause-under-conjunction-1.json +0 -0
- package/test/result-fixtures/in-clause-under-conjunction-2.json +0 -0
- package/test/result-fixtures/multi-instances/concepts-sg.json +0 -0
- package/test/result-fixtures/multi-instances/concepts-soderstornsmodellen.json +0 -0
- package/test/result-fixtures/multi-instances/datapoints-sg.json +0 -0
- package/test/result-fixtures/multi-instances/datapoints-soderstornsmodellen.json +0 -0
- package/test/result-fixtures/multi-instances/entities-sg.json +0 -0
- package/test/result-fixtures/multi-instances/entities-soderstornsmodellen.json +0 -0
- package/test/result-fixtures/multi-instances/schema-sg.json +0 -0
- package/test/result-fixtures/multi-instances/schema-soderstornsmodellen.json +0 -0
- package/test/schema.spec.ts +0 -0
- package/test/tslint.json +0 -0
- package/tsconfig-web.json +1 -1
- package/tsconfig.json +0 -0
- package/tslint.json +0 -0
- package/.nyc_output/795caf0b3acca8aacbd9d932d086ab81.json +0 -1
- package/coverage/src/file-readers/github-path-adapter.ts.html +0 -228
|
@@ -1,31 +1,30 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.InClauseUnderConjunction = void 0;
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
function InClauseUnderConjunction(queryParam, options) {
|
|
4
|
+
const path = require("path");
|
|
5
|
+
const head = require("lodash.head");
|
|
6
|
+
const values = require("lodash.values");
|
|
7
|
+
const keys = require("lodash.keys");
|
|
8
|
+
const get = require("lodash.get");
|
|
9
|
+
const flattenDeep = require("lodash.flattendeep");
|
|
10
|
+
const isEmpty = require("lodash.isempty");
|
|
11
|
+
const startsWith = require("lodash.startswith");
|
|
12
|
+
const includes = require("lodash.includes");
|
|
13
|
+
const compact = require("lodash.compact");
|
|
14
|
+
const ddfcsv_error_1 = require("../ddfcsv-error");
|
|
15
|
+
const ddf_query_validator_1 = require("ddf-query-validator");
|
|
16
|
+
const Papa = require('papaparse');
|
|
17
|
+
const WHERE_KEYWORD = 'where';
|
|
18
|
+
const JOIN_KEYWORD = 'join';
|
|
19
|
+
const KEY_IN = '$in';
|
|
20
|
+
const KEY_NIN = '$nin';
|
|
21
|
+
const KEY_AND = '$and';
|
|
22
|
+
const KEY_OR = '$or';
|
|
23
|
+
const getFirstConditionClause = clause => head(values(clause));
|
|
24
|
+
const getFirstKey = obj => head(keys(obj));
|
|
25
|
+
const isOneKeyBased = obj => keys(obj).length === 1;
|
|
26
|
+
class InClauseUnderConjunction {
|
|
27
|
+
constructor(queryParam, options) {
|
|
29
28
|
this.options = options;
|
|
30
29
|
this.flow = {};
|
|
31
30
|
this.fileReader = options.fileReader;
|
|
@@ -34,53 +33,41 @@ var InClauseUnderConjunction = (function () {
|
|
|
34
33
|
this.datapackage = options.datapackage;
|
|
35
34
|
this.conceptsLookup = options.conceptsLookup;
|
|
36
35
|
}
|
|
37
|
-
|
|
38
|
-
var _this = this;
|
|
36
|
+
isMatched() {
|
|
39
37
|
this.flow.joinObject = get(this.query, JOIN_KEYWORD);
|
|
40
|
-
|
|
38
|
+
const relatedFeatures = compact(ddf_query_validator_1.featureDetectors.map(detector => detector(this.query, this.conceptsLookup)));
|
|
41
39
|
return this.query.from === "datapoints";
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
case 5:
|
|
74
|
-
message = "Plugin \"InClauseUnderConjunction\" is not matched!";
|
|
75
|
-
warning(message);
|
|
76
|
-
throw new ddfcsv_error_1.DdfCsvError(message, 'InClauseUnderConjunction plugin');
|
|
77
|
-
}
|
|
78
|
-
});
|
|
79
|
-
});
|
|
80
|
-
};
|
|
81
|
-
InClauseUnderConjunction.prototype.fillResourceToFileHash = function () {
|
|
82
|
-
this.flow.resourceToFile = get(this.datapackage, 'resources', []).reduce(function (hash, resource) {
|
|
83
|
-
var constraints = resource.schema.fields.reduce(function (result, field) {
|
|
40
|
+
}
|
|
41
|
+
async getRecommendedFilesSet() {
|
|
42
|
+
const { debug, error, warning } = this.options.diagnostic.prepareDiagnosticFor('getRecommendedFilesSet');
|
|
43
|
+
if (this.isMatched()) {
|
|
44
|
+
debug('plugin matched');
|
|
45
|
+
let result;
|
|
46
|
+
try {
|
|
47
|
+
this.fillResourceToFileHash();
|
|
48
|
+
this.collectProcessableClauses();
|
|
49
|
+
this.collectEntityFilesNames();
|
|
50
|
+
const data = await this.collectEntities();
|
|
51
|
+
this.fillEntityValuesHash(data);
|
|
52
|
+
this.getFilesGroupsQueryClause();
|
|
53
|
+
result = this.getOptimalFilesGroup();
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
error('wrong data processing', err);
|
|
57
|
+
return [];
|
|
58
|
+
}
|
|
59
|
+
debug('recommended files found', result);
|
|
60
|
+
return result;
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
const message = `Plugin "InClauseUnderConjunction" is not matched!`;
|
|
64
|
+
warning(message);
|
|
65
|
+
throw new ddfcsv_error_1.DdfCsvError(message, 'InClauseUnderConjunction plugin');
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
fillResourceToFileHash() {
|
|
69
|
+
this.flow.resourceToFile = get(this.datapackage, 'resources', []).reduce((hash, resource) => {
|
|
70
|
+
const constraints = resource.schema.fields.reduce((result, field) => {
|
|
84
71
|
var _a;
|
|
85
72
|
if ((_a = field.constraints) === null || _a === void 0 ? void 0 : _a.enum) {
|
|
86
73
|
result.set(field.name, field.constraints.enum);
|
|
@@ -89,340 +76,174 @@ var InClauseUnderConjunction = (function () {
|
|
|
89
76
|
}, new Map());
|
|
90
77
|
hash.set(resource.name, {
|
|
91
78
|
path: resource.path,
|
|
92
|
-
constraints
|
|
79
|
+
constraints
|
|
93
80
|
});
|
|
94
81
|
return hash;
|
|
95
82
|
}, new Map());
|
|
96
83
|
return this;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
var _this = this;
|
|
101
|
-
var joinKeys = keys(this.flow.joinObject);
|
|
84
|
+
}
|
|
85
|
+
collectProcessableClauses() {
|
|
86
|
+
const joinKeys = keys(this.flow.joinObject);
|
|
102
87
|
this.flow.processableClauses = [];
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
if (this.singleAndField(where)) {
|
|
108
|
-
(_b = this.flow.processableClauses).push.apply(_b, tslib_1.__spreadArray([], tslib_1.__read(flattenDeep(where[KEY_AND].map(function (el) { return _this.getProcessableClauses(el); }))), false));
|
|
109
|
-
}
|
|
110
|
-
else if (this.singleOrField(where)) {
|
|
111
|
-
(_c = this.flow.processableClauses).push.apply(_c, tslib_1.__spreadArray([], tslib_1.__read(flattenDeep(where[KEY_OR].map(function (el) { return _this.getProcessableClauses(el); }))), false));
|
|
112
|
-
}
|
|
113
|
-
else {
|
|
114
|
-
(_d = this.flow.processableClauses).push.apply(_d, tslib_1.__spreadArray([], tslib_1.__read(this.getProcessableClauses(where)), false));
|
|
115
|
-
}
|
|
88
|
+
for (const joinKey of joinKeys) {
|
|
89
|
+
const where = get(this.flow.joinObject, `${joinKey}.${WHERE_KEYWORD}`, {});
|
|
90
|
+
if (this.singleAndField(where)) {
|
|
91
|
+
this.flow.processableClauses.push(...flattenDeep(where[KEY_AND].map(el => this.getProcessableClauses(el))));
|
|
116
92
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
93
|
+
else if (this.singleOrField(where)) {
|
|
94
|
+
this.flow.processableClauses.push(...flattenDeep(where[KEY_OR].map(el => this.getProcessableClauses(el))));
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
this.flow.processableClauses.push(...this.getProcessableClauses(where));
|
|
122
98
|
}
|
|
123
|
-
finally { if (e_1) throw e_1.error; }
|
|
124
99
|
}
|
|
125
100
|
return this;
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
var e_2, _a, e_3, _b, e_4, _c;
|
|
101
|
+
}
|
|
102
|
+
collectEntityFilesNames() {
|
|
129
103
|
this.flow.entityFilesNames = new Set();
|
|
130
104
|
this.flow.fileNameToPrimaryKeyHash = new Map();
|
|
131
|
-
|
|
132
|
-
for (
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
for (
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
try {
|
|
140
|
-
for (var _h = (e_4 = void 0, tslib_1.__values(schemaResourceRecord.resources)), _j = _h.next(); !_j.done; _j = _h.next()) {
|
|
141
|
-
var resourceName = _j.value;
|
|
142
|
-
var file = this.flow.resourceToFile.get(resourceName).path;
|
|
143
|
-
this.flow.entityFilesNames.add(file);
|
|
144
|
-
this.flow.fileNameToPrimaryKeyHash.set(file, primaryKey);
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
catch (e_4_1) { e_4 = { error: e_4_1 }; }
|
|
148
|
-
finally {
|
|
149
|
-
try {
|
|
150
|
-
if (_j && !_j.done && (_c = _h.return)) _c.call(_h);
|
|
151
|
-
}
|
|
152
|
-
finally { if (e_4) throw e_4.error; }
|
|
153
|
-
}
|
|
154
|
-
}
|
|
105
|
+
for (const schemaResourceRecord of this.datapackage.ddfSchema.entities) {
|
|
106
|
+
for (const clause of this.flow.processableClauses) {
|
|
107
|
+
const primaryKey = getFirstKey(clause);
|
|
108
|
+
if (head(schemaResourceRecord.primaryKey) === primaryKey) {
|
|
109
|
+
for (const resourceName of schemaResourceRecord.resources) {
|
|
110
|
+
const file = this.flow.resourceToFile.get(resourceName).path;
|
|
111
|
+
this.flow.entityFilesNames.add(file);
|
|
112
|
+
this.flow.fileNameToPrimaryKeyHash.set(file, primaryKey);
|
|
155
113
|
}
|
|
156
114
|
}
|
|
157
|
-
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
158
|
-
finally {
|
|
159
|
-
try {
|
|
160
|
-
if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
|
|
161
|
-
}
|
|
162
|
-
finally { if (e_3) throw e_3.error; }
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
167
|
-
finally {
|
|
168
|
-
try {
|
|
169
|
-
if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
|
|
170
115
|
}
|
|
171
|
-
finally { if (e_2) throw e_2.error; }
|
|
172
116
|
}
|
|
173
117
|
return this;
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
self.fileReader.readText(path.join(self.datasetPath, file),
|
|
118
|
+
}
|
|
119
|
+
collectEntities() {
|
|
120
|
+
const self = this;
|
|
121
|
+
const actions = [...self.flow.entityFilesNames].map(file => new Promise((actResolve, actReject) => {
|
|
122
|
+
self.fileReader.readText(path.join(self.datasetPath, file), (err, text) => {
|
|
179
123
|
if (err) {
|
|
180
124
|
return actReject(err);
|
|
181
125
|
}
|
|
182
126
|
Papa.parse(text, {
|
|
183
127
|
header: true,
|
|
184
128
|
skipEmptyLines: true,
|
|
185
|
-
complete:
|
|
186
|
-
error:
|
|
129
|
+
complete: result => actResolve({ file, result }),
|
|
130
|
+
error: error => actReject(error)
|
|
187
131
|
});
|
|
188
132
|
});
|
|
189
|
-
})
|
|
133
|
+
}));
|
|
190
134
|
return Promise.all(actions);
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
.
|
|
196
|
-
.map(function (key) { return key.replace(/^is--/, ''); })); };
|
|
135
|
+
}
|
|
136
|
+
fillEntityValuesHash(entitiesData) {
|
|
137
|
+
const getSubdomainsFromRecord = record => compact(keys(record)
|
|
138
|
+
.filter(key => startsWith(key, 'is--') && (record[key] === 'TRUE' || record[key] === 'true'))
|
|
139
|
+
.map(key => key.replace(/^is--/, '')));
|
|
197
140
|
this.flow.entityValueToFileHash = new Map();
|
|
198
141
|
this.flow.entityValueToDomainHash = new Map();
|
|
199
|
-
|
|
200
|
-
for (
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
}
|
|
210
|
-
var primaryKeyCellValue = entityRecord[primaryKeyForThisFile] || entityRecord[domainsForCurrentRecord[0]];
|
|
211
|
-
this.flow.entityValueToDomainHash.set(primaryKeyCellValue, domainsForCurrentRecord);
|
|
212
|
-
this.flow.entityValueToFileHash.set(primaryKeyCellValue, entityFileDescriptor.file);
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
catch (e_6_1) { e_6 = { error: e_6_1 }; }
|
|
216
|
-
finally {
|
|
217
|
-
try {
|
|
218
|
-
if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
|
|
219
|
-
}
|
|
220
|
-
finally { if (e_6) throw e_6.error; }
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
catch (e_5_1) { e_5 = { error: e_5_1 }; }
|
|
225
|
-
finally {
|
|
226
|
-
try {
|
|
227
|
-
if (entitiesData_1_1 && !entitiesData_1_1.done && (_a = entitiesData_1.return)) _a.call(entitiesData_1);
|
|
142
|
+
for (const entityFileDescriptor of entitiesData) {
|
|
143
|
+
for (const entityRecord of entityFileDescriptor.result.data) {
|
|
144
|
+
const primaryKeyForThisFile = this.flow.fileNameToPrimaryKeyHash.get(entityFileDescriptor.file);
|
|
145
|
+
const domainsForCurrentRecord = [...getSubdomainsFromRecord(entityRecord)];
|
|
146
|
+
if (isEmpty(domainsForCurrentRecord)) {
|
|
147
|
+
domainsForCurrentRecord.push(primaryKeyForThisFile);
|
|
148
|
+
}
|
|
149
|
+
const primaryKeyCellValue = entityRecord[primaryKeyForThisFile] || entityRecord[domainsForCurrentRecord[0]];
|
|
150
|
+
this.flow.entityValueToDomainHash.set(primaryKeyCellValue, domainsForCurrentRecord);
|
|
151
|
+
this.flow.entityValueToFileHash.set(primaryKeyCellValue, entityFileDescriptor.file);
|
|
228
152
|
}
|
|
229
|
-
finally { if (e_5) throw e_5.error; }
|
|
230
153
|
}
|
|
231
154
|
return this;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
try {
|
|
240
|
-
for (var _b = tslib_1.__values(_this.flow.entityValueToDomainHash.keys()), _c = _b.next(); !_c.done; _c = _b.next()) {
|
|
241
|
-
var entityKey = _c.value;
|
|
242
|
-
if (!includes(entityValuesToExclude, entityKey)) {
|
|
243
|
-
result.push(entityKey);
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
catch (e_14_1) { e_14 = { error: e_14_1 }; }
|
|
248
|
-
finally {
|
|
249
|
-
try {
|
|
250
|
-
if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
|
|
155
|
+
}
|
|
156
|
+
getFilesGroupsQueryClause() {
|
|
157
|
+
const getEntitiesExcept = (entityValuesToExclude) => {
|
|
158
|
+
const result = [];
|
|
159
|
+
for (const entityKey of this.flow.entityValueToDomainHash.keys()) {
|
|
160
|
+
if (!includes(entityValuesToExclude, entityKey)) {
|
|
161
|
+
result.push(entityKey);
|
|
251
162
|
}
|
|
252
|
-
finally { if (e_14) throw e_14.error; }
|
|
253
163
|
}
|
|
254
164
|
return result;
|
|
255
165
|
};
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
for (
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
var schemaResourceRecord = _l.value;
|
|
277
|
-
try {
|
|
278
|
-
for (var _m = (e_11 = void 0, tslib_1.__values(schemaResourceRecord.resources)), _o = _m.next(); !_o.done; _o = _m.next()) {
|
|
279
|
-
var resourceName = _o.value;
|
|
280
|
-
var file = this.flow.resourceToFile.get(resourceName);
|
|
281
|
-
if (includes(schemaResourceRecord.primaryKey, entityByQuery)) {
|
|
282
|
-
var constraint = file.constraints.get(entityByQuery);
|
|
283
|
-
if (constraint) {
|
|
284
|
-
if (constraint.includes(entityValueFromClause)) {
|
|
285
|
-
filesGroupByClause.datapoints.add(file.path);
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
else {
|
|
289
|
-
filesGroupByClause.datapoints.add(file.path);
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
catch (e_11_1) { e_11 = { error: e_11_1 }; }
|
|
295
|
-
finally {
|
|
296
|
-
try {
|
|
297
|
-
if (_o && !_o.done && (_e = _m.return)) _e.call(_m);
|
|
298
|
-
}
|
|
299
|
-
finally { if (e_11) throw e_11.error; }
|
|
300
|
-
}
|
|
166
|
+
const filesGroupsByClause = new Map();
|
|
167
|
+
for (const clause of this.flow.processableClauses) {
|
|
168
|
+
const filesGroupByClause = {
|
|
169
|
+
entities: this.flow.entityFilesNames,
|
|
170
|
+
datapoints: new Set(),
|
|
171
|
+
concepts: new Set()
|
|
172
|
+
};
|
|
173
|
+
const firstConditionClause = getFirstConditionClause(clause);
|
|
174
|
+
const entityValuesFromClause = firstConditionClause[KEY_IN] || getEntitiesExcept(firstConditionClause[KEY_NIN]);
|
|
175
|
+
for (const entityValueFromClause of entityValuesFromClause) {
|
|
176
|
+
const entitiesByQuery = this.flow.entityValueToDomainHash.get(entityValueFromClause);
|
|
177
|
+
for (const entityByQuery of entitiesByQuery) {
|
|
178
|
+
for (const schemaResourceRecord of this.datapackage.ddfSchema.datapoints) {
|
|
179
|
+
for (const resourceName of schemaResourceRecord.resources) {
|
|
180
|
+
const file = this.flow.resourceToFile.get(resourceName);
|
|
181
|
+
if (includes(schemaResourceRecord.primaryKey, entityByQuery)) {
|
|
182
|
+
const constraint = file.constraints.get(entityByQuery);
|
|
183
|
+
if (constraint) {
|
|
184
|
+
if (constraint.includes(entityValueFromClause)) {
|
|
185
|
+
filesGroupByClause.datapoints.add(file.path);
|
|
301
186
|
}
|
|
302
187
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
try {
|
|
306
|
-
if (_l && !_l.done && (_d = _k.return)) _d.call(_k);
|
|
307
|
-
}
|
|
308
|
-
finally { if (e_10) throw e_10.error; }
|
|
188
|
+
else {
|
|
189
|
+
filesGroupByClause.datapoints.add(file.path);
|
|
309
190
|
}
|
|
310
191
|
}
|
|
311
192
|
}
|
|
312
|
-
catch (e_9_1) { e_9 = { error: e_9_1 }; }
|
|
313
|
-
finally {
|
|
314
|
-
try {
|
|
315
|
-
if (entitiesByQuery_1_1 && !entitiesByQuery_1_1.done && (_c = entitiesByQuery_1.return)) _c.call(entitiesByQuery_1);
|
|
316
|
-
}
|
|
317
|
-
finally { if (e_9) throw e_9.error; }
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
}
|
|
321
|
-
catch (e_8_1) { e_8 = { error: e_8_1 }; }
|
|
322
|
-
finally {
|
|
323
|
-
try {
|
|
324
|
-
if (entityValuesFromClause_1_1 && !entityValuesFromClause_1_1.done && (_b = entityValuesFromClause_1.return)) _b.call(entityValuesFromClause_1);
|
|
325
|
-
}
|
|
326
|
-
finally { if (e_8) throw e_8.error; }
|
|
327
|
-
}
|
|
328
|
-
try {
|
|
329
|
-
for (var _p = (e_12 = void 0, tslib_1.__values(this.datapackage.ddfSchema.concepts)), _q = _p.next(); !_q.done; _q = _p.next()) {
|
|
330
|
-
var schemaResourceRecord = _q.value;
|
|
331
|
-
try {
|
|
332
|
-
for (var _r = (e_13 = void 0, tslib_1.__values(schemaResourceRecord.resources)), _s = _r.next(); !_s.done; _s = _r.next()) {
|
|
333
|
-
var resourceName = _s.value;
|
|
334
|
-
filesGroupByClause.concepts.add(this.flow.resourceToFile.get(resourceName).path);
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
catch (e_13_1) { e_13 = { error: e_13_1 }; }
|
|
338
|
-
finally {
|
|
339
|
-
try {
|
|
340
|
-
if (_s && !_s.done && (_g = _r.return)) _g.call(_r);
|
|
341
|
-
}
|
|
342
|
-
finally { if (e_13) throw e_13.error; }
|
|
343
|
-
}
|
|
344
193
|
}
|
|
345
194
|
}
|
|
346
|
-
catch (e_12_1) { e_12 = { error: e_12_1 }; }
|
|
347
|
-
finally {
|
|
348
|
-
try {
|
|
349
|
-
if (_q && !_q.done && (_f = _p.return)) _f.call(_p);
|
|
350
|
-
}
|
|
351
|
-
finally { if (e_12) throw e_12.error; }
|
|
352
|
-
}
|
|
353
|
-
filesGroupsByClause.set(clause, filesGroupByClause);
|
|
354
195
|
}
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
if (_j && !_j.done && (_a = _h.return)) _a.call(_h);
|
|
196
|
+
for (const schemaResourceRecord of this.datapackage.ddfSchema.concepts) {
|
|
197
|
+
for (const resourceName of schemaResourceRecord.resources) {
|
|
198
|
+
filesGroupByClause.concepts.add(this.flow.resourceToFile.get(resourceName).path);
|
|
199
|
+
}
|
|
360
200
|
}
|
|
361
|
-
|
|
201
|
+
filesGroupsByClause.set(clause, filesGroupByClause);
|
|
362
202
|
}
|
|
363
203
|
this.flow.filesGroupsByClause = filesGroupsByClause;
|
|
364
204
|
return this;
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
if (!appropriateClauseKey || size < appropriateClauseSize) {
|
|
378
|
-
appropriateClauseKey = key;
|
|
379
|
-
appropriateClauseSize = size;
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
|
-
}
|
|
383
|
-
catch (e_15_1) { e_15 = { error: e_15_1 }; }
|
|
384
|
-
finally {
|
|
385
|
-
try {
|
|
386
|
-
if (clauseKeys_1_1 && !clauseKeys_1_1.done && (_a = clauseKeys_1.return)) _a.call(clauseKeys_1);
|
|
205
|
+
}
|
|
206
|
+
getOptimalFilesGroup() {
|
|
207
|
+
const clauseKeys = this.flow.filesGroupsByClause.keys();
|
|
208
|
+
let appropriateClauseKey;
|
|
209
|
+
let appropriateClauseSize;
|
|
210
|
+
for (const key of clauseKeys) {
|
|
211
|
+
const size = this.flow.filesGroupsByClause.get(key).datapoints.size +
|
|
212
|
+
this.flow.filesGroupsByClause.get(key).entities.size +
|
|
213
|
+
this.flow.filesGroupsByClause.get(key).concepts.size;
|
|
214
|
+
if (!appropriateClauseKey || size < appropriateClauseSize) {
|
|
215
|
+
appropriateClauseKey = key;
|
|
216
|
+
appropriateClauseSize = size;
|
|
387
217
|
}
|
|
388
|
-
finally { if (e_15) throw e_15.error; }
|
|
389
218
|
}
|
|
390
219
|
if (!this.flow.filesGroupsByClause.get(appropriateClauseKey)) {
|
|
391
220
|
return [];
|
|
392
221
|
}
|
|
393
|
-
return
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
222
|
+
return [
|
|
223
|
+
...Array.from(this.flow.filesGroupsByClause.get(appropriateClauseKey).concepts),
|
|
224
|
+
...Array.from(this.flow.filesGroupsByClause.get(appropriateClauseKey).entities),
|
|
225
|
+
...Array.from(this.flow.filesGroupsByClause.get(appropriateClauseKey).datapoints)
|
|
226
|
+
];
|
|
227
|
+
}
|
|
228
|
+
getProcessableClauses(clause) {
|
|
229
|
+
const result = [];
|
|
230
|
+
const clauseKeys = keys(clause);
|
|
231
|
+
for (const key of clauseKeys) {
|
|
232
|
+
if (!startsWith(key, '$') && isOneKeyBased(clause[key])) {
|
|
233
|
+
const conditionKey = head(keys(clause[key]));
|
|
234
|
+
if (conditionKey === KEY_IN || conditionKey === KEY_NIN) {
|
|
235
|
+
result.push(clause);
|
|
407
236
|
}
|
|
408
237
|
}
|
|
409
238
|
}
|
|
410
|
-
catch (e_16_1) { e_16 = { error: e_16_1 }; }
|
|
411
|
-
finally {
|
|
412
|
-
try {
|
|
413
|
-
if (clauseKeys_2_1 && !clauseKeys_2_1.done && (_a = clauseKeys_2.return)) _a.call(clauseKeys_2);
|
|
414
|
-
}
|
|
415
|
-
finally { if (e_16) throw e_16.error; }
|
|
416
|
-
}
|
|
417
239
|
return result;
|
|
418
|
-
}
|
|
419
|
-
|
|
240
|
+
}
|
|
241
|
+
singleAndField(clause) {
|
|
420
242
|
return isOneKeyBased(clause) && !!get(clause, KEY_AND);
|
|
421
|
-
}
|
|
422
|
-
|
|
243
|
+
}
|
|
244
|
+
singleOrField(clause) {
|
|
423
245
|
return isOneKeyBased(clause) && !!get(clause, KEY_OR);
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
}());
|
|
246
|
+
}
|
|
247
|
+
}
|
|
427
248
|
exports.InClauseUnderConjunction = InClauseUnderConjunction;
|
|
428
249
|
//# sourceMappingURL=in-clause-under-conjunction.js.map
|