@vizabi/reader-ddfcsv 4.1.1 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.nyc_output/728a89bae7b247a7244820c1b6596c8b.json +1 -0
- package/.nyc_output/{b4e1a7f86ae858c85e0528842263588d.json → e875174b7b217be31b58605962f05304.json} +0 -0
- package/.travis.yml +0 -0
- package/CHANGELOG.md +0 -0
- package/LICENSE +0 -0
- package/README.md +0 -0
- package/coverage/base.css +0 -0
- package/coverage/block-navigation.js +0 -0
- package/coverage/index.html +39 -39
- package/coverage/prettify.css +0 -0
- package/coverage/prettify.js +0 -0
- package/coverage/sort-arrow-sprite.png +0 -0
- package/coverage/sorter.js +0 -0
- package/coverage/src/ddf-csv.ts.html +377 -377
- package/coverage/src/ddfcsv-error.ts.html +6 -6
- package/coverage/src/ddfcsv-reader.ts.html +52 -106
- package/coverage/src/file-readers/backend-file-reader.ts.html +15 -27
- package/coverage/src/file-readers/github-file-reader.ts.html +168 -0
- package/coverage/src/file-readers/index.html +26 -26
- package/coverage/src/index.html +29 -29
- package/coverage/src/index.ts.html +19 -4
- package/coverage/src/resource-selection-optimizer/in-clause-under-conjunction.ts.html +236 -236
- package/coverage/src/resource-selection-optimizer/index.html +20 -20
- package/coverage/src/resource-selection-optimizer/index.ts.html +3 -3
- package/coverage/src/test-cases/concepts.ts.html +1 -1
- package/coverage/src/test-cases/entities.ts.html +1 -1
- package/coverage/src/test-cases/index.html +1 -1
- package/deploy.js +0 -0
- package/dist/reader-ddfcsv-polyfill.js +1 -16
- package/dist/reader-ddfcsv-polyfill.js.map +1 -1
- package/dist/reader-ddfcsv.js +1 -16
- package/dist/reader-ddfcsv.js.map +1 -1
- package/icon.png +0 -0
- package/lib/src/ddf-csv.d.ts +0 -0
- package/lib/src/ddf-csv.js +16 -18
- package/lib/src/ddf-csv.js.map +1 -1
- package/lib/src/ddfcsv-error.d.ts +0 -0
- package/lib/src/ddfcsv-error.js +1 -0
- package/lib/src/ddfcsv-error.js.map +1 -1
- package/lib/src/ddfcsv-reader.d.ts +0 -0
- package/lib/src/ddfcsv-reader.js +6 -5
- package/lib/src/ddfcsv-reader.js.map +1 -1
- package/lib/src/file-readers/backend-file-reader.d.ts +0 -0
- package/lib/src/file-readers/backend-file-reader.js +1 -0
- package/lib/src/file-readers/backend-file-reader.js.map +1 -1
- package/lib/src/file-readers/github-path-adapter.d.ts +0 -0
- package/lib/src/file-readers/github-path-adapter.js +1 -0
- package/lib/src/file-readers/github-path-adapter.js.map +1 -1
- package/lib/src/index.d.ts +0 -0
- package/lib/src/index.js +5 -4
- package/lib/src/index.js.map +1 -1
- package/lib/src/interfaces.d.ts +0 -0
- package/lib/src/interfaces.js +0 -0
- package/lib/src/interfaces.js.map +0 -0
- package/lib/src/resource-selection-optimizer/in-clause-under-conjunction.d.ts +1 -0
- package/lib/src/resource-selection-optimizer/in-clause-under-conjunction.js +37 -12
- package/lib/src/resource-selection-optimizer/in-clause-under-conjunction.js.map +1 -1
- package/lib/src/resource-selection-optimizer/index.d.ts +0 -0
- package/lib/src/resource-selection-optimizer/index.js +1 -0
- package/lib/src/resource-selection-optimizer/index.js.map +1 -1
- package/lib-web/src/ddf-csv.d.ts +0 -0
- package/lib-web/src/ddf-csv.js +272 -388
- package/lib-web/src/ddf-csv.js.map +1 -1
- package/lib-web/src/ddfcsv-error.d.ts +0 -0
- package/lib-web/src/ddfcsv-error.js +9 -12
- package/lib-web/src/ddfcsv-error.js.map +1 -1
- package/lib-web/src/ddfcsv-reader.d.ts +0 -0
- package/lib-web/src/ddfcsv-reader.js +84 -125
- package/lib-web/src/ddfcsv-reader.js.map +1 -1
- package/lib-web/src/file-readers/frontend-file-reader.d.ts +0 -0
- package/lib-web/src/file-readers/frontend-file-reader.js +14 -16
- package/lib-web/src/file-readers/frontend-file-reader.js.map +1 -1
- package/lib-web/src/file-readers/github-path-adapter.d.ts +0 -0
- package/lib-web/src/file-readers/github-path-adapter.js +6 -5
- package/lib-web/src/file-readers/github-path-adapter.js.map +1 -1
- package/lib-web/src/index-web.d.ts +0 -0
- package/lib-web/src/index-web.js +6 -5
- package/lib-web/src/index-web.js.map +1 -1
- package/lib-web/src/interfaces.d.ts +0 -0
- package/lib-web/src/interfaces.js +0 -0
- package/lib-web/src/interfaces.js.map +0 -0
- package/lib-web/src/resource-selection-optimizer/in-clause-under-conjunction.d.ts +1 -0
- package/lib-web/src/resource-selection-optimizer/in-clause-under-conjunction.js +181 -335
- package/lib-web/src/resource-selection-optimizer/in-clause-under-conjunction.js.map +1 -1
- package/lib-web/src/resource-selection-optimizer/index.d.ts +0 -0
- package/lib-web/src/resource-selection-optimizer/index.js +5 -4
- package/lib-web/src/resource-selection-optimizer/index.js.map +1 -1
- package/package.json +6 -4
- package/scripts/set-own-version.js +0 -0
- package/src/ddf-csv.ts +11 -10
- package/src/ddfcsv-error.ts +0 -0
- package/src/ddfcsv-reader.ts +0 -0
- package/src/file-readers/backend-file-reader.ts +0 -0
- package/src/file-readers/frontend-file-reader.ts +0 -0
- package/src/file-readers/github-path-adapter.ts +0 -0
- package/src/index-web.ts +0 -0
- package/src/index.ts +0 -0
- package/src/interfaces.ts +0 -0
- package/src/resource-selection-optimizer/in-clause-under-conjunction.ts +40 -12
- package/src/resource-selection-optimizer/index.ts +0 -0
- package/src/test-cases/concepts.ts +0 -0
- package/src/test-cases/entities.ts +0 -0
- package/test/assets-fixtures/world-50m.json +0 -0
- package/test/assets.spec.ts +0 -0
- package/test/common.ts +0 -0
- package/test/definition/concepts-definition.spec.ts +0 -0
- package/test/definition/datapoints-definition.spec.ts +0 -0
- package/test/definition/entities-definition.spec.ts +0 -0
- package/test/definition/schema-definition.spec.ts +0 -0
- package/test/diagnostics.spec.ts +0 -0
- package/test/features-service.spec.ts +0 -0
- package/test/high-load.spec.ts +0 -0
- package/test/main.spec.ts +0 -0
- package/test/mocha.opts +0 -0
- package/test/multi-instances.spec.ts +0 -0
- package/test/result-fixtures/datapoints-assets.json +0 -0
- package/test/result-fixtures/in-clause-under-conjunction-1.json +0 -0
- package/test/result-fixtures/in-clause-under-conjunction-2.json +0 -0
- package/test/result-fixtures/multi-instances/concepts-sg.json +0 -0
- package/test/result-fixtures/multi-instances/concepts-soderstornsmodellen.json +0 -0
- package/test/result-fixtures/multi-instances/datapoints-sg.json +0 -0
- package/test/result-fixtures/multi-instances/datapoints-soderstornsmodellen.json +0 -0
- package/test/result-fixtures/multi-instances/entities-sg.json +0 -0
- package/test/result-fixtures/multi-instances/entities-soderstornsmodellen.json +0 -0
- package/test/result-fixtures/multi-instances/schema-sg.json +0 -0
- package/test/result-fixtures/multi-instances/schema-soderstornsmodellen.json +0 -0
- package/test/schema.spec.ts +0 -0
- package/test/tslint.json +0 -0
- package/tsconfig-web.json +5 -2
- package/tsconfig.json +1 -1
- package/tslint.json +0 -0
- package/.nyc_output/795caf0b3acca8aacbd9d932d086ab81.json +0 -1
- package/coverage/src/file-readers/github-path-adapter.ts.html +0 -228
|
@@ -1,29 +1,30 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
3
|
+
exports.InClauseUnderConjunction = void 0;
|
|
4
|
+
const path = require("path");
|
|
5
|
+
const head = require("lodash.head");
|
|
6
|
+
const values = require("lodash.values");
|
|
7
|
+
const keys = require("lodash.keys");
|
|
8
|
+
const get = require("lodash.get");
|
|
9
|
+
const flattenDeep = require("lodash.flattendeep");
|
|
10
|
+
const isEmpty = require("lodash.isempty");
|
|
11
|
+
const startsWith = require("lodash.startswith");
|
|
12
|
+
const includes = require("lodash.includes");
|
|
13
|
+
const compact = require("lodash.compact");
|
|
14
|
+
const ddfcsv_error_1 = require("../ddfcsv-error");
|
|
15
|
+
const ddf_query_validator_1 = require("ddf-query-validator");
|
|
16
|
+
const Papa = require('papaparse');
|
|
17
|
+
const WHERE_KEYWORD = 'where';
|
|
18
|
+
const JOIN_KEYWORD = 'join';
|
|
19
|
+
const KEY_IN = '$in';
|
|
20
|
+
const KEY_NIN = '$nin';
|
|
21
|
+
const KEY_AND = '$and';
|
|
22
|
+
const KEY_OR = '$or';
|
|
23
|
+
const getFirstConditionClause = clause => head(values(clause));
|
|
24
|
+
const getFirstKey = obj => head(keys(obj));
|
|
25
|
+
const isOneKeyBased = obj => keys(obj).length === 1;
|
|
26
|
+
class InClauseUnderConjunction {
|
|
27
|
+
constructor(queryParam, options) {
|
|
27
28
|
this.options = options;
|
|
28
29
|
this.flow = {};
|
|
29
30
|
this.fileReader = options.fileReader;
|
|
@@ -32,372 +33,217 @@ var InClauseUnderConjunction = (function () {
|
|
|
32
33
|
this.datapackage = options.datapackage;
|
|
33
34
|
this.conceptsLookup = options.conceptsLookup;
|
|
34
35
|
}
|
|
35
|
-
|
|
36
|
-
var _this = this;
|
|
36
|
+
isMatched() {
|
|
37
37
|
this.flow.joinObject = get(this.query, JOIN_KEYWORD);
|
|
38
|
-
|
|
39
|
-
return
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
38
|
+
const relatedFeatures = compact(ddf_query_validator_1.featureDetectors.map(detector => detector(this.query, this.conceptsLookup)));
|
|
39
|
+
return this.query.from === "datapoints";
|
|
40
|
+
}
|
|
41
|
+
async getRecommendedFilesSet() {
|
|
42
|
+
const { debug, error, warning } = this.options.diagnostic.prepareDiagnosticFor('getRecommendedFilesSet');
|
|
43
|
+
if (this.isMatched()) {
|
|
44
|
+
debug('plugin matched');
|
|
45
|
+
let result;
|
|
46
|
+
try {
|
|
47
|
+
this.fillResourceToFileHash();
|
|
48
|
+
this.collectProcessableClauses();
|
|
49
|
+
this.collectEntityFilesNames();
|
|
50
|
+
const data = await this.collectEntities();
|
|
51
|
+
this.fillEntityValuesHash(data);
|
|
52
|
+
this.getFilesGroupsQueryClause();
|
|
53
|
+
result = this.getOptimalFilesGroup();
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
error('wrong data processing', err);
|
|
57
|
+
return [];
|
|
58
|
+
}
|
|
59
|
+
debug('recommended files found', result);
|
|
60
|
+
return result;
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
const message = `Plugin "InClauseUnderConjunction" is not matched!`;
|
|
64
|
+
warning(message);
|
|
65
|
+
throw new ddfcsv_error_1.DdfCsvError(message, 'InClauseUnderConjunction plugin');
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
fillResourceToFileHash() {
|
|
69
|
+
this.flow.resourceToFile = get(this.datapackage, 'resources', []).reduce((hash, resource) => {
|
|
70
|
+
const constraints = resource.schema.fields.reduce((result, field) => {
|
|
71
|
+
var _a;
|
|
72
|
+
if ((_a = field.constraints) === null || _a === void 0 ? void 0 : _a.enum) {
|
|
73
|
+
result.set(field.name, field.constraints.enum);
|
|
74
|
+
}
|
|
75
|
+
return result;
|
|
76
|
+
}, new Map());
|
|
77
|
+
hash.set(resource.name, {
|
|
78
|
+
path: resource.path,
|
|
79
|
+
constraints
|
|
77
80
|
});
|
|
78
|
-
});
|
|
79
|
-
};
|
|
80
|
-
InClauseUnderConjunction.prototype.fillResourceToFileHash = function () {
|
|
81
|
-
this.flow.resourceToFile = get(this.datapackage, 'resources', []).reduce(function (hash, resource) {
|
|
82
|
-
hash.set(resource.name, resource.path);
|
|
83
81
|
return hash;
|
|
84
82
|
}, new Map());
|
|
85
83
|
return this;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
var e_1, _a, _b, _c;
|
|
90
|
-
var joinKeys = keys(this.flow.joinObject);
|
|
84
|
+
}
|
|
85
|
+
collectProcessableClauses() {
|
|
86
|
+
const joinKeys = keys(this.flow.joinObject);
|
|
91
87
|
this.flow.processableClauses = [];
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
if (this.singleAndField(where)) {
|
|
97
|
-
(_b = this.flow.processableClauses).push.apply(_b, tslib_1.__spread(flattenDeep(where[KEY_AND].map(function (el) { return _this.getProcessableClauses(el); }))));
|
|
98
|
-
}
|
|
99
|
-
else {
|
|
100
|
-
(_c = this.flow.processableClauses).push.apply(_c, tslib_1.__spread(this.getProcessableClauses(where)));
|
|
101
|
-
}
|
|
88
|
+
for (const joinKey of joinKeys) {
|
|
89
|
+
const where = get(this.flow.joinObject, `${joinKey}.${WHERE_KEYWORD}`, {});
|
|
90
|
+
if (this.singleAndField(where)) {
|
|
91
|
+
this.flow.processableClauses.push(...flattenDeep(where[KEY_AND].map(el => this.getProcessableClauses(el))));
|
|
102
92
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
93
|
+
else if (this.singleOrField(where)) {
|
|
94
|
+
this.flow.processableClauses.push(...flattenDeep(where[KEY_OR].map(el => this.getProcessableClauses(el))));
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
this.flow.processableClauses.push(...this.getProcessableClauses(where));
|
|
108
98
|
}
|
|
109
|
-
finally { if (e_1) throw e_1.error; }
|
|
110
99
|
}
|
|
111
100
|
return this;
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
this.flow.entityFilesNames = [];
|
|
101
|
+
}
|
|
102
|
+
collectEntityFilesNames() {
|
|
103
|
+
this.flow.entityFilesNames = new Set();
|
|
116
104
|
this.flow.fileNameToPrimaryKeyHash = new Map();
|
|
117
|
-
|
|
118
|
-
for (
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
for (
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
try {
|
|
126
|
-
for (var _h = tslib_1.__values(schemaResourceRecord.resources), _j = _h.next(); !_j.done; _j = _h.next()) {
|
|
127
|
-
var resourceName = _j.value;
|
|
128
|
-
var file = this.flow.resourceToFile.get(resourceName);
|
|
129
|
-
this.flow.entityFilesNames.push(file);
|
|
130
|
-
this.flow.fileNameToPrimaryKeyHash.set(file, primaryKey);
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
catch (e_4_1) { e_4 = { error: e_4_1 }; }
|
|
134
|
-
finally {
|
|
135
|
-
try {
|
|
136
|
-
if (_j && !_j.done && (_c = _h.return)) _c.call(_h);
|
|
137
|
-
}
|
|
138
|
-
finally { if (e_4) throw e_4.error; }
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
144
|
-
finally {
|
|
145
|
-
try {
|
|
146
|
-
if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
|
|
105
|
+
for (const schemaResourceRecord of this.datapackage.ddfSchema.entities) {
|
|
106
|
+
for (const clause of this.flow.processableClauses) {
|
|
107
|
+
const primaryKey = getFirstKey(clause);
|
|
108
|
+
if (head(schemaResourceRecord.primaryKey) === primaryKey) {
|
|
109
|
+
for (const resourceName of schemaResourceRecord.resources) {
|
|
110
|
+
const file = this.flow.resourceToFile.get(resourceName).path;
|
|
111
|
+
this.flow.entityFilesNames.add(file);
|
|
112
|
+
this.flow.fileNameToPrimaryKeyHash.set(file, primaryKey);
|
|
147
113
|
}
|
|
148
|
-
finally { if (e_3) throw e_3.error; }
|
|
149
114
|
}
|
|
150
115
|
}
|
|
151
116
|
}
|
|
152
|
-
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
153
|
-
finally {
|
|
154
|
-
try {
|
|
155
|
-
if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
|
|
156
|
-
}
|
|
157
|
-
finally { if (e_2) throw e_2.error; }
|
|
158
|
-
}
|
|
159
117
|
return this;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
self.fileReader.readText(path.join(self.datasetPath, file),
|
|
118
|
+
}
|
|
119
|
+
collectEntities() {
|
|
120
|
+
const self = this;
|
|
121
|
+
const actions = [...self.flow.entityFilesNames].map(file => new Promise((actResolve, actReject) => {
|
|
122
|
+
self.fileReader.readText(path.join(self.datasetPath, file), (err, text) => {
|
|
165
123
|
if (err) {
|
|
166
124
|
return actReject(err);
|
|
167
125
|
}
|
|
168
126
|
Papa.parse(text, {
|
|
169
127
|
header: true,
|
|
170
128
|
skipEmptyLines: true,
|
|
171
|
-
complete:
|
|
172
|
-
error:
|
|
129
|
+
complete: result => actResolve({ file, result }),
|
|
130
|
+
error: error => actReject(error)
|
|
173
131
|
});
|
|
174
132
|
});
|
|
175
|
-
})
|
|
133
|
+
}));
|
|
176
134
|
return Promise.all(actions);
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
.
|
|
182
|
-
.map(function (key) { return key.replace(/^is--/, ''); })); };
|
|
135
|
+
}
|
|
136
|
+
fillEntityValuesHash(entitiesData) {
|
|
137
|
+
const getSubdomainsFromRecord = record => compact(keys(record)
|
|
138
|
+
.filter(key => startsWith(key, 'is--') && (record[key] === 'TRUE' || record[key] === 'true'))
|
|
139
|
+
.map(key => key.replace(/^is--/, '')));
|
|
183
140
|
this.flow.entityValueToFileHash = new Map();
|
|
184
141
|
this.flow.entityValueToDomainHash = new Map();
|
|
185
|
-
|
|
186
|
-
for (
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
domainsForCurrentRecord.push(primaryKeyForThisFile);
|
|
196
|
-
}
|
|
197
|
-
this.flow.entityValueToDomainHash.set(primaryKeyCellValue, domainsForCurrentRecord);
|
|
198
|
-
this.flow.entityValueToFileHash.set(primaryKeyCellValue, entityFileDescriptor.file);
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
catch (e_6_1) { e_6 = { error: e_6_1 }; }
|
|
202
|
-
finally {
|
|
203
|
-
try {
|
|
204
|
-
if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
|
|
205
|
-
}
|
|
206
|
-
finally { if (e_6) throw e_6.error; }
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
catch (e_5_1) { e_5 = { error: e_5_1 }; }
|
|
211
|
-
finally {
|
|
212
|
-
try {
|
|
213
|
-
if (entitiesData_1_1 && !entitiesData_1_1.done && (_a = entitiesData_1.return)) _a.call(entitiesData_1);
|
|
142
|
+
for (const entityFileDescriptor of entitiesData) {
|
|
143
|
+
for (const entityRecord of entityFileDescriptor.result.data) {
|
|
144
|
+
const primaryKeyForThisFile = this.flow.fileNameToPrimaryKeyHash.get(entityFileDescriptor.file);
|
|
145
|
+
const domainsForCurrentRecord = [...getSubdomainsFromRecord(entityRecord)];
|
|
146
|
+
if (isEmpty(domainsForCurrentRecord)) {
|
|
147
|
+
domainsForCurrentRecord.push(primaryKeyForThisFile);
|
|
148
|
+
}
|
|
149
|
+
const primaryKeyCellValue = entityRecord[primaryKeyForThisFile] || entityRecord[domainsForCurrentRecord[0]];
|
|
150
|
+
this.flow.entityValueToDomainHash.set(primaryKeyCellValue, domainsForCurrentRecord);
|
|
151
|
+
this.flow.entityValueToFileHash.set(primaryKeyCellValue, entityFileDescriptor.file);
|
|
214
152
|
}
|
|
215
|
-
finally { if (e_5) throw e_5.error; }
|
|
216
153
|
}
|
|
217
154
|
return this;
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
try {
|
|
226
|
-
for (var _b = tslib_1.__values(_this.flow.entityValueToDomainHash.keys()), _c = _b.next(); !_c.done; _c = _b.next()) {
|
|
227
|
-
var entityKey = _c.value;
|
|
228
|
-
if (!includes(entityValuesToExclude, entityKey)) {
|
|
229
|
-
result.push(entityKey);
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
catch (e_14_1) { e_14 = { error: e_14_1 }; }
|
|
234
|
-
finally {
|
|
235
|
-
try {
|
|
236
|
-
if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
|
|
155
|
+
}
|
|
156
|
+
getFilesGroupsQueryClause() {
|
|
157
|
+
const getEntitiesExcept = (entityValuesToExclude) => {
|
|
158
|
+
const result = [];
|
|
159
|
+
for (const entityKey of this.flow.entityValueToDomainHash.keys()) {
|
|
160
|
+
if (!includes(entityValuesToExclude, entityKey)) {
|
|
161
|
+
result.push(entityKey);
|
|
237
162
|
}
|
|
238
|
-
finally { if (e_14) throw e_14.error; }
|
|
239
163
|
}
|
|
240
164
|
return result;
|
|
241
165
|
};
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
for (
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
for (var _k = tslib_1.__values(this.datapackage.ddfSchema.datapoints), _l = _k.next(); !_l.done; _l = _k.next()) {
|
|
263
|
-
var schemaResourceRecord = _l.value;
|
|
264
|
-
try {
|
|
265
|
-
for (var _m = tslib_1.__values(schemaResourceRecord.resources), _o = _m.next(); !_o.done; _o = _m.next()) {
|
|
266
|
-
var resourceName = _o.value;
|
|
267
|
-
if (includes(schemaResourceRecord.primaryKey, entityByQuery)) {
|
|
268
|
-
filesGroupByClause.datapoints.add(this.flow.resourceToFile.get(resourceName));
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
catch (e_11_1) { e_11 = { error: e_11_1 }; }
|
|
273
|
-
finally {
|
|
274
|
-
try {
|
|
275
|
-
if (_o && !_o.done && (_e = _m.return)) _e.call(_m);
|
|
276
|
-
}
|
|
277
|
-
finally { if (e_11) throw e_11.error; }
|
|
278
|
-
}
|
|
166
|
+
const filesGroupsByClause = new Map();
|
|
167
|
+
for (const clause of this.flow.processableClauses) {
|
|
168
|
+
const filesGroupByClause = {
|
|
169
|
+
entities: this.flow.entityFilesNames,
|
|
170
|
+
datapoints: new Set(),
|
|
171
|
+
concepts: new Set()
|
|
172
|
+
};
|
|
173
|
+
const firstConditionClause = getFirstConditionClause(clause);
|
|
174
|
+
const entityValuesFromClause = firstConditionClause[KEY_IN] || getEntitiesExcept(firstConditionClause[KEY_NIN]);
|
|
175
|
+
for (const entityValueFromClause of entityValuesFromClause) {
|
|
176
|
+
const entitiesByQuery = this.flow.entityValueToDomainHash.get(entityValueFromClause);
|
|
177
|
+
for (const entityByQuery of entitiesByQuery) {
|
|
178
|
+
for (const schemaResourceRecord of this.datapackage.ddfSchema.datapoints) {
|
|
179
|
+
for (const resourceName of schemaResourceRecord.resources) {
|
|
180
|
+
const file = this.flow.resourceToFile.get(resourceName);
|
|
181
|
+
if (includes(schemaResourceRecord.primaryKey, entityByQuery)) {
|
|
182
|
+
const constraint = file.constraints.get(entityByQuery);
|
|
183
|
+
if (constraint) {
|
|
184
|
+
if (constraint.includes(entityValueFromClause)) {
|
|
185
|
+
filesGroupByClause.datapoints.add(file.path);
|
|
279
186
|
}
|
|
280
187
|
}
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
try {
|
|
284
|
-
if (_l && !_l.done && (_d = _k.return)) _d.call(_k);
|
|
285
|
-
}
|
|
286
|
-
finally { if (e_10) throw e_10.error; }
|
|
188
|
+
else {
|
|
189
|
+
filesGroupByClause.datapoints.add(file.path);
|
|
287
190
|
}
|
|
288
191
|
}
|
|
289
192
|
}
|
|
290
|
-
catch (e_9_1) { e_9 = { error: e_9_1 }; }
|
|
291
|
-
finally {
|
|
292
|
-
try {
|
|
293
|
-
if (entitiesByQuery_1_1 && !entitiesByQuery_1_1.done && (_c = entitiesByQuery_1.return)) _c.call(entitiesByQuery_1);
|
|
294
|
-
}
|
|
295
|
-
finally { if (e_9) throw e_9.error; }
|
|
296
|
-
}
|
|
297
193
|
}
|
|
298
194
|
}
|
|
299
|
-
catch (e_8_1) { e_8 = { error: e_8_1 }; }
|
|
300
|
-
finally {
|
|
301
|
-
try {
|
|
302
|
-
if (entityValuesFromClause_1_1 && !entityValuesFromClause_1_1.done && (_b = entityValuesFromClause_1.return)) _b.call(entityValuesFromClause_1);
|
|
303
|
-
}
|
|
304
|
-
finally { if (e_8) throw e_8.error; }
|
|
305
|
-
}
|
|
306
|
-
try {
|
|
307
|
-
for (var _p = tslib_1.__values(this.datapackage.ddfSchema.concepts), _q = _p.next(); !_q.done; _q = _p.next()) {
|
|
308
|
-
var schemaResourceRecord = _q.value;
|
|
309
|
-
try {
|
|
310
|
-
for (var _r = tslib_1.__values(schemaResourceRecord.resources), _s = _r.next(); !_s.done; _s = _r.next()) {
|
|
311
|
-
var resourceName = _s.value;
|
|
312
|
-
filesGroupByClause.concepts.add(this.flow.resourceToFile.get(resourceName));
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
catch (e_13_1) { e_13 = { error: e_13_1 }; }
|
|
316
|
-
finally {
|
|
317
|
-
try {
|
|
318
|
-
if (_s && !_s.done && (_g = _r.return)) _g.call(_r);
|
|
319
|
-
}
|
|
320
|
-
finally { if (e_13) throw e_13.error; }
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
catch (e_12_1) { e_12 = { error: e_12_1 }; }
|
|
325
|
-
finally {
|
|
326
|
-
try {
|
|
327
|
-
if (_q && !_q.done && (_f = _p.return)) _f.call(_p);
|
|
328
|
-
}
|
|
329
|
-
finally { if (e_12) throw e_12.error; }
|
|
330
|
-
}
|
|
331
|
-
filesGroupsByClause.set(clause, filesGroupByClause);
|
|
332
195
|
}
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
if (_j && !_j.done && (_a = _h.return)) _a.call(_h);
|
|
196
|
+
for (const schemaResourceRecord of this.datapackage.ddfSchema.concepts) {
|
|
197
|
+
for (const resourceName of schemaResourceRecord.resources) {
|
|
198
|
+
filesGroupByClause.concepts.add(this.flow.resourceToFile.get(resourceName).path);
|
|
199
|
+
}
|
|
338
200
|
}
|
|
339
|
-
|
|
201
|
+
filesGroupsByClause.set(clause, filesGroupByClause);
|
|
340
202
|
}
|
|
341
203
|
this.flow.filesGroupsByClause = filesGroupsByClause;
|
|
342
204
|
return this;
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
if (!appropriateClauseKey || size < appropriateClauseSize) {
|
|
356
|
-
appropriateClauseKey = key;
|
|
357
|
-
appropriateClauseSize = size;
|
|
358
|
-
}
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
|
-
catch (e_15_1) { e_15 = { error: e_15_1 }; }
|
|
362
|
-
finally {
|
|
363
|
-
try {
|
|
364
|
-
if (clauseKeys_1_1 && !clauseKeys_1_1.done && (_a = clauseKeys_1.return)) _a.call(clauseKeys_1);
|
|
205
|
+
}
|
|
206
|
+
getOptimalFilesGroup() {
|
|
207
|
+
const clauseKeys = this.flow.filesGroupsByClause.keys();
|
|
208
|
+
let appropriateClauseKey;
|
|
209
|
+
let appropriateClauseSize;
|
|
210
|
+
for (const key of clauseKeys) {
|
|
211
|
+
const size = this.flow.filesGroupsByClause.get(key).datapoints.size +
|
|
212
|
+
this.flow.filesGroupsByClause.get(key).entities.size +
|
|
213
|
+
this.flow.filesGroupsByClause.get(key).concepts.size;
|
|
214
|
+
if (!appropriateClauseKey || size < appropriateClauseSize) {
|
|
215
|
+
appropriateClauseKey = key;
|
|
216
|
+
appropriateClauseSize = size;
|
|
365
217
|
}
|
|
366
|
-
finally { if (e_15) throw e_15.error; }
|
|
367
218
|
}
|
|
368
219
|
if (!this.flow.filesGroupsByClause.get(appropriateClauseKey)) {
|
|
369
220
|
return [];
|
|
370
221
|
}
|
|
371
|
-
return
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
222
|
+
return [
|
|
223
|
+
...Array.from(this.flow.filesGroupsByClause.get(appropriateClauseKey).concepts),
|
|
224
|
+
...Array.from(this.flow.filesGroupsByClause.get(appropriateClauseKey).entities),
|
|
225
|
+
...Array.from(this.flow.filesGroupsByClause.get(appropriateClauseKey).datapoints)
|
|
226
|
+
];
|
|
227
|
+
}
|
|
228
|
+
getProcessableClauses(clause) {
|
|
229
|
+
const result = [];
|
|
230
|
+
const clauseKeys = keys(clause);
|
|
231
|
+
for (const key of clauseKeys) {
|
|
232
|
+
if (!startsWith(key, '$') && isOneKeyBased(clause[key])) {
|
|
233
|
+
const conditionKey = head(keys(clause[key]));
|
|
234
|
+
if (conditionKey === KEY_IN || conditionKey === KEY_NIN) {
|
|
235
|
+
result.push(clause);
|
|
385
236
|
}
|
|
386
237
|
}
|
|
387
238
|
}
|
|
388
|
-
catch (e_16_1) { e_16 = { error: e_16_1 }; }
|
|
389
|
-
finally {
|
|
390
|
-
try {
|
|
391
|
-
if (clauseKeys_2_1 && !clauseKeys_2_1.done && (_a = clauseKeys_2.return)) _a.call(clauseKeys_2);
|
|
392
|
-
}
|
|
393
|
-
finally { if (e_16) throw e_16.error; }
|
|
394
|
-
}
|
|
395
239
|
return result;
|
|
396
|
-
}
|
|
397
|
-
|
|
240
|
+
}
|
|
241
|
+
singleAndField(clause) {
|
|
398
242
|
return isOneKeyBased(clause) && !!get(clause, KEY_AND);
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
|
|
243
|
+
}
|
|
244
|
+
singleOrField(clause) {
|
|
245
|
+
return isOneKeyBased(clause) && !!get(clause, KEY_OR);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
402
248
|
exports.InClauseUnderConjunction = InClauseUnderConjunction;
|
|
403
249
|
//# sourceMappingURL=in-clause-under-conjunction.js.map
|