convention_builder 1.4.4 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/module/index.js +1511 -59
- package/dist/node/index.js +1511 -59
- package/package.json +6 -4
package/dist/module/index.js
CHANGED
|
@@ -1180,12 +1180,12 @@ var require_util = __commonJS({
|
|
|
1180
1180
|
__name(alwaysValidSchema, "alwaysValidSchema");
|
|
1181
1181
|
exports.alwaysValidSchema = alwaysValidSchema;
|
|
1182
1182
|
function checkUnknownRules(it, schema = it.schema) {
|
|
1183
|
-
const { opts, self } = it;
|
|
1183
|
+
const { opts, self: self2 } = it;
|
|
1184
1184
|
if (!opts.strictSchema)
|
|
1185
1185
|
return;
|
|
1186
1186
|
if (typeof schema === "boolean")
|
|
1187
1187
|
return;
|
|
1188
|
-
const rules =
|
|
1188
|
+
const rules = self2.RULES.keywords;
|
|
1189
1189
|
for (const key in schema) {
|
|
1190
1190
|
if (!rules[key])
|
|
1191
1191
|
checkStrictMode(it, `unknown keyword: "${key}"`);
|
|
@@ -1595,8 +1595,8 @@ var require_applicability = __commonJS({
|
|
|
1595
1595
|
"use strict";
|
|
1596
1596
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1597
1597
|
exports.shouldUseRule = exports.shouldUseGroup = exports.schemaHasRulesForType = void 0;
|
|
1598
|
-
function schemaHasRulesForType({ schema, self }, type) {
|
|
1599
|
-
const group =
|
|
1598
|
+
function schemaHasRulesForType({ schema, self: self2 }, type) {
|
|
1599
|
+
const group = self2.RULES.types[type];
|
|
1600
1600
|
return group && group !== true && shouldUseGroup(schema, group);
|
|
1601
1601
|
}
|
|
1602
1602
|
__name(schemaHasRulesForType, "schemaHasRulesForType");
|
|
@@ -2104,7 +2104,7 @@ var require_keyword = __commonJS({
|
|
|
2104
2104
|
}
|
|
2105
2105
|
__name(validSchemaType, "validSchemaType");
|
|
2106
2106
|
exports.validSchemaType = validSchemaType;
|
|
2107
|
-
function validateKeywordUsage({ schema, opts, self, errSchemaPath }, def, keyword) {
|
|
2107
|
+
function validateKeywordUsage({ schema, opts, self: self2, errSchemaPath }, def, keyword) {
|
|
2108
2108
|
if (Array.isArray(def.keyword) ? !def.keyword.includes(keyword) : def.keyword !== keyword) {
|
|
2109
2109
|
throw new Error("ajv implementation error");
|
|
2110
2110
|
}
|
|
@@ -2115,9 +2115,9 @@ var require_keyword = __commonJS({
|
|
|
2115
2115
|
if (def.validateSchema) {
|
|
2116
2116
|
const valid = def.validateSchema(schema[keyword]);
|
|
2117
2117
|
if (!valid) {
|
|
2118
|
-
const msg = `keyword "${keyword}" value is invalid at path "${errSchemaPath}": ` +
|
|
2118
|
+
const msg = `keyword "${keyword}" value is invalid at path "${errSchemaPath}": ` + self2.errorsText(def.validateSchema.errors);
|
|
2119
2119
|
if (opts.validateSchema === "log")
|
|
2120
|
-
|
|
2120
|
+
self2.logger.error(msg);
|
|
2121
2121
|
else
|
|
2122
2122
|
throw new Error(msg);
|
|
2123
2123
|
}
|
|
@@ -2621,11 +2621,11 @@ var require_validate = __commonJS({
|
|
|
2621
2621
|
(0, boolSchema_1.boolOrEmptySchema)(it, valid);
|
|
2622
2622
|
}
|
|
2623
2623
|
__name(subschemaCode, "subschemaCode");
|
|
2624
|
-
function schemaCxtHasRules({ schema, self }) {
|
|
2624
|
+
function schemaCxtHasRules({ schema, self: self2 }) {
|
|
2625
2625
|
if (typeof schema == "boolean")
|
|
2626
2626
|
return !schema;
|
|
2627
2627
|
for (const key in schema)
|
|
2628
|
-
if (
|
|
2628
|
+
if (self2.RULES.all[key])
|
|
2629
2629
|
return true;
|
|
2630
2630
|
return false;
|
|
2631
2631
|
}
|
|
@@ -2659,9 +2659,9 @@ var require_validate = __commonJS({
|
|
|
2659
2659
|
}
|
|
2660
2660
|
__name(typeAndKeywords, "typeAndKeywords");
|
|
2661
2661
|
function checkRefsAndKeywords(it) {
|
|
2662
|
-
const { schema, errSchemaPath, opts, self } = it;
|
|
2663
|
-
if (schema.$ref && opts.ignoreKeywordsWithRef && (0, util_1.schemaHasRulesButRef)(schema,
|
|
2664
|
-
|
|
2662
|
+
const { schema, errSchemaPath, opts, self: self2 } = it;
|
|
2663
|
+
if (schema.$ref && opts.ignoreKeywordsWithRef && (0, util_1.schemaHasRulesButRef)(schema, self2.RULES)) {
|
|
2664
|
+
self2.logger.warn(`$ref: keywords ignored in schema at path "${errSchemaPath}"`);
|
|
2665
2665
|
}
|
|
2666
2666
|
}
|
|
2667
2667
|
__name(checkRefsAndKeywords, "checkRefsAndKeywords");
|
|
@@ -2714,8 +2714,8 @@ var require_validate = __commonJS({
|
|
|
2714
2714
|
}
|
|
2715
2715
|
__name(assignEvaluated, "assignEvaluated");
|
|
2716
2716
|
function schemaKeywords(it, types, typeErrors, errsCount) {
|
|
2717
|
-
const { gen, schema, data, allErrors, opts, self } = it;
|
|
2718
|
-
const { RULES } =
|
|
2717
|
+
const { gen, schema, data, allErrors, opts, self: self2 } = it;
|
|
2718
|
+
const { RULES } = self2;
|
|
2719
2719
|
if (schema.$ref && (opts.ignoreKeywordsWithRef || !(0, util_1.schemaHasRulesButRef)(schema, RULES))) {
|
|
2720
2720
|
gen.block(() => keywordCode(it, "$ref", RULES.all.$ref.definition));
|
|
2721
2721
|
return;
|
|
@@ -5103,11 +5103,11 @@ var require_ref = __commonJS({
|
|
|
5103
5103
|
schemaType: "string",
|
|
5104
5104
|
code(cxt) {
|
|
5105
5105
|
const { gen, schema: $ref, it } = cxt;
|
|
5106
|
-
const { baseId, schemaEnv: env, validateName, opts, self } = it;
|
|
5106
|
+
const { baseId, schemaEnv: env, validateName, opts, self: self2 } = it;
|
|
5107
5107
|
const { root } = env;
|
|
5108
5108
|
if (($ref === "#" || $ref === "#/") && baseId === root.baseId)
|
|
5109
5109
|
return callRootRef();
|
|
5110
|
-
const schOrEnv = compile_1.resolveRef.call(
|
|
5110
|
+
const schOrEnv = compile_1.resolveRef.call(self2, root, baseId, $ref);
|
|
5111
5111
|
if (schOrEnv === void 0)
|
|
5112
5112
|
throw new ref_error_1.default(it.opts.uriResolver, baseId, $ref);
|
|
5113
5113
|
if (schOrEnv instanceof compile_1.SchemaEnv)
|
|
@@ -6678,11 +6678,11 @@ var require_dynamicAnchor = __commonJS({
|
|
|
6678
6678
|
__name(dynamicAnchor, "dynamicAnchor");
|
|
6679
6679
|
exports.dynamicAnchor = dynamicAnchor;
|
|
6680
6680
|
function _getValidate(cxt) {
|
|
6681
|
-
const { schemaEnv, schema, self } = cxt.it;
|
|
6681
|
+
const { schemaEnv, schema, self: self2 } = cxt.it;
|
|
6682
6682
|
const { root, baseId, localRefs, meta } = schemaEnv.root;
|
|
6683
|
-
const { schemaId } =
|
|
6683
|
+
const { schemaId } = self2.opts;
|
|
6684
6684
|
const sch = new compile_1.SchemaEnv({ schema, schemaId, root, baseId, localRefs, meta });
|
|
6685
|
-
compile_1.compileSchema.call(
|
|
6685
|
+
compile_1.compileSchema.call(self2, sch);
|
|
6686
6686
|
return (0, ref_1.getValidate)(cxt, sch);
|
|
6687
6687
|
}
|
|
6688
6688
|
__name(_getValidate, "_getValidate");
|
|
@@ -6999,7 +6999,7 @@ var require_format = __commonJS({
|
|
|
6999
6999
|
error,
|
|
7000
7000
|
code(cxt, ruleType) {
|
|
7001
7001
|
const { gen, data, $data, schema, schemaCode, it } = cxt;
|
|
7002
|
-
const { opts, errSchemaPath, schemaEnv, self } = it;
|
|
7002
|
+
const { opts, errSchemaPath, schemaEnv, self: self2 } = it;
|
|
7003
7003
|
if (!opts.validateFormats)
|
|
7004
7004
|
return;
|
|
7005
7005
|
if ($data)
|
|
@@ -7008,7 +7008,7 @@ var require_format = __commonJS({
|
|
|
7008
7008
|
validateFormat();
|
|
7009
7009
|
function validate$DataFormat() {
|
|
7010
7010
|
const fmts = gen.scopeValue("formats", {
|
|
7011
|
-
ref:
|
|
7011
|
+
ref: self2.formats,
|
|
7012
7012
|
code: opts.code.formats
|
|
7013
7013
|
});
|
|
7014
7014
|
const fDef = gen.const("fDef", (0, codegen_1._)`${fmts}[${schemaCode}]`);
|
|
@@ -7031,7 +7031,7 @@ var require_format = __commonJS({
|
|
|
7031
7031
|
}
|
|
7032
7032
|
__name(validate$DataFormat, "validate$DataFormat");
|
|
7033
7033
|
function validateFormat() {
|
|
7034
|
-
const formatDef =
|
|
7034
|
+
const formatDef = self2.formats[schema];
|
|
7035
7035
|
if (!formatDef) {
|
|
7036
7036
|
unknownFormat();
|
|
7037
7037
|
return;
|
|
@@ -7043,7 +7043,7 @@ var require_format = __commonJS({
|
|
|
7043
7043
|
cxt.pass(validCondition());
|
|
7044
7044
|
function unknownFormat() {
|
|
7045
7045
|
if (opts.strictSchema === false) {
|
|
7046
|
-
|
|
7046
|
+
self2.logger.warn(unknownMsg());
|
|
7047
7047
|
return;
|
|
7048
7048
|
}
|
|
7049
7049
|
throw new Error(unknownMsg());
|
|
@@ -8196,17 +8196,17 @@ var require_limit = __commonJS({
|
|
|
8196
8196
|
error,
|
|
8197
8197
|
code(cxt) {
|
|
8198
8198
|
const { gen, data, schemaCode, keyword, it } = cxt;
|
|
8199
|
-
const { opts, self } = it;
|
|
8199
|
+
const { opts, self: self2 } = it;
|
|
8200
8200
|
if (!opts.validateFormats)
|
|
8201
8201
|
return;
|
|
8202
|
-
const fCxt = new ajv_1.KeywordCxt(it,
|
|
8202
|
+
const fCxt = new ajv_1.KeywordCxt(it, self2.RULES.all.format.definition, "format");
|
|
8203
8203
|
if (fCxt.$data)
|
|
8204
8204
|
validate$DataFormat();
|
|
8205
8205
|
else
|
|
8206
8206
|
validateFormat();
|
|
8207
8207
|
function validate$DataFormat() {
|
|
8208
8208
|
const fmts = gen.scopeValue("formats", {
|
|
8209
|
-
ref:
|
|
8209
|
+
ref: self2.formats,
|
|
8210
8210
|
code: opts.code.formats
|
|
8211
8211
|
});
|
|
8212
8212
|
const fmt = gen.const("fmt", codegen_1._`${fmts}[${fCxt.schemaCode}]`);
|
|
@@ -8215,7 +8215,7 @@ var require_limit = __commonJS({
|
|
|
8215
8215
|
__name(validate$DataFormat, "validate$DataFormat");
|
|
8216
8216
|
function validateFormat() {
|
|
8217
8217
|
const format = fCxt.schema;
|
|
8218
|
-
const fmtDef =
|
|
8218
|
+
const fmtDef = self2.formats[format];
|
|
8219
8219
|
if (!fmtDef || fmtDef === true)
|
|
8220
8220
|
return;
|
|
8221
8221
|
if (typeof fmtDef != "object" || fmtDef instanceof RegExp || typeof fmtDef.compare != "function") {
|
|
@@ -8614,7 +8614,12 @@ var require_schema_utilities = __commonJS({
|
|
|
8614
8614
|
__name(apiComposeTaxonomiesTransformation, "apiComposeTaxonomiesTransformation");
|
|
8615
8615
|
function exampleImporter(filename, basePath = `${__dirname}/../../../definitions/examples/raw_examples_and_scripts/`, schemataSource = `${__dirname}/../../../input/collection`) {
|
|
8616
8616
|
let rawData = JSON.parse(fs.readFileSync(`${basePath}/${filename}`)).map((obj) => {
|
|
8617
|
-
let output = obj.entity;
|
|
8617
|
+
let output = obj.entity ? obj.entity : obj;
|
|
8618
|
+
if (!output) {
|
|
8619
|
+
console.log(obj);
|
|
8620
|
+
console.log(output);
|
|
8621
|
+
}
|
|
8622
|
+
;
|
|
8618
8623
|
if (output.attributes.relationships) {
|
|
8619
8624
|
output.relationships = output.attributes.relationships;
|
|
8620
8625
|
delete output.attributes.relationships;
|
|
@@ -8631,7 +8636,7 @@ var require_schema_utilities = __commonJS({
|
|
|
8631
8636
|
});
|
|
8632
8637
|
let newEntities = [];
|
|
8633
8638
|
let entities = rawData.flatMap((entity) => {
|
|
8634
|
-
let transformedData = apiComposeTaxonomiesTransformation(entity);
|
|
8639
|
+
let transformedData = entity.relationships ? apiComposeTaxonomiesTransformation(entity) : { mainEntity: entity, newEntities: [] };
|
|
8635
8640
|
return [transformedData.mainEntity, ...transformedData.newEntities];
|
|
8636
8641
|
});
|
|
8637
8642
|
return entities;
|
|
@@ -9052,6 +9057,20 @@ var require_schema_utilities = __commonJS({
|
|
|
9052
9057
|
return output;
|
|
9053
9058
|
}
|
|
9054
9059
|
__name(findAllNonObjectAttributesInTree, "findAllNonObjectAttributesInTree");
|
|
9060
|
+
function gitlabConventionURLTemplate({ conventionName, projectID }) {
|
|
9061
|
+
return `https://gitlab.com/api/v4/projects/${projectID}/jobs/artifacts/main/raw/output/collection/conventions/${conventionName}/schema.json?job=copy_schemas`;
|
|
9062
|
+
}
|
|
9063
|
+
__name(gitlabConventionURLTemplate, "gitlabConventionURLTemplate");
|
|
9064
|
+
function gitlabOverlayURLTemplate({ overlayName, projectID, baseEntityTypeAndbundle = "" }) {
|
|
9065
|
+
return `https://gitlab.com/api/v4/projects/${projectID}/jobs/artifacts/main/raw/input/collection/${overlayName}/schema.json?job=copy_schemas`;
|
|
9066
|
+
}
|
|
9067
|
+
__name(gitlabOverlayURLTemplate, "gitlabOverlayURLTemplate");
|
|
9068
|
+
function basicSchemaURLTemplate({ baseEntityTypeAndBundle, projectID }) {
|
|
9069
|
+
let type = baseEntityTypeAndBundle.split("--")[0];
|
|
9070
|
+
let bundle = baseEntityTypeAndBundle.split("--")[1];
|
|
9071
|
+
return `https://gitlab.com/api/v4/projects/${projectID}/jobs/artifacts/main/raw/input/collection/${type}/${bundle}/schema.json?job=copy_schemas`;
|
|
9072
|
+
}
|
|
9073
|
+
__name(basicSchemaURLTemplate, "basicSchemaURLTemplate");
|
|
9055
9074
|
exports.dig = dig;
|
|
9056
9075
|
exports.bury = bury;
|
|
9057
9076
|
exports.fixRelationshipDataField = fixRelationshipDataField;
|
|
@@ -9074,6 +9093,1347 @@ var require_schema_utilities = __commonJS({
|
|
|
9074
9093
|
exports.listAttributesAndSubAttributes = listAttributesAndSubAttributes;
|
|
9075
9094
|
exports.findAllObjectAttributesInTree = findAllObjectAttributesInTree;
|
|
9076
9095
|
exports.findAllNonObjectAttributesInTree = findAllNonObjectAttributesInTree;
|
|
9096
|
+
exports.gitlabConventionURLTemplate = gitlabConventionURLTemplate;
|
|
9097
|
+
exports.gitlabOverlayURLTemplate = gitlabOverlayURLTemplate;
|
|
9098
|
+
exports.basicSchemaURLTemplate = basicSchemaURLTemplate;
|
|
9099
|
+
}
|
|
9100
|
+
});
|
|
9101
|
+
|
|
9102
|
+
// node_modules/papaparse/papaparse.js
|
|
9103
|
+
var require_papaparse = __commonJS({
|
|
9104
|
+
"node_modules/papaparse/papaparse.js"(exports, module) {
|
|
9105
|
+
(function(root, factory) {
|
|
9106
|
+
if (typeof define === "function" && define.amd) {
|
|
9107
|
+
define([], factory);
|
|
9108
|
+
} else if (typeof module === "object" && typeof exports !== "undefined") {
|
|
9109
|
+
module.exports = factory();
|
|
9110
|
+
} else {
|
|
9111
|
+
root.Papa = factory();
|
|
9112
|
+
}
|
|
9113
|
+
})(exports, /* @__PURE__ */ __name(function moduleFactory() {
|
|
9114
|
+
"use strict";
|
|
9115
|
+
var global = function() {
|
|
9116
|
+
if (typeof self !== "undefined") {
|
|
9117
|
+
return self;
|
|
9118
|
+
}
|
|
9119
|
+
if (typeof window !== "undefined") {
|
|
9120
|
+
return window;
|
|
9121
|
+
}
|
|
9122
|
+
if (typeof global !== "undefined") {
|
|
9123
|
+
return global;
|
|
9124
|
+
}
|
|
9125
|
+
return {};
|
|
9126
|
+
}();
|
|
9127
|
+
function getWorkerBlob() {
|
|
9128
|
+
var URL = global.URL || global.webkitURL || null;
|
|
9129
|
+
var code = moduleFactory.toString();
|
|
9130
|
+
return Papa.BLOB_URL || (Papa.BLOB_URL = URL.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ", "(", code, ")();"], { type: "text/javascript" })));
|
|
9131
|
+
}
|
|
9132
|
+
__name(getWorkerBlob, "getWorkerBlob");
|
|
9133
|
+
var IS_WORKER = !global.document && !!global.postMessage, IS_PAPA_WORKER = global.IS_PAPA_WORKER || false;
|
|
9134
|
+
var workers = {}, workerIdCounter = 0;
|
|
9135
|
+
var Papa = {};
|
|
9136
|
+
Papa.parse = CsvToJson;
|
|
9137
|
+
Papa.unparse = JsonToCsv;
|
|
9138
|
+
Papa.RECORD_SEP = String.fromCharCode(30);
|
|
9139
|
+
Papa.UNIT_SEP = String.fromCharCode(31);
|
|
9140
|
+
Papa.BYTE_ORDER_MARK = "\uFEFF";
|
|
9141
|
+
Papa.BAD_DELIMITERS = ["\r", "\n", '"', Papa.BYTE_ORDER_MARK];
|
|
9142
|
+
Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker;
|
|
9143
|
+
Papa.NODE_STREAM_INPUT = 1;
|
|
9144
|
+
Papa.LocalChunkSize = 1024 * 1024 * 10;
|
|
9145
|
+
Papa.RemoteChunkSize = 1024 * 1024 * 5;
|
|
9146
|
+
Papa.DefaultDelimiter = ",";
|
|
9147
|
+
Papa.Parser = Parser;
|
|
9148
|
+
Papa.ParserHandle = ParserHandle;
|
|
9149
|
+
Papa.NetworkStreamer = NetworkStreamer;
|
|
9150
|
+
Papa.FileStreamer = FileStreamer;
|
|
9151
|
+
Papa.StringStreamer = StringStreamer;
|
|
9152
|
+
Papa.ReadableStreamStreamer = ReadableStreamStreamer;
|
|
9153
|
+
if (typeof PAPA_BROWSER_CONTEXT === "undefined") {
|
|
9154
|
+
Papa.DuplexStreamStreamer = DuplexStreamStreamer;
|
|
9155
|
+
}
|
|
9156
|
+
if (global.jQuery) {
|
|
9157
|
+
var $ = global.jQuery;
|
|
9158
|
+
$.fn.parse = function(options) {
|
|
9159
|
+
var config = options.config || {};
|
|
9160
|
+
var queue = [];
|
|
9161
|
+
this.each(function(idx) {
|
|
9162
|
+
var supported = $(this).prop("tagName").toUpperCase() === "INPUT" && $(this).attr("type").toLowerCase() === "file" && global.FileReader;
|
|
9163
|
+
if (!supported || !this.files || this.files.length === 0)
|
|
9164
|
+
return true;
|
|
9165
|
+
for (var i = 0; i < this.files.length; i++) {
|
|
9166
|
+
queue.push({
|
|
9167
|
+
file: this.files[i],
|
|
9168
|
+
inputElem: this,
|
|
9169
|
+
instanceConfig: $.extend({}, config)
|
|
9170
|
+
});
|
|
9171
|
+
}
|
|
9172
|
+
});
|
|
9173
|
+
parseNextFile();
|
|
9174
|
+
return this;
|
|
9175
|
+
function parseNextFile() {
|
|
9176
|
+
if (queue.length === 0) {
|
|
9177
|
+
if (isFunction(options.complete))
|
|
9178
|
+
options.complete();
|
|
9179
|
+
return;
|
|
9180
|
+
}
|
|
9181
|
+
var f = queue[0];
|
|
9182
|
+
if (isFunction(options.before)) {
|
|
9183
|
+
var returned = options.before(f.file, f.inputElem);
|
|
9184
|
+
if (typeof returned === "object") {
|
|
9185
|
+
if (returned.action === "abort") {
|
|
9186
|
+
error("AbortError", f.file, f.inputElem, returned.reason);
|
|
9187
|
+
return;
|
|
9188
|
+
} else if (returned.action === "skip") {
|
|
9189
|
+
fileComplete();
|
|
9190
|
+
return;
|
|
9191
|
+
} else if (typeof returned.config === "object")
|
|
9192
|
+
f.instanceConfig = $.extend(f.instanceConfig, returned.config);
|
|
9193
|
+
} else if (returned === "skip") {
|
|
9194
|
+
fileComplete();
|
|
9195
|
+
return;
|
|
9196
|
+
}
|
|
9197
|
+
}
|
|
9198
|
+
var userCompleteFunc = f.instanceConfig.complete;
|
|
9199
|
+
f.instanceConfig.complete = function(results) {
|
|
9200
|
+
if (isFunction(userCompleteFunc))
|
|
9201
|
+
userCompleteFunc(results, f.file, f.inputElem);
|
|
9202
|
+
fileComplete();
|
|
9203
|
+
};
|
|
9204
|
+
Papa.parse(f.file, f.instanceConfig);
|
|
9205
|
+
}
|
|
9206
|
+
__name(parseNextFile, "parseNextFile");
|
|
9207
|
+
function error(name, file, elem, reason) {
|
|
9208
|
+
if (isFunction(options.error))
|
|
9209
|
+
options.error({ name }, file, elem, reason);
|
|
9210
|
+
}
|
|
9211
|
+
__name(error, "error");
|
|
9212
|
+
function fileComplete() {
|
|
9213
|
+
queue.splice(0, 1);
|
|
9214
|
+
parseNextFile();
|
|
9215
|
+
}
|
|
9216
|
+
__name(fileComplete, "fileComplete");
|
|
9217
|
+
};
|
|
9218
|
+
}
|
|
9219
|
+
if (IS_PAPA_WORKER) {
|
|
9220
|
+
global.onmessage = workerThreadReceivedMessage;
|
|
9221
|
+
}
|
|
9222
|
+
function CsvToJson(_input, _config) {
|
|
9223
|
+
_config = _config || {};
|
|
9224
|
+
var dynamicTyping = _config.dynamicTyping || false;
|
|
9225
|
+
if (isFunction(dynamicTyping)) {
|
|
9226
|
+
_config.dynamicTypingFunction = dynamicTyping;
|
|
9227
|
+
dynamicTyping = {};
|
|
9228
|
+
}
|
|
9229
|
+
_config.dynamicTyping = dynamicTyping;
|
|
9230
|
+
_config.transform = isFunction(_config.transform) ? _config.transform : false;
|
|
9231
|
+
if (_config.worker && Papa.WORKERS_SUPPORTED) {
|
|
9232
|
+
var w = newWorker();
|
|
9233
|
+
w.userStep = _config.step;
|
|
9234
|
+
w.userChunk = _config.chunk;
|
|
9235
|
+
w.userComplete = _config.complete;
|
|
9236
|
+
w.userError = _config.error;
|
|
9237
|
+
_config.step = isFunction(_config.step);
|
|
9238
|
+
_config.chunk = isFunction(_config.chunk);
|
|
9239
|
+
_config.complete = isFunction(_config.complete);
|
|
9240
|
+
_config.error = isFunction(_config.error);
|
|
9241
|
+
delete _config.worker;
|
|
9242
|
+
w.postMessage({
|
|
9243
|
+
input: _input,
|
|
9244
|
+
config: _config,
|
|
9245
|
+
workerId: w.id
|
|
9246
|
+
});
|
|
9247
|
+
return;
|
|
9248
|
+
}
|
|
9249
|
+
var streamer = null;
|
|
9250
|
+
if (_input === Papa.NODE_STREAM_INPUT && typeof PAPA_BROWSER_CONTEXT === "undefined") {
|
|
9251
|
+
streamer = new DuplexStreamStreamer(_config);
|
|
9252
|
+
return streamer.getStream();
|
|
9253
|
+
} else if (typeof _input === "string") {
|
|
9254
|
+
_input = stripBom(_input);
|
|
9255
|
+
if (_config.download)
|
|
9256
|
+
streamer = new NetworkStreamer(_config);
|
|
9257
|
+
else
|
|
9258
|
+
streamer = new StringStreamer(_config);
|
|
9259
|
+
} else if (_input.readable === true && isFunction(_input.read) && isFunction(_input.on)) {
|
|
9260
|
+
streamer = new ReadableStreamStreamer(_config);
|
|
9261
|
+
} else if (global.File && _input instanceof File || _input instanceof Object)
|
|
9262
|
+
streamer = new FileStreamer(_config);
|
|
9263
|
+
return streamer.stream(_input);
|
|
9264
|
+
function stripBom(string) {
|
|
9265
|
+
if (string.charCodeAt(0) === 65279) {
|
|
9266
|
+
return string.slice(1);
|
|
9267
|
+
}
|
|
9268
|
+
return string;
|
|
9269
|
+
}
|
|
9270
|
+
__name(stripBom, "stripBom");
|
|
9271
|
+
}
|
|
9272
|
+
__name(CsvToJson, "CsvToJson");
|
|
9273
|
+
function JsonToCsv(_input, _config) {
|
|
9274
|
+
var _quotes = false;
|
|
9275
|
+
var _writeHeader = true;
|
|
9276
|
+
var _delimiter = ",";
|
|
9277
|
+
var _newline = "\r\n";
|
|
9278
|
+
var _quoteChar = '"';
|
|
9279
|
+
var _escapedQuote = _quoteChar + _quoteChar;
|
|
9280
|
+
var _skipEmptyLines = false;
|
|
9281
|
+
var _columns = null;
|
|
9282
|
+
var _escapeFormulae = false;
|
|
9283
|
+
unpackConfig();
|
|
9284
|
+
var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), "g");
|
|
9285
|
+
if (typeof _input === "string")
|
|
9286
|
+
_input = JSON.parse(_input);
|
|
9287
|
+
if (Array.isArray(_input)) {
|
|
9288
|
+
if (!_input.length || Array.isArray(_input[0]))
|
|
9289
|
+
return serialize(null, _input, _skipEmptyLines);
|
|
9290
|
+
else if (typeof _input[0] === "object")
|
|
9291
|
+
return serialize(_columns || Object.keys(_input[0]), _input, _skipEmptyLines);
|
|
9292
|
+
} else if (typeof _input === "object") {
|
|
9293
|
+
if (typeof _input.data === "string")
|
|
9294
|
+
_input.data = JSON.parse(_input.data);
|
|
9295
|
+
if (Array.isArray(_input.data)) {
|
|
9296
|
+
if (!_input.fields)
|
|
9297
|
+
_input.fields = _input.meta && _input.meta.fields || _columns;
|
|
9298
|
+
if (!_input.fields)
|
|
9299
|
+
_input.fields = Array.isArray(_input.data[0]) ? _input.fields : typeof _input.data[0] === "object" ? Object.keys(_input.data[0]) : [];
|
|
9300
|
+
if (!Array.isArray(_input.data[0]) && typeof _input.data[0] !== "object")
|
|
9301
|
+
_input.data = [_input.data];
|
|
9302
|
+
}
|
|
9303
|
+
return serialize(_input.fields || [], _input.data || [], _skipEmptyLines);
|
|
9304
|
+
}
|
|
9305
|
+
throw new Error("Unable to serialize unrecognized input");
|
|
9306
|
+
function unpackConfig() {
|
|
9307
|
+
if (typeof _config !== "object")
|
|
9308
|
+
return;
|
|
9309
|
+
if (typeof _config.delimiter === "string" && !Papa.BAD_DELIMITERS.filter(function(value) {
|
|
9310
|
+
return _config.delimiter.indexOf(value) !== -1;
|
|
9311
|
+
}).length) {
|
|
9312
|
+
_delimiter = _config.delimiter;
|
|
9313
|
+
}
|
|
9314
|
+
if (typeof _config.quotes === "boolean" || typeof _config.quotes === "function" || Array.isArray(_config.quotes))
|
|
9315
|
+
_quotes = _config.quotes;
|
|
9316
|
+
if (typeof _config.skipEmptyLines === "boolean" || typeof _config.skipEmptyLines === "string")
|
|
9317
|
+
_skipEmptyLines = _config.skipEmptyLines;
|
|
9318
|
+
if (typeof _config.newline === "string")
|
|
9319
|
+
_newline = _config.newline;
|
|
9320
|
+
if (typeof _config.quoteChar === "string")
|
|
9321
|
+
_quoteChar = _config.quoteChar;
|
|
9322
|
+
if (typeof _config.header === "boolean")
|
|
9323
|
+
_writeHeader = _config.header;
|
|
9324
|
+
if (Array.isArray(_config.columns)) {
|
|
9325
|
+
if (_config.columns.length === 0)
|
|
9326
|
+
throw new Error("Option columns is empty");
|
|
9327
|
+
_columns = _config.columns;
|
|
9328
|
+
}
|
|
9329
|
+
if (_config.escapeChar !== void 0) {
|
|
9330
|
+
_escapedQuote = _config.escapeChar + _quoteChar;
|
|
9331
|
+
}
|
|
9332
|
+
if (typeof _config.escapeFormulae === "boolean" || _config.escapeFormulae instanceof RegExp) {
|
|
9333
|
+
_escapeFormulae = _config.escapeFormulae instanceof RegExp ? _config.escapeFormulae : /^[=+\-@\t\r].*$/;
|
|
9334
|
+
}
|
|
9335
|
+
}
|
|
9336
|
+
__name(unpackConfig, "unpackConfig");
|
|
9337
|
+
function serialize(fields, data, skipEmptyLines) {
|
|
9338
|
+
var csv = "";
|
|
9339
|
+
if (typeof fields === "string")
|
|
9340
|
+
fields = JSON.parse(fields);
|
|
9341
|
+
if (typeof data === "string")
|
|
9342
|
+
data = JSON.parse(data);
|
|
9343
|
+
var hasHeader = Array.isArray(fields) && fields.length > 0;
|
|
9344
|
+
var dataKeyedByField = !Array.isArray(data[0]);
|
|
9345
|
+
if (hasHeader && _writeHeader) {
|
|
9346
|
+
for (var i = 0; i < fields.length; i++) {
|
|
9347
|
+
if (i > 0)
|
|
9348
|
+
csv += _delimiter;
|
|
9349
|
+
csv += safe(fields[i], i);
|
|
9350
|
+
}
|
|
9351
|
+
if (data.length > 0)
|
|
9352
|
+
csv += _newline;
|
|
9353
|
+
}
|
|
9354
|
+
for (var row = 0; row < data.length; row++) {
|
|
9355
|
+
var maxCol = hasHeader ? fields.length : data[row].length;
|
|
9356
|
+
var emptyLine = false;
|
|
9357
|
+
var nullLine = hasHeader ? Object.keys(data[row]).length === 0 : data[row].length === 0;
|
|
9358
|
+
if (skipEmptyLines && !hasHeader) {
|
|
9359
|
+
emptyLine = skipEmptyLines === "greedy" ? data[row].join("").trim() === "" : data[row].length === 1 && data[row][0].length === 0;
|
|
9360
|
+
}
|
|
9361
|
+
if (skipEmptyLines === "greedy" && hasHeader) {
|
|
9362
|
+
var line = [];
|
|
9363
|
+
for (var c = 0; c < maxCol; c++) {
|
|
9364
|
+
var cx = dataKeyedByField ? fields[c] : c;
|
|
9365
|
+
line.push(data[row][cx]);
|
|
9366
|
+
}
|
|
9367
|
+
emptyLine = line.join("").trim() === "";
|
|
9368
|
+
}
|
|
9369
|
+
if (!emptyLine) {
|
|
9370
|
+
for (var col = 0; col < maxCol; col++) {
|
|
9371
|
+
if (col > 0 && !nullLine)
|
|
9372
|
+
csv += _delimiter;
|
|
9373
|
+
var colIdx = hasHeader && dataKeyedByField ? fields[col] : col;
|
|
9374
|
+
csv += safe(data[row][colIdx], col);
|
|
9375
|
+
}
|
|
9376
|
+
if (row < data.length - 1 && (!skipEmptyLines || maxCol > 0 && !nullLine)) {
|
|
9377
|
+
csv += _newline;
|
|
9378
|
+
}
|
|
9379
|
+
}
|
|
9380
|
+
}
|
|
9381
|
+
return csv;
|
|
9382
|
+
}
|
|
9383
|
+
__name(serialize, "serialize");
|
|
9384
|
+
function safe(str, col) {
|
|
9385
|
+
if (typeof str === "undefined" || str === null)
|
|
9386
|
+
return "";
|
|
9387
|
+
if (str.constructor === Date)
|
|
9388
|
+
return JSON.stringify(str).slice(1, 25);
|
|
9389
|
+
var needsQuotes = false;
|
|
9390
|
+
if (_escapeFormulae && typeof str === "string" && _escapeFormulae.test(str)) {
|
|
9391
|
+
str = "'" + str;
|
|
9392
|
+
needsQuotes = true;
|
|
9393
|
+
}
|
|
9394
|
+
var escapedQuoteStr = str.toString().replace(quoteCharRegex, _escapedQuote);
|
|
9395
|
+
needsQuotes = needsQuotes || _quotes === true || typeof _quotes === "function" && _quotes(str, col) || Array.isArray(_quotes) && _quotes[col] || hasAny(escapedQuoteStr, Papa.BAD_DELIMITERS) || escapedQuoteStr.indexOf(_delimiter) > -1 || escapedQuoteStr.charAt(0) === " " || escapedQuoteStr.charAt(escapedQuoteStr.length - 1) === " ";
|
|
9396
|
+
return needsQuotes ? _quoteChar + escapedQuoteStr + _quoteChar : escapedQuoteStr;
|
|
9397
|
+
}
|
|
9398
|
+
__name(safe, "safe");
|
|
9399
|
+
function hasAny(str, substrings) {
|
|
9400
|
+
for (var i = 0; i < substrings.length; i++)
|
|
9401
|
+
if (str.indexOf(substrings[i]) > -1)
|
|
9402
|
+
return true;
|
|
9403
|
+
return false;
|
|
9404
|
+
}
|
|
9405
|
+
__name(hasAny, "hasAny");
|
|
9406
|
+
}
|
|
9407
|
+
__name(JsonToCsv, "JsonToCsv");
|
|
9408
|
+
function ChunkStreamer(config) {
|
|
9409
|
+
this._handle = null;
|
|
9410
|
+
this._finished = false;
|
|
9411
|
+
this._completed = false;
|
|
9412
|
+
this._halted = false;
|
|
9413
|
+
this._input = null;
|
|
9414
|
+
this._baseIndex = 0;
|
|
9415
|
+
this._partialLine = "";
|
|
9416
|
+
this._rowCount = 0;
|
|
9417
|
+
this._start = 0;
|
|
9418
|
+
this._nextChunk = null;
|
|
9419
|
+
this.isFirstChunk = true;
|
|
9420
|
+
this._completeResults = {
|
|
9421
|
+
data: [],
|
|
9422
|
+
errors: [],
|
|
9423
|
+
meta: {}
|
|
9424
|
+
};
|
|
9425
|
+
replaceConfig.call(this, config);
|
|
9426
|
+
this.parseChunk = function(chunk, isFakeChunk) {
|
|
9427
|
+
if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) {
|
|
9428
|
+
var modifiedChunk = this._config.beforeFirstChunk(chunk);
|
|
9429
|
+
if (modifiedChunk !== void 0)
|
|
9430
|
+
chunk = modifiedChunk;
|
|
9431
|
+
}
|
|
9432
|
+
this.isFirstChunk = false;
|
|
9433
|
+
this._halted = false;
|
|
9434
|
+
var aggregate = this._partialLine + chunk;
|
|
9435
|
+
this._partialLine = "";
|
|
9436
|
+
var results = this._handle.parse(aggregate, this._baseIndex, !this._finished);
|
|
9437
|
+
if (this._handle.paused() || this._handle.aborted()) {
|
|
9438
|
+
this._halted = true;
|
|
9439
|
+
return;
|
|
9440
|
+
}
|
|
9441
|
+
var lastIndex = results.meta.cursor;
|
|
9442
|
+
if (!this._finished) {
|
|
9443
|
+
this._partialLine = aggregate.substring(lastIndex - this._baseIndex);
|
|
9444
|
+
this._baseIndex = lastIndex;
|
|
9445
|
+
}
|
|
9446
|
+
if (results && results.data)
|
|
9447
|
+
this._rowCount += results.data.length;
|
|
9448
|
+
var finishedIncludingPreview = this._finished || this._config.preview && this._rowCount >= this._config.preview;
|
|
9449
|
+
if (IS_PAPA_WORKER) {
|
|
9450
|
+
global.postMessage({
|
|
9451
|
+
results,
|
|
9452
|
+
workerId: Papa.WORKER_ID,
|
|
9453
|
+
finished: finishedIncludingPreview
|
|
9454
|
+
});
|
|
9455
|
+
} else if (isFunction(this._config.chunk) && !isFakeChunk) {
|
|
9456
|
+
this._config.chunk(results, this._handle);
|
|
9457
|
+
if (this._handle.paused() || this._handle.aborted()) {
|
|
9458
|
+
this._halted = true;
|
|
9459
|
+
return;
|
|
9460
|
+
}
|
|
9461
|
+
results = void 0;
|
|
9462
|
+
this._completeResults = void 0;
|
|
9463
|
+
}
|
|
9464
|
+
if (!this._config.step && !this._config.chunk) {
|
|
9465
|
+
this._completeResults.data = this._completeResults.data.concat(results.data);
|
|
9466
|
+
this._completeResults.errors = this._completeResults.errors.concat(results.errors);
|
|
9467
|
+
this._completeResults.meta = results.meta;
|
|
9468
|
+
}
|
|
9469
|
+
if (!this._completed && finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) {
|
|
9470
|
+
this._config.complete(this._completeResults, this._input);
|
|
9471
|
+
this._completed = true;
|
|
9472
|
+
}
|
|
9473
|
+
if (!finishedIncludingPreview && (!results || !results.meta.paused))
|
|
9474
|
+
this._nextChunk();
|
|
9475
|
+
return results;
|
|
9476
|
+
};
|
|
9477
|
+
this._sendError = function(error) {
|
|
9478
|
+
if (isFunction(this._config.error))
|
|
9479
|
+
this._config.error(error);
|
|
9480
|
+
else if (IS_PAPA_WORKER && this._config.error) {
|
|
9481
|
+
global.postMessage({
|
|
9482
|
+
workerId: Papa.WORKER_ID,
|
|
9483
|
+
error,
|
|
9484
|
+
finished: false
|
|
9485
|
+
});
|
|
9486
|
+
}
|
|
9487
|
+
};
|
|
9488
|
+
function replaceConfig(config2) {
|
|
9489
|
+
var configCopy = copy(config2);
|
|
9490
|
+
configCopy.chunkSize = parseInt(configCopy.chunkSize);
|
|
9491
|
+
if (!config2.step && !config2.chunk)
|
|
9492
|
+
configCopy.chunkSize = null;
|
|
9493
|
+
this._handle = new ParserHandle(configCopy);
|
|
9494
|
+
this._handle.streamer = this;
|
|
9495
|
+
this._config = configCopy;
|
|
9496
|
+
}
|
|
9497
|
+
__name(replaceConfig, "replaceConfig");
|
|
9498
|
+
}
|
|
9499
|
+
__name(ChunkStreamer, "ChunkStreamer");
|
|
9500
|
+
function NetworkStreamer(config) {
|
|
9501
|
+
config = config || {};
|
|
9502
|
+
if (!config.chunkSize)
|
|
9503
|
+
config.chunkSize = Papa.RemoteChunkSize;
|
|
9504
|
+
ChunkStreamer.call(this, config);
|
|
9505
|
+
var xhr;
|
|
9506
|
+
if (IS_WORKER) {
|
|
9507
|
+
this._nextChunk = function() {
|
|
9508
|
+
this._readChunk();
|
|
9509
|
+
this._chunkLoaded();
|
|
9510
|
+
};
|
|
9511
|
+
} else {
|
|
9512
|
+
this._nextChunk = function() {
|
|
9513
|
+
this._readChunk();
|
|
9514
|
+
};
|
|
9515
|
+
}
|
|
9516
|
+
this.stream = function(url) {
|
|
9517
|
+
this._input = url;
|
|
9518
|
+
this._nextChunk();
|
|
9519
|
+
};
|
|
9520
|
+
this._readChunk = function() {
|
|
9521
|
+
if (this._finished) {
|
|
9522
|
+
this._chunkLoaded();
|
|
9523
|
+
return;
|
|
9524
|
+
}
|
|
9525
|
+
xhr = new XMLHttpRequest();
|
|
9526
|
+
if (this._config.withCredentials) {
|
|
9527
|
+
xhr.withCredentials = this._config.withCredentials;
|
|
9528
|
+
}
|
|
9529
|
+
if (!IS_WORKER) {
|
|
9530
|
+
xhr.onload = bindFunction(this._chunkLoaded, this);
|
|
9531
|
+
xhr.onerror = bindFunction(this._chunkError, this);
|
|
9532
|
+
}
|
|
9533
|
+
xhr.open(this._config.downloadRequestBody ? "POST" : "GET", this._input, !IS_WORKER);
|
|
9534
|
+
if (this._config.downloadRequestHeaders) {
|
|
9535
|
+
var headers = this._config.downloadRequestHeaders;
|
|
9536
|
+
for (var headerName in headers) {
|
|
9537
|
+
xhr.setRequestHeader(headerName, headers[headerName]);
|
|
9538
|
+
}
|
|
9539
|
+
}
|
|
9540
|
+
if (this._config.chunkSize) {
|
|
9541
|
+
var end = this._start + this._config.chunkSize - 1;
|
|
9542
|
+
xhr.setRequestHeader("Range", "bytes=" + this._start + "-" + end);
|
|
9543
|
+
}
|
|
9544
|
+
try {
|
|
9545
|
+
xhr.send(this._config.downloadRequestBody);
|
|
9546
|
+
} catch (err) {
|
|
9547
|
+
this._chunkError(err.message);
|
|
9548
|
+
}
|
|
9549
|
+
if (IS_WORKER && xhr.status === 0)
|
|
9550
|
+
this._chunkError();
|
|
9551
|
+
};
|
|
9552
|
+
this._chunkLoaded = function() {
|
|
9553
|
+
if (xhr.readyState !== 4)
|
|
9554
|
+
return;
|
|
9555
|
+
if (xhr.status < 200 || xhr.status >= 400) {
|
|
9556
|
+
this._chunkError();
|
|
9557
|
+
return;
|
|
9558
|
+
}
|
|
9559
|
+
this._start += this._config.chunkSize ? this._config.chunkSize : xhr.responseText.length;
|
|
9560
|
+
this._finished = !this._config.chunkSize || this._start >= getFileSize(xhr);
|
|
9561
|
+
this.parseChunk(xhr.responseText);
|
|
9562
|
+
};
|
|
9563
|
+
this._chunkError = function(errorMessage) {
|
|
9564
|
+
var errorText = xhr.statusText || errorMessage;
|
|
9565
|
+
this._sendError(new Error(errorText));
|
|
9566
|
+
};
|
|
9567
|
+
function getFileSize(xhr2) {
|
|
9568
|
+
var contentRange = xhr2.getResponseHeader("Content-Range");
|
|
9569
|
+
if (contentRange === null) {
|
|
9570
|
+
return -1;
|
|
9571
|
+
}
|
|
9572
|
+
return parseInt(contentRange.substring(contentRange.lastIndexOf("/") + 1));
|
|
9573
|
+
}
|
|
9574
|
+
__name(getFileSize, "getFileSize");
|
|
9575
|
+
}
|
|
9576
|
+
__name(NetworkStreamer, "NetworkStreamer");
|
|
9577
|
+
NetworkStreamer.prototype = Object.create(ChunkStreamer.prototype);
|
|
9578
|
+
NetworkStreamer.prototype.constructor = NetworkStreamer;
|
|
9579
|
+
function FileStreamer(config) {
|
|
9580
|
+
config = config || {};
|
|
9581
|
+
if (!config.chunkSize)
|
|
9582
|
+
config.chunkSize = Papa.LocalChunkSize;
|
|
9583
|
+
ChunkStreamer.call(this, config);
|
|
9584
|
+
var reader, slice;
|
|
9585
|
+
var usingAsyncReader = typeof FileReader !== "undefined";
|
|
9586
|
+
this.stream = function(file) {
|
|
9587
|
+
this._input = file;
|
|
9588
|
+
slice = file.slice || file.webkitSlice || file.mozSlice;
|
|
9589
|
+
if (usingAsyncReader) {
|
|
9590
|
+
reader = new FileReader();
|
|
9591
|
+
reader.onload = bindFunction(this._chunkLoaded, this);
|
|
9592
|
+
reader.onerror = bindFunction(this._chunkError, this);
|
|
9593
|
+
} else
|
|
9594
|
+
reader = new FileReaderSync();
|
|
9595
|
+
this._nextChunk();
|
|
9596
|
+
};
|
|
9597
|
+
this._nextChunk = function() {
|
|
9598
|
+
if (!this._finished && (!this._config.preview || this._rowCount < this._config.preview))
|
|
9599
|
+
this._readChunk();
|
|
9600
|
+
};
|
|
9601
|
+
this._readChunk = function() {
|
|
9602
|
+
var input = this._input;
|
|
9603
|
+
if (this._config.chunkSize) {
|
|
9604
|
+
var end = Math.min(this._start + this._config.chunkSize, this._input.size);
|
|
9605
|
+
input = slice.call(input, this._start, end);
|
|
9606
|
+
}
|
|
9607
|
+
var txt = reader.readAsText(input, this._config.encoding);
|
|
9608
|
+
if (!usingAsyncReader)
|
|
9609
|
+
this._chunkLoaded({ target: { result: txt } });
|
|
9610
|
+
};
|
|
9611
|
+
this._chunkLoaded = function(event) {
|
|
9612
|
+
this._start += this._config.chunkSize;
|
|
9613
|
+
this._finished = !this._config.chunkSize || this._start >= this._input.size;
|
|
9614
|
+
this.parseChunk(event.target.result);
|
|
9615
|
+
};
|
|
9616
|
+
this._chunkError = function() {
|
|
9617
|
+
this._sendError(reader.error);
|
|
9618
|
+
};
|
|
9619
|
+
}
|
|
9620
|
+
__name(FileStreamer, "FileStreamer");
|
|
9621
|
+
FileStreamer.prototype = Object.create(ChunkStreamer.prototype);
|
|
9622
|
+
FileStreamer.prototype.constructor = FileStreamer;
|
|
9623
|
+
function StringStreamer(config) {
|
|
9624
|
+
config = config || {};
|
|
9625
|
+
ChunkStreamer.call(this, config);
|
|
9626
|
+
var remaining;
|
|
9627
|
+
this.stream = function(s) {
|
|
9628
|
+
remaining = s;
|
|
9629
|
+
return this._nextChunk();
|
|
9630
|
+
};
|
|
9631
|
+
this._nextChunk = function() {
|
|
9632
|
+
if (this._finished)
|
|
9633
|
+
return;
|
|
9634
|
+
var size = this._config.chunkSize;
|
|
9635
|
+
var chunk;
|
|
9636
|
+
if (size) {
|
|
9637
|
+
chunk = remaining.substring(0, size);
|
|
9638
|
+
remaining = remaining.substring(size);
|
|
9639
|
+
} else {
|
|
9640
|
+
chunk = remaining;
|
|
9641
|
+
remaining = "";
|
|
9642
|
+
}
|
|
9643
|
+
this._finished = !remaining;
|
|
9644
|
+
return this.parseChunk(chunk);
|
|
9645
|
+
};
|
|
9646
|
+
}
|
|
9647
|
+
__name(StringStreamer, "StringStreamer");
|
|
9648
|
+
StringStreamer.prototype = Object.create(StringStreamer.prototype);
|
|
9649
|
+
StringStreamer.prototype.constructor = StringStreamer;
|
|
9650
|
+
function ReadableStreamStreamer(config) {
|
|
9651
|
+
config = config || {};
|
|
9652
|
+
ChunkStreamer.call(this, config);
|
|
9653
|
+
var queue = [];
|
|
9654
|
+
var parseOnData = true;
|
|
9655
|
+
var streamHasEnded = false;
|
|
9656
|
+
this.pause = function() {
|
|
9657
|
+
ChunkStreamer.prototype.pause.apply(this, arguments);
|
|
9658
|
+
this._input.pause();
|
|
9659
|
+
};
|
|
9660
|
+
this.resume = function() {
|
|
9661
|
+
ChunkStreamer.prototype.resume.apply(this, arguments);
|
|
9662
|
+
this._input.resume();
|
|
9663
|
+
};
|
|
9664
|
+
this.stream = function(stream) {
|
|
9665
|
+
this._input = stream;
|
|
9666
|
+
this._input.on("data", this._streamData);
|
|
9667
|
+
this._input.on("end", this._streamEnd);
|
|
9668
|
+
this._input.on("error", this._streamError);
|
|
9669
|
+
};
|
|
9670
|
+
this._checkIsFinished = function() {
|
|
9671
|
+
if (streamHasEnded && queue.length === 1) {
|
|
9672
|
+
this._finished = true;
|
|
9673
|
+
}
|
|
9674
|
+
};
|
|
9675
|
+
this._nextChunk = function() {
|
|
9676
|
+
this._checkIsFinished();
|
|
9677
|
+
if (queue.length) {
|
|
9678
|
+
this.parseChunk(queue.shift());
|
|
9679
|
+
} else {
|
|
9680
|
+
parseOnData = true;
|
|
9681
|
+
}
|
|
9682
|
+
};
|
|
9683
|
+
this._streamData = bindFunction(function(chunk) {
|
|
9684
|
+
try {
|
|
9685
|
+
queue.push(typeof chunk === "string" ? chunk : chunk.toString(this._config.encoding));
|
|
9686
|
+
if (parseOnData) {
|
|
9687
|
+
parseOnData = false;
|
|
9688
|
+
this._checkIsFinished();
|
|
9689
|
+
this.parseChunk(queue.shift());
|
|
9690
|
+
}
|
|
9691
|
+
} catch (error) {
|
|
9692
|
+
this._streamError(error);
|
|
9693
|
+
}
|
|
9694
|
+
}, this);
|
|
9695
|
+
this._streamError = bindFunction(function(error) {
|
|
9696
|
+
this._streamCleanUp();
|
|
9697
|
+
this._sendError(error);
|
|
9698
|
+
}, this);
|
|
9699
|
+
this._streamEnd = bindFunction(function() {
|
|
9700
|
+
this._streamCleanUp();
|
|
9701
|
+
streamHasEnded = true;
|
|
9702
|
+
this._streamData("");
|
|
9703
|
+
}, this);
|
|
9704
|
+
this._streamCleanUp = bindFunction(function() {
|
|
9705
|
+
this._input.removeListener("data", this._streamData);
|
|
9706
|
+
this._input.removeListener("end", this._streamEnd);
|
|
9707
|
+
this._input.removeListener("error", this._streamError);
|
|
9708
|
+
}, this);
|
|
9709
|
+
}
|
|
9710
|
+
__name(ReadableStreamStreamer, "ReadableStreamStreamer");
|
|
9711
|
+
ReadableStreamStreamer.prototype = Object.create(ChunkStreamer.prototype);
|
|
9712
|
+
ReadableStreamStreamer.prototype.constructor = ReadableStreamStreamer;
|
|
9713
|
+
function DuplexStreamStreamer(_config) {
|
|
9714
|
+
var Duplex = __require("stream").Duplex;
|
|
9715
|
+
var config = copy(_config);
|
|
9716
|
+
var parseOnWrite = true;
|
|
9717
|
+
var writeStreamHasFinished = false;
|
|
9718
|
+
var parseCallbackQueue = [];
|
|
9719
|
+
var stream = null;
|
|
9720
|
+
this._onCsvData = function(results) {
|
|
9721
|
+
var data = results.data;
|
|
9722
|
+
if (!stream.push(data) && !this._handle.paused()) {
|
|
9723
|
+
this._handle.pause();
|
|
9724
|
+
}
|
|
9725
|
+
};
|
|
9726
|
+
this._onCsvComplete = function() {
|
|
9727
|
+
stream.push(null);
|
|
9728
|
+
};
|
|
9729
|
+
config.step = bindFunction(this._onCsvData, this);
|
|
9730
|
+
config.complete = bindFunction(this._onCsvComplete, this);
|
|
9731
|
+
ChunkStreamer.call(this, config);
|
|
9732
|
+
this._nextChunk = function() {
|
|
9733
|
+
if (writeStreamHasFinished && parseCallbackQueue.length === 1) {
|
|
9734
|
+
this._finished = true;
|
|
9735
|
+
}
|
|
9736
|
+
if (parseCallbackQueue.length) {
|
|
9737
|
+
parseCallbackQueue.shift()();
|
|
9738
|
+
} else {
|
|
9739
|
+
parseOnWrite = true;
|
|
9740
|
+
}
|
|
9741
|
+
};
|
|
9742
|
+
this._addToParseQueue = function(chunk, callback) {
|
|
9743
|
+
parseCallbackQueue.push(bindFunction(function() {
|
|
9744
|
+
this.parseChunk(typeof chunk === "string" ? chunk : chunk.toString(config.encoding));
|
|
9745
|
+
if (isFunction(callback)) {
|
|
9746
|
+
return callback();
|
|
9747
|
+
}
|
|
9748
|
+
}, this));
|
|
9749
|
+
if (parseOnWrite) {
|
|
9750
|
+
parseOnWrite = false;
|
|
9751
|
+
this._nextChunk();
|
|
9752
|
+
}
|
|
9753
|
+
};
|
|
9754
|
+
this._onRead = function() {
|
|
9755
|
+
if (this._handle.paused()) {
|
|
9756
|
+
this._handle.resume();
|
|
9757
|
+
}
|
|
9758
|
+
};
|
|
9759
|
+
this._onWrite = function(chunk, encoding, callback) {
|
|
9760
|
+
this._addToParseQueue(chunk, callback);
|
|
9761
|
+
};
|
|
9762
|
+
this._onWriteComplete = function() {
|
|
9763
|
+
writeStreamHasFinished = true;
|
|
9764
|
+
this._addToParseQueue("");
|
|
9765
|
+
};
|
|
9766
|
+
this.getStream = function() {
|
|
9767
|
+
return stream;
|
|
9768
|
+
};
|
|
9769
|
+
stream = new Duplex({
|
|
9770
|
+
readableObjectMode: true,
|
|
9771
|
+
decodeStrings: false,
|
|
9772
|
+
read: bindFunction(this._onRead, this),
|
|
9773
|
+
write: bindFunction(this._onWrite, this)
|
|
9774
|
+
});
|
|
9775
|
+
stream.once("finish", bindFunction(this._onWriteComplete, this));
|
|
9776
|
+
}
|
|
9777
|
+
__name(DuplexStreamStreamer, "DuplexStreamStreamer");
|
|
9778
|
+
if (typeof PAPA_BROWSER_CONTEXT === "undefined") {
|
|
9779
|
+
DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype);
|
|
9780
|
+
DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer;
|
|
9781
|
+
}
|
|
9782
|
+
function ParserHandle(_config) {
|
|
9783
|
+
var MAX_FLOAT = Math.pow(2, 53);
|
|
9784
|
+
var MIN_FLOAT = -MAX_FLOAT;
|
|
9785
|
+
var FLOAT = /^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/;
|
|
9786
|
+
var ISO_DATE = /^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/;
|
|
9787
|
+
var self2 = this;
|
|
9788
|
+
var _stepCounter = 0;
|
|
9789
|
+
var _rowCounter = 0;
|
|
9790
|
+
var _input;
|
|
9791
|
+
var _parser;
|
|
9792
|
+
var _paused = false;
|
|
9793
|
+
var _aborted = false;
|
|
9794
|
+
var _delimiterError;
|
|
9795
|
+
var _fields = [];
|
|
9796
|
+
var _results = {
|
|
9797
|
+
// The last results returned from the parser
|
|
9798
|
+
data: [],
|
|
9799
|
+
errors: [],
|
|
9800
|
+
meta: {}
|
|
9801
|
+
};
|
|
9802
|
+
if (isFunction(_config.step)) {
|
|
9803
|
+
var userStep = _config.step;
|
|
9804
|
+
_config.step = function(results) {
|
|
9805
|
+
_results = results;
|
|
9806
|
+
if (needsHeaderRow())
|
|
9807
|
+
processResults();
|
|
9808
|
+
else {
|
|
9809
|
+
processResults();
|
|
9810
|
+
if (_results.data.length === 0)
|
|
9811
|
+
return;
|
|
9812
|
+
_stepCounter += results.data.length;
|
|
9813
|
+
if (_config.preview && _stepCounter > _config.preview)
|
|
9814
|
+
_parser.abort();
|
|
9815
|
+
else {
|
|
9816
|
+
_results.data = _results.data[0];
|
|
9817
|
+
userStep(_results, self2);
|
|
9818
|
+
}
|
|
9819
|
+
}
|
|
9820
|
+
};
|
|
9821
|
+
}
|
|
9822
|
+
this.parse = function(input, baseIndex, ignoreLastRow) {
|
|
9823
|
+
var quoteChar = _config.quoteChar || '"';
|
|
9824
|
+
if (!_config.newline)
|
|
9825
|
+
_config.newline = guessLineEndings(input, quoteChar);
|
|
9826
|
+
_delimiterError = false;
|
|
9827
|
+
if (!_config.delimiter) {
|
|
9828
|
+
var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments, _config.delimitersToGuess);
|
|
9829
|
+
if (delimGuess.successful)
|
|
9830
|
+
_config.delimiter = delimGuess.bestDelimiter;
|
|
9831
|
+
else {
|
|
9832
|
+
_delimiterError = true;
|
|
9833
|
+
_config.delimiter = Papa.DefaultDelimiter;
|
|
9834
|
+
}
|
|
9835
|
+
_results.meta.delimiter = _config.delimiter;
|
|
9836
|
+
} else if (isFunction(_config.delimiter)) {
|
|
9837
|
+
_config.delimiter = _config.delimiter(input);
|
|
9838
|
+
_results.meta.delimiter = _config.delimiter;
|
|
9839
|
+
}
|
|
9840
|
+
var parserConfig = copy(_config);
|
|
9841
|
+
if (_config.preview && _config.header)
|
|
9842
|
+
parserConfig.preview++;
|
|
9843
|
+
_input = input;
|
|
9844
|
+
_parser = new Parser(parserConfig);
|
|
9845
|
+
_results = _parser.parse(_input, baseIndex, ignoreLastRow);
|
|
9846
|
+
processResults();
|
|
9847
|
+
return _paused ? { meta: { paused: true } } : _results || { meta: { paused: false } };
|
|
9848
|
+
};
|
|
9849
|
+
this.paused = function() {
|
|
9850
|
+
return _paused;
|
|
9851
|
+
};
|
|
9852
|
+
this.pause = function() {
|
|
9853
|
+
_paused = true;
|
|
9854
|
+
_parser.abort();
|
|
9855
|
+
_input = isFunction(_config.chunk) ? "" : _input.substring(_parser.getCharIndex());
|
|
9856
|
+
};
|
|
9857
|
+
this.resume = function() {
|
|
9858
|
+
if (self2.streamer._halted) {
|
|
9859
|
+
_paused = false;
|
|
9860
|
+
self2.streamer.parseChunk(_input, true);
|
|
9861
|
+
} else {
|
|
9862
|
+
setTimeout(self2.resume, 3);
|
|
9863
|
+
}
|
|
9864
|
+
};
|
|
9865
|
+
this.aborted = function() {
|
|
9866
|
+
return _aborted;
|
|
9867
|
+
};
|
|
9868
|
+
this.abort = function() {
|
|
9869
|
+
_aborted = true;
|
|
9870
|
+
_parser.abort();
|
|
9871
|
+
_results.meta.aborted = true;
|
|
9872
|
+
if (isFunction(_config.complete))
|
|
9873
|
+
_config.complete(_results);
|
|
9874
|
+
_input = "";
|
|
9875
|
+
};
|
|
9876
|
+
function testEmptyLine(s) {
|
|
9877
|
+
return _config.skipEmptyLines === "greedy" ? s.join("").trim() === "" : s.length === 1 && s[0].length === 0;
|
|
9878
|
+
}
|
|
9879
|
+
__name(testEmptyLine, "testEmptyLine");
|
|
9880
|
+
function testFloat(s) {
|
|
9881
|
+
if (FLOAT.test(s)) {
|
|
9882
|
+
var floatValue = parseFloat(s);
|
|
9883
|
+
if (floatValue > MIN_FLOAT && floatValue < MAX_FLOAT) {
|
|
9884
|
+
return true;
|
|
9885
|
+
}
|
|
9886
|
+
}
|
|
9887
|
+
return false;
|
|
9888
|
+
}
|
|
9889
|
+
__name(testFloat, "testFloat");
|
|
9890
|
+
function processResults() {
|
|
9891
|
+
if (_results && _delimiterError) {
|
|
9892
|
+
addError("Delimiter", "UndetectableDelimiter", "Unable to auto-detect delimiting character; defaulted to '" + Papa.DefaultDelimiter + "'");
|
|
9893
|
+
_delimiterError = false;
|
|
9894
|
+
}
|
|
9895
|
+
if (_config.skipEmptyLines) {
|
|
9896
|
+
_results.data = _results.data.filter(function(d) {
|
|
9897
|
+
return !testEmptyLine(d);
|
|
9898
|
+
});
|
|
9899
|
+
}
|
|
9900
|
+
if (needsHeaderRow())
|
|
9901
|
+
fillHeaderFields();
|
|
9902
|
+
return applyHeaderAndDynamicTypingAndTransformation();
|
|
9903
|
+
}
|
|
9904
|
+
__name(processResults, "processResults");
|
|
9905
|
+
function needsHeaderRow() {
|
|
9906
|
+
return _config.header && _fields.length === 0;
|
|
9907
|
+
}
|
|
9908
|
+
__name(needsHeaderRow, "needsHeaderRow");
|
|
9909
|
+
function fillHeaderFields() {
|
|
9910
|
+
if (!_results)
|
|
9911
|
+
return;
|
|
9912
|
+
function addHeader(header, i2) {
|
|
9913
|
+
if (isFunction(_config.transformHeader))
|
|
9914
|
+
header = _config.transformHeader(header, i2);
|
|
9915
|
+
_fields.push(header);
|
|
9916
|
+
}
|
|
9917
|
+
__name(addHeader, "addHeader");
|
|
9918
|
+
if (Array.isArray(_results.data[0])) {
|
|
9919
|
+
for (var i = 0; needsHeaderRow() && i < _results.data.length; i++)
|
|
9920
|
+
_results.data[i].forEach(addHeader);
|
|
9921
|
+
_results.data.splice(0, 1);
|
|
9922
|
+
} else
|
|
9923
|
+
_results.data.forEach(addHeader);
|
|
9924
|
+
}
|
|
9925
|
+
__name(fillHeaderFields, "fillHeaderFields");
|
|
9926
|
+
function shouldApplyDynamicTyping(field) {
|
|
9927
|
+
if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === void 0) {
|
|
9928
|
+
_config.dynamicTyping[field] = _config.dynamicTypingFunction(field);
|
|
9929
|
+
}
|
|
9930
|
+
return (_config.dynamicTyping[field] || _config.dynamicTyping) === true;
|
|
9931
|
+
}
|
|
9932
|
+
__name(shouldApplyDynamicTyping, "shouldApplyDynamicTyping");
|
|
9933
|
+
function parseDynamic(field, value) {
|
|
9934
|
+
if (shouldApplyDynamicTyping(field)) {
|
|
9935
|
+
if (value === "true" || value === "TRUE")
|
|
9936
|
+
return true;
|
|
9937
|
+
else if (value === "false" || value === "FALSE")
|
|
9938
|
+
return false;
|
|
9939
|
+
else if (testFloat(value))
|
|
9940
|
+
return parseFloat(value);
|
|
9941
|
+
else if (ISO_DATE.test(value))
|
|
9942
|
+
return new Date(value);
|
|
9943
|
+
else
|
|
9944
|
+
return value === "" ? null : value;
|
|
9945
|
+
}
|
|
9946
|
+
return value;
|
|
9947
|
+
}
|
|
9948
|
+
__name(parseDynamic, "parseDynamic");
|
|
9949
|
+
function applyHeaderAndDynamicTypingAndTransformation() {
|
|
9950
|
+
if (!_results || !_config.header && !_config.dynamicTyping && !_config.transform)
|
|
9951
|
+
return _results;
|
|
9952
|
+
function processRow(rowSource, i) {
|
|
9953
|
+
var row = _config.header ? {} : [];
|
|
9954
|
+
var j;
|
|
9955
|
+
for (j = 0; j < rowSource.length; j++) {
|
|
9956
|
+
var field = j;
|
|
9957
|
+
var value = rowSource[j];
|
|
9958
|
+
if (_config.header)
|
|
9959
|
+
field = j >= _fields.length ? "__parsed_extra" : _fields[j];
|
|
9960
|
+
if (_config.transform)
|
|
9961
|
+
value = _config.transform(value, field);
|
|
9962
|
+
value = parseDynamic(field, value);
|
|
9963
|
+
if (field === "__parsed_extra") {
|
|
9964
|
+
row[field] = row[field] || [];
|
|
9965
|
+
row[field].push(value);
|
|
9966
|
+
} else
|
|
9967
|
+
row[field] = value;
|
|
9968
|
+
}
|
|
9969
|
+
if (_config.header) {
|
|
9970
|
+
if (j > _fields.length)
|
|
9971
|
+
addError("FieldMismatch", "TooManyFields", "Too many fields: expected " + _fields.length + " fields but parsed " + j, _rowCounter + i);
|
|
9972
|
+
else if (j < _fields.length)
|
|
9973
|
+
addError("FieldMismatch", "TooFewFields", "Too few fields: expected " + _fields.length + " fields but parsed " + j, _rowCounter + i);
|
|
9974
|
+
}
|
|
9975
|
+
return row;
|
|
9976
|
+
}
|
|
9977
|
+
__name(processRow, "processRow");
|
|
9978
|
+
var incrementBy = 1;
|
|
9979
|
+
if (!_results.data.length || Array.isArray(_results.data[0])) {
|
|
9980
|
+
_results.data = _results.data.map(processRow);
|
|
9981
|
+
incrementBy = _results.data.length;
|
|
9982
|
+
} else
|
|
9983
|
+
_results.data = processRow(_results.data, 0);
|
|
9984
|
+
if (_config.header && _results.meta)
|
|
9985
|
+
_results.meta.fields = _fields;
|
|
9986
|
+
_rowCounter += incrementBy;
|
|
9987
|
+
return _results;
|
|
9988
|
+
}
|
|
9989
|
+
__name(applyHeaderAndDynamicTypingAndTransformation, "applyHeaderAndDynamicTypingAndTransformation");
|
|
9990
|
+
function guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess) {
|
|
9991
|
+
var bestDelim, bestDelta, fieldCountPrevRow, maxFieldCount;
|
|
9992
|
+
delimitersToGuess = delimitersToGuess || [",", " ", "|", ";", Papa.RECORD_SEP, Papa.UNIT_SEP];
|
|
9993
|
+
for (var i = 0; i < delimitersToGuess.length; i++) {
|
|
9994
|
+
var delim = delimitersToGuess[i];
|
|
9995
|
+
var delta = 0, avgFieldCount = 0, emptyLinesCount = 0;
|
|
9996
|
+
fieldCountPrevRow = void 0;
|
|
9997
|
+
var preview = new Parser({
|
|
9998
|
+
comments,
|
|
9999
|
+
delimiter: delim,
|
|
10000
|
+
newline,
|
|
10001
|
+
preview: 10
|
|
10002
|
+
}).parse(input);
|
|
10003
|
+
for (var j = 0; j < preview.data.length; j++) {
|
|
10004
|
+
if (skipEmptyLines && testEmptyLine(preview.data[j])) {
|
|
10005
|
+
emptyLinesCount++;
|
|
10006
|
+
continue;
|
|
10007
|
+
}
|
|
10008
|
+
var fieldCount = preview.data[j].length;
|
|
10009
|
+
avgFieldCount += fieldCount;
|
|
10010
|
+
if (typeof fieldCountPrevRow === "undefined") {
|
|
10011
|
+
fieldCountPrevRow = fieldCount;
|
|
10012
|
+
continue;
|
|
10013
|
+
} else if (fieldCount > 0) {
|
|
10014
|
+
delta += Math.abs(fieldCount - fieldCountPrevRow);
|
|
10015
|
+
fieldCountPrevRow = fieldCount;
|
|
10016
|
+
}
|
|
10017
|
+
}
|
|
10018
|
+
if (preview.data.length > 0)
|
|
10019
|
+
avgFieldCount /= preview.data.length - emptyLinesCount;
|
|
10020
|
+
if ((typeof bestDelta === "undefined" || delta <= bestDelta) && (typeof maxFieldCount === "undefined" || avgFieldCount > maxFieldCount) && avgFieldCount > 1.99) {
|
|
10021
|
+
bestDelta = delta;
|
|
10022
|
+
bestDelim = delim;
|
|
10023
|
+
maxFieldCount = avgFieldCount;
|
|
10024
|
+
}
|
|
10025
|
+
}
|
|
10026
|
+
_config.delimiter = bestDelim;
|
|
10027
|
+
return {
|
|
10028
|
+
successful: !!bestDelim,
|
|
10029
|
+
bestDelimiter: bestDelim
|
|
10030
|
+
};
|
|
10031
|
+
}
|
|
10032
|
+
__name(guessDelimiter, "guessDelimiter");
|
|
10033
|
+
function guessLineEndings(input, quoteChar) {
|
|
10034
|
+
input = input.substring(0, 1024 * 1024);
|
|
10035
|
+
var re = new RegExp(escapeRegExp(quoteChar) + "([^]*?)" + escapeRegExp(quoteChar), "gm");
|
|
10036
|
+
input = input.replace(re, "");
|
|
10037
|
+
var r = input.split("\r");
|
|
10038
|
+
var n = input.split("\n");
|
|
10039
|
+
var nAppearsFirst = n.length > 1 && n[0].length < r[0].length;
|
|
10040
|
+
if (r.length === 1 || nAppearsFirst)
|
|
10041
|
+
return "\n";
|
|
10042
|
+
var numWithN = 0;
|
|
10043
|
+
for (var i = 0; i < r.length; i++) {
|
|
10044
|
+
if (r[i][0] === "\n")
|
|
10045
|
+
numWithN++;
|
|
10046
|
+
}
|
|
10047
|
+
return numWithN >= r.length / 2 ? "\r\n" : "\r";
|
|
10048
|
+
}
|
|
10049
|
+
__name(guessLineEndings, "guessLineEndings");
|
|
10050
|
+
function addError(type, code, msg, row) {
|
|
10051
|
+
var error = {
|
|
10052
|
+
type,
|
|
10053
|
+
code,
|
|
10054
|
+
message: msg
|
|
10055
|
+
};
|
|
10056
|
+
if (row !== void 0) {
|
|
10057
|
+
error.row = row;
|
|
10058
|
+
}
|
|
10059
|
+
_results.errors.push(error);
|
|
10060
|
+
}
|
|
10061
|
+
__name(addError, "addError");
|
|
10062
|
+
}
|
|
10063
|
+
__name(ParserHandle, "ParserHandle");
|
|
10064
|
+
function escapeRegExp(string) {
|
|
10065
|
+
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
10066
|
+
}
|
|
10067
|
+
__name(escapeRegExp, "escapeRegExp");
|
|
10068
|
+
function Parser(config) {
|
|
10069
|
+
config = config || {};
|
|
10070
|
+
var delim = config.delimiter;
|
|
10071
|
+
var newline = config.newline;
|
|
10072
|
+
var comments = config.comments;
|
|
10073
|
+
var step = config.step;
|
|
10074
|
+
var preview = config.preview;
|
|
10075
|
+
var fastMode = config.fastMode;
|
|
10076
|
+
var quoteChar;
|
|
10077
|
+
if (config.quoteChar === void 0 || config.quoteChar === null) {
|
|
10078
|
+
quoteChar = '"';
|
|
10079
|
+
} else {
|
|
10080
|
+
quoteChar = config.quoteChar;
|
|
10081
|
+
}
|
|
10082
|
+
var escapeChar = quoteChar;
|
|
10083
|
+
if (config.escapeChar !== void 0) {
|
|
10084
|
+
escapeChar = config.escapeChar;
|
|
10085
|
+
}
|
|
10086
|
+
if (typeof delim !== "string" || Papa.BAD_DELIMITERS.indexOf(delim) > -1)
|
|
10087
|
+
delim = ",";
|
|
10088
|
+
if (comments === delim)
|
|
10089
|
+
throw new Error("Comment character same as delimiter");
|
|
10090
|
+
else if (comments === true)
|
|
10091
|
+
comments = "#";
|
|
10092
|
+
else if (typeof comments !== "string" || Papa.BAD_DELIMITERS.indexOf(comments) > -1)
|
|
10093
|
+
comments = false;
|
|
10094
|
+
if (newline !== "\n" && newline !== "\r" && newline !== "\r\n")
|
|
10095
|
+
newline = "\n";
|
|
10096
|
+
var cursor = 0;
|
|
10097
|
+
var aborted = false;
|
|
10098
|
+
this.parse = function(input, baseIndex, ignoreLastRow) {
|
|
10099
|
+
if (typeof input !== "string")
|
|
10100
|
+
throw new Error("Input must be a string");
|
|
10101
|
+
var inputLen = input.length, delimLen = delim.length, newlineLen = newline.length, commentsLen = comments.length;
|
|
10102
|
+
var stepIsFunction = isFunction(step);
|
|
10103
|
+
cursor = 0;
|
|
10104
|
+
var data = [], errors = [], row = [], lastCursor = 0;
|
|
10105
|
+
if (!input)
|
|
10106
|
+
return returnable();
|
|
10107
|
+
if (config.header && !baseIndex) {
|
|
10108
|
+
var firstLine = input.split(newline)[0];
|
|
10109
|
+
var headers = firstLine.split(delim);
|
|
10110
|
+
var separator = "_";
|
|
10111
|
+
var headerMap = [];
|
|
10112
|
+
var headerCount = {};
|
|
10113
|
+
var duplicateHeaders = false;
|
|
10114
|
+
for (var j in headers) {
|
|
10115
|
+
var header = headers[j];
|
|
10116
|
+
if (isFunction(config.transformHeader))
|
|
10117
|
+
header = config.transformHeader(header, j);
|
|
10118
|
+
var headerName = header;
|
|
10119
|
+
var count = headerCount[header] || 0;
|
|
10120
|
+
if (count > 0) {
|
|
10121
|
+
duplicateHeaders = true;
|
|
10122
|
+
headerName = header + separator + count;
|
|
10123
|
+
}
|
|
10124
|
+
headerCount[header] = count + 1;
|
|
10125
|
+
while (headerMap.includes(headerName)) {
|
|
10126
|
+
headerName = headerName + separator + count;
|
|
10127
|
+
}
|
|
10128
|
+
headerMap.push(headerName);
|
|
10129
|
+
}
|
|
10130
|
+
if (duplicateHeaders) {
|
|
10131
|
+
var editedInput = input.split(newline);
|
|
10132
|
+
editedInput[0] = headerMap.join(delim);
|
|
10133
|
+
input = editedInput.join(newline);
|
|
10134
|
+
}
|
|
10135
|
+
}
|
|
10136
|
+
if (fastMode || fastMode !== false && input.indexOf(quoteChar) === -1) {
|
|
10137
|
+
var rows = input.split(newline);
|
|
10138
|
+
for (var i = 0; i < rows.length; i++) {
|
|
10139
|
+
row = rows[i];
|
|
10140
|
+
cursor += row.length;
|
|
10141
|
+
if (i !== rows.length - 1)
|
|
10142
|
+
cursor += newline.length;
|
|
10143
|
+
else if (ignoreLastRow)
|
|
10144
|
+
return returnable();
|
|
10145
|
+
if (comments && row.substring(0, commentsLen) === comments)
|
|
10146
|
+
continue;
|
|
10147
|
+
if (stepIsFunction) {
|
|
10148
|
+
data = [];
|
|
10149
|
+
pushRow(row.split(delim));
|
|
10150
|
+
doStep();
|
|
10151
|
+
if (aborted)
|
|
10152
|
+
return returnable();
|
|
10153
|
+
} else
|
|
10154
|
+
pushRow(row.split(delim));
|
|
10155
|
+
if (preview && i >= preview) {
|
|
10156
|
+
data = data.slice(0, preview);
|
|
10157
|
+
return returnable(true);
|
|
10158
|
+
}
|
|
10159
|
+
}
|
|
10160
|
+
return returnable();
|
|
10161
|
+
}
|
|
10162
|
+
var nextDelim = input.indexOf(delim, cursor);
|
|
10163
|
+
var nextNewline = input.indexOf(newline, cursor);
|
|
10164
|
+
var quoteCharRegex = new RegExp(escapeRegExp(escapeChar) + escapeRegExp(quoteChar), "g");
|
|
10165
|
+
var quoteSearch = input.indexOf(quoteChar, cursor);
|
|
10166
|
+
for (; ; ) {
|
|
10167
|
+
if (input[cursor] === quoteChar) {
|
|
10168
|
+
quoteSearch = cursor;
|
|
10169
|
+
cursor++;
|
|
10170
|
+
for (; ; ) {
|
|
10171
|
+
quoteSearch = input.indexOf(quoteChar, quoteSearch + 1);
|
|
10172
|
+
if (quoteSearch === -1) {
|
|
10173
|
+
if (!ignoreLastRow) {
|
|
10174
|
+
errors.push({
|
|
10175
|
+
type: "Quotes",
|
|
10176
|
+
code: "MissingQuotes",
|
|
10177
|
+
message: "Quoted field unterminated",
|
|
10178
|
+
row: data.length,
|
|
10179
|
+
// row has yet to be inserted
|
|
10180
|
+
index: cursor
|
|
10181
|
+
});
|
|
10182
|
+
}
|
|
10183
|
+
return finish();
|
|
10184
|
+
}
|
|
10185
|
+
if (quoteSearch === inputLen - 1) {
|
|
10186
|
+
var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar);
|
|
10187
|
+
return finish(value);
|
|
10188
|
+
}
|
|
10189
|
+
if (quoteChar === escapeChar && input[quoteSearch + 1] === escapeChar) {
|
|
10190
|
+
quoteSearch++;
|
|
10191
|
+
continue;
|
|
10192
|
+
}
|
|
10193
|
+
if (quoteChar !== escapeChar && quoteSearch !== 0 && input[quoteSearch - 1] === escapeChar) {
|
|
10194
|
+
continue;
|
|
10195
|
+
}
|
|
10196
|
+
if (nextDelim !== -1 && nextDelim < quoteSearch + 1) {
|
|
10197
|
+
nextDelim = input.indexOf(delim, quoteSearch + 1);
|
|
10198
|
+
}
|
|
10199
|
+
if (nextNewline !== -1 && nextNewline < quoteSearch + 1) {
|
|
10200
|
+
nextNewline = input.indexOf(newline, quoteSearch + 1);
|
|
10201
|
+
}
|
|
10202
|
+
var checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline);
|
|
10203
|
+
var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo);
|
|
10204
|
+
if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndDelimiter, delimLen) === delim) {
|
|
10205
|
+
row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));
|
|
10206
|
+
cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen;
|
|
10207
|
+
if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen] !== quoteChar) {
|
|
10208
|
+
quoteSearch = input.indexOf(quoteChar, cursor);
|
|
10209
|
+
}
|
|
10210
|
+
nextDelim = input.indexOf(delim, cursor);
|
|
10211
|
+
nextNewline = input.indexOf(newline, cursor);
|
|
10212
|
+
break;
|
|
10213
|
+
}
|
|
10214
|
+
var spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline);
|
|
10215
|
+
if (input.substring(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen) === newline) {
|
|
10216
|
+
row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));
|
|
10217
|
+
saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen);
|
|
10218
|
+
nextDelim = input.indexOf(delim, cursor);
|
|
10219
|
+
quoteSearch = input.indexOf(quoteChar, cursor);
|
|
10220
|
+
if (stepIsFunction) {
|
|
10221
|
+
doStep();
|
|
10222
|
+
if (aborted)
|
|
10223
|
+
return returnable();
|
|
10224
|
+
}
|
|
10225
|
+
if (preview && data.length >= preview)
|
|
10226
|
+
return returnable(true);
|
|
10227
|
+
break;
|
|
10228
|
+
}
|
|
10229
|
+
errors.push({
|
|
10230
|
+
type: "Quotes",
|
|
10231
|
+
code: "InvalidQuotes",
|
|
10232
|
+
message: "Trailing quote on quoted field is malformed",
|
|
10233
|
+
row: data.length,
|
|
10234
|
+
// row has yet to be inserted
|
|
10235
|
+
index: cursor
|
|
10236
|
+
});
|
|
10237
|
+
quoteSearch++;
|
|
10238
|
+
continue;
|
|
10239
|
+
}
|
|
10240
|
+
continue;
|
|
10241
|
+
}
|
|
10242
|
+
if (comments && row.length === 0 && input.substring(cursor, cursor + commentsLen) === comments) {
|
|
10243
|
+
if (nextNewline === -1)
|
|
10244
|
+
return returnable();
|
|
10245
|
+
cursor = nextNewline + newlineLen;
|
|
10246
|
+
nextNewline = input.indexOf(newline, cursor);
|
|
10247
|
+
nextDelim = input.indexOf(delim, cursor);
|
|
10248
|
+
continue;
|
|
10249
|
+
}
|
|
10250
|
+
if (nextDelim !== -1 && (nextDelim < nextNewline || nextNewline === -1)) {
|
|
10251
|
+
row.push(input.substring(cursor, nextDelim));
|
|
10252
|
+
cursor = nextDelim + delimLen;
|
|
10253
|
+
nextDelim = input.indexOf(delim, cursor);
|
|
10254
|
+
continue;
|
|
10255
|
+
}
|
|
10256
|
+
if (nextNewline !== -1) {
|
|
10257
|
+
row.push(input.substring(cursor, nextNewline));
|
|
10258
|
+
saveRow(nextNewline + newlineLen);
|
|
10259
|
+
if (stepIsFunction) {
|
|
10260
|
+
doStep();
|
|
10261
|
+
if (aborted)
|
|
10262
|
+
return returnable();
|
|
10263
|
+
}
|
|
10264
|
+
if (preview && data.length >= preview)
|
|
10265
|
+
return returnable(true);
|
|
10266
|
+
continue;
|
|
10267
|
+
}
|
|
10268
|
+
break;
|
|
10269
|
+
}
|
|
10270
|
+
return finish();
|
|
10271
|
+
function pushRow(row2) {
|
|
10272
|
+
data.push(row2);
|
|
10273
|
+
lastCursor = cursor;
|
|
10274
|
+
}
|
|
10275
|
+
__name(pushRow, "pushRow");
|
|
10276
|
+
function extraSpaces(index) {
|
|
10277
|
+
var spaceLength = 0;
|
|
10278
|
+
if (index !== -1) {
|
|
10279
|
+
var textBetweenClosingQuoteAndIndex = input.substring(quoteSearch + 1, index);
|
|
10280
|
+
if (textBetweenClosingQuoteAndIndex && textBetweenClosingQuoteAndIndex.trim() === "") {
|
|
10281
|
+
spaceLength = textBetweenClosingQuoteAndIndex.length;
|
|
10282
|
+
}
|
|
10283
|
+
}
|
|
10284
|
+
return spaceLength;
|
|
10285
|
+
}
|
|
10286
|
+
__name(extraSpaces, "extraSpaces");
|
|
10287
|
+
function finish(value2) {
|
|
10288
|
+
if (ignoreLastRow)
|
|
10289
|
+
return returnable();
|
|
10290
|
+
if (typeof value2 === "undefined")
|
|
10291
|
+
value2 = input.substring(cursor);
|
|
10292
|
+
row.push(value2);
|
|
10293
|
+
cursor = inputLen;
|
|
10294
|
+
pushRow(row);
|
|
10295
|
+
if (stepIsFunction)
|
|
10296
|
+
doStep();
|
|
10297
|
+
return returnable();
|
|
10298
|
+
}
|
|
10299
|
+
__name(finish, "finish");
|
|
10300
|
+
function saveRow(newCursor) {
|
|
10301
|
+
cursor = newCursor;
|
|
10302
|
+
pushRow(row);
|
|
10303
|
+
row = [];
|
|
10304
|
+
nextNewline = input.indexOf(newline, cursor);
|
|
10305
|
+
}
|
|
10306
|
+
__name(saveRow, "saveRow");
|
|
10307
|
+
function returnable(stopped) {
|
|
10308
|
+
return {
|
|
10309
|
+
data,
|
|
10310
|
+
errors,
|
|
10311
|
+
meta: {
|
|
10312
|
+
delimiter: delim,
|
|
10313
|
+
linebreak: newline,
|
|
10314
|
+
aborted,
|
|
10315
|
+
truncated: !!stopped,
|
|
10316
|
+
cursor: lastCursor + (baseIndex || 0)
|
|
10317
|
+
}
|
|
10318
|
+
};
|
|
10319
|
+
}
|
|
10320
|
+
__name(returnable, "returnable");
|
|
10321
|
+
function doStep() {
|
|
10322
|
+
step(returnable());
|
|
10323
|
+
data = [];
|
|
10324
|
+
errors = [];
|
|
10325
|
+
}
|
|
10326
|
+
__name(doStep, "doStep");
|
|
10327
|
+
};
|
|
10328
|
+
this.abort = function() {
|
|
10329
|
+
aborted = true;
|
|
10330
|
+
};
|
|
10331
|
+
this.getCharIndex = function() {
|
|
10332
|
+
return cursor;
|
|
10333
|
+
};
|
|
10334
|
+
}
|
|
10335
|
+
__name(Parser, "Parser");
|
|
10336
|
+
function newWorker() {
|
|
10337
|
+
if (!Papa.WORKERS_SUPPORTED)
|
|
10338
|
+
return false;
|
|
10339
|
+
var workerUrl = getWorkerBlob();
|
|
10340
|
+
var w = new global.Worker(workerUrl);
|
|
10341
|
+
w.onmessage = mainThreadReceivedMessage;
|
|
10342
|
+
w.id = workerIdCounter++;
|
|
10343
|
+
workers[w.id] = w;
|
|
10344
|
+
return w;
|
|
10345
|
+
}
|
|
10346
|
+
__name(newWorker, "newWorker");
|
|
10347
|
+
function mainThreadReceivedMessage(e) {
|
|
10348
|
+
var msg = e.data;
|
|
10349
|
+
var worker = workers[msg.workerId];
|
|
10350
|
+
var aborted = false;
|
|
10351
|
+
if (msg.error)
|
|
10352
|
+
worker.userError(msg.error, msg.file);
|
|
10353
|
+
else if (msg.results && msg.results.data) {
|
|
10354
|
+
var abort = /* @__PURE__ */ __name(function() {
|
|
10355
|
+
aborted = true;
|
|
10356
|
+
completeWorker(msg.workerId, { data: [], errors: [], meta: { aborted: true } });
|
|
10357
|
+
}, "abort");
|
|
10358
|
+
var handle = {
|
|
10359
|
+
abort,
|
|
10360
|
+
pause: notImplemented,
|
|
10361
|
+
resume: notImplemented
|
|
10362
|
+
};
|
|
10363
|
+
if (isFunction(worker.userStep)) {
|
|
10364
|
+
for (var i = 0; i < msg.results.data.length; i++) {
|
|
10365
|
+
worker.userStep({
|
|
10366
|
+
data: msg.results.data[i],
|
|
10367
|
+
errors: msg.results.errors,
|
|
10368
|
+
meta: msg.results.meta
|
|
10369
|
+
}, handle);
|
|
10370
|
+
if (aborted)
|
|
10371
|
+
break;
|
|
10372
|
+
}
|
|
10373
|
+
delete msg.results;
|
|
10374
|
+
} else if (isFunction(worker.userChunk)) {
|
|
10375
|
+
worker.userChunk(msg.results, handle, msg.file);
|
|
10376
|
+
delete msg.results;
|
|
10377
|
+
}
|
|
10378
|
+
}
|
|
10379
|
+
if (msg.finished && !aborted)
|
|
10380
|
+
completeWorker(msg.workerId, msg.results);
|
|
10381
|
+
}
|
|
10382
|
+
__name(mainThreadReceivedMessage, "mainThreadReceivedMessage");
|
|
10383
|
+
function completeWorker(workerId, results) {
|
|
10384
|
+
var worker = workers[workerId];
|
|
10385
|
+
if (isFunction(worker.userComplete))
|
|
10386
|
+
worker.userComplete(results);
|
|
10387
|
+
worker.terminate();
|
|
10388
|
+
delete workers[workerId];
|
|
10389
|
+
}
|
|
10390
|
+
__name(completeWorker, "completeWorker");
|
|
10391
|
+
function notImplemented() {
|
|
10392
|
+
throw new Error("Not implemented.");
|
|
10393
|
+
}
|
|
10394
|
+
__name(notImplemented, "notImplemented");
|
|
10395
|
+
function workerThreadReceivedMessage(e) {
|
|
10396
|
+
var msg = e.data;
|
|
10397
|
+
if (typeof Papa.WORKER_ID === "undefined" && msg)
|
|
10398
|
+
Papa.WORKER_ID = msg.workerId;
|
|
10399
|
+
if (typeof msg.input === "string") {
|
|
10400
|
+
global.postMessage({
|
|
10401
|
+
workerId: Papa.WORKER_ID,
|
|
10402
|
+
results: Papa.parse(msg.input, msg.config),
|
|
10403
|
+
finished: true
|
|
10404
|
+
});
|
|
10405
|
+
} else if (global.File && msg.input instanceof File || msg.input instanceof Object) {
|
|
10406
|
+
var results = Papa.parse(msg.input, msg.config);
|
|
10407
|
+
if (results)
|
|
10408
|
+
global.postMessage({
|
|
10409
|
+
workerId: Papa.WORKER_ID,
|
|
10410
|
+
results,
|
|
10411
|
+
finished: true
|
|
10412
|
+
});
|
|
10413
|
+
}
|
|
10414
|
+
}
|
|
10415
|
+
__name(workerThreadReceivedMessage, "workerThreadReceivedMessage");
|
|
10416
|
+
function copy(obj) {
|
|
10417
|
+
if (typeof obj !== "object" || obj === null)
|
|
10418
|
+
return obj;
|
|
10419
|
+
var cpy = Array.isArray(obj) ? [] : {};
|
|
10420
|
+
for (var key in obj)
|
|
10421
|
+
cpy[key] = copy(obj[key]);
|
|
10422
|
+
return cpy;
|
|
10423
|
+
}
|
|
10424
|
+
__name(copy, "copy");
|
|
10425
|
+
function bindFunction(f, self2) {
|
|
10426
|
+
return function() {
|
|
10427
|
+
f.apply(self2, arguments);
|
|
10428
|
+
};
|
|
10429
|
+
}
|
|
10430
|
+
__name(bindFunction, "bindFunction");
|
|
10431
|
+
function isFunction(func) {
|
|
10432
|
+
return typeof func === "function";
|
|
10433
|
+
}
|
|
10434
|
+
__name(isFunction, "isFunction");
|
|
10435
|
+
return Papa;
|
|
10436
|
+
}, "moduleFactory"));
|
|
9077
10437
|
}
|
|
9078
10438
|
});
|
|
9079
10439
|
|
|
@@ -9088,8 +10448,14 @@ var require_convention_builder = __commonJS({
|
|
|
9088
10448
|
bury,
|
|
9089
10449
|
listAttributesAndSubAttributes,
|
|
9090
10450
|
findAllObjectAttributesInTree,
|
|
9091
|
-
findAllNonObjectAttributesInTree
|
|
10451
|
+
findAllNonObjectAttributesInTree,
|
|
10452
|
+
gitlabOverlayURLTemplate,
|
|
10453
|
+
gitlabConventionURLTemplate,
|
|
10454
|
+
basicSchemaURLTemplate,
|
|
10455
|
+
arrayToStructuredConvention,
|
|
10456
|
+
flattenJSONSchema
|
|
9092
10457
|
} = require_schema_utilities();
|
|
10458
|
+
var papa = require_papaparse();
|
|
9093
10459
|
function getAttributes(root, convention) {
|
|
9094
10460
|
let branches = convention.relationships.filter((rel) => rel.containerEntity == root).map((d) => d.mentionedEntity);
|
|
9095
10461
|
let output = {};
|
|
@@ -9103,21 +10469,24 @@ var require_convention_builder = __commonJS({
|
|
|
9103
10469
|
name,
|
|
9104
10470
|
validExamples,
|
|
9105
10471
|
erroredExamples,
|
|
9106
|
-
|
|
10472
|
+
repoURLTemplateFunction = gitlabOverlayURLTemplate,
|
|
9107
10473
|
storageFolder = `${__dirname}/../../../../output/collection/overlays`,
|
|
9108
10474
|
strictEnums = true,
|
|
9109
10475
|
overlay = false,
|
|
9110
10476
|
originalSrcScript,
|
|
9111
10477
|
trivial = false,
|
|
9112
10478
|
baseSchemataFolder = `${__dirname}/../../../../input/collection`,
|
|
10479
|
+
baseSchema,
|
|
10480
|
+
gitlabProjectId = process.env.CI_PROJECT_ID,
|
|
9113
10481
|
requiredFields = [],
|
|
9114
|
-
modifiedAttributes
|
|
10482
|
+
modifiedAttributes,
|
|
10483
|
+
deletedAttributes = []
|
|
9115
10484
|
}) {
|
|
9116
10485
|
this.name = name;
|
|
9117
10486
|
this.typeAndBundle = typeAndBundle;
|
|
9118
10487
|
this.validExamples = validExamples;
|
|
9119
10488
|
this.erroredExamples = erroredExamples;
|
|
9120
|
-
this.
|
|
10489
|
+
this.repoURLTemplateFunction = repoURLTemplateFunction;
|
|
9121
10490
|
this.schemaName = `${typeAndBundle}--${name}`;
|
|
9122
10491
|
this.storageFolder = storageFolder;
|
|
9123
10492
|
this.strictEnums = strictEnums;
|
|
@@ -9131,15 +10500,17 @@ var require_convention_builder = __commonJS({
|
|
|
9131
10500
|
;
|
|
9132
10501
|
this.type = typeAndBundle.split("--")[0];
|
|
9133
10502
|
this.bundle = typeAndBundle.split("--")[1];
|
|
9134
|
-
|
|
9135
|
-
|
|
9136
|
-
|
|
9137
|
-
|
|
9138
|
-
|
|
9139
|
-
|
|
9140
|
-
|
|
9141
|
-
|
|
9142
|
-
|
|
10503
|
+
if (!baseSchema) {
|
|
10504
|
+
try {
|
|
10505
|
+
baseSchema = JSON.parse(fs.readFileSync(`${baseSchemataFolder}/${this.type}/${this.bundle}/schema.json`));
|
|
10506
|
+
delete baseSchema.schema;
|
|
10507
|
+
} catch (e) {
|
|
10508
|
+
if (e.code == "ENOENT") {
|
|
10509
|
+
throw new Error(`No schema for ${typeAndBundle} was found. This might be due to the schema not existing in the farm, or the current "input/collection" folder structure being out of date. You can update it using the "getAllSchemas.js" script, in the script folder. Search path was: ${baseSchemataFolder}/${this.type}/${this.bundle}/`);
|
|
10510
|
+
} else {
|
|
10511
|
+
throw e;
|
|
10512
|
+
}
|
|
10513
|
+
;
|
|
9143
10514
|
}
|
|
9144
10515
|
;
|
|
9145
10516
|
}
|
|
@@ -9147,6 +10518,7 @@ var require_convention_builder = __commonJS({
|
|
|
9147
10518
|
this.baseSchema = baseSchema;
|
|
9148
10519
|
this.unmodifiedAttributes = listAttributesAndSubAttributes(this.baseSchema);
|
|
9149
10520
|
this.modifiedAttributes = modifiedAttributes ? modifiedAttributes : /* @__PURE__ */ new Set([]);
|
|
10521
|
+
this.deletedAttributes = deletedAttributes;
|
|
9150
10522
|
if (!overlay) {
|
|
9151
10523
|
this.overlay = {
|
|
9152
10524
|
properties: {
|
|
@@ -9178,8 +10550,8 @@ var require_convention_builder = __commonJS({
|
|
|
9178
10550
|
*/
|
|
9179
10551
|
updateSchema() {
|
|
9180
10552
|
this.schema = {};
|
|
9181
|
-
this.schema.$id = `${this.repoURL}/schemata/${this.typeAndBundle.replace("--", "/")}/schema.json`;
|
|
9182
10553
|
Object.assign(this.schema, this.baseSchema);
|
|
10554
|
+
this.schema.$id = this.repoURLTemplateFunction({ overlayName: this.name, projectID: this.gitlabProjectId, baseEntityTypeAndBundle: this.typeAndBundle });
|
|
9183
10555
|
let allModifiedAttributes = listAttributesAndSubAttributes(this.overlay);
|
|
9184
10556
|
this.modifiedAttributes = allModifiedAttributes;
|
|
9185
10557
|
this.unmodifiedAttributes = listAttributesAndSubAttributes(this.baseSchema);
|
|
@@ -9188,6 +10560,7 @@ var require_convention_builder = __commonJS({
|
|
|
9188
10560
|
let currentValue = dig(this.overlay.properties.attributes.properties, attr);
|
|
9189
10561
|
bury(this.schema, `properties.attributes.properties.${attr}`, currentValue, false);
|
|
9190
10562
|
});
|
|
10563
|
+
this.deletedAttributes.forEach((deletedAttr) => delete this.schema.properties.attributes.properties[deletedAttr]);
|
|
9191
10564
|
let originalRequiredFields = this.schema.properties.attributes.required ? this.schema.properties.attributes.required : [];
|
|
9192
10565
|
let allRequiredSet = Array.from(/* @__PURE__ */ new Set([...originalRequiredFields, ...Array.from(this.requiredFields)]));
|
|
9193
10566
|
this.schema.properties.attributes.required = allRequiredSet;
|
|
@@ -9290,6 +10663,22 @@ var require_convention_builder = __commonJS({
|
|
|
9290
10663
|
;
|
|
9291
10664
|
this.setGenericSection({ attribute, section: subschema, description });
|
|
9292
10665
|
}
|
|
10666
|
+
/**
|
|
10667
|
+
* Mark an attribute to be omitted from the resulting schema.
|
|
10668
|
+
* @param {} attribute
|
|
10669
|
+
*/
|
|
10670
|
+
deleteAttribute(attribute) {
|
|
10671
|
+
this.deletedAttributes.push(attribute);
|
|
10672
|
+
delete this.overlay.properties.attributes.properties[attribute];
|
|
10673
|
+
this.updateAttributeStatus(attribute);
|
|
10674
|
+
}
|
|
10675
|
+
/**
|
|
10676
|
+
* This method takes this object's schema and returns an array of fixed attributes objects, which is equivalent to a row by row encoded excel spreadsheet and is mostly meant at allowing users to work on said schema via a tabular structure.
|
|
10677
|
+
* The output will include one row for each data entry path (not all of the paths) and will synthesize all of its properties into columns, such as name, Title, Type, etc.
|
|
10678
|
+
*/
|
|
10679
|
+
getTabularRepresentation() {
|
|
10680
|
+
return flattenJSONSchema(this.schema, false);
|
|
10681
|
+
}
|
|
9293
10682
|
/**
|
|
9294
10683
|
* Build an AJV validator and ensure all valid examples are accepted and all error examples are rejected. Returns an array attribute for each set of examples, plus a general success attribute indicating wether all examples resulted as expected and a failedExamples array only listing entities for which there was no success (meanin unrejected error examples and rejected valid examples).
|
|
9295
10684
|
*/
|
|
@@ -9380,16 +10769,18 @@ var require_convention_builder = __commonJS({
|
|
|
9380
10769
|
constructor({
|
|
9381
10770
|
typeAndBundle,
|
|
9382
10771
|
attributeOverlayName,
|
|
9383
|
-
|
|
10772
|
+
repoURLTemplateFunction = basicSchemaURLTemplate,
|
|
9384
10773
|
strictEnums = true,
|
|
9385
|
-
baseSchemataFolder = `${__dirname}/../../../../input/collection
|
|
10774
|
+
baseSchemataFolder = `${__dirname}/../../../../input/collection`,
|
|
10775
|
+
gitlabProjectId = process.env.CI_PROJECT_ID
|
|
9386
10776
|
}) {
|
|
9387
10777
|
super({
|
|
9388
10778
|
typeAndBundle,
|
|
9389
|
-
|
|
10779
|
+
repoURLTemplateFunction,
|
|
9390
10780
|
storageFolder: baseSchemataFolder,
|
|
9391
10781
|
baseSchemataFolder,
|
|
9392
|
-
strictEnums
|
|
10782
|
+
strictEnums,
|
|
10783
|
+
gitlabProjectId
|
|
9393
10784
|
});
|
|
9394
10785
|
this.attributeOverlayName = attributeOverlayName;
|
|
9395
10786
|
}
|
|
@@ -9417,12 +10808,13 @@ var require_convention_builder = __commonJS({
|
|
|
9417
10808
|
title,
|
|
9418
10809
|
schemaName,
|
|
9419
10810
|
version,
|
|
9420
|
-
|
|
10811
|
+
repoURLTemplateFunction = gitlabConventionURLTemplate,
|
|
9421
10812
|
description,
|
|
9422
10813
|
validExamples = [],
|
|
9423
10814
|
erroredExamples = [],
|
|
9424
10815
|
storageFolder = `${__dirname}/../../../../output/collection`,
|
|
9425
10816
|
baseSchemataFolder = `${__dirname}/../../../../input/collection`,
|
|
10817
|
+
gitlabProjectId = process.env.CI_PROJECT_ID,
|
|
9426
10818
|
strictEnums = false,
|
|
9427
10819
|
overlays = false,
|
|
9428
10820
|
relationships = false,
|
|
@@ -9437,13 +10829,14 @@ var require_convention_builder = __commonJS({
|
|
|
9437
10829
|
this.description = description;
|
|
9438
10830
|
this.schemaName = schemaName;
|
|
9439
10831
|
this.version = version;
|
|
9440
|
-
this.
|
|
10832
|
+
this.repoURLTemplateFunction = repoURLTemplateFunction, this.validExamples = validExamples;
|
|
9441
10833
|
this.erroredExamples = erroredExamples;
|
|
9442
10834
|
this.overlays = {};
|
|
9443
10835
|
this.relationships = [];
|
|
9444
10836
|
this.required = [];
|
|
9445
10837
|
this.storageFolder = storageFolder;
|
|
9446
10838
|
this.baseSchemataFolder = baseSchemataFolder;
|
|
10839
|
+
this.gitlabProjectId = gitlabProjectId;
|
|
9447
10840
|
if (overlays) {
|
|
9448
10841
|
let overlayNames = Object.keys(overlays);
|
|
9449
10842
|
this.overlays = {};
|
|
@@ -9519,7 +10912,7 @@ var require_convention_builder = __commonJS({
|
|
|
9519
10912
|
this.schema = {
|
|
9520
10913
|
title: this.title,
|
|
9521
10914
|
type: "object",
|
|
9522
|
-
$id:
|
|
10915
|
+
$id: this.repoURLTemplateFunction({ conventionName: this.schemaName, projectID: this.gitlabProjectId }),
|
|
9523
10916
|
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
9524
10917
|
properties: {},
|
|
9525
10918
|
description: this.description,
|
|
@@ -9775,8 +11168,13 @@ import TabItem from '@theme/TabItem';
|
|
|
9775
11168
|
* @param {string} path -- A string representing an absolute path, leading to either a JSON file containing an individual example or a folder containing several examples.
|
|
9776
11169
|
* @param {object|array} object -- Either an array of examples or an individual example.
|
|
9777
11170
|
* @param {boolean} is_valid_example -- Whether to store the provided examples as valid examples or errors, for the tests.
|
|
11171
|
+
* @param {boolean} structured_as_array -- Mark as "true" if the provided example is structured as a loose array of entities, as FarmOS delivers them (instead of structured as a convention object in which every entity is a property). The method will try to structure the entity and return meaningful information if it is not successful in doing so.
|
|
9778
11172
|
*/
|
|
9779
|
-
addExample({ path, object, is_valid_example }) {
|
|
11173
|
+
addExample({ path, object, is_valid_example, structured_as_array }) {
|
|
11174
|
+
let output = {
|
|
11175
|
+
success: false,
|
|
11176
|
+
errors: []
|
|
11177
|
+
};
|
|
9780
11178
|
if (!path && !object) {
|
|
9781
11179
|
throw new Error(`Either an object or a path leading to a folder or a json file need to be provided. Both arguments are empty.`);
|
|
9782
11180
|
}
|
|
@@ -9793,12 +11191,43 @@ import TabItem from '@theme/TabItem';
|
|
|
9793
11191
|
newExamples = this.getExamplesFromFile({ path });
|
|
9794
11192
|
}
|
|
9795
11193
|
;
|
|
11194
|
+
if (structured_as_array) {
|
|
11195
|
+
newExamples = newExamples.map((exampleArray) => {
|
|
11196
|
+
let exampleObject = arrayToStructuredConvention({
|
|
11197
|
+
entitiesArray: exampleArray,
|
|
11198
|
+
conventionObject: this
|
|
11199
|
+
});
|
|
11200
|
+
if (!exampleObject.is_valid && is_valid_example) {
|
|
11201
|
+
output.errors.push("The `arrayToStructuredConvention` function wasn't able to rebuild a valid instance of this convention's schema from the provided array of loose entities. Look into the '.structuringAttempt' attribute for more information.");
|
|
11202
|
+
}
|
|
11203
|
+
;
|
|
11204
|
+
if (!is_valid_example) {
|
|
11205
|
+
output.warning = "Since the example is not marked as 'valid', we can't check it is being properly parsed. Please review the output 'assembled_entity' to check it looks as it should.";
|
|
11206
|
+
}
|
|
11207
|
+
;
|
|
11208
|
+
output.structuringAttempt = exampleObject;
|
|
11209
|
+
return exampleObject.assembled_entity;
|
|
11210
|
+
});
|
|
11211
|
+
}
|
|
11212
|
+
;
|
|
9796
11213
|
if (is_valid_example) {
|
|
9797
11214
|
this.validExamples.push(...newExamples);
|
|
9798
11215
|
} else {
|
|
9799
11216
|
this.erroredExamples.push(...newExamples);
|
|
9800
11217
|
}
|
|
9801
11218
|
;
|
|
11219
|
+
if (output.errors.length == 0) {
|
|
11220
|
+
output.success = true;
|
|
11221
|
+
}
|
|
11222
|
+
;
|
|
11223
|
+
return output;
|
|
11224
|
+
}
|
|
11225
|
+
/**
|
|
11226
|
+
* This method takes this object's schema and returns an array of fixed attributes objects, which is equivalent to a row by row encoded excel spreadsheet and is mostly meant at allowing users to work on said schema via a tabular structure.
|
|
11227
|
+
* The output will include one row for each data entry path (not all of the paths) and will synthesize all of its properties into columns, such as name, Title, Type, etc.
|
|
11228
|
+
*/
|
|
11229
|
+
getTabularRepresentation() {
|
|
11230
|
+
return flattenJSONSchema(this.schema, true);
|
|
9802
11231
|
}
|
|
9803
11232
|
/**
|
|
9804
11233
|
* Build an AJV validator and ensure all valid examples are accepted and all error examples are rejected. Returns an array attribute for each set of examples, plus a general success attribute indicating wether all examples resulted as expected and a failedExamples array only listing entities for which there was no success (meanin unrejected error examples and rejected valid examples).
|
|
@@ -9881,6 +11310,8 @@ import TabItem from '@theme/TabItem';
|
|
|
9881
11310
|
fs.mkdirSync(`${targetPath}/examples/correct`, { recursive: true }, console.error);
|
|
9882
11311
|
fs.mkdirSync(`${targetPath}/examples/incorrect`, { recursive: true }, console.error);
|
|
9883
11312
|
fs.writeFileSync(`${targetPath}/schema.json`, JSON.stringify(this.schema, null, " "), console.error);
|
|
11313
|
+
let flatTableRepresentation = this.getTabularRepresentation();
|
|
11314
|
+
fs.writeFileSync(`${targetPath}/flat_schema.csv`, papa.unparse(flatTableRepresentation), console.error);
|
|
9884
11315
|
this.validExamples.forEach((example, i) => {
|
|
9885
11316
|
fs.writeFileSync(`${targetPath}/examples/correct/example_${i + 1}.json`, JSON.stringify(example, null, " "), console.error);
|
|
9886
11317
|
});
|
|
@@ -9916,7 +11347,7 @@ import TabItem from '@theme/TabItem';
|
|
|
9916
11347
|
};
|
|
9917
11348
|
__name(_ConventionSchema, "ConventionSchema");
|
|
9918
11349
|
var ConventionSchema = _ConventionSchema;
|
|
9919
|
-
var
|
|
11350
|
+
var _InputConventionChanges = class _InputConventionChanges {
|
|
9920
11351
|
constructor({
|
|
9921
11352
|
type,
|
|
9922
11353
|
bundle,
|
|
@@ -9924,7 +11355,8 @@ import TabItem from '@theme/TabItem';
|
|
|
9924
11355
|
baseSchemataFolder = `${__dirname}/../../../../input/collection`,
|
|
9925
11356
|
sourceSchemataFile = `${__dirname}/../../../../input/farmos.json`,
|
|
9926
11357
|
overlayName,
|
|
9927
|
-
overlayDescription
|
|
11358
|
+
overlayDescription,
|
|
11359
|
+
gitlabProjectId = process.env.CI_PROJECT_ID
|
|
9928
11360
|
}) {
|
|
9929
11361
|
let integralSchemataFile = JSON.parse(fs.readFileSync(sourceSchemataFile));
|
|
9930
11362
|
this.type = type;
|
|
@@ -9935,6 +11367,7 @@ import TabItem from '@theme/TabItem';
|
|
|
9935
11367
|
this.sourceSchemata = integralSchemataFile;
|
|
9936
11368
|
this.lexicon = [...findAllObjectAttributesInTree(this.sourceSchemata), ...findAllNonObjectAttributesInTree(this.sourceSchemata)];
|
|
9937
11369
|
this.baseSchemataFolder = baseSchemataFolder;
|
|
11370
|
+
this.gitlabProjectId = gitlabProjectId;
|
|
9938
11371
|
let involvedSchemata = this.lexicon.filter((entry) => this.attribute == entry.attribute).flatMap((entry) => entry.instances).filter((instance) => this.type ? instance.type == this.type : true).filter((instance) => this.bundle ? instance.bundle == this.bundle : true).map((d) => d.schema);
|
|
9939
11372
|
this.targetSchemata = Array.from(new Set(involvedSchemata));
|
|
9940
11373
|
this.overlays = [];
|
|
@@ -9942,8 +11375,10 @@ import TabItem from '@theme/TabItem';
|
|
|
9942
11375
|
let currentOverlay = new AttributePatchingSchemaOverlay({
|
|
9943
11376
|
typeAndBundle: target,
|
|
9944
11377
|
attributeOverlayName: this.overlayName,
|
|
9945
|
-
baseSchemataFolder: this.baseSchemataFolder
|
|
11378
|
+
baseSchemataFolder: this.baseSchemataFolder,
|
|
11379
|
+
gitlabProjectId: this.gitlabProjectId
|
|
9946
11380
|
});
|
|
11381
|
+
currentOverlay.gitlabProjectId = this.gitlabProjectId;
|
|
9947
11382
|
this.overlays.push(currentOverlay);
|
|
9948
11383
|
});
|
|
9949
11384
|
}
|
|
@@ -10004,6 +11439,15 @@ import TabItem from '@theme/TabItem';
|
|
|
10004
11439
|
});
|
|
10005
11440
|
});
|
|
10006
11441
|
}
|
|
11442
|
+
/**
|
|
11443
|
+
* Mark an attribute to be omitted from the resulting schema.
|
|
11444
|
+
* @param {} attribute
|
|
11445
|
+
*/
|
|
11446
|
+
deleteAttribute() {
|
|
11447
|
+
this.overlays.forEach((overlay) => {
|
|
11448
|
+
overlay.deleteAttribute(this.attribute);
|
|
11449
|
+
});
|
|
11450
|
+
}
|
|
10007
11451
|
/**
|
|
10008
11452
|
* Stores all involved schemas into the folder structure for `base schemata` (provided as the parameter `baseSchemataFolder`) which will be consumed by our conventions later.
|
|
10009
11453
|
* @returns {} -- It returns objects containing all paths.
|
|
@@ -10016,21 +11460,21 @@ import TabItem from '@theme/TabItem';
|
|
|
10016
11460
|
return output;
|
|
10017
11461
|
}
|
|
10018
11462
|
};
|
|
10019
|
-
__name(
|
|
10020
|
-
var
|
|
11463
|
+
__name(_InputConventionChanges, "InputConventionChanges");
|
|
11464
|
+
var InputConventionChanges = _InputConventionChanges;
|
|
10021
11465
|
exports.SchemaOverlay = SchemaOverlay;
|
|
10022
11466
|
exports.ConventionSchema = ConventionSchema;
|
|
10023
|
-
exports.
|
|
11467
|
+
exports.InputConventionChanges = InputConventionChanges;
|
|
10024
11468
|
}
|
|
10025
11469
|
});
|
|
10026
11470
|
|
|
10027
11471
|
// src/index.js
|
|
10028
11472
|
var require_src = __commonJS({
|
|
10029
11473
|
"src/index.js"(exports) {
|
|
10030
|
-
var { SchemaOverlay, ConventionSchema,
|
|
11474
|
+
var { SchemaOverlay, ConventionSchema, InputConventionChanges } = require_convention_builder();
|
|
10031
11475
|
var schemaUtils = require_schema_utilities();
|
|
10032
11476
|
exports.SchemaOverlay = SchemaOverlay;
|
|
10033
|
-
exports.
|
|
11477
|
+
exports.InputConventionChanges = InputConventionChanges;
|
|
10034
11478
|
exports.ConventionSchema = ConventionSchema;
|
|
10035
11479
|
exports.fixRelationshipDataField = schemaUtils.fixRelationshipDataField;
|
|
10036
11480
|
exports.buildValidator = schemaUtils.buildValidator;
|
|
@@ -10054,5 +11498,13 @@ export default require_src();
|
|
|
10054
11498
|
|
|
10055
11499
|
uri-js/dist/es5/uri.all.js:
|
|
10056
11500
|
(** @license URI.js v4.4.1 (c) 2011 Gary Court. License: http://github.com/garycourt/uri-js *)
|
|
11501
|
+
|
|
11502
|
+
papaparse/papaparse.js:
|
|
11503
|
+
(* @license
|
|
11504
|
+
Papa Parse
|
|
11505
|
+
v5.4.1
|
|
11506
|
+
https://github.com/mholt/PapaParse
|
|
11507
|
+
License: MIT
|
|
11508
|
+
*)
|
|
10057
11509
|
*/
|
|
10058
11510
|
//# sourceMappingURL=index.js.map
|