convention_builder 1.4.4 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/module/index.js +1511 -59
- package/dist/node/index.js +1511 -59
- package/package.json +6 -4
package/dist/node/index.js
CHANGED
|
@@ -1173,12 +1173,12 @@ var require_util = __commonJS({
|
|
|
1173
1173
|
__name(alwaysValidSchema, "alwaysValidSchema");
|
|
1174
1174
|
exports2.alwaysValidSchema = alwaysValidSchema;
|
|
1175
1175
|
function checkUnknownRules(it, schema = it.schema) {
|
|
1176
|
-
const { opts, self } = it;
|
|
1176
|
+
const { opts, self: self2 } = it;
|
|
1177
1177
|
if (!opts.strictSchema)
|
|
1178
1178
|
return;
|
|
1179
1179
|
if (typeof schema === "boolean")
|
|
1180
1180
|
return;
|
|
1181
|
-
const rules =
|
|
1181
|
+
const rules = self2.RULES.keywords;
|
|
1182
1182
|
for (const key in schema) {
|
|
1183
1183
|
if (!rules[key])
|
|
1184
1184
|
checkStrictMode(it, `unknown keyword: "${key}"`);
|
|
@@ -1588,8 +1588,8 @@ var require_applicability = __commonJS({
|
|
|
1588
1588
|
"use strict";
|
|
1589
1589
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
1590
1590
|
exports2.shouldUseRule = exports2.shouldUseGroup = exports2.schemaHasRulesForType = void 0;
|
|
1591
|
-
function schemaHasRulesForType({ schema, self }, type) {
|
|
1592
|
-
const group =
|
|
1591
|
+
function schemaHasRulesForType({ schema, self: self2 }, type) {
|
|
1592
|
+
const group = self2.RULES.types[type];
|
|
1593
1593
|
return group && group !== true && shouldUseGroup(schema, group);
|
|
1594
1594
|
}
|
|
1595
1595
|
__name(schemaHasRulesForType, "schemaHasRulesForType");
|
|
@@ -2097,7 +2097,7 @@ var require_keyword = __commonJS({
|
|
|
2097
2097
|
}
|
|
2098
2098
|
__name(validSchemaType, "validSchemaType");
|
|
2099
2099
|
exports2.validSchemaType = validSchemaType;
|
|
2100
|
-
function validateKeywordUsage({ schema, opts, self, errSchemaPath }, def, keyword) {
|
|
2100
|
+
function validateKeywordUsage({ schema, opts, self: self2, errSchemaPath }, def, keyword) {
|
|
2101
2101
|
if (Array.isArray(def.keyword) ? !def.keyword.includes(keyword) : def.keyword !== keyword) {
|
|
2102
2102
|
throw new Error("ajv implementation error");
|
|
2103
2103
|
}
|
|
@@ -2108,9 +2108,9 @@ var require_keyword = __commonJS({
|
|
|
2108
2108
|
if (def.validateSchema) {
|
|
2109
2109
|
const valid = def.validateSchema(schema[keyword]);
|
|
2110
2110
|
if (!valid) {
|
|
2111
|
-
const msg = `keyword "${keyword}" value is invalid at path "${errSchemaPath}": ` +
|
|
2111
|
+
const msg = `keyword "${keyword}" value is invalid at path "${errSchemaPath}": ` + self2.errorsText(def.validateSchema.errors);
|
|
2112
2112
|
if (opts.validateSchema === "log")
|
|
2113
|
-
|
|
2113
|
+
self2.logger.error(msg);
|
|
2114
2114
|
else
|
|
2115
2115
|
throw new Error(msg);
|
|
2116
2116
|
}
|
|
@@ -2614,11 +2614,11 @@ var require_validate = __commonJS({
|
|
|
2614
2614
|
(0, boolSchema_1.boolOrEmptySchema)(it, valid);
|
|
2615
2615
|
}
|
|
2616
2616
|
__name(subschemaCode, "subschemaCode");
|
|
2617
|
-
function schemaCxtHasRules({ schema, self }) {
|
|
2617
|
+
function schemaCxtHasRules({ schema, self: self2 }) {
|
|
2618
2618
|
if (typeof schema == "boolean")
|
|
2619
2619
|
return !schema;
|
|
2620
2620
|
for (const key in schema)
|
|
2621
|
-
if (
|
|
2621
|
+
if (self2.RULES.all[key])
|
|
2622
2622
|
return true;
|
|
2623
2623
|
return false;
|
|
2624
2624
|
}
|
|
@@ -2652,9 +2652,9 @@ var require_validate = __commonJS({
|
|
|
2652
2652
|
}
|
|
2653
2653
|
__name(typeAndKeywords, "typeAndKeywords");
|
|
2654
2654
|
function checkRefsAndKeywords(it) {
|
|
2655
|
-
const { schema, errSchemaPath, opts, self } = it;
|
|
2656
|
-
if (schema.$ref && opts.ignoreKeywordsWithRef && (0, util_1.schemaHasRulesButRef)(schema,
|
|
2657
|
-
|
|
2655
|
+
const { schema, errSchemaPath, opts, self: self2 } = it;
|
|
2656
|
+
if (schema.$ref && opts.ignoreKeywordsWithRef && (0, util_1.schemaHasRulesButRef)(schema, self2.RULES)) {
|
|
2657
|
+
self2.logger.warn(`$ref: keywords ignored in schema at path "${errSchemaPath}"`);
|
|
2658
2658
|
}
|
|
2659
2659
|
}
|
|
2660
2660
|
__name(checkRefsAndKeywords, "checkRefsAndKeywords");
|
|
@@ -2707,8 +2707,8 @@ var require_validate = __commonJS({
|
|
|
2707
2707
|
}
|
|
2708
2708
|
__name(assignEvaluated, "assignEvaluated");
|
|
2709
2709
|
function schemaKeywords(it, types, typeErrors, errsCount) {
|
|
2710
|
-
const { gen, schema, data, allErrors, opts, self } = it;
|
|
2711
|
-
const { RULES } =
|
|
2710
|
+
const { gen, schema, data, allErrors, opts, self: self2 } = it;
|
|
2711
|
+
const { RULES } = self2;
|
|
2712
2712
|
if (schema.$ref && (opts.ignoreKeywordsWithRef || !(0, util_1.schemaHasRulesButRef)(schema, RULES))) {
|
|
2713
2713
|
gen.block(() => keywordCode(it, "$ref", RULES.all.$ref.definition));
|
|
2714
2714
|
return;
|
|
@@ -5096,11 +5096,11 @@ var require_ref = __commonJS({
|
|
|
5096
5096
|
schemaType: "string",
|
|
5097
5097
|
code(cxt) {
|
|
5098
5098
|
const { gen, schema: $ref, it } = cxt;
|
|
5099
|
-
const { baseId, schemaEnv: env, validateName, opts, self } = it;
|
|
5099
|
+
const { baseId, schemaEnv: env, validateName, opts, self: self2 } = it;
|
|
5100
5100
|
const { root } = env;
|
|
5101
5101
|
if (($ref === "#" || $ref === "#/") && baseId === root.baseId)
|
|
5102
5102
|
return callRootRef();
|
|
5103
|
-
const schOrEnv = compile_1.resolveRef.call(
|
|
5103
|
+
const schOrEnv = compile_1.resolveRef.call(self2, root, baseId, $ref);
|
|
5104
5104
|
if (schOrEnv === void 0)
|
|
5105
5105
|
throw new ref_error_1.default(it.opts.uriResolver, baseId, $ref);
|
|
5106
5106
|
if (schOrEnv instanceof compile_1.SchemaEnv)
|
|
@@ -6671,11 +6671,11 @@ var require_dynamicAnchor = __commonJS({
|
|
|
6671
6671
|
__name(dynamicAnchor, "dynamicAnchor");
|
|
6672
6672
|
exports2.dynamicAnchor = dynamicAnchor;
|
|
6673
6673
|
function _getValidate(cxt) {
|
|
6674
|
-
const { schemaEnv, schema, self } = cxt.it;
|
|
6674
|
+
const { schemaEnv, schema, self: self2 } = cxt.it;
|
|
6675
6675
|
const { root, baseId, localRefs, meta } = schemaEnv.root;
|
|
6676
|
-
const { schemaId } =
|
|
6676
|
+
const { schemaId } = self2.opts;
|
|
6677
6677
|
const sch = new compile_1.SchemaEnv({ schema, schemaId, root, baseId, localRefs, meta });
|
|
6678
|
-
compile_1.compileSchema.call(
|
|
6678
|
+
compile_1.compileSchema.call(self2, sch);
|
|
6679
6679
|
return (0, ref_1.getValidate)(cxt, sch);
|
|
6680
6680
|
}
|
|
6681
6681
|
__name(_getValidate, "_getValidate");
|
|
@@ -6992,7 +6992,7 @@ var require_format = __commonJS({
|
|
|
6992
6992
|
error,
|
|
6993
6993
|
code(cxt, ruleType) {
|
|
6994
6994
|
const { gen, data, $data, schema, schemaCode, it } = cxt;
|
|
6995
|
-
const { opts, errSchemaPath, schemaEnv, self } = it;
|
|
6995
|
+
const { opts, errSchemaPath, schemaEnv, self: self2 } = it;
|
|
6996
6996
|
if (!opts.validateFormats)
|
|
6997
6997
|
return;
|
|
6998
6998
|
if ($data)
|
|
@@ -7001,7 +7001,7 @@ var require_format = __commonJS({
|
|
|
7001
7001
|
validateFormat();
|
|
7002
7002
|
function validate$DataFormat() {
|
|
7003
7003
|
const fmts = gen.scopeValue("formats", {
|
|
7004
|
-
ref:
|
|
7004
|
+
ref: self2.formats,
|
|
7005
7005
|
code: opts.code.formats
|
|
7006
7006
|
});
|
|
7007
7007
|
const fDef = gen.const("fDef", (0, codegen_1._)`${fmts}[${schemaCode}]`);
|
|
@@ -7024,7 +7024,7 @@ var require_format = __commonJS({
|
|
|
7024
7024
|
}
|
|
7025
7025
|
__name(validate$DataFormat, "validate$DataFormat");
|
|
7026
7026
|
function validateFormat() {
|
|
7027
|
-
const formatDef =
|
|
7027
|
+
const formatDef = self2.formats[schema];
|
|
7028
7028
|
if (!formatDef) {
|
|
7029
7029
|
unknownFormat();
|
|
7030
7030
|
return;
|
|
@@ -7036,7 +7036,7 @@ var require_format = __commonJS({
|
|
|
7036
7036
|
cxt.pass(validCondition());
|
|
7037
7037
|
function unknownFormat() {
|
|
7038
7038
|
if (opts.strictSchema === false) {
|
|
7039
|
-
|
|
7039
|
+
self2.logger.warn(unknownMsg());
|
|
7040
7040
|
return;
|
|
7041
7041
|
}
|
|
7042
7042
|
throw new Error(unknownMsg());
|
|
@@ -8189,17 +8189,17 @@ var require_limit = __commonJS({
|
|
|
8189
8189
|
error,
|
|
8190
8190
|
code(cxt) {
|
|
8191
8191
|
const { gen, data, schemaCode, keyword, it } = cxt;
|
|
8192
|
-
const { opts, self } = it;
|
|
8192
|
+
const { opts, self: self2 } = it;
|
|
8193
8193
|
if (!opts.validateFormats)
|
|
8194
8194
|
return;
|
|
8195
|
-
const fCxt = new ajv_1.KeywordCxt(it,
|
|
8195
|
+
const fCxt = new ajv_1.KeywordCxt(it, self2.RULES.all.format.definition, "format");
|
|
8196
8196
|
if (fCxt.$data)
|
|
8197
8197
|
validate$DataFormat();
|
|
8198
8198
|
else
|
|
8199
8199
|
validateFormat();
|
|
8200
8200
|
function validate$DataFormat() {
|
|
8201
8201
|
const fmts = gen.scopeValue("formats", {
|
|
8202
|
-
ref:
|
|
8202
|
+
ref: self2.formats,
|
|
8203
8203
|
code: opts.code.formats
|
|
8204
8204
|
});
|
|
8205
8205
|
const fmt = gen.const("fmt", codegen_1._`${fmts}[${fCxt.schemaCode}]`);
|
|
@@ -8208,7 +8208,7 @@ var require_limit = __commonJS({
|
|
|
8208
8208
|
__name(validate$DataFormat, "validate$DataFormat");
|
|
8209
8209
|
function validateFormat() {
|
|
8210
8210
|
const format = fCxt.schema;
|
|
8211
|
-
const fmtDef =
|
|
8211
|
+
const fmtDef = self2.formats[format];
|
|
8212
8212
|
if (!fmtDef || fmtDef === true)
|
|
8213
8213
|
return;
|
|
8214
8214
|
if (typeof fmtDef != "object" || fmtDef instanceof RegExp || typeof fmtDef.compare != "function") {
|
|
@@ -8607,7 +8607,12 @@ var require_schema_utilities = __commonJS({
|
|
|
8607
8607
|
__name(apiComposeTaxonomiesTransformation, "apiComposeTaxonomiesTransformation");
|
|
8608
8608
|
function exampleImporter(filename, basePath = `${__dirname}/../../../definitions/examples/raw_examples_and_scripts/`, schemataSource = `${__dirname}/../../../input/collection`) {
|
|
8609
8609
|
let rawData = JSON.parse(fs.readFileSync(`${basePath}/${filename}`)).map((obj) => {
|
|
8610
|
-
let output = obj.entity;
|
|
8610
|
+
let output = obj.entity ? obj.entity : obj;
|
|
8611
|
+
if (!output) {
|
|
8612
|
+
console.log(obj);
|
|
8613
|
+
console.log(output);
|
|
8614
|
+
}
|
|
8615
|
+
;
|
|
8611
8616
|
if (output.attributes.relationships) {
|
|
8612
8617
|
output.relationships = output.attributes.relationships;
|
|
8613
8618
|
delete output.attributes.relationships;
|
|
@@ -8624,7 +8629,7 @@ var require_schema_utilities = __commonJS({
|
|
|
8624
8629
|
});
|
|
8625
8630
|
let newEntities = [];
|
|
8626
8631
|
let entities = rawData.flatMap((entity) => {
|
|
8627
|
-
let transformedData = apiComposeTaxonomiesTransformation(entity);
|
|
8632
|
+
let transformedData = entity.relationships ? apiComposeTaxonomiesTransformation(entity) : { mainEntity: entity, newEntities: [] };
|
|
8628
8633
|
return [transformedData.mainEntity, ...transformedData.newEntities];
|
|
8629
8634
|
});
|
|
8630
8635
|
return entities;
|
|
@@ -9045,6 +9050,20 @@ var require_schema_utilities = __commonJS({
|
|
|
9045
9050
|
return output;
|
|
9046
9051
|
}
|
|
9047
9052
|
__name(findAllNonObjectAttributesInTree, "findAllNonObjectAttributesInTree");
|
|
9053
|
+
function gitlabConventionURLTemplate({ conventionName, projectID }) {
|
|
9054
|
+
return `https://gitlab.com/api/v4/projects/${projectID}/jobs/artifacts/main/raw/output/collection/conventions/${conventionName}/schema.json?job=copy_schemas`;
|
|
9055
|
+
}
|
|
9056
|
+
__name(gitlabConventionURLTemplate, "gitlabConventionURLTemplate");
|
|
9057
|
+
function gitlabOverlayURLTemplate({ overlayName, projectID, baseEntityTypeAndbundle = "" }) {
|
|
9058
|
+
return `https://gitlab.com/api/v4/projects/${projectID}/jobs/artifacts/main/raw/input/collection/${overlayName}/schema.json?job=copy_schemas`;
|
|
9059
|
+
}
|
|
9060
|
+
__name(gitlabOverlayURLTemplate, "gitlabOverlayURLTemplate");
|
|
9061
|
+
function basicSchemaURLTemplate({ baseEntityTypeAndBundle, projectID }) {
|
|
9062
|
+
let type = baseEntityTypeAndBundle.split("--")[0];
|
|
9063
|
+
let bundle = baseEntityTypeAndBundle.split("--")[1];
|
|
9064
|
+
return `https://gitlab.com/api/v4/projects/${projectID}/jobs/artifacts/main/raw/input/collection/${type}/${bundle}/schema.json?job=copy_schemas`;
|
|
9065
|
+
}
|
|
9066
|
+
__name(basicSchemaURLTemplate, "basicSchemaURLTemplate");
|
|
9048
9067
|
exports2.dig = dig;
|
|
9049
9068
|
exports2.bury = bury;
|
|
9050
9069
|
exports2.fixRelationshipDataField = fixRelationshipDataField;
|
|
@@ -9067,6 +9086,1347 @@ var require_schema_utilities = __commonJS({
|
|
|
9067
9086
|
exports2.listAttributesAndSubAttributes = listAttributesAndSubAttributes;
|
|
9068
9087
|
exports2.findAllObjectAttributesInTree = findAllObjectAttributesInTree;
|
|
9069
9088
|
exports2.findAllNonObjectAttributesInTree = findAllNonObjectAttributesInTree;
|
|
9089
|
+
exports2.gitlabConventionURLTemplate = gitlabConventionURLTemplate;
|
|
9090
|
+
exports2.gitlabOverlayURLTemplate = gitlabOverlayURLTemplate;
|
|
9091
|
+
exports2.basicSchemaURLTemplate = basicSchemaURLTemplate;
|
|
9092
|
+
}
|
|
9093
|
+
});
|
|
9094
|
+
|
|
9095
|
+
// node_modules/papaparse/papaparse.js
|
|
9096
|
+
var require_papaparse = __commonJS({
|
|
9097
|
+
"node_modules/papaparse/papaparse.js"(exports2, module2) {
|
|
9098
|
+
(function(root, factory) {
|
|
9099
|
+
if (typeof define === "function" && define.amd) {
|
|
9100
|
+
define([], factory);
|
|
9101
|
+
} else if (typeof module2 === "object" && typeof exports2 !== "undefined") {
|
|
9102
|
+
module2.exports = factory();
|
|
9103
|
+
} else {
|
|
9104
|
+
root.Papa = factory();
|
|
9105
|
+
}
|
|
9106
|
+
})(exports2, /* @__PURE__ */ __name(function moduleFactory() {
|
|
9107
|
+
"use strict";
|
|
9108
|
+
var global = function() {
|
|
9109
|
+
if (typeof self !== "undefined") {
|
|
9110
|
+
return self;
|
|
9111
|
+
}
|
|
9112
|
+
if (typeof window !== "undefined") {
|
|
9113
|
+
return window;
|
|
9114
|
+
}
|
|
9115
|
+
if (typeof global !== "undefined") {
|
|
9116
|
+
return global;
|
|
9117
|
+
}
|
|
9118
|
+
return {};
|
|
9119
|
+
}();
|
|
9120
|
+
function getWorkerBlob() {
|
|
9121
|
+
var URL = global.URL || global.webkitURL || null;
|
|
9122
|
+
var code = moduleFactory.toString();
|
|
9123
|
+
return Papa.BLOB_URL || (Papa.BLOB_URL = URL.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ", "(", code, ")();"], { type: "text/javascript" })));
|
|
9124
|
+
}
|
|
9125
|
+
__name(getWorkerBlob, "getWorkerBlob");
|
|
9126
|
+
var IS_WORKER = !global.document && !!global.postMessage, IS_PAPA_WORKER = global.IS_PAPA_WORKER || false;
|
|
9127
|
+
var workers = {}, workerIdCounter = 0;
|
|
9128
|
+
var Papa = {};
|
|
9129
|
+
Papa.parse = CsvToJson;
|
|
9130
|
+
Papa.unparse = JsonToCsv;
|
|
9131
|
+
Papa.RECORD_SEP = String.fromCharCode(30);
|
|
9132
|
+
Papa.UNIT_SEP = String.fromCharCode(31);
|
|
9133
|
+
Papa.BYTE_ORDER_MARK = "\uFEFF";
|
|
9134
|
+
Papa.BAD_DELIMITERS = ["\r", "\n", '"', Papa.BYTE_ORDER_MARK];
|
|
9135
|
+
Papa.WORKERS_SUPPORTED = !IS_WORKER && !!global.Worker;
|
|
9136
|
+
Papa.NODE_STREAM_INPUT = 1;
|
|
9137
|
+
Papa.LocalChunkSize = 1024 * 1024 * 10;
|
|
9138
|
+
Papa.RemoteChunkSize = 1024 * 1024 * 5;
|
|
9139
|
+
Papa.DefaultDelimiter = ",";
|
|
9140
|
+
Papa.Parser = Parser;
|
|
9141
|
+
Papa.ParserHandle = ParserHandle;
|
|
9142
|
+
Papa.NetworkStreamer = NetworkStreamer;
|
|
9143
|
+
Papa.FileStreamer = FileStreamer;
|
|
9144
|
+
Papa.StringStreamer = StringStreamer;
|
|
9145
|
+
Papa.ReadableStreamStreamer = ReadableStreamStreamer;
|
|
9146
|
+
if (typeof PAPA_BROWSER_CONTEXT === "undefined") {
|
|
9147
|
+
Papa.DuplexStreamStreamer = DuplexStreamStreamer;
|
|
9148
|
+
}
|
|
9149
|
+
if (global.jQuery) {
|
|
9150
|
+
var $ = global.jQuery;
|
|
9151
|
+
$.fn.parse = function(options) {
|
|
9152
|
+
var config = options.config || {};
|
|
9153
|
+
var queue = [];
|
|
9154
|
+
this.each(function(idx) {
|
|
9155
|
+
var supported = $(this).prop("tagName").toUpperCase() === "INPUT" && $(this).attr("type").toLowerCase() === "file" && global.FileReader;
|
|
9156
|
+
if (!supported || !this.files || this.files.length === 0)
|
|
9157
|
+
return true;
|
|
9158
|
+
for (var i = 0; i < this.files.length; i++) {
|
|
9159
|
+
queue.push({
|
|
9160
|
+
file: this.files[i],
|
|
9161
|
+
inputElem: this,
|
|
9162
|
+
instanceConfig: $.extend({}, config)
|
|
9163
|
+
});
|
|
9164
|
+
}
|
|
9165
|
+
});
|
|
9166
|
+
parseNextFile();
|
|
9167
|
+
return this;
|
|
9168
|
+
function parseNextFile() {
|
|
9169
|
+
if (queue.length === 0) {
|
|
9170
|
+
if (isFunction(options.complete))
|
|
9171
|
+
options.complete();
|
|
9172
|
+
return;
|
|
9173
|
+
}
|
|
9174
|
+
var f = queue[0];
|
|
9175
|
+
if (isFunction(options.before)) {
|
|
9176
|
+
var returned = options.before(f.file, f.inputElem);
|
|
9177
|
+
if (typeof returned === "object") {
|
|
9178
|
+
if (returned.action === "abort") {
|
|
9179
|
+
error("AbortError", f.file, f.inputElem, returned.reason);
|
|
9180
|
+
return;
|
|
9181
|
+
} else if (returned.action === "skip") {
|
|
9182
|
+
fileComplete();
|
|
9183
|
+
return;
|
|
9184
|
+
} else if (typeof returned.config === "object")
|
|
9185
|
+
f.instanceConfig = $.extend(f.instanceConfig, returned.config);
|
|
9186
|
+
} else if (returned === "skip") {
|
|
9187
|
+
fileComplete();
|
|
9188
|
+
return;
|
|
9189
|
+
}
|
|
9190
|
+
}
|
|
9191
|
+
var userCompleteFunc = f.instanceConfig.complete;
|
|
9192
|
+
f.instanceConfig.complete = function(results) {
|
|
9193
|
+
if (isFunction(userCompleteFunc))
|
|
9194
|
+
userCompleteFunc(results, f.file, f.inputElem);
|
|
9195
|
+
fileComplete();
|
|
9196
|
+
};
|
|
9197
|
+
Papa.parse(f.file, f.instanceConfig);
|
|
9198
|
+
}
|
|
9199
|
+
__name(parseNextFile, "parseNextFile");
|
|
9200
|
+
function error(name, file, elem, reason) {
|
|
9201
|
+
if (isFunction(options.error))
|
|
9202
|
+
options.error({ name }, file, elem, reason);
|
|
9203
|
+
}
|
|
9204
|
+
__name(error, "error");
|
|
9205
|
+
function fileComplete() {
|
|
9206
|
+
queue.splice(0, 1);
|
|
9207
|
+
parseNextFile();
|
|
9208
|
+
}
|
|
9209
|
+
__name(fileComplete, "fileComplete");
|
|
9210
|
+
};
|
|
9211
|
+
}
|
|
9212
|
+
if (IS_PAPA_WORKER) {
|
|
9213
|
+
global.onmessage = workerThreadReceivedMessage;
|
|
9214
|
+
}
|
|
9215
|
+
function CsvToJson(_input, _config) {
|
|
9216
|
+
_config = _config || {};
|
|
9217
|
+
var dynamicTyping = _config.dynamicTyping || false;
|
|
9218
|
+
if (isFunction(dynamicTyping)) {
|
|
9219
|
+
_config.dynamicTypingFunction = dynamicTyping;
|
|
9220
|
+
dynamicTyping = {};
|
|
9221
|
+
}
|
|
9222
|
+
_config.dynamicTyping = dynamicTyping;
|
|
9223
|
+
_config.transform = isFunction(_config.transform) ? _config.transform : false;
|
|
9224
|
+
if (_config.worker && Papa.WORKERS_SUPPORTED) {
|
|
9225
|
+
var w = newWorker();
|
|
9226
|
+
w.userStep = _config.step;
|
|
9227
|
+
w.userChunk = _config.chunk;
|
|
9228
|
+
w.userComplete = _config.complete;
|
|
9229
|
+
w.userError = _config.error;
|
|
9230
|
+
_config.step = isFunction(_config.step);
|
|
9231
|
+
_config.chunk = isFunction(_config.chunk);
|
|
9232
|
+
_config.complete = isFunction(_config.complete);
|
|
9233
|
+
_config.error = isFunction(_config.error);
|
|
9234
|
+
delete _config.worker;
|
|
9235
|
+
w.postMessage({
|
|
9236
|
+
input: _input,
|
|
9237
|
+
config: _config,
|
|
9238
|
+
workerId: w.id
|
|
9239
|
+
});
|
|
9240
|
+
return;
|
|
9241
|
+
}
|
|
9242
|
+
var streamer = null;
|
|
9243
|
+
if (_input === Papa.NODE_STREAM_INPUT && typeof PAPA_BROWSER_CONTEXT === "undefined") {
|
|
9244
|
+
streamer = new DuplexStreamStreamer(_config);
|
|
9245
|
+
return streamer.getStream();
|
|
9246
|
+
} else if (typeof _input === "string") {
|
|
9247
|
+
_input = stripBom(_input);
|
|
9248
|
+
if (_config.download)
|
|
9249
|
+
streamer = new NetworkStreamer(_config);
|
|
9250
|
+
else
|
|
9251
|
+
streamer = new StringStreamer(_config);
|
|
9252
|
+
} else if (_input.readable === true && isFunction(_input.read) && isFunction(_input.on)) {
|
|
9253
|
+
streamer = new ReadableStreamStreamer(_config);
|
|
9254
|
+
} else if (global.File && _input instanceof File || _input instanceof Object)
|
|
9255
|
+
streamer = new FileStreamer(_config);
|
|
9256
|
+
return streamer.stream(_input);
|
|
9257
|
+
function stripBom(string) {
|
|
9258
|
+
if (string.charCodeAt(0) === 65279) {
|
|
9259
|
+
return string.slice(1);
|
|
9260
|
+
}
|
|
9261
|
+
return string;
|
|
9262
|
+
}
|
|
9263
|
+
__name(stripBom, "stripBom");
|
|
9264
|
+
}
|
|
9265
|
+
__name(CsvToJson, "CsvToJson");
|
|
9266
|
+
function JsonToCsv(_input, _config) {
|
|
9267
|
+
var _quotes = false;
|
|
9268
|
+
var _writeHeader = true;
|
|
9269
|
+
var _delimiter = ",";
|
|
9270
|
+
var _newline = "\r\n";
|
|
9271
|
+
var _quoteChar = '"';
|
|
9272
|
+
var _escapedQuote = _quoteChar + _quoteChar;
|
|
9273
|
+
var _skipEmptyLines = false;
|
|
9274
|
+
var _columns = null;
|
|
9275
|
+
var _escapeFormulae = false;
|
|
9276
|
+
unpackConfig();
|
|
9277
|
+
var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), "g");
|
|
9278
|
+
if (typeof _input === "string")
|
|
9279
|
+
_input = JSON.parse(_input);
|
|
9280
|
+
if (Array.isArray(_input)) {
|
|
9281
|
+
if (!_input.length || Array.isArray(_input[0]))
|
|
9282
|
+
return serialize(null, _input, _skipEmptyLines);
|
|
9283
|
+
else if (typeof _input[0] === "object")
|
|
9284
|
+
return serialize(_columns || Object.keys(_input[0]), _input, _skipEmptyLines);
|
|
9285
|
+
} else if (typeof _input === "object") {
|
|
9286
|
+
if (typeof _input.data === "string")
|
|
9287
|
+
_input.data = JSON.parse(_input.data);
|
|
9288
|
+
if (Array.isArray(_input.data)) {
|
|
9289
|
+
if (!_input.fields)
|
|
9290
|
+
_input.fields = _input.meta && _input.meta.fields || _columns;
|
|
9291
|
+
if (!_input.fields)
|
|
9292
|
+
_input.fields = Array.isArray(_input.data[0]) ? _input.fields : typeof _input.data[0] === "object" ? Object.keys(_input.data[0]) : [];
|
|
9293
|
+
if (!Array.isArray(_input.data[0]) && typeof _input.data[0] !== "object")
|
|
9294
|
+
_input.data = [_input.data];
|
|
9295
|
+
}
|
|
9296
|
+
return serialize(_input.fields || [], _input.data || [], _skipEmptyLines);
|
|
9297
|
+
}
|
|
9298
|
+
throw new Error("Unable to serialize unrecognized input");
|
|
9299
|
+
function unpackConfig() {
|
|
9300
|
+
if (typeof _config !== "object")
|
|
9301
|
+
return;
|
|
9302
|
+
if (typeof _config.delimiter === "string" && !Papa.BAD_DELIMITERS.filter(function(value) {
|
|
9303
|
+
return _config.delimiter.indexOf(value) !== -1;
|
|
9304
|
+
}).length) {
|
|
9305
|
+
_delimiter = _config.delimiter;
|
|
9306
|
+
}
|
|
9307
|
+
if (typeof _config.quotes === "boolean" || typeof _config.quotes === "function" || Array.isArray(_config.quotes))
|
|
9308
|
+
_quotes = _config.quotes;
|
|
9309
|
+
if (typeof _config.skipEmptyLines === "boolean" || typeof _config.skipEmptyLines === "string")
|
|
9310
|
+
_skipEmptyLines = _config.skipEmptyLines;
|
|
9311
|
+
if (typeof _config.newline === "string")
|
|
9312
|
+
_newline = _config.newline;
|
|
9313
|
+
if (typeof _config.quoteChar === "string")
|
|
9314
|
+
_quoteChar = _config.quoteChar;
|
|
9315
|
+
if (typeof _config.header === "boolean")
|
|
9316
|
+
_writeHeader = _config.header;
|
|
9317
|
+
if (Array.isArray(_config.columns)) {
|
|
9318
|
+
if (_config.columns.length === 0)
|
|
9319
|
+
throw new Error("Option columns is empty");
|
|
9320
|
+
_columns = _config.columns;
|
|
9321
|
+
}
|
|
9322
|
+
if (_config.escapeChar !== void 0) {
|
|
9323
|
+
_escapedQuote = _config.escapeChar + _quoteChar;
|
|
9324
|
+
}
|
|
9325
|
+
if (typeof _config.escapeFormulae === "boolean" || _config.escapeFormulae instanceof RegExp) {
|
|
9326
|
+
_escapeFormulae = _config.escapeFormulae instanceof RegExp ? _config.escapeFormulae : /^[=+\-@\t\r].*$/;
|
|
9327
|
+
}
|
|
9328
|
+
}
|
|
9329
|
+
__name(unpackConfig, "unpackConfig");
|
|
9330
|
+
function serialize(fields, data, skipEmptyLines) {
|
|
9331
|
+
var csv = "";
|
|
9332
|
+
if (typeof fields === "string")
|
|
9333
|
+
fields = JSON.parse(fields);
|
|
9334
|
+
if (typeof data === "string")
|
|
9335
|
+
data = JSON.parse(data);
|
|
9336
|
+
var hasHeader = Array.isArray(fields) && fields.length > 0;
|
|
9337
|
+
var dataKeyedByField = !Array.isArray(data[0]);
|
|
9338
|
+
if (hasHeader && _writeHeader) {
|
|
9339
|
+
for (var i = 0; i < fields.length; i++) {
|
|
9340
|
+
if (i > 0)
|
|
9341
|
+
csv += _delimiter;
|
|
9342
|
+
csv += safe(fields[i], i);
|
|
9343
|
+
}
|
|
9344
|
+
if (data.length > 0)
|
|
9345
|
+
csv += _newline;
|
|
9346
|
+
}
|
|
9347
|
+
for (var row = 0; row < data.length; row++) {
|
|
9348
|
+
var maxCol = hasHeader ? fields.length : data[row].length;
|
|
9349
|
+
var emptyLine = false;
|
|
9350
|
+
var nullLine = hasHeader ? Object.keys(data[row]).length === 0 : data[row].length === 0;
|
|
9351
|
+
if (skipEmptyLines && !hasHeader) {
|
|
9352
|
+
emptyLine = skipEmptyLines === "greedy" ? data[row].join("").trim() === "" : data[row].length === 1 && data[row][0].length === 0;
|
|
9353
|
+
}
|
|
9354
|
+
if (skipEmptyLines === "greedy" && hasHeader) {
|
|
9355
|
+
var line = [];
|
|
9356
|
+
for (var c = 0; c < maxCol; c++) {
|
|
9357
|
+
var cx = dataKeyedByField ? fields[c] : c;
|
|
9358
|
+
line.push(data[row][cx]);
|
|
9359
|
+
}
|
|
9360
|
+
emptyLine = line.join("").trim() === "";
|
|
9361
|
+
}
|
|
9362
|
+
if (!emptyLine) {
|
|
9363
|
+
for (var col = 0; col < maxCol; col++) {
|
|
9364
|
+
if (col > 0 && !nullLine)
|
|
9365
|
+
csv += _delimiter;
|
|
9366
|
+
var colIdx = hasHeader && dataKeyedByField ? fields[col] : col;
|
|
9367
|
+
csv += safe(data[row][colIdx], col);
|
|
9368
|
+
}
|
|
9369
|
+
if (row < data.length - 1 && (!skipEmptyLines || maxCol > 0 && !nullLine)) {
|
|
9370
|
+
csv += _newline;
|
|
9371
|
+
}
|
|
9372
|
+
}
|
|
9373
|
+
}
|
|
9374
|
+
return csv;
|
|
9375
|
+
}
|
|
9376
|
+
__name(serialize, "serialize");
|
|
9377
|
+
function safe(str, col) {
|
|
9378
|
+
if (typeof str === "undefined" || str === null)
|
|
9379
|
+
return "";
|
|
9380
|
+
if (str.constructor === Date)
|
|
9381
|
+
return JSON.stringify(str).slice(1, 25);
|
|
9382
|
+
var needsQuotes = false;
|
|
9383
|
+
if (_escapeFormulae && typeof str === "string" && _escapeFormulae.test(str)) {
|
|
9384
|
+
str = "'" + str;
|
|
9385
|
+
needsQuotes = true;
|
|
9386
|
+
}
|
|
9387
|
+
var escapedQuoteStr = str.toString().replace(quoteCharRegex, _escapedQuote);
|
|
9388
|
+
needsQuotes = needsQuotes || _quotes === true || typeof _quotes === "function" && _quotes(str, col) || Array.isArray(_quotes) && _quotes[col] || hasAny(escapedQuoteStr, Papa.BAD_DELIMITERS) || escapedQuoteStr.indexOf(_delimiter) > -1 || escapedQuoteStr.charAt(0) === " " || escapedQuoteStr.charAt(escapedQuoteStr.length - 1) === " ";
|
|
9389
|
+
return needsQuotes ? _quoteChar + escapedQuoteStr + _quoteChar : escapedQuoteStr;
|
|
9390
|
+
}
|
|
9391
|
+
__name(safe, "safe");
|
|
9392
|
+
function hasAny(str, substrings) {
|
|
9393
|
+
for (var i = 0; i < substrings.length; i++)
|
|
9394
|
+
if (str.indexOf(substrings[i]) > -1)
|
|
9395
|
+
return true;
|
|
9396
|
+
return false;
|
|
9397
|
+
}
|
|
9398
|
+
__name(hasAny, "hasAny");
|
|
9399
|
+
}
|
|
9400
|
+
__name(JsonToCsv, "JsonToCsv");
|
|
9401
|
+
function ChunkStreamer(config) {
|
|
9402
|
+
this._handle = null;
|
|
9403
|
+
this._finished = false;
|
|
9404
|
+
this._completed = false;
|
|
9405
|
+
this._halted = false;
|
|
9406
|
+
this._input = null;
|
|
9407
|
+
this._baseIndex = 0;
|
|
9408
|
+
this._partialLine = "";
|
|
9409
|
+
this._rowCount = 0;
|
|
9410
|
+
this._start = 0;
|
|
9411
|
+
this._nextChunk = null;
|
|
9412
|
+
this.isFirstChunk = true;
|
|
9413
|
+
this._completeResults = {
|
|
9414
|
+
data: [],
|
|
9415
|
+
errors: [],
|
|
9416
|
+
meta: {}
|
|
9417
|
+
};
|
|
9418
|
+
replaceConfig.call(this, config);
|
|
9419
|
+
this.parseChunk = function(chunk, isFakeChunk) {
|
|
9420
|
+
if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) {
|
|
9421
|
+
var modifiedChunk = this._config.beforeFirstChunk(chunk);
|
|
9422
|
+
if (modifiedChunk !== void 0)
|
|
9423
|
+
chunk = modifiedChunk;
|
|
9424
|
+
}
|
|
9425
|
+
this.isFirstChunk = false;
|
|
9426
|
+
this._halted = false;
|
|
9427
|
+
var aggregate = this._partialLine + chunk;
|
|
9428
|
+
this._partialLine = "";
|
|
9429
|
+
var results = this._handle.parse(aggregate, this._baseIndex, !this._finished);
|
|
9430
|
+
if (this._handle.paused() || this._handle.aborted()) {
|
|
9431
|
+
this._halted = true;
|
|
9432
|
+
return;
|
|
9433
|
+
}
|
|
9434
|
+
var lastIndex = results.meta.cursor;
|
|
9435
|
+
if (!this._finished) {
|
|
9436
|
+
this._partialLine = aggregate.substring(lastIndex - this._baseIndex);
|
|
9437
|
+
this._baseIndex = lastIndex;
|
|
9438
|
+
}
|
|
9439
|
+
if (results && results.data)
|
|
9440
|
+
this._rowCount += results.data.length;
|
|
9441
|
+
var finishedIncludingPreview = this._finished || this._config.preview && this._rowCount >= this._config.preview;
|
|
9442
|
+
if (IS_PAPA_WORKER) {
|
|
9443
|
+
global.postMessage({
|
|
9444
|
+
results,
|
|
9445
|
+
workerId: Papa.WORKER_ID,
|
|
9446
|
+
finished: finishedIncludingPreview
|
|
9447
|
+
});
|
|
9448
|
+
} else if (isFunction(this._config.chunk) && !isFakeChunk) {
|
|
9449
|
+
this._config.chunk(results, this._handle);
|
|
9450
|
+
if (this._handle.paused() || this._handle.aborted()) {
|
|
9451
|
+
this._halted = true;
|
|
9452
|
+
return;
|
|
9453
|
+
}
|
|
9454
|
+
results = void 0;
|
|
9455
|
+
this._completeResults = void 0;
|
|
9456
|
+
}
|
|
9457
|
+
if (!this._config.step && !this._config.chunk) {
|
|
9458
|
+
this._completeResults.data = this._completeResults.data.concat(results.data);
|
|
9459
|
+
this._completeResults.errors = this._completeResults.errors.concat(results.errors);
|
|
9460
|
+
this._completeResults.meta = results.meta;
|
|
9461
|
+
}
|
|
9462
|
+
if (!this._completed && finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) {
|
|
9463
|
+
this._config.complete(this._completeResults, this._input);
|
|
9464
|
+
this._completed = true;
|
|
9465
|
+
}
|
|
9466
|
+
if (!finishedIncludingPreview && (!results || !results.meta.paused))
|
|
9467
|
+
this._nextChunk();
|
|
9468
|
+
return results;
|
|
9469
|
+
};
|
|
9470
|
+
this._sendError = function(error) {
|
|
9471
|
+
if (isFunction(this._config.error))
|
|
9472
|
+
this._config.error(error);
|
|
9473
|
+
else if (IS_PAPA_WORKER && this._config.error) {
|
|
9474
|
+
global.postMessage({
|
|
9475
|
+
workerId: Papa.WORKER_ID,
|
|
9476
|
+
error,
|
|
9477
|
+
finished: false
|
|
9478
|
+
});
|
|
9479
|
+
}
|
|
9480
|
+
};
|
|
9481
|
+
function replaceConfig(config2) {
|
|
9482
|
+
var configCopy = copy(config2);
|
|
9483
|
+
configCopy.chunkSize = parseInt(configCopy.chunkSize);
|
|
9484
|
+
if (!config2.step && !config2.chunk)
|
|
9485
|
+
configCopy.chunkSize = null;
|
|
9486
|
+
this._handle = new ParserHandle(configCopy);
|
|
9487
|
+
this._handle.streamer = this;
|
|
9488
|
+
this._config = configCopy;
|
|
9489
|
+
}
|
|
9490
|
+
__name(replaceConfig, "replaceConfig");
|
|
9491
|
+
}
|
|
9492
|
+
__name(ChunkStreamer, "ChunkStreamer");
|
|
9493
|
+
function NetworkStreamer(config) {
|
|
9494
|
+
config = config || {};
|
|
9495
|
+
if (!config.chunkSize)
|
|
9496
|
+
config.chunkSize = Papa.RemoteChunkSize;
|
|
9497
|
+
ChunkStreamer.call(this, config);
|
|
9498
|
+
var xhr;
|
|
9499
|
+
if (IS_WORKER) {
|
|
9500
|
+
this._nextChunk = function() {
|
|
9501
|
+
this._readChunk();
|
|
9502
|
+
this._chunkLoaded();
|
|
9503
|
+
};
|
|
9504
|
+
} else {
|
|
9505
|
+
this._nextChunk = function() {
|
|
9506
|
+
this._readChunk();
|
|
9507
|
+
};
|
|
9508
|
+
}
|
|
9509
|
+
this.stream = function(url) {
|
|
9510
|
+
this._input = url;
|
|
9511
|
+
this._nextChunk();
|
|
9512
|
+
};
|
|
9513
|
+
this._readChunk = function() {
|
|
9514
|
+
if (this._finished) {
|
|
9515
|
+
this._chunkLoaded();
|
|
9516
|
+
return;
|
|
9517
|
+
}
|
|
9518
|
+
xhr = new XMLHttpRequest();
|
|
9519
|
+
if (this._config.withCredentials) {
|
|
9520
|
+
xhr.withCredentials = this._config.withCredentials;
|
|
9521
|
+
}
|
|
9522
|
+
if (!IS_WORKER) {
|
|
9523
|
+
xhr.onload = bindFunction(this._chunkLoaded, this);
|
|
9524
|
+
xhr.onerror = bindFunction(this._chunkError, this);
|
|
9525
|
+
}
|
|
9526
|
+
xhr.open(this._config.downloadRequestBody ? "POST" : "GET", this._input, !IS_WORKER);
|
|
9527
|
+
if (this._config.downloadRequestHeaders) {
|
|
9528
|
+
var headers = this._config.downloadRequestHeaders;
|
|
9529
|
+
for (var headerName in headers) {
|
|
9530
|
+
xhr.setRequestHeader(headerName, headers[headerName]);
|
|
9531
|
+
}
|
|
9532
|
+
}
|
|
9533
|
+
if (this._config.chunkSize) {
|
|
9534
|
+
var end = this._start + this._config.chunkSize - 1;
|
|
9535
|
+
xhr.setRequestHeader("Range", "bytes=" + this._start + "-" + end);
|
|
9536
|
+
}
|
|
9537
|
+
try {
|
|
9538
|
+
xhr.send(this._config.downloadRequestBody);
|
|
9539
|
+
} catch (err) {
|
|
9540
|
+
this._chunkError(err.message);
|
|
9541
|
+
}
|
|
9542
|
+
if (IS_WORKER && xhr.status === 0)
|
|
9543
|
+
this._chunkError();
|
|
9544
|
+
};
|
|
9545
|
+
this._chunkLoaded = function() {
|
|
9546
|
+
if (xhr.readyState !== 4)
|
|
9547
|
+
return;
|
|
9548
|
+
if (xhr.status < 200 || xhr.status >= 400) {
|
|
9549
|
+
this._chunkError();
|
|
9550
|
+
return;
|
|
9551
|
+
}
|
|
9552
|
+
this._start += this._config.chunkSize ? this._config.chunkSize : xhr.responseText.length;
|
|
9553
|
+
this._finished = !this._config.chunkSize || this._start >= getFileSize(xhr);
|
|
9554
|
+
this.parseChunk(xhr.responseText);
|
|
9555
|
+
};
|
|
9556
|
+
this._chunkError = function(errorMessage) {
|
|
9557
|
+
var errorText = xhr.statusText || errorMessage;
|
|
9558
|
+
this._sendError(new Error(errorText));
|
|
9559
|
+
};
|
|
9560
|
+
function getFileSize(xhr2) {
|
|
9561
|
+
var contentRange = xhr2.getResponseHeader("Content-Range");
|
|
9562
|
+
if (contentRange === null) {
|
|
9563
|
+
return -1;
|
|
9564
|
+
}
|
|
9565
|
+
return parseInt(contentRange.substring(contentRange.lastIndexOf("/") + 1));
|
|
9566
|
+
}
|
|
9567
|
+
__name(getFileSize, "getFileSize");
|
|
9568
|
+
}
|
|
9569
|
+
__name(NetworkStreamer, "NetworkStreamer");
|
|
9570
|
+
NetworkStreamer.prototype = Object.create(ChunkStreamer.prototype);
|
|
9571
|
+
NetworkStreamer.prototype.constructor = NetworkStreamer;
|
|
9572
|
+
function FileStreamer(config) {
|
|
9573
|
+
config = config || {};
|
|
9574
|
+
if (!config.chunkSize)
|
|
9575
|
+
config.chunkSize = Papa.LocalChunkSize;
|
|
9576
|
+
ChunkStreamer.call(this, config);
|
|
9577
|
+
var reader, slice;
|
|
9578
|
+
var usingAsyncReader = typeof FileReader !== "undefined";
|
|
9579
|
+
this.stream = function(file) {
|
|
9580
|
+
this._input = file;
|
|
9581
|
+
slice = file.slice || file.webkitSlice || file.mozSlice;
|
|
9582
|
+
if (usingAsyncReader) {
|
|
9583
|
+
reader = new FileReader();
|
|
9584
|
+
reader.onload = bindFunction(this._chunkLoaded, this);
|
|
9585
|
+
reader.onerror = bindFunction(this._chunkError, this);
|
|
9586
|
+
} else
|
|
9587
|
+
reader = new FileReaderSync();
|
|
9588
|
+
this._nextChunk();
|
|
9589
|
+
};
|
|
9590
|
+
this._nextChunk = function() {
|
|
9591
|
+
if (!this._finished && (!this._config.preview || this._rowCount < this._config.preview))
|
|
9592
|
+
this._readChunk();
|
|
9593
|
+
};
|
|
9594
|
+
this._readChunk = function() {
|
|
9595
|
+
var input = this._input;
|
|
9596
|
+
if (this._config.chunkSize) {
|
|
9597
|
+
var end = Math.min(this._start + this._config.chunkSize, this._input.size);
|
|
9598
|
+
input = slice.call(input, this._start, end);
|
|
9599
|
+
}
|
|
9600
|
+
var txt = reader.readAsText(input, this._config.encoding);
|
|
9601
|
+
if (!usingAsyncReader)
|
|
9602
|
+
this._chunkLoaded({ target: { result: txt } });
|
|
9603
|
+
};
|
|
9604
|
+
this._chunkLoaded = function(event) {
|
|
9605
|
+
this._start += this._config.chunkSize;
|
|
9606
|
+
this._finished = !this._config.chunkSize || this._start >= this._input.size;
|
|
9607
|
+
this.parseChunk(event.target.result);
|
|
9608
|
+
};
|
|
9609
|
+
this._chunkError = function() {
|
|
9610
|
+
this._sendError(reader.error);
|
|
9611
|
+
};
|
|
9612
|
+
}
|
|
9613
|
+
__name(FileStreamer, "FileStreamer");
|
|
9614
|
+
FileStreamer.prototype = Object.create(ChunkStreamer.prototype);
|
|
9615
|
+
FileStreamer.prototype.constructor = FileStreamer;
|
|
9616
|
+
function StringStreamer(config) {
|
|
9617
|
+
config = config || {};
|
|
9618
|
+
ChunkStreamer.call(this, config);
|
|
9619
|
+
var remaining;
|
|
9620
|
+
this.stream = function(s) {
|
|
9621
|
+
remaining = s;
|
|
9622
|
+
return this._nextChunk();
|
|
9623
|
+
};
|
|
9624
|
+
this._nextChunk = function() {
|
|
9625
|
+
if (this._finished)
|
|
9626
|
+
return;
|
|
9627
|
+
var size = this._config.chunkSize;
|
|
9628
|
+
var chunk;
|
|
9629
|
+
if (size) {
|
|
9630
|
+
chunk = remaining.substring(0, size);
|
|
9631
|
+
remaining = remaining.substring(size);
|
|
9632
|
+
} else {
|
|
9633
|
+
chunk = remaining;
|
|
9634
|
+
remaining = "";
|
|
9635
|
+
}
|
|
9636
|
+
this._finished = !remaining;
|
|
9637
|
+
return this.parseChunk(chunk);
|
|
9638
|
+
};
|
|
9639
|
+
}
|
|
9640
|
+
__name(StringStreamer, "StringStreamer");
|
|
9641
|
+
StringStreamer.prototype = Object.create(StringStreamer.prototype);
|
|
9642
|
+
StringStreamer.prototype.constructor = StringStreamer;
|
|
9643
|
+
function ReadableStreamStreamer(config) {
|
|
9644
|
+
config = config || {};
|
|
9645
|
+
ChunkStreamer.call(this, config);
|
|
9646
|
+
var queue = [];
|
|
9647
|
+
var parseOnData = true;
|
|
9648
|
+
var streamHasEnded = false;
|
|
9649
|
+
this.pause = function() {
|
|
9650
|
+
ChunkStreamer.prototype.pause.apply(this, arguments);
|
|
9651
|
+
this._input.pause();
|
|
9652
|
+
};
|
|
9653
|
+
this.resume = function() {
|
|
9654
|
+
ChunkStreamer.prototype.resume.apply(this, arguments);
|
|
9655
|
+
this._input.resume();
|
|
9656
|
+
};
|
|
9657
|
+
this.stream = function(stream) {
|
|
9658
|
+
this._input = stream;
|
|
9659
|
+
this._input.on("data", this._streamData);
|
|
9660
|
+
this._input.on("end", this._streamEnd);
|
|
9661
|
+
this._input.on("error", this._streamError);
|
|
9662
|
+
};
|
|
9663
|
+
this._checkIsFinished = function() {
|
|
9664
|
+
if (streamHasEnded && queue.length === 1) {
|
|
9665
|
+
this._finished = true;
|
|
9666
|
+
}
|
|
9667
|
+
};
|
|
9668
|
+
this._nextChunk = function() {
|
|
9669
|
+
this._checkIsFinished();
|
|
9670
|
+
if (queue.length) {
|
|
9671
|
+
this.parseChunk(queue.shift());
|
|
9672
|
+
} else {
|
|
9673
|
+
parseOnData = true;
|
|
9674
|
+
}
|
|
9675
|
+
};
|
|
9676
|
+
this._streamData = bindFunction(function(chunk) {
|
|
9677
|
+
try {
|
|
9678
|
+
queue.push(typeof chunk === "string" ? chunk : chunk.toString(this._config.encoding));
|
|
9679
|
+
if (parseOnData) {
|
|
9680
|
+
parseOnData = false;
|
|
9681
|
+
this._checkIsFinished();
|
|
9682
|
+
this.parseChunk(queue.shift());
|
|
9683
|
+
}
|
|
9684
|
+
} catch (error) {
|
|
9685
|
+
this._streamError(error);
|
|
9686
|
+
}
|
|
9687
|
+
}, this);
|
|
9688
|
+
this._streamError = bindFunction(function(error) {
|
|
9689
|
+
this._streamCleanUp();
|
|
9690
|
+
this._sendError(error);
|
|
9691
|
+
}, this);
|
|
9692
|
+
this._streamEnd = bindFunction(function() {
|
|
9693
|
+
this._streamCleanUp();
|
|
9694
|
+
streamHasEnded = true;
|
|
9695
|
+
this._streamData("");
|
|
9696
|
+
}, this);
|
|
9697
|
+
this._streamCleanUp = bindFunction(function() {
|
|
9698
|
+
this._input.removeListener("data", this._streamData);
|
|
9699
|
+
this._input.removeListener("end", this._streamEnd);
|
|
9700
|
+
this._input.removeListener("error", this._streamError);
|
|
9701
|
+
}, this);
|
|
9702
|
+
}
|
|
9703
|
+
__name(ReadableStreamStreamer, "ReadableStreamStreamer");
|
|
9704
|
+
ReadableStreamStreamer.prototype = Object.create(ChunkStreamer.prototype);
|
|
9705
|
+
ReadableStreamStreamer.prototype.constructor = ReadableStreamStreamer;
|
|
9706
|
+
function DuplexStreamStreamer(_config) {
|
|
9707
|
+
var Duplex = require("stream").Duplex;
|
|
9708
|
+
var config = copy(_config);
|
|
9709
|
+
var parseOnWrite = true;
|
|
9710
|
+
var writeStreamHasFinished = false;
|
|
9711
|
+
var parseCallbackQueue = [];
|
|
9712
|
+
var stream = null;
|
|
9713
|
+
this._onCsvData = function(results) {
|
|
9714
|
+
var data = results.data;
|
|
9715
|
+
if (!stream.push(data) && !this._handle.paused()) {
|
|
9716
|
+
this._handle.pause();
|
|
9717
|
+
}
|
|
9718
|
+
};
|
|
9719
|
+
this._onCsvComplete = function() {
|
|
9720
|
+
stream.push(null);
|
|
9721
|
+
};
|
|
9722
|
+
config.step = bindFunction(this._onCsvData, this);
|
|
9723
|
+
config.complete = bindFunction(this._onCsvComplete, this);
|
|
9724
|
+
ChunkStreamer.call(this, config);
|
|
9725
|
+
this._nextChunk = function() {
|
|
9726
|
+
if (writeStreamHasFinished && parseCallbackQueue.length === 1) {
|
|
9727
|
+
this._finished = true;
|
|
9728
|
+
}
|
|
9729
|
+
if (parseCallbackQueue.length) {
|
|
9730
|
+
parseCallbackQueue.shift()();
|
|
9731
|
+
} else {
|
|
9732
|
+
parseOnWrite = true;
|
|
9733
|
+
}
|
|
9734
|
+
};
|
|
9735
|
+
this._addToParseQueue = function(chunk, callback) {
|
|
9736
|
+
parseCallbackQueue.push(bindFunction(function() {
|
|
9737
|
+
this.parseChunk(typeof chunk === "string" ? chunk : chunk.toString(config.encoding));
|
|
9738
|
+
if (isFunction(callback)) {
|
|
9739
|
+
return callback();
|
|
9740
|
+
}
|
|
9741
|
+
}, this));
|
|
9742
|
+
if (parseOnWrite) {
|
|
9743
|
+
parseOnWrite = false;
|
|
9744
|
+
this._nextChunk();
|
|
9745
|
+
}
|
|
9746
|
+
};
|
|
9747
|
+
this._onRead = function() {
|
|
9748
|
+
if (this._handle.paused()) {
|
|
9749
|
+
this._handle.resume();
|
|
9750
|
+
}
|
|
9751
|
+
};
|
|
9752
|
+
this._onWrite = function(chunk, encoding, callback) {
|
|
9753
|
+
this._addToParseQueue(chunk, callback);
|
|
9754
|
+
};
|
|
9755
|
+
this._onWriteComplete = function() {
|
|
9756
|
+
writeStreamHasFinished = true;
|
|
9757
|
+
this._addToParseQueue("");
|
|
9758
|
+
};
|
|
9759
|
+
this.getStream = function() {
|
|
9760
|
+
return stream;
|
|
9761
|
+
};
|
|
9762
|
+
stream = new Duplex({
|
|
9763
|
+
readableObjectMode: true,
|
|
9764
|
+
decodeStrings: false,
|
|
9765
|
+
read: bindFunction(this._onRead, this),
|
|
9766
|
+
write: bindFunction(this._onWrite, this)
|
|
9767
|
+
});
|
|
9768
|
+
stream.once("finish", bindFunction(this._onWriteComplete, this));
|
|
9769
|
+
}
|
|
9770
|
+
__name(DuplexStreamStreamer, "DuplexStreamStreamer");
|
|
9771
|
+
if (typeof PAPA_BROWSER_CONTEXT === "undefined") {
|
|
9772
|
+
DuplexStreamStreamer.prototype = Object.create(ChunkStreamer.prototype);
|
|
9773
|
+
DuplexStreamStreamer.prototype.constructor = DuplexStreamStreamer;
|
|
9774
|
+
}
|
|
9775
|
+
function ParserHandle(_config) {
|
|
9776
|
+
var MAX_FLOAT = Math.pow(2, 53);
|
|
9777
|
+
var MIN_FLOAT = -MAX_FLOAT;
|
|
9778
|
+
var FLOAT = /^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/;
|
|
9779
|
+
var ISO_DATE = /^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/;
|
|
9780
|
+
var self2 = this;
|
|
9781
|
+
var _stepCounter = 0;
|
|
9782
|
+
var _rowCounter = 0;
|
|
9783
|
+
var _input;
|
|
9784
|
+
var _parser;
|
|
9785
|
+
var _paused = false;
|
|
9786
|
+
var _aborted = false;
|
|
9787
|
+
var _delimiterError;
|
|
9788
|
+
var _fields = [];
|
|
9789
|
+
var _results = {
|
|
9790
|
+
// The last results returned from the parser
|
|
9791
|
+
data: [],
|
|
9792
|
+
errors: [],
|
|
9793
|
+
meta: {}
|
|
9794
|
+
};
|
|
9795
|
+
if (isFunction(_config.step)) {
|
|
9796
|
+
var userStep = _config.step;
|
|
9797
|
+
_config.step = function(results) {
|
|
9798
|
+
_results = results;
|
|
9799
|
+
if (needsHeaderRow())
|
|
9800
|
+
processResults();
|
|
9801
|
+
else {
|
|
9802
|
+
processResults();
|
|
9803
|
+
if (_results.data.length === 0)
|
|
9804
|
+
return;
|
|
9805
|
+
_stepCounter += results.data.length;
|
|
9806
|
+
if (_config.preview && _stepCounter > _config.preview)
|
|
9807
|
+
_parser.abort();
|
|
9808
|
+
else {
|
|
9809
|
+
_results.data = _results.data[0];
|
|
9810
|
+
userStep(_results, self2);
|
|
9811
|
+
}
|
|
9812
|
+
}
|
|
9813
|
+
};
|
|
9814
|
+
}
|
|
9815
|
+
this.parse = function(input, baseIndex, ignoreLastRow) {
|
|
9816
|
+
var quoteChar = _config.quoteChar || '"';
|
|
9817
|
+
if (!_config.newline)
|
|
9818
|
+
_config.newline = guessLineEndings(input, quoteChar);
|
|
9819
|
+
_delimiterError = false;
|
|
9820
|
+
if (!_config.delimiter) {
|
|
9821
|
+
var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments, _config.delimitersToGuess);
|
|
9822
|
+
if (delimGuess.successful)
|
|
9823
|
+
_config.delimiter = delimGuess.bestDelimiter;
|
|
9824
|
+
else {
|
|
9825
|
+
_delimiterError = true;
|
|
9826
|
+
_config.delimiter = Papa.DefaultDelimiter;
|
|
9827
|
+
}
|
|
9828
|
+
_results.meta.delimiter = _config.delimiter;
|
|
9829
|
+
} else if (isFunction(_config.delimiter)) {
|
|
9830
|
+
_config.delimiter = _config.delimiter(input);
|
|
9831
|
+
_results.meta.delimiter = _config.delimiter;
|
|
9832
|
+
}
|
|
9833
|
+
var parserConfig = copy(_config);
|
|
9834
|
+
if (_config.preview && _config.header)
|
|
9835
|
+
parserConfig.preview++;
|
|
9836
|
+
_input = input;
|
|
9837
|
+
_parser = new Parser(parserConfig);
|
|
9838
|
+
_results = _parser.parse(_input, baseIndex, ignoreLastRow);
|
|
9839
|
+
processResults();
|
|
9840
|
+
return _paused ? { meta: { paused: true } } : _results || { meta: { paused: false } };
|
|
9841
|
+
};
|
|
9842
|
+
this.paused = function() {
|
|
9843
|
+
return _paused;
|
|
9844
|
+
};
|
|
9845
|
+
this.pause = function() {
|
|
9846
|
+
_paused = true;
|
|
9847
|
+
_parser.abort();
|
|
9848
|
+
_input = isFunction(_config.chunk) ? "" : _input.substring(_parser.getCharIndex());
|
|
9849
|
+
};
|
|
9850
|
+
this.resume = function() {
|
|
9851
|
+
if (self2.streamer._halted) {
|
|
9852
|
+
_paused = false;
|
|
9853
|
+
self2.streamer.parseChunk(_input, true);
|
|
9854
|
+
} else {
|
|
9855
|
+
setTimeout(self2.resume, 3);
|
|
9856
|
+
}
|
|
9857
|
+
};
|
|
9858
|
+
this.aborted = function() {
|
|
9859
|
+
return _aborted;
|
|
9860
|
+
};
|
|
9861
|
+
this.abort = function() {
|
|
9862
|
+
_aborted = true;
|
|
9863
|
+
_parser.abort();
|
|
9864
|
+
_results.meta.aborted = true;
|
|
9865
|
+
if (isFunction(_config.complete))
|
|
9866
|
+
_config.complete(_results);
|
|
9867
|
+
_input = "";
|
|
9868
|
+
};
|
|
9869
|
+
function testEmptyLine(s) {
|
|
9870
|
+
return _config.skipEmptyLines === "greedy" ? s.join("").trim() === "" : s.length === 1 && s[0].length === 0;
|
|
9871
|
+
}
|
|
9872
|
+
__name(testEmptyLine, "testEmptyLine");
|
|
9873
|
+
function testFloat(s) {
|
|
9874
|
+
if (FLOAT.test(s)) {
|
|
9875
|
+
var floatValue = parseFloat(s);
|
|
9876
|
+
if (floatValue > MIN_FLOAT && floatValue < MAX_FLOAT) {
|
|
9877
|
+
return true;
|
|
9878
|
+
}
|
|
9879
|
+
}
|
|
9880
|
+
return false;
|
|
9881
|
+
}
|
|
9882
|
+
__name(testFloat, "testFloat");
|
|
9883
|
+
function processResults() {
|
|
9884
|
+
if (_results && _delimiterError) {
|
|
9885
|
+
addError("Delimiter", "UndetectableDelimiter", "Unable to auto-detect delimiting character; defaulted to '" + Papa.DefaultDelimiter + "'");
|
|
9886
|
+
_delimiterError = false;
|
|
9887
|
+
}
|
|
9888
|
+
if (_config.skipEmptyLines) {
|
|
9889
|
+
_results.data = _results.data.filter(function(d) {
|
|
9890
|
+
return !testEmptyLine(d);
|
|
9891
|
+
});
|
|
9892
|
+
}
|
|
9893
|
+
if (needsHeaderRow())
|
|
9894
|
+
fillHeaderFields();
|
|
9895
|
+
return applyHeaderAndDynamicTypingAndTransformation();
|
|
9896
|
+
}
|
|
9897
|
+
__name(processResults, "processResults");
|
|
9898
|
+
function needsHeaderRow() {
|
|
9899
|
+
return _config.header && _fields.length === 0;
|
|
9900
|
+
}
|
|
9901
|
+
__name(needsHeaderRow, "needsHeaderRow");
|
|
9902
|
+
function fillHeaderFields() {
|
|
9903
|
+
if (!_results)
|
|
9904
|
+
return;
|
|
9905
|
+
function addHeader(header, i2) {
|
|
9906
|
+
if (isFunction(_config.transformHeader))
|
|
9907
|
+
header = _config.transformHeader(header, i2);
|
|
9908
|
+
_fields.push(header);
|
|
9909
|
+
}
|
|
9910
|
+
__name(addHeader, "addHeader");
|
|
9911
|
+
if (Array.isArray(_results.data[0])) {
|
|
9912
|
+
for (var i = 0; needsHeaderRow() && i < _results.data.length; i++)
|
|
9913
|
+
_results.data[i].forEach(addHeader);
|
|
9914
|
+
_results.data.splice(0, 1);
|
|
9915
|
+
} else
|
|
9916
|
+
_results.data.forEach(addHeader);
|
|
9917
|
+
}
|
|
9918
|
+
__name(fillHeaderFields, "fillHeaderFields");
|
|
9919
|
+
function shouldApplyDynamicTyping(field) {
|
|
9920
|
+
if (_config.dynamicTypingFunction && _config.dynamicTyping[field] === void 0) {
|
|
9921
|
+
_config.dynamicTyping[field] = _config.dynamicTypingFunction(field);
|
|
9922
|
+
}
|
|
9923
|
+
return (_config.dynamicTyping[field] || _config.dynamicTyping) === true;
|
|
9924
|
+
}
|
|
9925
|
+
__name(shouldApplyDynamicTyping, "shouldApplyDynamicTyping");
|
|
9926
|
+
function parseDynamic(field, value) {
|
|
9927
|
+
if (shouldApplyDynamicTyping(field)) {
|
|
9928
|
+
if (value === "true" || value === "TRUE")
|
|
9929
|
+
return true;
|
|
9930
|
+
else if (value === "false" || value === "FALSE")
|
|
9931
|
+
return false;
|
|
9932
|
+
else if (testFloat(value))
|
|
9933
|
+
return parseFloat(value);
|
|
9934
|
+
else if (ISO_DATE.test(value))
|
|
9935
|
+
return new Date(value);
|
|
9936
|
+
else
|
|
9937
|
+
return value === "" ? null : value;
|
|
9938
|
+
}
|
|
9939
|
+
return value;
|
|
9940
|
+
}
|
|
9941
|
+
__name(parseDynamic, "parseDynamic");
|
|
9942
|
+
function applyHeaderAndDynamicTypingAndTransformation() {
|
|
9943
|
+
if (!_results || !_config.header && !_config.dynamicTyping && !_config.transform)
|
|
9944
|
+
return _results;
|
|
9945
|
+
function processRow(rowSource, i) {
|
|
9946
|
+
var row = _config.header ? {} : [];
|
|
9947
|
+
var j;
|
|
9948
|
+
for (j = 0; j < rowSource.length; j++) {
|
|
9949
|
+
var field = j;
|
|
9950
|
+
var value = rowSource[j];
|
|
9951
|
+
if (_config.header)
|
|
9952
|
+
field = j >= _fields.length ? "__parsed_extra" : _fields[j];
|
|
9953
|
+
if (_config.transform)
|
|
9954
|
+
value = _config.transform(value, field);
|
|
9955
|
+
value = parseDynamic(field, value);
|
|
9956
|
+
if (field === "__parsed_extra") {
|
|
9957
|
+
row[field] = row[field] || [];
|
|
9958
|
+
row[field].push(value);
|
|
9959
|
+
} else
|
|
9960
|
+
row[field] = value;
|
|
9961
|
+
}
|
|
9962
|
+
if (_config.header) {
|
|
9963
|
+
if (j > _fields.length)
|
|
9964
|
+
addError("FieldMismatch", "TooManyFields", "Too many fields: expected " + _fields.length + " fields but parsed " + j, _rowCounter + i);
|
|
9965
|
+
else if (j < _fields.length)
|
|
9966
|
+
addError("FieldMismatch", "TooFewFields", "Too few fields: expected " + _fields.length + " fields but parsed " + j, _rowCounter + i);
|
|
9967
|
+
}
|
|
9968
|
+
return row;
|
|
9969
|
+
}
|
|
9970
|
+
__name(processRow, "processRow");
|
|
9971
|
+
var incrementBy = 1;
|
|
9972
|
+
if (!_results.data.length || Array.isArray(_results.data[0])) {
|
|
9973
|
+
_results.data = _results.data.map(processRow);
|
|
9974
|
+
incrementBy = _results.data.length;
|
|
9975
|
+
} else
|
|
9976
|
+
_results.data = processRow(_results.data, 0);
|
|
9977
|
+
if (_config.header && _results.meta)
|
|
9978
|
+
_results.meta.fields = _fields;
|
|
9979
|
+
_rowCounter += incrementBy;
|
|
9980
|
+
return _results;
|
|
9981
|
+
}
|
|
9982
|
+
__name(applyHeaderAndDynamicTypingAndTransformation, "applyHeaderAndDynamicTypingAndTransformation");
|
|
9983
|
+
function guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess) {
|
|
9984
|
+
var bestDelim, bestDelta, fieldCountPrevRow, maxFieldCount;
|
|
9985
|
+
delimitersToGuess = delimitersToGuess || [",", " ", "|", ";", Papa.RECORD_SEP, Papa.UNIT_SEP];
|
|
9986
|
+
for (var i = 0; i < delimitersToGuess.length; i++) {
|
|
9987
|
+
var delim = delimitersToGuess[i];
|
|
9988
|
+
var delta = 0, avgFieldCount = 0, emptyLinesCount = 0;
|
|
9989
|
+
fieldCountPrevRow = void 0;
|
|
9990
|
+
var preview = new Parser({
|
|
9991
|
+
comments,
|
|
9992
|
+
delimiter: delim,
|
|
9993
|
+
newline,
|
|
9994
|
+
preview: 10
|
|
9995
|
+
}).parse(input);
|
|
9996
|
+
for (var j = 0; j < preview.data.length; j++) {
|
|
9997
|
+
if (skipEmptyLines && testEmptyLine(preview.data[j])) {
|
|
9998
|
+
emptyLinesCount++;
|
|
9999
|
+
continue;
|
|
10000
|
+
}
|
|
10001
|
+
var fieldCount = preview.data[j].length;
|
|
10002
|
+
avgFieldCount += fieldCount;
|
|
10003
|
+
if (typeof fieldCountPrevRow === "undefined") {
|
|
10004
|
+
fieldCountPrevRow = fieldCount;
|
|
10005
|
+
continue;
|
|
10006
|
+
} else if (fieldCount > 0) {
|
|
10007
|
+
delta += Math.abs(fieldCount - fieldCountPrevRow);
|
|
10008
|
+
fieldCountPrevRow = fieldCount;
|
|
10009
|
+
}
|
|
10010
|
+
}
|
|
10011
|
+
if (preview.data.length > 0)
|
|
10012
|
+
avgFieldCount /= preview.data.length - emptyLinesCount;
|
|
10013
|
+
if ((typeof bestDelta === "undefined" || delta <= bestDelta) && (typeof maxFieldCount === "undefined" || avgFieldCount > maxFieldCount) && avgFieldCount > 1.99) {
|
|
10014
|
+
bestDelta = delta;
|
|
10015
|
+
bestDelim = delim;
|
|
10016
|
+
maxFieldCount = avgFieldCount;
|
|
10017
|
+
}
|
|
10018
|
+
}
|
|
10019
|
+
_config.delimiter = bestDelim;
|
|
10020
|
+
return {
|
|
10021
|
+
successful: !!bestDelim,
|
|
10022
|
+
bestDelimiter: bestDelim
|
|
10023
|
+
};
|
|
10024
|
+
}
|
|
10025
|
+
__name(guessDelimiter, "guessDelimiter");
|
|
10026
|
+
function guessLineEndings(input, quoteChar) {
|
|
10027
|
+
input = input.substring(0, 1024 * 1024);
|
|
10028
|
+
var re = new RegExp(escapeRegExp(quoteChar) + "([^]*?)" + escapeRegExp(quoteChar), "gm");
|
|
10029
|
+
input = input.replace(re, "");
|
|
10030
|
+
var r = input.split("\r");
|
|
10031
|
+
var n = input.split("\n");
|
|
10032
|
+
var nAppearsFirst = n.length > 1 && n[0].length < r[0].length;
|
|
10033
|
+
if (r.length === 1 || nAppearsFirst)
|
|
10034
|
+
return "\n";
|
|
10035
|
+
var numWithN = 0;
|
|
10036
|
+
for (var i = 0; i < r.length; i++) {
|
|
10037
|
+
if (r[i][0] === "\n")
|
|
10038
|
+
numWithN++;
|
|
10039
|
+
}
|
|
10040
|
+
return numWithN >= r.length / 2 ? "\r\n" : "\r";
|
|
10041
|
+
}
|
|
10042
|
+
__name(guessLineEndings, "guessLineEndings");
|
|
10043
|
+
function addError(type, code, msg, row) {
|
|
10044
|
+
var error = {
|
|
10045
|
+
type,
|
|
10046
|
+
code,
|
|
10047
|
+
message: msg
|
|
10048
|
+
};
|
|
10049
|
+
if (row !== void 0) {
|
|
10050
|
+
error.row = row;
|
|
10051
|
+
}
|
|
10052
|
+
_results.errors.push(error);
|
|
10053
|
+
}
|
|
10054
|
+
__name(addError, "addError");
|
|
10055
|
+
}
|
|
10056
|
+
__name(ParserHandle, "ParserHandle");
|
|
10057
|
+
function escapeRegExp(string) {
|
|
10058
|
+
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
10059
|
+
}
|
|
10060
|
+
__name(escapeRegExp, "escapeRegExp");
|
|
10061
|
+
function Parser(config) {
|
|
10062
|
+
config = config || {};
|
|
10063
|
+
var delim = config.delimiter;
|
|
10064
|
+
var newline = config.newline;
|
|
10065
|
+
var comments = config.comments;
|
|
10066
|
+
var step = config.step;
|
|
10067
|
+
var preview = config.preview;
|
|
10068
|
+
var fastMode = config.fastMode;
|
|
10069
|
+
var quoteChar;
|
|
10070
|
+
if (config.quoteChar === void 0 || config.quoteChar === null) {
|
|
10071
|
+
quoteChar = '"';
|
|
10072
|
+
} else {
|
|
10073
|
+
quoteChar = config.quoteChar;
|
|
10074
|
+
}
|
|
10075
|
+
var escapeChar = quoteChar;
|
|
10076
|
+
if (config.escapeChar !== void 0) {
|
|
10077
|
+
escapeChar = config.escapeChar;
|
|
10078
|
+
}
|
|
10079
|
+
if (typeof delim !== "string" || Papa.BAD_DELIMITERS.indexOf(delim) > -1)
|
|
10080
|
+
delim = ",";
|
|
10081
|
+
if (comments === delim)
|
|
10082
|
+
throw new Error("Comment character same as delimiter");
|
|
10083
|
+
else if (comments === true)
|
|
10084
|
+
comments = "#";
|
|
10085
|
+
else if (typeof comments !== "string" || Papa.BAD_DELIMITERS.indexOf(comments) > -1)
|
|
10086
|
+
comments = false;
|
|
10087
|
+
if (newline !== "\n" && newline !== "\r" && newline !== "\r\n")
|
|
10088
|
+
newline = "\n";
|
|
10089
|
+
var cursor = 0;
|
|
10090
|
+
var aborted = false;
|
|
10091
|
+
this.parse = function(input, baseIndex, ignoreLastRow) {
|
|
10092
|
+
if (typeof input !== "string")
|
|
10093
|
+
throw new Error("Input must be a string");
|
|
10094
|
+
var inputLen = input.length, delimLen = delim.length, newlineLen = newline.length, commentsLen = comments.length;
|
|
10095
|
+
var stepIsFunction = isFunction(step);
|
|
10096
|
+
cursor = 0;
|
|
10097
|
+
var data = [], errors = [], row = [], lastCursor = 0;
|
|
10098
|
+
if (!input)
|
|
10099
|
+
return returnable();
|
|
10100
|
+
if (config.header && !baseIndex) {
|
|
10101
|
+
var firstLine = input.split(newline)[0];
|
|
10102
|
+
var headers = firstLine.split(delim);
|
|
10103
|
+
var separator = "_";
|
|
10104
|
+
var headerMap = [];
|
|
10105
|
+
var headerCount = {};
|
|
10106
|
+
var duplicateHeaders = false;
|
|
10107
|
+
for (var j in headers) {
|
|
10108
|
+
var header = headers[j];
|
|
10109
|
+
if (isFunction(config.transformHeader))
|
|
10110
|
+
header = config.transformHeader(header, j);
|
|
10111
|
+
var headerName = header;
|
|
10112
|
+
var count = headerCount[header] || 0;
|
|
10113
|
+
if (count > 0) {
|
|
10114
|
+
duplicateHeaders = true;
|
|
10115
|
+
headerName = header + separator + count;
|
|
10116
|
+
}
|
|
10117
|
+
headerCount[header] = count + 1;
|
|
10118
|
+
while (headerMap.includes(headerName)) {
|
|
10119
|
+
headerName = headerName + separator + count;
|
|
10120
|
+
}
|
|
10121
|
+
headerMap.push(headerName);
|
|
10122
|
+
}
|
|
10123
|
+
if (duplicateHeaders) {
|
|
10124
|
+
var editedInput = input.split(newline);
|
|
10125
|
+
editedInput[0] = headerMap.join(delim);
|
|
10126
|
+
input = editedInput.join(newline);
|
|
10127
|
+
}
|
|
10128
|
+
}
|
|
10129
|
+
if (fastMode || fastMode !== false && input.indexOf(quoteChar) === -1) {
|
|
10130
|
+
var rows = input.split(newline);
|
|
10131
|
+
for (var i = 0; i < rows.length; i++) {
|
|
10132
|
+
row = rows[i];
|
|
10133
|
+
cursor += row.length;
|
|
10134
|
+
if (i !== rows.length - 1)
|
|
10135
|
+
cursor += newline.length;
|
|
10136
|
+
else if (ignoreLastRow)
|
|
10137
|
+
return returnable();
|
|
10138
|
+
if (comments && row.substring(0, commentsLen) === comments)
|
|
10139
|
+
continue;
|
|
10140
|
+
if (stepIsFunction) {
|
|
10141
|
+
data = [];
|
|
10142
|
+
pushRow(row.split(delim));
|
|
10143
|
+
doStep();
|
|
10144
|
+
if (aborted)
|
|
10145
|
+
return returnable();
|
|
10146
|
+
} else
|
|
10147
|
+
pushRow(row.split(delim));
|
|
10148
|
+
if (preview && i >= preview) {
|
|
10149
|
+
data = data.slice(0, preview);
|
|
10150
|
+
return returnable(true);
|
|
10151
|
+
}
|
|
10152
|
+
}
|
|
10153
|
+
return returnable();
|
|
10154
|
+
}
|
|
10155
|
+
var nextDelim = input.indexOf(delim, cursor);
|
|
10156
|
+
var nextNewline = input.indexOf(newline, cursor);
|
|
10157
|
+
var quoteCharRegex = new RegExp(escapeRegExp(escapeChar) + escapeRegExp(quoteChar), "g");
|
|
10158
|
+
var quoteSearch = input.indexOf(quoteChar, cursor);
|
|
10159
|
+
for (; ; ) {
|
|
10160
|
+
if (input[cursor] === quoteChar) {
|
|
10161
|
+
quoteSearch = cursor;
|
|
10162
|
+
cursor++;
|
|
10163
|
+
for (; ; ) {
|
|
10164
|
+
quoteSearch = input.indexOf(quoteChar, quoteSearch + 1);
|
|
10165
|
+
if (quoteSearch === -1) {
|
|
10166
|
+
if (!ignoreLastRow) {
|
|
10167
|
+
errors.push({
|
|
10168
|
+
type: "Quotes",
|
|
10169
|
+
code: "MissingQuotes",
|
|
10170
|
+
message: "Quoted field unterminated",
|
|
10171
|
+
row: data.length,
|
|
10172
|
+
// row has yet to be inserted
|
|
10173
|
+
index: cursor
|
|
10174
|
+
});
|
|
10175
|
+
}
|
|
10176
|
+
return finish();
|
|
10177
|
+
}
|
|
10178
|
+
if (quoteSearch === inputLen - 1) {
|
|
10179
|
+
var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar);
|
|
10180
|
+
return finish(value);
|
|
10181
|
+
}
|
|
10182
|
+
if (quoteChar === escapeChar && input[quoteSearch + 1] === escapeChar) {
|
|
10183
|
+
quoteSearch++;
|
|
10184
|
+
continue;
|
|
10185
|
+
}
|
|
10186
|
+
if (quoteChar !== escapeChar && quoteSearch !== 0 && input[quoteSearch - 1] === escapeChar) {
|
|
10187
|
+
continue;
|
|
10188
|
+
}
|
|
10189
|
+
if (nextDelim !== -1 && nextDelim < quoteSearch + 1) {
|
|
10190
|
+
nextDelim = input.indexOf(delim, quoteSearch + 1);
|
|
10191
|
+
}
|
|
10192
|
+
if (nextNewline !== -1 && nextNewline < quoteSearch + 1) {
|
|
10193
|
+
nextNewline = input.indexOf(newline, quoteSearch + 1);
|
|
10194
|
+
}
|
|
10195
|
+
var checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline);
|
|
10196
|
+
var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo);
|
|
10197
|
+
if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndDelimiter, delimLen) === delim) {
|
|
10198
|
+
row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));
|
|
10199
|
+
cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen;
|
|
10200
|
+
if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen] !== quoteChar) {
|
|
10201
|
+
quoteSearch = input.indexOf(quoteChar, cursor);
|
|
10202
|
+
}
|
|
10203
|
+
nextDelim = input.indexOf(delim, cursor);
|
|
10204
|
+
nextNewline = input.indexOf(newline, cursor);
|
|
10205
|
+
break;
|
|
10206
|
+
}
|
|
10207
|
+
var spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline);
|
|
10208
|
+
if (input.substring(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen) === newline) {
|
|
10209
|
+
row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));
|
|
10210
|
+
saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen);
|
|
10211
|
+
nextDelim = input.indexOf(delim, cursor);
|
|
10212
|
+
quoteSearch = input.indexOf(quoteChar, cursor);
|
|
10213
|
+
if (stepIsFunction) {
|
|
10214
|
+
doStep();
|
|
10215
|
+
if (aborted)
|
|
10216
|
+
return returnable();
|
|
10217
|
+
}
|
|
10218
|
+
if (preview && data.length >= preview)
|
|
10219
|
+
return returnable(true);
|
|
10220
|
+
break;
|
|
10221
|
+
}
|
|
10222
|
+
errors.push({
|
|
10223
|
+
type: "Quotes",
|
|
10224
|
+
code: "InvalidQuotes",
|
|
10225
|
+
message: "Trailing quote on quoted field is malformed",
|
|
10226
|
+
row: data.length,
|
|
10227
|
+
// row has yet to be inserted
|
|
10228
|
+
index: cursor
|
|
10229
|
+
});
|
|
10230
|
+
quoteSearch++;
|
|
10231
|
+
continue;
|
|
10232
|
+
}
|
|
10233
|
+
continue;
|
|
10234
|
+
}
|
|
10235
|
+
if (comments && row.length === 0 && input.substring(cursor, cursor + commentsLen) === comments) {
|
|
10236
|
+
if (nextNewline === -1)
|
|
10237
|
+
return returnable();
|
|
10238
|
+
cursor = nextNewline + newlineLen;
|
|
10239
|
+
nextNewline = input.indexOf(newline, cursor);
|
|
10240
|
+
nextDelim = input.indexOf(delim, cursor);
|
|
10241
|
+
continue;
|
|
10242
|
+
}
|
|
10243
|
+
if (nextDelim !== -1 && (nextDelim < nextNewline || nextNewline === -1)) {
|
|
10244
|
+
row.push(input.substring(cursor, nextDelim));
|
|
10245
|
+
cursor = nextDelim + delimLen;
|
|
10246
|
+
nextDelim = input.indexOf(delim, cursor);
|
|
10247
|
+
continue;
|
|
10248
|
+
}
|
|
10249
|
+
if (nextNewline !== -1) {
|
|
10250
|
+
row.push(input.substring(cursor, nextNewline));
|
|
10251
|
+
saveRow(nextNewline + newlineLen);
|
|
10252
|
+
if (stepIsFunction) {
|
|
10253
|
+
doStep();
|
|
10254
|
+
if (aborted)
|
|
10255
|
+
return returnable();
|
|
10256
|
+
}
|
|
10257
|
+
if (preview && data.length >= preview)
|
|
10258
|
+
return returnable(true);
|
|
10259
|
+
continue;
|
|
10260
|
+
}
|
|
10261
|
+
break;
|
|
10262
|
+
}
|
|
10263
|
+
return finish();
|
|
10264
|
+
function pushRow(row2) {
|
|
10265
|
+
data.push(row2);
|
|
10266
|
+
lastCursor = cursor;
|
|
10267
|
+
}
|
|
10268
|
+
__name(pushRow, "pushRow");
|
|
10269
|
+
function extraSpaces(index) {
|
|
10270
|
+
var spaceLength = 0;
|
|
10271
|
+
if (index !== -1) {
|
|
10272
|
+
var textBetweenClosingQuoteAndIndex = input.substring(quoteSearch + 1, index);
|
|
10273
|
+
if (textBetweenClosingQuoteAndIndex && textBetweenClosingQuoteAndIndex.trim() === "") {
|
|
10274
|
+
spaceLength = textBetweenClosingQuoteAndIndex.length;
|
|
10275
|
+
}
|
|
10276
|
+
}
|
|
10277
|
+
return spaceLength;
|
|
10278
|
+
}
|
|
10279
|
+
__name(extraSpaces, "extraSpaces");
|
|
10280
|
+
function finish(value2) {
|
|
10281
|
+
if (ignoreLastRow)
|
|
10282
|
+
return returnable();
|
|
10283
|
+
if (typeof value2 === "undefined")
|
|
10284
|
+
value2 = input.substring(cursor);
|
|
10285
|
+
row.push(value2);
|
|
10286
|
+
cursor = inputLen;
|
|
10287
|
+
pushRow(row);
|
|
10288
|
+
if (stepIsFunction)
|
|
10289
|
+
doStep();
|
|
10290
|
+
return returnable();
|
|
10291
|
+
}
|
|
10292
|
+
__name(finish, "finish");
|
|
10293
|
+
function saveRow(newCursor) {
|
|
10294
|
+
cursor = newCursor;
|
|
10295
|
+
pushRow(row);
|
|
10296
|
+
row = [];
|
|
10297
|
+
nextNewline = input.indexOf(newline, cursor);
|
|
10298
|
+
}
|
|
10299
|
+
__name(saveRow, "saveRow");
|
|
10300
|
+
function returnable(stopped) {
|
|
10301
|
+
return {
|
|
10302
|
+
data,
|
|
10303
|
+
errors,
|
|
10304
|
+
meta: {
|
|
10305
|
+
delimiter: delim,
|
|
10306
|
+
linebreak: newline,
|
|
10307
|
+
aborted,
|
|
10308
|
+
truncated: !!stopped,
|
|
10309
|
+
cursor: lastCursor + (baseIndex || 0)
|
|
10310
|
+
}
|
|
10311
|
+
};
|
|
10312
|
+
}
|
|
10313
|
+
__name(returnable, "returnable");
|
|
10314
|
+
function doStep() {
|
|
10315
|
+
step(returnable());
|
|
10316
|
+
data = [];
|
|
10317
|
+
errors = [];
|
|
10318
|
+
}
|
|
10319
|
+
__name(doStep, "doStep");
|
|
10320
|
+
};
|
|
10321
|
+
this.abort = function() {
|
|
10322
|
+
aborted = true;
|
|
10323
|
+
};
|
|
10324
|
+
this.getCharIndex = function() {
|
|
10325
|
+
return cursor;
|
|
10326
|
+
};
|
|
10327
|
+
}
|
|
10328
|
+
__name(Parser, "Parser");
|
|
10329
|
+
function newWorker() {
|
|
10330
|
+
if (!Papa.WORKERS_SUPPORTED)
|
|
10331
|
+
return false;
|
|
10332
|
+
var workerUrl = getWorkerBlob();
|
|
10333
|
+
var w = new global.Worker(workerUrl);
|
|
10334
|
+
w.onmessage = mainThreadReceivedMessage;
|
|
10335
|
+
w.id = workerIdCounter++;
|
|
10336
|
+
workers[w.id] = w;
|
|
10337
|
+
return w;
|
|
10338
|
+
}
|
|
10339
|
+
__name(newWorker, "newWorker");
|
|
10340
|
+
function mainThreadReceivedMessage(e) {
|
|
10341
|
+
var msg = e.data;
|
|
10342
|
+
var worker = workers[msg.workerId];
|
|
10343
|
+
var aborted = false;
|
|
10344
|
+
if (msg.error)
|
|
10345
|
+
worker.userError(msg.error, msg.file);
|
|
10346
|
+
else if (msg.results && msg.results.data) {
|
|
10347
|
+
var abort = /* @__PURE__ */ __name(function() {
|
|
10348
|
+
aborted = true;
|
|
10349
|
+
completeWorker(msg.workerId, { data: [], errors: [], meta: { aborted: true } });
|
|
10350
|
+
}, "abort");
|
|
10351
|
+
var handle = {
|
|
10352
|
+
abort,
|
|
10353
|
+
pause: notImplemented,
|
|
10354
|
+
resume: notImplemented
|
|
10355
|
+
};
|
|
10356
|
+
if (isFunction(worker.userStep)) {
|
|
10357
|
+
for (var i = 0; i < msg.results.data.length; i++) {
|
|
10358
|
+
worker.userStep({
|
|
10359
|
+
data: msg.results.data[i],
|
|
10360
|
+
errors: msg.results.errors,
|
|
10361
|
+
meta: msg.results.meta
|
|
10362
|
+
}, handle);
|
|
10363
|
+
if (aborted)
|
|
10364
|
+
break;
|
|
10365
|
+
}
|
|
10366
|
+
delete msg.results;
|
|
10367
|
+
} else if (isFunction(worker.userChunk)) {
|
|
10368
|
+
worker.userChunk(msg.results, handle, msg.file);
|
|
10369
|
+
delete msg.results;
|
|
10370
|
+
}
|
|
10371
|
+
}
|
|
10372
|
+
if (msg.finished && !aborted)
|
|
10373
|
+
completeWorker(msg.workerId, msg.results);
|
|
10374
|
+
}
|
|
10375
|
+
__name(mainThreadReceivedMessage, "mainThreadReceivedMessage");
|
|
10376
|
+
function completeWorker(workerId, results) {
|
|
10377
|
+
var worker = workers[workerId];
|
|
10378
|
+
if (isFunction(worker.userComplete))
|
|
10379
|
+
worker.userComplete(results);
|
|
10380
|
+
worker.terminate();
|
|
10381
|
+
delete workers[workerId];
|
|
10382
|
+
}
|
|
10383
|
+
__name(completeWorker, "completeWorker");
|
|
10384
|
+
function notImplemented() {
|
|
10385
|
+
throw new Error("Not implemented.");
|
|
10386
|
+
}
|
|
10387
|
+
__name(notImplemented, "notImplemented");
|
|
10388
|
+
function workerThreadReceivedMessage(e) {
|
|
10389
|
+
var msg = e.data;
|
|
10390
|
+
if (typeof Papa.WORKER_ID === "undefined" && msg)
|
|
10391
|
+
Papa.WORKER_ID = msg.workerId;
|
|
10392
|
+
if (typeof msg.input === "string") {
|
|
10393
|
+
global.postMessage({
|
|
10394
|
+
workerId: Papa.WORKER_ID,
|
|
10395
|
+
results: Papa.parse(msg.input, msg.config),
|
|
10396
|
+
finished: true
|
|
10397
|
+
});
|
|
10398
|
+
} else if (global.File && msg.input instanceof File || msg.input instanceof Object) {
|
|
10399
|
+
var results = Papa.parse(msg.input, msg.config);
|
|
10400
|
+
if (results)
|
|
10401
|
+
global.postMessage({
|
|
10402
|
+
workerId: Papa.WORKER_ID,
|
|
10403
|
+
results,
|
|
10404
|
+
finished: true
|
|
10405
|
+
});
|
|
10406
|
+
}
|
|
10407
|
+
}
|
|
10408
|
+
__name(workerThreadReceivedMessage, "workerThreadReceivedMessage");
|
|
10409
|
+
function copy(obj) {
|
|
10410
|
+
if (typeof obj !== "object" || obj === null)
|
|
10411
|
+
return obj;
|
|
10412
|
+
var cpy = Array.isArray(obj) ? [] : {};
|
|
10413
|
+
for (var key in obj)
|
|
10414
|
+
cpy[key] = copy(obj[key]);
|
|
10415
|
+
return cpy;
|
|
10416
|
+
}
|
|
10417
|
+
__name(copy, "copy");
|
|
10418
|
+
function bindFunction(f, self2) {
|
|
10419
|
+
return function() {
|
|
10420
|
+
f.apply(self2, arguments);
|
|
10421
|
+
};
|
|
10422
|
+
}
|
|
10423
|
+
__name(bindFunction, "bindFunction");
|
|
10424
|
+
function isFunction(func) {
|
|
10425
|
+
return typeof func === "function";
|
|
10426
|
+
}
|
|
10427
|
+
__name(isFunction, "isFunction");
|
|
10428
|
+
return Papa;
|
|
10429
|
+
}, "moduleFactory"));
|
|
9070
10430
|
}
|
|
9071
10431
|
});
|
|
9072
10432
|
|
|
@@ -9081,8 +10441,14 @@ var require_convention_builder = __commonJS({
|
|
|
9081
10441
|
bury,
|
|
9082
10442
|
listAttributesAndSubAttributes,
|
|
9083
10443
|
findAllObjectAttributesInTree,
|
|
9084
|
-
findAllNonObjectAttributesInTree
|
|
10444
|
+
findAllNonObjectAttributesInTree,
|
|
10445
|
+
gitlabOverlayURLTemplate,
|
|
10446
|
+
gitlabConventionURLTemplate,
|
|
10447
|
+
basicSchemaURLTemplate,
|
|
10448
|
+
arrayToStructuredConvention,
|
|
10449
|
+
flattenJSONSchema
|
|
9085
10450
|
} = require_schema_utilities();
|
|
10451
|
+
var papa = require_papaparse();
|
|
9086
10452
|
function getAttributes(root, convention) {
|
|
9087
10453
|
let branches = convention.relationships.filter((rel) => rel.containerEntity == root).map((d) => d.mentionedEntity);
|
|
9088
10454
|
let output = {};
|
|
@@ -9096,21 +10462,24 @@ var require_convention_builder = __commonJS({
|
|
|
9096
10462
|
name,
|
|
9097
10463
|
validExamples,
|
|
9098
10464
|
erroredExamples,
|
|
9099
|
-
|
|
10465
|
+
repoURLTemplateFunction = gitlabOverlayURLTemplate,
|
|
9100
10466
|
storageFolder = `${__dirname}/../../../../output/collection/overlays`,
|
|
9101
10467
|
strictEnums = true,
|
|
9102
10468
|
overlay = false,
|
|
9103
10469
|
originalSrcScript,
|
|
9104
10470
|
trivial = false,
|
|
9105
10471
|
baseSchemataFolder = `${__dirname}/../../../../input/collection`,
|
|
10472
|
+
baseSchema,
|
|
10473
|
+
gitlabProjectId = process.env.CI_PROJECT_ID,
|
|
9106
10474
|
requiredFields = [],
|
|
9107
|
-
modifiedAttributes
|
|
10475
|
+
modifiedAttributes,
|
|
10476
|
+
deletedAttributes = []
|
|
9108
10477
|
}) {
|
|
9109
10478
|
this.name = name;
|
|
9110
10479
|
this.typeAndBundle = typeAndBundle;
|
|
9111
10480
|
this.validExamples = validExamples;
|
|
9112
10481
|
this.erroredExamples = erroredExamples;
|
|
9113
|
-
this.
|
|
10482
|
+
this.repoURLTemplateFunction = repoURLTemplateFunction;
|
|
9114
10483
|
this.schemaName = `${typeAndBundle}--${name}`;
|
|
9115
10484
|
this.storageFolder = storageFolder;
|
|
9116
10485
|
this.strictEnums = strictEnums;
|
|
@@ -9124,15 +10493,17 @@ var require_convention_builder = __commonJS({
|
|
|
9124
10493
|
;
|
|
9125
10494
|
this.type = typeAndBundle.split("--")[0];
|
|
9126
10495
|
this.bundle = typeAndBundle.split("--")[1];
|
|
9127
|
-
|
|
9128
|
-
|
|
9129
|
-
|
|
9130
|
-
|
|
9131
|
-
|
|
9132
|
-
|
|
9133
|
-
|
|
9134
|
-
|
|
9135
|
-
|
|
10496
|
+
if (!baseSchema) {
|
|
10497
|
+
try {
|
|
10498
|
+
baseSchema = JSON.parse(fs.readFileSync(`${baseSchemataFolder}/${this.type}/${this.bundle}/schema.json`));
|
|
10499
|
+
delete baseSchema.schema;
|
|
10500
|
+
} catch (e) {
|
|
10501
|
+
if (e.code == "ENOENT") {
|
|
10502
|
+
throw new Error(`No schema for ${typeAndBundle} was found. This might be due to the schema not existing in the farm, or the current "input/collection" folder structure being out of date. You can update it using the "getAllSchemas.js" script, in the script folder. Search path was: ${baseSchemataFolder}/${this.type}/${this.bundle}/`);
|
|
10503
|
+
} else {
|
|
10504
|
+
throw e;
|
|
10505
|
+
}
|
|
10506
|
+
;
|
|
9136
10507
|
}
|
|
9137
10508
|
;
|
|
9138
10509
|
}
|
|
@@ -9140,6 +10511,7 @@ var require_convention_builder = __commonJS({
|
|
|
9140
10511
|
this.baseSchema = baseSchema;
|
|
9141
10512
|
this.unmodifiedAttributes = listAttributesAndSubAttributes(this.baseSchema);
|
|
9142
10513
|
this.modifiedAttributes = modifiedAttributes ? modifiedAttributes : /* @__PURE__ */ new Set([]);
|
|
10514
|
+
this.deletedAttributes = deletedAttributes;
|
|
9143
10515
|
if (!overlay) {
|
|
9144
10516
|
this.overlay = {
|
|
9145
10517
|
properties: {
|
|
@@ -9171,8 +10543,8 @@ var require_convention_builder = __commonJS({
|
|
|
9171
10543
|
*/
|
|
9172
10544
|
updateSchema() {
|
|
9173
10545
|
this.schema = {};
|
|
9174
|
-
this.schema.$id = `${this.repoURL}/schemata/${this.typeAndBundle.replace("--", "/")}/schema.json`;
|
|
9175
10546
|
Object.assign(this.schema, this.baseSchema);
|
|
10547
|
+
this.schema.$id = this.repoURLTemplateFunction({ overlayName: this.name, projectID: this.gitlabProjectId, baseEntityTypeAndBundle: this.typeAndBundle });
|
|
9176
10548
|
let allModifiedAttributes = listAttributesAndSubAttributes(this.overlay);
|
|
9177
10549
|
this.modifiedAttributes = allModifiedAttributes;
|
|
9178
10550
|
this.unmodifiedAttributes = listAttributesAndSubAttributes(this.baseSchema);
|
|
@@ -9181,6 +10553,7 @@ var require_convention_builder = __commonJS({
|
|
|
9181
10553
|
let currentValue = dig(this.overlay.properties.attributes.properties, attr);
|
|
9182
10554
|
bury(this.schema, `properties.attributes.properties.${attr}`, currentValue, false);
|
|
9183
10555
|
});
|
|
10556
|
+
this.deletedAttributes.forEach((deletedAttr) => delete this.schema.properties.attributes.properties[deletedAttr]);
|
|
9184
10557
|
let originalRequiredFields = this.schema.properties.attributes.required ? this.schema.properties.attributes.required : [];
|
|
9185
10558
|
let allRequiredSet = Array.from(/* @__PURE__ */ new Set([...originalRequiredFields, ...Array.from(this.requiredFields)]));
|
|
9186
10559
|
this.schema.properties.attributes.required = allRequiredSet;
|
|
@@ -9283,6 +10656,22 @@ var require_convention_builder = __commonJS({
|
|
|
9283
10656
|
;
|
|
9284
10657
|
this.setGenericSection({ attribute, section: subschema, description });
|
|
9285
10658
|
}
|
|
10659
|
+
/**
|
|
10660
|
+
* Mark an attribute to be omitted from the resulting schema.
|
|
10661
|
+
* @param {} attribute
|
|
10662
|
+
*/
|
|
10663
|
+
deleteAttribute(attribute) {
|
|
10664
|
+
this.deletedAttributes.push(attribute);
|
|
10665
|
+
delete this.overlay.properties.attributes.properties[attribute];
|
|
10666
|
+
this.updateAttributeStatus(attribute);
|
|
10667
|
+
}
|
|
10668
|
+
/**
|
|
10669
|
+
* This method takes this object's schema and returns an array of fixed attributes objects, which is equivalent to a row by row encoded excel spreadsheet and is mostly meant at allowing users to work on said schema via a tabular structure.
|
|
10670
|
+
* The output will include one row for each data entry path (not all of the paths) and will synthesize all of its properties into columns, such as name, Title, Type, etc.
|
|
10671
|
+
*/
|
|
10672
|
+
getTabularRepresentation() {
|
|
10673
|
+
return flattenJSONSchema(this.schema, false);
|
|
10674
|
+
}
|
|
9286
10675
|
/**
|
|
9287
10676
|
* Build an AJV validator and ensure all valid examples are accepted and all error examples are rejected. Returns an array attribute for each set of examples, plus a general success attribute indicating wether all examples resulted as expected and a failedExamples array only listing entities for which there was no success (meanin unrejected error examples and rejected valid examples).
|
|
9288
10677
|
*/
|
|
@@ -9373,16 +10762,18 @@ var require_convention_builder = __commonJS({
|
|
|
9373
10762
|
constructor({
|
|
9374
10763
|
typeAndBundle,
|
|
9375
10764
|
attributeOverlayName,
|
|
9376
|
-
|
|
10765
|
+
repoURLTemplateFunction = basicSchemaURLTemplate,
|
|
9377
10766
|
strictEnums = true,
|
|
9378
|
-
baseSchemataFolder = `${__dirname}/../../../../input/collection
|
|
10767
|
+
baseSchemataFolder = `${__dirname}/../../../../input/collection`,
|
|
10768
|
+
gitlabProjectId = process.env.CI_PROJECT_ID
|
|
9379
10769
|
}) {
|
|
9380
10770
|
super({
|
|
9381
10771
|
typeAndBundle,
|
|
9382
|
-
|
|
10772
|
+
repoURLTemplateFunction,
|
|
9383
10773
|
storageFolder: baseSchemataFolder,
|
|
9384
10774
|
baseSchemataFolder,
|
|
9385
|
-
strictEnums
|
|
10775
|
+
strictEnums,
|
|
10776
|
+
gitlabProjectId
|
|
9386
10777
|
});
|
|
9387
10778
|
this.attributeOverlayName = attributeOverlayName;
|
|
9388
10779
|
}
|
|
@@ -9410,12 +10801,13 @@ var require_convention_builder = __commonJS({
|
|
|
9410
10801
|
title,
|
|
9411
10802
|
schemaName,
|
|
9412
10803
|
version,
|
|
9413
|
-
|
|
10804
|
+
repoURLTemplateFunction = gitlabConventionURLTemplate,
|
|
9414
10805
|
description,
|
|
9415
10806
|
validExamples = [],
|
|
9416
10807
|
erroredExamples = [],
|
|
9417
10808
|
storageFolder = `${__dirname}/../../../../output/collection`,
|
|
9418
10809
|
baseSchemataFolder = `${__dirname}/../../../../input/collection`,
|
|
10810
|
+
gitlabProjectId = process.env.CI_PROJECT_ID,
|
|
9419
10811
|
strictEnums = false,
|
|
9420
10812
|
overlays = false,
|
|
9421
10813
|
relationships = false,
|
|
@@ -9430,13 +10822,14 @@ var require_convention_builder = __commonJS({
|
|
|
9430
10822
|
this.description = description;
|
|
9431
10823
|
this.schemaName = schemaName;
|
|
9432
10824
|
this.version = version;
|
|
9433
|
-
this.
|
|
10825
|
+
this.repoURLTemplateFunction = repoURLTemplateFunction, this.validExamples = validExamples;
|
|
9434
10826
|
this.erroredExamples = erroredExamples;
|
|
9435
10827
|
this.overlays = {};
|
|
9436
10828
|
this.relationships = [];
|
|
9437
10829
|
this.required = [];
|
|
9438
10830
|
this.storageFolder = storageFolder;
|
|
9439
10831
|
this.baseSchemataFolder = baseSchemataFolder;
|
|
10832
|
+
this.gitlabProjectId = gitlabProjectId;
|
|
9440
10833
|
if (overlays) {
|
|
9441
10834
|
let overlayNames = Object.keys(overlays);
|
|
9442
10835
|
this.overlays = {};
|
|
@@ -9512,7 +10905,7 @@ var require_convention_builder = __commonJS({
|
|
|
9512
10905
|
this.schema = {
|
|
9513
10906
|
title: this.title,
|
|
9514
10907
|
type: "object",
|
|
9515
|
-
$id:
|
|
10908
|
+
$id: this.repoURLTemplateFunction({ conventionName: this.schemaName, projectID: this.gitlabProjectId }),
|
|
9516
10909
|
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
9517
10910
|
properties: {},
|
|
9518
10911
|
description: this.description,
|
|
@@ -9768,8 +11161,13 @@ import TabItem from '@theme/TabItem';
|
|
|
9768
11161
|
* @param {string} path -- A string representing an absolute path, leading to either a JSON file containing an individual example or a folder containing several examples.
|
|
9769
11162
|
* @param {object|array} object -- Either an array of examples or an individual example.
|
|
9770
11163
|
* @param {boolean} is_valid_example -- Whether to store the provided examples as valid examples or errors, for the tests.
|
|
11164
|
+
* @param {boolean} structured_as_array -- Mark as "true" if the provided example is structured as a loose array of entities, as FarmOS delivers them (instead of structured as a convention object in which every entity is a property). The method will try to structure the entity and return meaningful information if it is not successful in doing so.
|
|
9771
11165
|
*/
|
|
9772
|
-
addExample({ path, object, is_valid_example }) {
|
|
11166
|
+
addExample({ path, object, is_valid_example, structured_as_array }) {
|
|
11167
|
+
let output = {
|
|
11168
|
+
success: false,
|
|
11169
|
+
errors: []
|
|
11170
|
+
};
|
|
9773
11171
|
if (!path && !object) {
|
|
9774
11172
|
throw new Error(`Either an object or a path leading to a folder or a json file need to be provided. Both arguments are empty.`);
|
|
9775
11173
|
}
|
|
@@ -9786,12 +11184,43 @@ import TabItem from '@theme/TabItem';
|
|
|
9786
11184
|
newExamples = this.getExamplesFromFile({ path });
|
|
9787
11185
|
}
|
|
9788
11186
|
;
|
|
11187
|
+
if (structured_as_array) {
|
|
11188
|
+
newExamples = newExamples.map((exampleArray) => {
|
|
11189
|
+
let exampleObject = arrayToStructuredConvention({
|
|
11190
|
+
entitiesArray: exampleArray,
|
|
11191
|
+
conventionObject: this
|
|
11192
|
+
});
|
|
11193
|
+
if (!exampleObject.is_valid && is_valid_example) {
|
|
11194
|
+
output.errors.push("The `arrayToStructuredConvention` function wasn't able to rebuild a valid instance of this convention's schema from the provided array of loose entities. Look into the '.structuringAttempt' attribute for more information.");
|
|
11195
|
+
}
|
|
11196
|
+
;
|
|
11197
|
+
if (!is_valid_example) {
|
|
11198
|
+
output.warning = "Since the example is not marked as 'valid', we can't check it is being properly parsed. Please review the output 'assembled_entity' to check it looks as it should.";
|
|
11199
|
+
}
|
|
11200
|
+
;
|
|
11201
|
+
output.structuringAttempt = exampleObject;
|
|
11202
|
+
return exampleObject.assembled_entity;
|
|
11203
|
+
});
|
|
11204
|
+
}
|
|
11205
|
+
;
|
|
9789
11206
|
if (is_valid_example) {
|
|
9790
11207
|
this.validExamples.push(...newExamples);
|
|
9791
11208
|
} else {
|
|
9792
11209
|
this.erroredExamples.push(...newExamples);
|
|
9793
11210
|
}
|
|
9794
11211
|
;
|
|
11212
|
+
if (output.errors.length == 0) {
|
|
11213
|
+
output.success = true;
|
|
11214
|
+
}
|
|
11215
|
+
;
|
|
11216
|
+
return output;
|
|
11217
|
+
}
|
|
11218
|
+
/**
|
|
11219
|
+
* This method takes this object's schema and returns an array of fixed attributes objects, which is equivalent to a row by row encoded excel spreadsheet and is mostly meant at allowing users to work on said schema via a tabular structure.
|
|
11220
|
+
* The output will include one row for each data entry path (not all of the paths) and will synthesize all of its properties into columns, such as name, Title, Type, etc.
|
|
11221
|
+
*/
|
|
11222
|
+
getTabularRepresentation() {
|
|
11223
|
+
return flattenJSONSchema(this.schema, true);
|
|
9795
11224
|
}
|
|
9796
11225
|
/**
|
|
9797
11226
|
* Build an AJV validator and ensure all valid examples are accepted and all error examples are rejected. Returns an array attribute for each set of examples, plus a general success attribute indicating wether all examples resulted as expected and a failedExamples array only listing entities for which there was no success (meanin unrejected error examples and rejected valid examples).
|
|
@@ -9874,6 +11303,8 @@ import TabItem from '@theme/TabItem';
|
|
|
9874
11303
|
fs.mkdirSync(`${targetPath}/examples/correct`, { recursive: true }, console.error);
|
|
9875
11304
|
fs.mkdirSync(`${targetPath}/examples/incorrect`, { recursive: true }, console.error);
|
|
9876
11305
|
fs.writeFileSync(`${targetPath}/schema.json`, JSON.stringify(this.schema, null, " "), console.error);
|
|
11306
|
+
let flatTableRepresentation = this.getTabularRepresentation();
|
|
11307
|
+
fs.writeFileSync(`${targetPath}/flat_schema.csv`, papa.unparse(flatTableRepresentation), console.error);
|
|
9877
11308
|
this.validExamples.forEach((example, i) => {
|
|
9878
11309
|
fs.writeFileSync(`${targetPath}/examples/correct/example_${i + 1}.json`, JSON.stringify(example, null, " "), console.error);
|
|
9879
11310
|
});
|
|
@@ -9909,7 +11340,7 @@ import TabItem from '@theme/TabItem';
|
|
|
9909
11340
|
};
|
|
9910
11341
|
__name(_ConventionSchema, "ConventionSchema");
|
|
9911
11342
|
var ConventionSchema2 = _ConventionSchema;
|
|
9912
|
-
var
|
|
11343
|
+
var _InputConventionChanges = class _InputConventionChanges {
|
|
9913
11344
|
constructor({
|
|
9914
11345
|
type,
|
|
9915
11346
|
bundle,
|
|
@@ -9917,7 +11348,8 @@ import TabItem from '@theme/TabItem';
|
|
|
9917
11348
|
baseSchemataFolder = `${__dirname}/../../../../input/collection`,
|
|
9918
11349
|
sourceSchemataFile = `${__dirname}/../../../../input/farmos.json`,
|
|
9919
11350
|
overlayName,
|
|
9920
|
-
overlayDescription
|
|
11351
|
+
overlayDescription,
|
|
11352
|
+
gitlabProjectId = process.env.CI_PROJECT_ID
|
|
9921
11353
|
}) {
|
|
9922
11354
|
let integralSchemataFile = JSON.parse(fs.readFileSync(sourceSchemataFile));
|
|
9923
11355
|
this.type = type;
|
|
@@ -9928,6 +11360,7 @@ import TabItem from '@theme/TabItem';
|
|
|
9928
11360
|
this.sourceSchemata = integralSchemataFile;
|
|
9929
11361
|
this.lexicon = [...findAllObjectAttributesInTree(this.sourceSchemata), ...findAllNonObjectAttributesInTree(this.sourceSchemata)];
|
|
9930
11362
|
this.baseSchemataFolder = baseSchemataFolder;
|
|
11363
|
+
this.gitlabProjectId = gitlabProjectId;
|
|
9931
11364
|
let involvedSchemata = this.lexicon.filter((entry) => this.attribute == entry.attribute).flatMap((entry) => entry.instances).filter((instance) => this.type ? instance.type == this.type : true).filter((instance) => this.bundle ? instance.bundle == this.bundle : true).map((d) => d.schema);
|
|
9932
11365
|
this.targetSchemata = Array.from(new Set(involvedSchemata));
|
|
9933
11366
|
this.overlays = [];
|
|
@@ -9935,8 +11368,10 @@ import TabItem from '@theme/TabItem';
|
|
|
9935
11368
|
let currentOverlay = new AttributePatchingSchemaOverlay({
|
|
9936
11369
|
typeAndBundle: target,
|
|
9937
11370
|
attributeOverlayName: this.overlayName,
|
|
9938
|
-
baseSchemataFolder: this.baseSchemataFolder
|
|
11371
|
+
baseSchemataFolder: this.baseSchemataFolder,
|
|
11372
|
+
gitlabProjectId: this.gitlabProjectId
|
|
9939
11373
|
});
|
|
11374
|
+
currentOverlay.gitlabProjectId = this.gitlabProjectId;
|
|
9940
11375
|
this.overlays.push(currentOverlay);
|
|
9941
11376
|
});
|
|
9942
11377
|
}
|
|
@@ -9997,6 +11432,15 @@ import TabItem from '@theme/TabItem';
|
|
|
9997
11432
|
});
|
|
9998
11433
|
});
|
|
9999
11434
|
}
|
|
11435
|
+
/**
|
|
11436
|
+
* Mark an attribute to be omitted from the resulting schema.
|
|
11437
|
+
* @param {} attribute
|
|
11438
|
+
*/
|
|
11439
|
+
deleteAttribute() {
|
|
11440
|
+
this.overlays.forEach((overlay) => {
|
|
11441
|
+
overlay.deleteAttribute(this.attribute);
|
|
11442
|
+
});
|
|
11443
|
+
}
|
|
10000
11444
|
/**
|
|
10001
11445
|
* Stores all involved schemas into the folder structure for `base schemata` (provided as the parameter `baseSchemataFolder`) which will be consumed by our conventions later.
|
|
10002
11446
|
* @returns {} -- It returns objects containing all paths.
|
|
@@ -10009,19 +11453,19 @@ import TabItem from '@theme/TabItem';
|
|
|
10009
11453
|
return output;
|
|
10010
11454
|
}
|
|
10011
11455
|
};
|
|
10012
|
-
__name(
|
|
10013
|
-
var
|
|
11456
|
+
__name(_InputConventionChanges, "InputConventionChanges");
|
|
11457
|
+
var InputConventionChanges2 = _InputConventionChanges;
|
|
10014
11458
|
exports2.SchemaOverlay = SchemaOverlay2;
|
|
10015
11459
|
exports2.ConventionSchema = ConventionSchema2;
|
|
10016
|
-
exports2.
|
|
11460
|
+
exports2.InputConventionChanges = InputConventionChanges2;
|
|
10017
11461
|
}
|
|
10018
11462
|
});
|
|
10019
11463
|
|
|
10020
11464
|
// src/index.js
|
|
10021
|
-
var { SchemaOverlay, ConventionSchema,
|
|
11465
|
+
var { SchemaOverlay, ConventionSchema, InputConventionChanges } = require_convention_builder();
|
|
10022
11466
|
var schemaUtils = require_schema_utilities();
|
|
10023
11467
|
exports.SchemaOverlay = SchemaOverlay;
|
|
10024
|
-
exports.
|
|
11468
|
+
exports.InputConventionChanges = InputConventionChanges;
|
|
10025
11469
|
exports.ConventionSchema = ConventionSchema;
|
|
10026
11470
|
exports.fixRelationshipDataField = schemaUtils.fixRelationshipDataField;
|
|
10027
11471
|
exports.buildValidator = schemaUtils.buildValidator;
|
|
@@ -10042,5 +11486,13 @@ exports.flattenJSONSchema = schemaUtils.flattenJSONSchema;
|
|
|
10042
11486
|
|
|
10043
11487
|
uri-js/dist/es5/uri.all.js:
|
|
10044
11488
|
(** @license URI.js v4.4.1 (c) 2011 Gary Court. License: http://github.com/garycourt/uri-js *)
|
|
11489
|
+
|
|
11490
|
+
papaparse/papaparse.js:
|
|
11491
|
+
(* @license
|
|
11492
|
+
Papa Parse
|
|
11493
|
+
v5.4.1
|
|
11494
|
+
https://github.com/mholt/PapaParse
|
|
11495
|
+
License: MIT
|
|
11496
|
+
*)
|
|
10045
11497
|
*/
|
|
10046
11498
|
//# sourceMappingURL=index.js.map
|