@graphty/graph-io 0.3.40 → 0.3.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/chunks/{children-CL3Cy0ez.js → children-CA4coBi0.js} +36 -27
- package/dist/chunks/children-CA4coBi0.js.map +1 -0
- package/dist/chunks/{choose-CS3fvGT1.js → choose-DvpbdRuz.js} +2 -2
- package/dist/chunks/{choose-CS3fvGT1.js.map → choose-DvpbdRuz.js.map} +1 -1
- package/dist/chunks/{constants-DqKZT6cH.js → constants-fT3E8JYY.js} +2 -2
- package/dist/chunks/{constants-DqKZT6cH.js.map → constants-fT3E8JYY.js.map} +1 -1
- package/dist/chunks/{escape-BJRLd3vJ.js → escape-B62ZGM_a.js} +2 -2
- package/dist/chunks/{escape-BJRLd3vJ.js.map → escape-B62ZGM_a.js.map} +1 -1
- package/dist/chunks/{export-B-aVCwyG.js → export-rKLeGQMo.js} +404 -243
- package/dist/chunks/export-rKLeGQMo.js.map +1 -0
- package/dist/chunks/{exporter-6ocL_9x_.js → exporter-8hLNemFk.js} +4 -4
- package/dist/chunks/{exporter-6ocL_9x_.js.map → exporter-8hLNemFk.js.map} +1 -1
- package/dist/chunks/{exporter-BVXnV9-9.js → exporter-BWwEuOPT.js} +29 -17
- package/dist/chunks/exporter-BWwEuOPT.js.map +1 -0
- package/dist/chunks/{exporter-Bcp-l4Bo.js → exporter-CpRyHZ3H.js} +574 -475
- package/dist/chunks/exporter-CpRyHZ3H.js.map +1 -0
- package/dist/chunks/{exporter-SFYkDzD0.js → exporter-yLxKBvLE.js} +23 -20
- package/dist/chunks/{exporter-SFYkDzD0.js.map → exporter-yLxKBvLE.js.map} +1 -1
- package/dist/chunks/{importer-DcKB97kW.js → importer-BbNb7eNg.js} +111 -71
- package/dist/chunks/importer-BbNb7eNg.js.map +1 -0
- package/dist/chunks/{importer-D_srHyj6.js → importer-C98dpSH4.js} +5 -5
- package/dist/chunks/{importer-D_srHyj6.js.map → importer-C98dpSH4.js.map} +1 -1
- package/dist/chunks/{importer-BJlGAOht.js → importer-D794XUTe.js} +3 -3
- package/dist/chunks/{importer-BJlGAOht.js.map → importer-D794XUTe.js.map} +1 -1
- package/dist/chunks/{importer-AjwOJrn2.js → importer-Dz7tI8Sa.js} +202 -181
- package/dist/chunks/importer-Dz7tI8Sa.js.map +1 -0
- package/dist/chunks/{importer-DEp_fYjf.js → importer-Ht7LNg8L.js} +92 -64
- package/dist/chunks/importer-Ht7LNg8L.js.map +1 -0
- package/dist/chunks/{importer-CPqnIY8m.js → importer-WrGRZb0N.js} +42 -29
- package/dist/chunks/importer-WrGRZb0N.js.map +1 -0
- package/dist/chunks/{importer-BpuS3J8C.js → importer-xZ-3suY1.js} +74 -61
- package/dist/chunks/importer-xZ-3suY1.js.map +1 -0
- package/dist/chunks/{json-elements-DpO6AXIc.js → json-elements-DKSqpNW1.js} +3 -3
- package/dist/chunks/{json-elements-DpO6AXIc.js.map → json-elements-DKSqpNW1.js.map} +1 -1
- package/dist/chunks/{ontology-export-50L9DOV4.js → ontology-export-CtRXjg_i.js} +2 -2
- package/dist/chunks/{ontology-export-50L9DOV4.js.map → ontology-export-CtRXjg_i.js.map} +1 -1
- package/dist/chunks/{records-r3SFHz11.js → records-Cu4bxYOm.js} +69 -36
- package/dist/chunks/records-Cu4bxYOm.js.map +1 -0
- package/dist/chunks/{text-Bjfa1xYL.js → text-DHOJXnED.js} +20 -11
- package/dist/chunks/text-DHOJXnED.js.map +1 -0
- package/dist/csv.js +323 -241
- package/dist/csv.js.map +1 -1
- package/dist/cx.js +1 -1
- package/dist/cx2.js +2 -2
- package/dist/cys.js +1 -1
- package/dist/dot.js +1 -1
- package/dist/gexf.js +195 -97
- package/dist/gexf.js.map +1 -1
- package/dist/gml.js +170 -149
- package/dist/gml.js.map +1 -1
- package/dist/graph-io.js +27 -27
- package/dist/graphml.js +2 -2
- package/dist/json.js +1 -1
- package/dist/neo4j.js +103 -65
- package/dist/neo4j.js.map +1 -1
- package/dist/obo.js +1 -1
- package/dist/pajek.js +1 -1
- package/dist/src/children.d.ts.map +1 -1
- package/dist/src/children.js +50 -27
- package/dist/src/children.js.map +1 -1
- package/dist/src/common/attributes.d.ts.map +1 -1
- package/dist/src/common/attributes.js +86 -42
- package/dist/src/common/attributes.js.map +1 -1
- package/dist/src/common/input.d.ts.map +1 -1
- package/dist/src/common/input.js +358 -184
- package/dist/src/common/input.js.map +1 -1
- package/dist/src/common/lists.js +30 -15
- package/dist/src/common/lists.js.map +1 -1
- package/dist/src/common/text.d.ts +7 -0
- package/dist/src/common/text.d.ts.map +1 -1
- package/dist/src/common/text.js +15 -7
- package/dist/src/common/text.js.map +1 -1
- package/dist/src/common/zip.d.ts.map +1 -1
- package/dist/src/common/zip.js +36 -19
- package/dist/src/common/zip.js.map +1 -1
- package/dist/src/formats/csv/exporter.d.ts.map +1 -1
- package/dist/src/formats/csv/exporter.js +288 -146
- package/dist/src/formats/csv/exporter.js.map +1 -1
- package/dist/src/formats/csv/importer.d.ts.map +1 -1
- package/dist/src/formats/csv/importer.js +129 -80
- package/dist/src/formats/csv/importer.js.map +1 -1
- package/dist/src/formats/csv/records.d.ts.map +1 -1
- package/dist/src/formats/csv/records.js +78 -36
- package/dist/src/formats/csv/records.js.map +1 -1
- package/dist/src/formats/cys/tables.d.ts.map +1 -1
- package/dist/src/formats/cys/tables.js +11 -8
- package/dist/src/formats/cys/tables.js.map +1 -1
- package/dist/src/formats/dot/tokenizer.d.ts +39 -0
- package/dist/src/formats/dot/tokenizer.d.ts.map +1 -1
- package/dist/src/formats/dot/tokenizer.js +112 -66
- package/dist/src/formats/dot/tokenizer.js.map +1 -1
- package/dist/src/formats/gexf/exporter.d.ts.map +1 -1
- package/dist/src/formats/gexf/exporter.js +65 -37
- package/dist/src/formats/gexf/exporter.js.map +1 -1
- package/dist/src/formats/gexf/importer.d.ts.map +1 -1
- package/dist/src/formats/gexf/importer.js +138 -50
- package/dist/src/formats/gexf/importer.js.map +1 -1
- package/dist/src/formats/gml/exporter.d.ts.map +1 -1
- package/dist/src/formats/gml/exporter.js +51 -35
- package/dist/src/formats/gml/exporter.js.map +1 -1
- package/dist/src/formats/gml/importer.d.ts.map +1 -1
- package/dist/src/formats/gml/importer.js +103 -58
- package/dist/src/formats/gml/importer.js.map +1 -1
- package/dist/src/formats/gml/syntax.d.ts.map +1 -1
- package/dist/src/formats/gml/syntax.js +17 -8
- package/dist/src/formats/gml/syntax.js.map +1 -1
- package/dist/src/formats/graphml/exporter.d.ts.map +1 -1
- package/dist/src/formats/graphml/exporter.js +34 -24
- package/dist/src/formats/graphml/exporter.js.map +1 -1
- package/dist/src/formats/graphml/importer.d.ts.map +1 -1
- package/dist/src/formats/graphml/importer.js +17 -13
- package/dist/src/formats/graphml/importer.js.map +1 -1
- package/dist/src/formats/graphml/tree.js +23 -14
- package/dist/src/formats/graphml/tree.js.map +1 -1
- package/dist/src/formats/graphml/yfiles.d.ts.map +1 -1
- package/dist/src/formats/graphml/yfiles.js +22 -10
- package/dist/src/formats/graphml/yfiles.js.map +1 -1
- package/dist/src/formats/json/dialect.d.ts.map +1 -1
- package/dist/src/formats/json/dialect.js +54 -20
- package/dist/src/formats/json/dialect.js.map +1 -1
- package/dist/src/formats/json/exporter.d.ts.map +1 -1
- package/dist/src/formats/json/exporter.js +38 -21
- package/dist/src/formats/json/exporter.js.map +1 -1
- package/dist/src/formats/json/importer.d.ts.map +1 -1
- package/dist/src/formats/json/importer.js +536 -343
- package/dist/src/formats/json/importer.js.map +1 -1
- package/dist/src/formats/json/obographs.js +29 -17
- package/dist/src/formats/json/obographs.js.map +1 -1
- package/dist/src/formats/neo4j/exporter.d.ts.map +1 -1
- package/dist/src/formats/neo4j/exporter.js +73 -43
- package/dist/src/formats/neo4j/exporter.js.map +1 -1
- package/dist/src/formats/neo4j/header.d.ts.map +1 -1
- package/dist/src/formats/neo4j/header.js +37 -16
- package/dist/src/formats/neo4j/header.js.map +1 -1
- package/dist/src/formats/neo4j/importer.d.ts.map +1 -1
- package/dist/src/formats/neo4j/importer.js +21 -11
- package/dist/src/formats/neo4j/importer.js.map +1 -1
- package/dist/src/formats/obo/exporter.d.ts.map +1 -1
- package/dist/src/formats/obo/exporter.js +18 -7
- package/dist/src/formats/obo/exporter.js.map +1 -1
- package/dist/src/formats/pajek/exporter.d.ts.map +1 -1
- package/dist/src/formats/pajek/exporter.js +142 -73
- package/dist/src/formats/pajek/exporter.js.map +1 -1
- package/dist/src/formats/pajek/importer.d.ts.map +1 -1
- package/dist/src/formats/pajek/importer.js +26 -15
- package/dist/src/formats/pajek/importer.js.map +1 -1
- package/dist/src/formats/pajek/syntax.d.ts.map +1 -1
- package/dist/src/formats/pajek/syntax.js +85 -58
- package/dist/src/formats/pajek/syntax.js.map +1 -1
- package/dist/src/formats/xgmml/emit.d.ts +7 -0
- package/dist/src/formats/xgmml/emit.d.ts.map +1 -1
- package/dist/src/formats/xgmml/emit.js +79 -40
- package/dist/src/formats/xgmml/emit.js.map +1 -1
- package/dist/src/formats/xgmml/exporter.d.ts.map +1 -1
- package/dist/src/formats/xgmml/exporter.js +38 -12
- package/dist/src/formats/xgmml/exporter.js.map +1 -1
- package/dist/src/formats/xgmml/importer.d.ts.map +1 -1
- package/dist/src/formats/xgmml/importer.js +38 -18
- package/dist/src/formats/xgmml/importer.js.map +1 -1
- package/dist/src/sniff.js +109 -57
- package/dist/src/sniff.js.map +1 -1
- package/dist/xgmml.js +2 -2
- package/package.json +2 -2
- package/src/children.ts +54 -27
- package/src/common/attributes.ts +118 -49
- package/src/common/input.ts +439 -197
- package/src/common/lists.ts +29 -15
- package/src/common/text.ts +24 -15
- package/src/common/zip.ts +47 -18
- package/src/formats/csv/exporter.ts +377 -175
- package/src/formats/csv/importer.ts +155 -84
- package/src/formats/csv/records.ts +90 -35
- package/src/formats/cys/tables.ts +11 -10
- package/src/formats/dot/tokenizer.ts +118 -68
- package/src/formats/gexf/exporter.ts +67 -36
- package/src/formats/gexf/importer.ts +173 -63
- package/src/formats/gml/exporter.ts +78 -82
- package/src/formats/gml/importer.ts +119 -68
- package/src/formats/gml/syntax.ts +18 -8
- package/src/formats/graphml/exporter.ts +44 -24
- package/src/formats/graphml/importer.ts +18 -11
- package/src/formats/graphml/tree.ts +22 -12
- package/src/formats/graphml/yfiles.ts +23 -10
- package/src/formats/json/dialect.ts +61 -22
- package/src/formats/json/exporter.ts +40 -21
- package/src/formats/json/importer.ts +701 -385
- package/src/formats/json/obographs.ts +35 -15
- package/src/formats/neo4j/exporter.ts +79 -41
- package/src/formats/neo4j/header.ts +50 -16
- package/src/formats/neo4j/importer.ts +21 -10
- package/src/formats/obo/exporter.ts +34 -17
- package/src/formats/pajek/exporter.ts +180 -117
- package/src/formats/pajek/importer.ts +27 -15
- package/src/formats/pajek/syntax.ts +99 -58
- package/src/formats/xgmml/emit.ts +88 -40
- package/src/formats/xgmml/exporter.ts +40 -12
- package/src/formats/xgmml/importer.ts +45 -23
- package/src/sniff.ts +115 -59
- package/dist/chunks/children-CL3Cy0ez.js.map +0 -1
- package/dist/chunks/export-B-aVCwyG.js.map +0 -1
- package/dist/chunks/exporter-BVXnV9-9.js.map +0 -1
- package/dist/chunks/exporter-Bcp-l4Bo.js.map +0 -1
- package/dist/chunks/importer-AjwOJrn2.js.map +0 -1
- package/dist/chunks/importer-BpuS3J8C.js.map +0 -1
- package/dist/chunks/importer-CPqnIY8m.js.map +0 -1
- package/dist/chunks/importer-DEp_fYjf.js.map +0 -1
- package/dist/chunks/importer-DcKB97kW.js.map +0 -1
- package/dist/chunks/records-r3SFHz11.js.map +0 -1
- package/dist/chunks/text-Bjfa1xYL.js.map +0 -1
|
@@ -638,21 +638,9 @@ function splitBracketed(text) {
|
|
|
638
638
|
}
|
|
639
639
|
let item = "";
|
|
640
640
|
if (i < n && (inner[i] === '"' || inner[i] === "'")) {
|
|
641
|
-
const
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
if (inner[i] === quote) {
|
|
645
|
-
if (inner[i + 1] === quote) {
|
|
646
|
-
item += quote;
|
|
647
|
-
i += 2;
|
|
648
|
-
continue;
|
|
649
|
-
}
|
|
650
|
-
i++;
|
|
651
|
-
break;
|
|
652
|
-
}
|
|
653
|
-
item += inner[i];
|
|
654
|
-
i++;
|
|
655
|
-
}
|
|
641
|
+
const quoted = readQuoted(inner, i);
|
|
642
|
+
item = quoted.text;
|
|
643
|
+
i = quoted.end;
|
|
656
644
|
}
|
|
657
645
|
let end = inner.indexOf(",", i);
|
|
658
646
|
if (end < 0) {
|
|
@@ -671,6 +659,24 @@ function splitBracketed(text) {
|
|
|
671
659
|
}
|
|
672
660
|
}
|
|
673
661
|
}
|
|
662
|
+
function readQuoted(inner, open) {
|
|
663
|
+
const quote = inner[open];
|
|
664
|
+
const n = inner.length;
|
|
665
|
+
let text = "";
|
|
666
|
+
let i = open + 1;
|
|
667
|
+
while (i < n) {
|
|
668
|
+
if (inner[i] !== quote) {
|
|
669
|
+
text += inner[i];
|
|
670
|
+
i++;
|
|
671
|
+
} else if (inner[i + 1] === quote) {
|
|
672
|
+
text += quote;
|
|
673
|
+
i += 2;
|
|
674
|
+
} else {
|
|
675
|
+
return { text, end: i + 1 };
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
return { text, end: i };
|
|
679
|
+
}
|
|
674
680
|
function isSpace(c) {
|
|
675
681
|
return c === 32 || c === 9 || c === 13 || c === 10;
|
|
676
682
|
}
|
|
@@ -686,22 +692,7 @@ const DICT_SAMPLE_ROWS = 1024;
|
|
|
686
692
|
function declareAttribute(input) {
|
|
687
693
|
const issues = [];
|
|
688
694
|
const listSyntax = input.listSyntax ?? "gexf";
|
|
689
|
-
let spec;
|
|
690
|
-
if (input.type === null) {
|
|
691
|
-
spec = stringSpec(null);
|
|
692
|
-
} else {
|
|
693
|
-
const mapped = mapDeclaredType(input.format, input.type, input.long);
|
|
694
|
-
if (mapped === null) {
|
|
695
|
-
spec = stringSpec(input.type);
|
|
696
|
-
issues.push({
|
|
697
|
-
category: "unsupported",
|
|
698
|
-
code: UNKNOWN_TYPE_CODE,
|
|
699
|
-
message: `${input.format} type "${input.type}" is not supported; values kept as text`
|
|
700
|
-
});
|
|
701
|
-
} else {
|
|
702
|
-
spec = mapped;
|
|
703
|
-
}
|
|
704
|
-
}
|
|
695
|
+
let spec = declaredSpec(input, issues);
|
|
705
696
|
const { name, renamed } = resolveName(input);
|
|
706
697
|
if (renamed) {
|
|
707
698
|
issues.push({
|
|
@@ -733,20 +724,7 @@ function declareAttribute(input) {
|
|
|
733
724
|
}
|
|
734
725
|
const optionsText = input.optionsText ?? null;
|
|
735
726
|
if (optionsText !== null) {
|
|
736
|
-
|
|
737
|
-
const options = splitListText(optionsText, listSyntax).map((item) => parseItem(item, spec));
|
|
738
|
-
decl.options = options;
|
|
739
|
-
if (spec.dtype === "string" && !spec.list && options.every((o) => typeof o === "string")) {
|
|
740
|
-
decl.dtype = "dict";
|
|
741
|
-
spec = { ...spec, dtype: "dict" };
|
|
742
|
-
}
|
|
743
|
-
} catch (err) {
|
|
744
|
-
issues.push({
|
|
745
|
-
category: "validation-error",
|
|
746
|
-
code: BAD_OPTIONS_CODE,
|
|
747
|
-
message: `options "${optionsText}" of attribute "${name}" do not parse as ${spec.declared}: ${messageOf$1(err)}`
|
|
748
|
-
});
|
|
749
|
-
}
|
|
727
|
+
spec = declareOptions(decl, spec, optionsText, listSyntax, issues);
|
|
750
728
|
}
|
|
751
729
|
const defaultText = input.defaultText ?? null;
|
|
752
730
|
if (defaultText !== null) {
|
|
@@ -763,6 +741,38 @@ function declareAttribute(input) {
|
|
|
763
741
|
const companion = spec.temporal !== null && !spec.list ? timeTextCompanion(name) : null;
|
|
764
742
|
return { decl, spec, listSyntax, companion, renamed, issues };
|
|
765
743
|
}
|
|
744
|
+
function declaredSpec(input, issues) {
|
|
745
|
+
if (input.type === null) {
|
|
746
|
+
return stringSpec(null);
|
|
747
|
+
}
|
|
748
|
+
const mapped = mapDeclaredType(input.format, input.type, input.long);
|
|
749
|
+
if (mapped !== null) {
|
|
750
|
+
return mapped;
|
|
751
|
+
}
|
|
752
|
+
issues.push({
|
|
753
|
+
category: "unsupported",
|
|
754
|
+
code: UNKNOWN_TYPE_CODE,
|
|
755
|
+
message: `${input.format} type "${input.type}" is not supported; values kept as text`
|
|
756
|
+
});
|
|
757
|
+
return stringSpec(input.type);
|
|
758
|
+
}
|
|
759
|
+
function declareOptions(decl, spec, optionsText, listSyntax, issues) {
|
|
760
|
+
try {
|
|
761
|
+
const options = splitListText(optionsText, listSyntax).map((item) => parseItem(item, spec));
|
|
762
|
+
decl.options = options;
|
|
763
|
+
if (spec.dtype === "string" && !spec.list && options.every((o) => typeof o === "string")) {
|
|
764
|
+
decl.dtype = "dict";
|
|
765
|
+
return { ...spec, dtype: "dict" };
|
|
766
|
+
}
|
|
767
|
+
} catch (err) {
|
|
768
|
+
issues.push({
|
|
769
|
+
category: "validation-error",
|
|
770
|
+
code: BAD_OPTIONS_CODE,
|
|
771
|
+
message: `options "${optionsText}" of attribute "${decl.name}" do not parse as ${spec.declared}: ${messageOf$1(err)}`
|
|
772
|
+
});
|
|
773
|
+
}
|
|
774
|
+
return spec;
|
|
775
|
+
}
|
|
766
776
|
function parseDeclaredValue(text, spec, listSyntax) {
|
|
767
777
|
if (spec.list) {
|
|
768
778
|
return splitListText(text, listSyntax).map((item) => parseItem(item, spec));
|
|
@@ -804,29 +814,12 @@ function declareResolved(sink, domain, decl, report, where) {
|
|
|
804
814
|
throw err;
|
|
805
815
|
}
|
|
806
816
|
if (err.code === "E_COLUMN_EXISTS" && !renamed) {
|
|
807
|
-
|
|
808
|
-
const id = current.origin?.id ?? null;
|
|
809
|
-
const name = uniqueColumnName(current.name, id, taken);
|
|
810
|
-
report.warning(
|
|
811
|
-
"coercion",
|
|
812
|
-
COLUMN_RENAMED_CODE,
|
|
813
|
-
`${domain} column "${current.name}" renamed to "${name}": the sink already holds that name with another shape`,
|
|
814
|
-
{ ...where, element: where?.element ?? current.name }
|
|
815
|
-
);
|
|
816
|
-
current = { ...current, name };
|
|
817
|
+
current = renameColumn(sink, domain, current, report, where);
|
|
817
818
|
renamed = true;
|
|
818
819
|
continue;
|
|
819
820
|
}
|
|
820
821
|
if (err.code === "E_DUPLICATE_ROLE" && current.role !== void 0 && !roleDropped) {
|
|
821
|
-
|
|
822
|
-
report.warning(
|
|
823
|
-
"coercion",
|
|
824
|
-
ROLE_TAKEN_CODE,
|
|
825
|
-
`${domain} column "${current.name}" loses the ${current.role} role: "${holder}" already holds it`,
|
|
826
|
-
{ ...where, element: where?.element ?? current.name }
|
|
827
|
-
);
|
|
828
|
-
const { role: _role, ...withoutRole } = current;
|
|
829
|
-
current = withoutRole;
|
|
822
|
+
current = dropRole(err, domain, current, current.role, report, where);
|
|
830
823
|
roleDropped = true;
|
|
831
824
|
continue;
|
|
832
825
|
}
|
|
@@ -834,6 +827,29 @@ function declareResolved(sink, domain, decl, report, where) {
|
|
|
834
827
|
}
|
|
835
828
|
}
|
|
836
829
|
}
|
|
830
|
+
function renameColumn(sink, domain, decl, report, where) {
|
|
831
|
+
const taken = takenIn(sink, domain);
|
|
832
|
+
const id = decl.origin?.id ?? null;
|
|
833
|
+
const name = uniqueColumnName(decl.name, id, taken);
|
|
834
|
+
report.warning(
|
|
835
|
+
"coercion",
|
|
836
|
+
COLUMN_RENAMED_CODE,
|
|
837
|
+
`${domain} column "${decl.name}" renamed to "${name}": the sink already holds that name with another shape`,
|
|
838
|
+
{ ...where, element: where?.element ?? decl.name }
|
|
839
|
+
);
|
|
840
|
+
return { ...decl, name };
|
|
841
|
+
}
|
|
842
|
+
function dropRole(err, domain, decl, role, report, where) {
|
|
843
|
+
const holder = typeof err.details.holder === "string" ? err.details.holder : "another column";
|
|
844
|
+
report.warning(
|
|
845
|
+
"coercion",
|
|
846
|
+
ROLE_TAKEN_CODE,
|
|
847
|
+
`${domain} column "${decl.name}" loses the ${role} role: "${holder}" already holds it`,
|
|
848
|
+
{ ...where, element: where?.element ?? decl.name }
|
|
849
|
+
);
|
|
850
|
+
const { role: _role, ...withoutRole } = decl;
|
|
851
|
+
return withoutRole;
|
|
852
|
+
}
|
|
837
853
|
function declareCompanion(sink, domain, companion, report) {
|
|
838
854
|
try {
|
|
839
855
|
return declareOn(sink, domain, companion);
|
|
@@ -1691,39 +1707,68 @@ class ByteDecoder {
|
|
|
1691
1707
|
const bad = this.encoding === "utf-8" ? invalidUtf8(all) : null;
|
|
1692
1708
|
const at = base + (bad === null ? 0 : bad.index);
|
|
1693
1709
|
if (this.mayFallBack && this.asciiSoFar) {
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
if (this.textFollows && bad?.cut === true) {
|
|
1698
|
-
return this.report.fail(
|
|
1699
|
-
INVALID_UTF8_CODE,
|
|
1700
|
-
`invalid UTF-8 at byte ${at}: a byte chunk ends inside a multi-byte sequence and a text chunk follows`,
|
|
1701
|
-
void 0,
|
|
1702
|
-
{ byteOffset: at }
|
|
1703
|
-
);
|
|
1704
|
-
}
|
|
1705
|
-
if (!stream && bad?.truncated === true) {
|
|
1706
|
-
return this.report.fail(
|
|
1707
|
-
INVALID_UTF8_CODE,
|
|
1708
|
-
`the input ends in the middle of a UTF-8 sequence at byte ${at} (it may be truncated)`,
|
|
1709
|
-
void 0,
|
|
1710
|
-
{ byteOffset: at }
|
|
1711
|
-
);
|
|
1712
|
-
}
|
|
1713
|
-
if (!startsWithUtf8(all)) {
|
|
1714
|
-
const cutNote = !stream && bad?.cut === true ? ": it ends inside a multi-byte UTF-8 sequence, so it may be truncated" : "";
|
|
1715
|
-
this.report.warning(
|
|
1716
|
-
"coercion",
|
|
1717
|
-
ENCODING_FALLBACK_CODE,
|
|
1718
|
-
this.declaredUtf8 ? `the input declares UTF-8 but is not valid UTF-8 (at byte ${at}); everything before was ASCII, so it is read as windows-1252 (pass the encoding option to choose another)` : `the input is not valid UTF-8 (at byte ${at}${cutNote}) and declares no encoding; read as windows-1252 (pass the encoding option to choose another)`
|
|
1719
|
-
);
|
|
1720
|
-
this.use("windows-1252", false);
|
|
1721
|
-
this.fellBack = true;
|
|
1722
|
-
this.mixedPending = true;
|
|
1723
|
-
this.reportUtf8After(all, base);
|
|
1724
|
-
return this.decoder.decode(all, { stream });
|
|
1710
|
+
const text = this.fallBack(all, bad, at, base, stream);
|
|
1711
|
+
if (text !== null) {
|
|
1712
|
+
return text;
|
|
1725
1713
|
}
|
|
1726
1714
|
}
|
|
1715
|
+
return this.failDecode(bytes, at, base, stream);
|
|
1716
|
+
}
|
|
1717
|
+
/**
|
|
1718
|
+
* Undeclared input that was ASCII so far failed to decode: fail on binary data or a cut
|
|
1719
|
+
* sequence, else switch to windows-1252 with a warning unless the bytes start with valid
|
|
1720
|
+
* non-ASCII UTF-8.
|
|
1721
|
+
* @param all - the held-back bytes followed by the bytes that failed
|
|
1722
|
+
* @param bad - the first invalid UTF-8 sequence, or null when the encoding is not UTF-8
|
|
1723
|
+
* @param at - the byte offset of the first invalid byte
|
|
1724
|
+
* @param base - the byte offset of `all`
|
|
1725
|
+
* @param stream - whether more bytes follow
|
|
1726
|
+
* @returns the bytes decoded as windows-1252, or null when they do not fall back
|
|
1727
|
+
*/
|
|
1728
|
+
fallBack(all, bad, at, base, stream) {
|
|
1729
|
+
if (this.control !== null) {
|
|
1730
|
+
return this.binary(this.control);
|
|
1731
|
+
}
|
|
1732
|
+
if (this.textFollows && bad?.cut === true) {
|
|
1733
|
+
return this.report.fail(
|
|
1734
|
+
INVALID_UTF8_CODE,
|
|
1735
|
+
`invalid UTF-8 at byte ${at}: a byte chunk ends inside a multi-byte sequence and a text chunk follows`,
|
|
1736
|
+
void 0,
|
|
1737
|
+
{ byteOffset: at }
|
|
1738
|
+
);
|
|
1739
|
+
}
|
|
1740
|
+
if (!stream && bad?.truncated === true) {
|
|
1741
|
+
return this.report.fail(
|
|
1742
|
+
INVALID_UTF8_CODE,
|
|
1743
|
+
`the input ends in the middle of a UTF-8 sequence at byte ${at} (it may be truncated)`,
|
|
1744
|
+
void 0,
|
|
1745
|
+
{ byteOffset: at }
|
|
1746
|
+
);
|
|
1747
|
+
}
|
|
1748
|
+
if (startsWithUtf8(all)) {
|
|
1749
|
+
return null;
|
|
1750
|
+
}
|
|
1751
|
+
const cutNote = !stream && bad?.cut === true ? ": it ends inside a multi-byte UTF-8 sequence, so it may be truncated" : "";
|
|
1752
|
+
this.report.warning(
|
|
1753
|
+
"coercion",
|
|
1754
|
+
ENCODING_FALLBACK_CODE,
|
|
1755
|
+
this.declaredUtf8 ? `the input declares UTF-8 but is not valid UTF-8 (at byte ${at}); everything before was ASCII, so it is read as windows-1252 (pass the encoding option to choose another)` : `the input is not valid UTF-8 (at byte ${at}${cutNote}) and declares no encoding; read as windows-1252 (pass the encoding option to choose another)`
|
|
1756
|
+
);
|
|
1757
|
+
this.use("windows-1252", false);
|
|
1758
|
+
this.fellBack = true;
|
|
1759
|
+
this.mixedPending = true;
|
|
1760
|
+
this.reportUtf8After(all, base);
|
|
1761
|
+
return this.decoder.decode(all, { stream });
|
|
1762
|
+
}
|
|
1763
|
+
/**
|
|
1764
|
+
* Fail the import on bytes that are not valid in the encoding.
|
|
1765
|
+
* @param bytes - the bytes that failed
|
|
1766
|
+
* @param at - the byte offset of the first invalid UTF-8 byte
|
|
1767
|
+
* @param base - the byte offset of the held-back bytes and these
|
|
1768
|
+
* @param stream - whether more bytes follow
|
|
1769
|
+
* @returns never: it throws
|
|
1770
|
+
*/
|
|
1771
|
+
failDecode(bytes, at, base, stream) {
|
|
1727
1772
|
if (this.encoding === "utf-8") {
|
|
1728
1773
|
const after = this.mayFallBack ? " after valid non-ASCII UTF-8 text; pass the encoding option" : "";
|
|
1729
1774
|
return this.report.fail(INVALID_UTF8_CODE, `invalid UTF-8 at byte ${at}${after}`, void 0, {
|
|
@@ -2300,124 +2345,175 @@ function reportTextEncoding(report, encoding) {
|
|
|
2300
2345
|
async function* textChunks(rawInput, report, options = {}) {
|
|
2301
2346
|
const input = normalizeInput(rawInput);
|
|
2302
2347
|
const signal = options.signal ?? null;
|
|
2303
|
-
const onProgress = options.onProgress ?? null;
|
|
2304
2348
|
const filter = new TextFilter(report, options);
|
|
2305
2349
|
throwIfAborted(signal);
|
|
2306
2350
|
if (typeof input === "string") {
|
|
2307
2351
|
reportTextEncoding(report, options.encoding);
|
|
2308
|
-
|
|
2309
|
-
let done2 = 0;
|
|
2310
|
-
let start = 0;
|
|
2311
|
-
while (start < input.length) {
|
|
2312
|
-
throwIfAborted(signal);
|
|
2313
|
-
let end = Math.min(start + DECODE_SLICE, input.length);
|
|
2314
|
-
const highSurrogate = (input.charCodeAt(end - 1) & 64512) === 55296;
|
|
2315
|
-
if (end < input.length && highSurrogate) {
|
|
2316
|
-
end++;
|
|
2317
|
-
}
|
|
2318
|
-
const piece = start === 0 && end === input.length ? input : input.slice(start, end);
|
|
2319
|
-
start = end;
|
|
2320
|
-
const text = filter.push(piece);
|
|
2321
|
-
if (text.length > 0) {
|
|
2322
|
-
yield text;
|
|
2323
|
-
}
|
|
2324
|
-
if (onProgress !== null) {
|
|
2325
|
-
done2 += utf8Length(piece);
|
|
2326
|
-
onProgress(done2, total);
|
|
2327
|
-
}
|
|
2328
|
-
}
|
|
2329
|
-
if (input.length === 0) {
|
|
2330
|
-
onProgress?.(0, 0);
|
|
2331
|
-
}
|
|
2332
|
-
filter.end();
|
|
2352
|
+
yield* stringTextChunks(input, filter, options);
|
|
2333
2353
|
return;
|
|
2334
2354
|
}
|
|
2335
2355
|
const decoder = new ByteDecoder(report, options);
|
|
2336
|
-
const headBytes = Math.max(options.declarationBytes ?? HEAD_BYTES, BOM_BYTES);
|
|
2337
2356
|
if (input instanceof Uint8Array) {
|
|
2338
|
-
|
|
2339
|
-
decoder.start(input.subarray(0, headBytes));
|
|
2340
|
-
for (let offset = 0; offset < length; offset += DECODE_SLICE) {
|
|
2341
|
-
throwIfAborted(signal);
|
|
2342
|
-
if (input.byteLength !== length) {
|
|
2343
|
-
throw new GraphFormatError(
|
|
2344
|
-
"E_UNSUPPORTED",
|
|
2345
|
-
"the input's buffer was detached (transferred) or resized during the import",
|
|
2346
|
-
{ reason: "detached" }
|
|
2347
|
-
);
|
|
2348
|
-
}
|
|
2349
|
-
const slice = input.subarray(offset, Math.min(offset + DECODE_SLICE, length));
|
|
2350
|
-
const text = filter.push(decoder.decode(slice, offset + DECODE_SLICE < length));
|
|
2351
|
-
if (text.length > 0) {
|
|
2352
|
-
yield text;
|
|
2353
|
-
}
|
|
2354
|
-
onProgress?.(Math.min(offset + DECODE_SLICE, length), length);
|
|
2355
|
-
}
|
|
2356
|
-
if (length === 0) {
|
|
2357
|
-
onProgress?.(0, 0);
|
|
2358
|
-
}
|
|
2359
|
-
filter.end();
|
|
2357
|
+
yield* byteTextChunks(input, filter, decoder, options);
|
|
2360
2358
|
return;
|
|
2361
2359
|
}
|
|
2362
|
-
const
|
|
2360
|
+
const stream = new StreamText(filter, decoder, options);
|
|
2361
|
+
for await (const chunk of isReadableStream(input) ? streamChunks(input, signal) : input) {
|
|
2362
|
+
throwIfAborted(signal);
|
|
2363
|
+
const text = stream.push(chunk);
|
|
2364
|
+
if (text.length > 0) {
|
|
2365
|
+
yield text;
|
|
2366
|
+
}
|
|
2367
|
+
options.onProgress?.(stream.done);
|
|
2368
|
+
}
|
|
2369
|
+
const tail = stream.end();
|
|
2370
|
+
if (tail.length > 0) {
|
|
2371
|
+
yield tail;
|
|
2372
|
+
}
|
|
2373
|
+
filter.end();
|
|
2374
|
+
options.onProgress?.(stream.done, stream.done);
|
|
2375
|
+
}
|
|
2376
|
+
function stringSliceEnd(input, start) {
|
|
2377
|
+
const end = Math.min(start + DECODE_SLICE, input.length);
|
|
2378
|
+
const highSurrogate = (input.charCodeAt(end - 1) & 64512) === 55296;
|
|
2379
|
+
return end < input.length && highSurrogate ? end + 1 : end;
|
|
2380
|
+
}
|
|
2381
|
+
function* stringTextChunks(input, filter, options) {
|
|
2382
|
+
const signal = options.signal ?? null;
|
|
2383
|
+
const onProgress = options.onProgress ?? null;
|
|
2384
|
+
const total = onProgress === null ? 0 : utf8Length(input);
|
|
2363
2385
|
let done = 0;
|
|
2364
|
-
let
|
|
2365
|
-
|
|
2366
|
-
const flushHead = (stream) => {
|
|
2367
|
-
const bytes = concatBytes(head);
|
|
2368
|
-
head = [];
|
|
2369
|
-
headLength = 0;
|
|
2370
|
-
decoder.start(bytes.subarray(0, headBytes));
|
|
2371
|
-
return decoder.decode(bytes, stream);
|
|
2372
|
-
};
|
|
2373
|
-
const chunks = isReadableStream(input) ? streamChunks(input, signal) : input;
|
|
2374
|
-
for await (const chunk of chunks) {
|
|
2386
|
+
let start = 0;
|
|
2387
|
+
while (start < input.length) {
|
|
2375
2388
|
throwIfAborted(signal);
|
|
2376
|
-
|
|
2377
|
-
const
|
|
2389
|
+
const end = stringSliceEnd(input, start);
|
|
2390
|
+
const piece = start === 0 && end === input.length ? input : input.slice(start, end);
|
|
2391
|
+
start = end;
|
|
2392
|
+
const text = filter.push(piece);
|
|
2393
|
+
if (text.length > 0) {
|
|
2394
|
+
yield text;
|
|
2395
|
+
}
|
|
2396
|
+
if (onProgress !== null) {
|
|
2397
|
+
done += utf8Length(piece);
|
|
2398
|
+
onProgress(done, total);
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
if (input.length === 0) {
|
|
2402
|
+
onProgress?.(0, 0);
|
|
2403
|
+
}
|
|
2404
|
+
filter.end();
|
|
2405
|
+
}
|
|
2406
|
+
function* byteTextChunks(input, filter, decoder, options) {
|
|
2407
|
+
const signal = options.signal ?? null;
|
|
2408
|
+
const length = input.byteLength;
|
|
2409
|
+
decoder.start(input.subarray(0, headByteCount(options)));
|
|
2410
|
+
for (let offset = 0; offset < length; offset += DECODE_SLICE) {
|
|
2411
|
+
throwIfAborted(signal);
|
|
2412
|
+
if (input.byteLength !== length) {
|
|
2413
|
+
throw new GraphFormatError(
|
|
2414
|
+
"E_UNSUPPORTED",
|
|
2415
|
+
"the input's buffer was detached (transferred) or resized during the import",
|
|
2416
|
+
{ reason: "detached" }
|
|
2417
|
+
);
|
|
2418
|
+
}
|
|
2419
|
+
const slice = input.subarray(offset, Math.min(offset + DECODE_SLICE, length));
|
|
2420
|
+
const text = filter.push(decoder.decode(slice, offset + DECODE_SLICE < length));
|
|
2421
|
+
if (text.length > 0) {
|
|
2422
|
+
yield text;
|
|
2423
|
+
}
|
|
2424
|
+
options.onProgress?.(Math.min(offset + DECODE_SLICE, length), length);
|
|
2425
|
+
}
|
|
2426
|
+
if (length === 0) {
|
|
2427
|
+
options.onProgress?.(0, 0);
|
|
2428
|
+
}
|
|
2429
|
+
filter.end();
|
|
2430
|
+
}
|
|
2431
|
+
function headByteCount(options) {
|
|
2432
|
+
return Math.max(options.declarationBytes ?? HEAD_BYTES, BOM_BYTES);
|
|
2433
|
+
}
|
|
2434
|
+
class StreamText {
|
|
2435
|
+
/**
|
|
2436
|
+
* Create the reader of one stream input.
|
|
2437
|
+
* @param filter - the text checks
|
|
2438
|
+
* @param decoder - the byte decoder
|
|
2439
|
+
* @param options - progress, the declaration reader and its size
|
|
2440
|
+
*/
|
|
2441
|
+
constructor(filter, decoder, options) {
|
|
2442
|
+
this.filter = filter;
|
|
2443
|
+
this.decoder = decoder;
|
|
2444
|
+
this.done = 0;
|
|
2445
|
+
this.head = [];
|
|
2446
|
+
this.headLength = 0;
|
|
2447
|
+
this.headBytes = headByteCount(options);
|
|
2448
|
+
this.needed = options.declaredEncoding === void 0 ? BOM_BYTES : this.headBytes;
|
|
2449
|
+
this.counting = (options.onProgress ?? null) !== null;
|
|
2450
|
+
}
|
|
2451
|
+
/**
|
|
2452
|
+
* Read the next chunk.
|
|
2453
|
+
* @param chunk - a string or a Uint8Array (anything else is E_UNSUPPORTED)
|
|
2454
|
+
* @returns its filtered text; empty while bytes are held back
|
|
2455
|
+
*/
|
|
2456
|
+
push(chunk) {
|
|
2378
2457
|
if (typeof chunk === "string") {
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
} else {
|
|
2385
|
-
decoder.start(new Uint8Array(0));
|
|
2386
|
-
}
|
|
2387
|
-
text = filter.push(pending + chunk);
|
|
2388
|
-
done += onProgress === null ? 0 : utf8Length(chunk);
|
|
2389
|
-
} else if (bytes !== null) {
|
|
2390
|
-
done += bytes.byteLength;
|
|
2391
|
-
if (decoder.started) {
|
|
2392
|
-
text = filter.push(decoder.decode(bytes, true));
|
|
2393
|
-
} else {
|
|
2394
|
-
head.push(bytes);
|
|
2395
|
-
headLength += bytes.byteLength;
|
|
2396
|
-
text = headLength >= needed ? filter.push(flushHead(true)) : "";
|
|
2397
|
-
}
|
|
2398
|
-
} else {
|
|
2458
|
+
this.done += this.counting ? utf8Length(chunk) : 0;
|
|
2459
|
+
return this.filter.push(this.beforeText() + chunk);
|
|
2460
|
+
}
|
|
2461
|
+
const bytes = asBytes(chunk);
|
|
2462
|
+
if (bytes === null) {
|
|
2399
2463
|
throw new GraphFormatError("E_UNSUPPORTED", "an input chunk must be a string or a Uint8Array", {
|
|
2400
2464
|
reason: "chunk type",
|
|
2401
2465
|
found: typeof chunk
|
|
2402
2466
|
});
|
|
2403
2467
|
}
|
|
2404
|
-
|
|
2405
|
-
|
|
2468
|
+
this.done += bytes.byteLength;
|
|
2469
|
+
if (this.decoder.started) {
|
|
2470
|
+
return this.filter.push(this.decoder.decode(bytes, true));
|
|
2406
2471
|
}
|
|
2407
|
-
|
|
2472
|
+
this.head.push(bytes);
|
|
2473
|
+
this.headLength += bytes.byteLength;
|
|
2474
|
+
return this.headLength >= this.needed ? this.filter.push(this.flushHead(true)) : "";
|
|
2408
2475
|
}
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2476
|
+
/**
|
|
2477
|
+
* The input ended: the filtered text of whatever is still held or pending.
|
|
2478
|
+
* @returns the text
|
|
2479
|
+
*/
|
|
2480
|
+
end() {
|
|
2481
|
+
let tail = "";
|
|
2482
|
+
if (this.headLength > 0) {
|
|
2483
|
+
tail = this.flushHead(false);
|
|
2484
|
+
} else if (this.decoder.started) {
|
|
2485
|
+
tail = this.decoder.decode(new Uint8Array(0), false);
|
|
2486
|
+
}
|
|
2487
|
+
return this.filter.push(tail);
|
|
2414
2488
|
}
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2489
|
+
/**
|
|
2490
|
+
* Start the decoder on the held bytes and decode them.
|
|
2491
|
+
* @param stream - whether more bytes follow
|
|
2492
|
+
* @returns their text
|
|
2493
|
+
*/
|
|
2494
|
+
flushHead(stream) {
|
|
2495
|
+
const bytes = concatBytes(this.head);
|
|
2496
|
+
this.head = [];
|
|
2497
|
+
this.headLength = 0;
|
|
2498
|
+
this.decoder.start(bytes.subarray(0, this.headBytes));
|
|
2499
|
+
return this.decoder.decode(bytes, stream);
|
|
2500
|
+
}
|
|
2501
|
+
/**
|
|
2502
|
+
* A text chunk follows: finish any byte sequence still pending in the decoder before switching
|
|
2503
|
+
* to text. The BOM and declaration checks apply to the start of the input only, so bytes after
|
|
2504
|
+
* text are never taken for a BOM.
|
|
2505
|
+
* @returns the text of the pending bytes
|
|
2506
|
+
*/
|
|
2507
|
+
beforeText() {
|
|
2508
|
+
if (this.headLength > 0) {
|
|
2509
|
+
return this.decoder.flushBeforeText(() => this.flushHead(false));
|
|
2510
|
+
}
|
|
2511
|
+
if (this.decoder.started) {
|
|
2512
|
+
return this.decoder.flushBeforeText(() => this.decoder.decode(new Uint8Array(0), false));
|
|
2513
|
+
}
|
|
2514
|
+
this.decoder.start(new Uint8Array(0));
|
|
2515
|
+
return "";
|
|
2418
2516
|
}
|
|
2419
|
-
filter.end();
|
|
2420
|
-
onProgress?.(done, done);
|
|
2421
2517
|
}
|
|
2422
2518
|
function isReadableStream(input) {
|
|
2423
2519
|
return typeof input.getReader === "function";
|
|
@@ -2538,76 +2634,141 @@ class LineReader {
|
|
|
2538
2634
|
* @returns nothing
|
|
2539
2635
|
*/
|
|
2540
2636
|
async *[Symbol.asyncIterator]() {
|
|
2541
|
-
const pending =
|
|
2542
|
-
let pendingLength = 0;
|
|
2637
|
+
const pending = new PendingLine((line) => tooLarge(this.report, this.lineTooLong(), line));
|
|
2543
2638
|
let trailingCr = false;
|
|
2544
2639
|
for await (const chunk of textChunks(this.input, this.report, this.options)) {
|
|
2545
2640
|
let start = 0;
|
|
2546
|
-
const end = chunk.length;
|
|
2547
2641
|
if (trailingCr) {
|
|
2548
2642
|
trailingCr = false;
|
|
2549
2643
|
this.lineNumber++;
|
|
2550
|
-
|
|
2551
|
-
pending.length = 0;
|
|
2552
|
-
pendingLength = 0;
|
|
2553
|
-
yield joined.slice(0, -1);
|
|
2644
|
+
yield pending.take().slice(0, -1);
|
|
2554
2645
|
if (chunk.charCodeAt(0) === 10) {
|
|
2555
2646
|
start = 1;
|
|
2556
2647
|
}
|
|
2557
2648
|
}
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
while (nl >= 0 || cr >= 0) {
|
|
2561
|
-
let cut;
|
|
2562
|
-
let next;
|
|
2563
|
-
if (cr >= 0 && (nl < 0 || cr < nl)) {
|
|
2564
|
-
if (cr === end - 1) {
|
|
2565
|
-
break;
|
|
2566
|
-
}
|
|
2567
|
-
cut = cr;
|
|
2568
|
-
next = chunk.charCodeAt(cr + 1) === 10 ? cr + 2 : cr + 1;
|
|
2569
|
-
} else {
|
|
2570
|
-
cut = nl;
|
|
2571
|
-
next = nl + 1;
|
|
2572
|
-
}
|
|
2649
|
+
const breaks = new LineBreaks(chunk, start);
|
|
2650
|
+
while (breaks.advance()) {
|
|
2573
2651
|
this.lineNumber++;
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
yield piece;
|
|
2577
|
-
} else {
|
|
2578
|
-
if (pendingLength + piece.length > MAX_TEXT_LENGTH) {
|
|
2579
|
-
tooLarge(this.report, this.lineTooLong(), this.lineNumber);
|
|
2580
|
-
}
|
|
2581
|
-
pending.push(piece);
|
|
2582
|
-
const joined = pending.join("");
|
|
2583
|
-
pending.length = 0;
|
|
2584
|
-
pendingLength = 0;
|
|
2585
|
-
yield joined;
|
|
2586
|
-
}
|
|
2587
|
-
start = next;
|
|
2588
|
-
if (nl >= 0 && nl < next) {
|
|
2589
|
-
nl = chunk.indexOf("\n", next);
|
|
2590
|
-
}
|
|
2591
|
-
if (cr >= 0 && cr < next) {
|
|
2592
|
-
cr = chunk.indexOf("\r", next);
|
|
2593
|
-
}
|
|
2652
|
+
yield pending.finish(chunk.slice(start, breaks.cut), this.lineNumber);
|
|
2653
|
+
start = breaks.next;
|
|
2594
2654
|
}
|
|
2595
|
-
if (start <
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
tooLarge(this.report, this.lineTooLong(), this.lineNumber + 1);
|
|
2599
|
-
}
|
|
2600
|
-
pending.push(start === 0 ? chunk : chunk.slice(start));
|
|
2601
|
-
trailingCr = chunk.charCodeAt(end - 1) === 13;
|
|
2655
|
+
if (start < chunk.length) {
|
|
2656
|
+
pending.appendRest(chunk, start, this.lineNumber + 1);
|
|
2657
|
+
trailingCr = chunk.codePointAt(chunk.length - 1) === 13;
|
|
2602
2658
|
}
|
|
2603
2659
|
}
|
|
2604
|
-
if (pending.
|
|
2660
|
+
if (!pending.empty) {
|
|
2605
2661
|
this.lineNumber++;
|
|
2606
|
-
const joined = pending.
|
|
2662
|
+
const joined = pending.take();
|
|
2607
2663
|
yield joined.endsWith("\r") ? joined.slice(0, -1) : joined;
|
|
2608
2664
|
}
|
|
2609
2665
|
}
|
|
2610
2666
|
}
|
|
2667
|
+
class PendingLine {
|
|
2668
|
+
/**
|
|
2669
|
+
* Start an empty line.
|
|
2670
|
+
* @param tooLong - fails the import when the line grows longer than one string can hold
|
|
2671
|
+
*/
|
|
2672
|
+
constructor(tooLong) {
|
|
2673
|
+
this.pieces = [];
|
|
2674
|
+
this.length = 0;
|
|
2675
|
+
this.tooLong = tooLong;
|
|
2676
|
+
}
|
|
2677
|
+
/**
|
|
2678
|
+
* Whether no piece is held.
|
|
2679
|
+
* @returns true when the line in progress is empty
|
|
2680
|
+
*/
|
|
2681
|
+
get empty() {
|
|
2682
|
+
return this.pieces.length === 0;
|
|
2683
|
+
}
|
|
2684
|
+
/**
|
|
2685
|
+
* Hold the rest of a chunk from `start`: the line goes on in the next chunk.
|
|
2686
|
+
* @param chunk - the chunk
|
|
2687
|
+
* @param start - where the unfinished line starts in it
|
|
2688
|
+
* @param line - the 1-based number of the line, for the too-long error
|
|
2689
|
+
*/
|
|
2690
|
+
appendRest(chunk, start, line) {
|
|
2691
|
+
this.append(start === 0 ? chunk : chunk.slice(start), line);
|
|
2692
|
+
}
|
|
2693
|
+
/**
|
|
2694
|
+
* End the line with its last piece.
|
|
2695
|
+
* @param piece - the text from the start of the chunk (or the last break) to the line break
|
|
2696
|
+
* @param line - the 1-based number of the line, for the too-long error
|
|
2697
|
+
* @returns the whole line
|
|
2698
|
+
*/
|
|
2699
|
+
finish(piece, line) {
|
|
2700
|
+
if (this.pieces.length === 0) {
|
|
2701
|
+
return piece;
|
|
2702
|
+
}
|
|
2703
|
+
this.append(piece, line);
|
|
2704
|
+
return this.take();
|
|
2705
|
+
}
|
|
2706
|
+
/**
|
|
2707
|
+
* Join the held pieces and start a new line.
|
|
2708
|
+
* @returns the joined text
|
|
2709
|
+
*/
|
|
2710
|
+
take() {
|
|
2711
|
+
const joined = this.pieces.join("");
|
|
2712
|
+
this.pieces.length = 0;
|
|
2713
|
+
this.length = 0;
|
|
2714
|
+
return joined;
|
|
2715
|
+
}
|
|
2716
|
+
/**
|
|
2717
|
+
* Hold one more piece.
|
|
2718
|
+
* @param piece - the piece
|
|
2719
|
+
* @param line - the 1-based number of the line, for the too-long error
|
|
2720
|
+
*/
|
|
2721
|
+
append(piece, line) {
|
|
2722
|
+
this.length += piece.length;
|
|
2723
|
+
if (this.length > MAX_TEXT_LENGTH) {
|
|
2724
|
+
this.tooLong(line);
|
|
2725
|
+
}
|
|
2726
|
+
this.pieces.push(piece);
|
|
2727
|
+
}
|
|
2728
|
+
}
|
|
2729
|
+
class LineBreaks {
|
|
2730
|
+
/**
|
|
2731
|
+
* Scan a chunk from a position.
|
|
2732
|
+
* @param chunk - the chunk
|
|
2733
|
+
* @param start - where scanning starts
|
|
2734
|
+
*/
|
|
2735
|
+
constructor(chunk, start) {
|
|
2736
|
+
this.cut = 0;
|
|
2737
|
+
this.chunk = chunk;
|
|
2738
|
+
this.next = start;
|
|
2739
|
+
this.nl = chunk.indexOf("\n", start);
|
|
2740
|
+
this.cr = chunk.indexOf("\r", start);
|
|
2741
|
+
}
|
|
2742
|
+
/**
|
|
2743
|
+
* Find the next line break.
|
|
2744
|
+
* @returns false when none is left, or the only one left is a trailing `\r` (it may be the first
|
|
2745
|
+
* half of a `\r\n` split across chunks)
|
|
2746
|
+
*/
|
|
2747
|
+
advance() {
|
|
2748
|
+
const { chunk, next } = this;
|
|
2749
|
+
if (this.nl >= 0 && this.nl < next) {
|
|
2750
|
+
this.nl = chunk.indexOf("\n", next);
|
|
2751
|
+
}
|
|
2752
|
+
if (this.cr >= 0 && this.cr < next) {
|
|
2753
|
+
this.cr = chunk.indexOf("\r", next);
|
|
2754
|
+
}
|
|
2755
|
+
const { nl, cr } = this;
|
|
2756
|
+
if (cr >= 0 && (nl < 0 || cr < nl)) {
|
|
2757
|
+
if (cr === chunk.length - 1) {
|
|
2758
|
+
return false;
|
|
2759
|
+
}
|
|
2760
|
+
this.cut = cr;
|
|
2761
|
+
this.next = chunk.codePointAt(cr + 1) === 10 ? cr + 2 : cr + 1;
|
|
2762
|
+
return true;
|
|
2763
|
+
}
|
|
2764
|
+
if (nl < 0) {
|
|
2765
|
+
return false;
|
|
2766
|
+
}
|
|
2767
|
+
this.cut = nl;
|
|
2768
|
+
this.next = nl + 1;
|
|
2769
|
+
return true;
|
|
2770
|
+
}
|
|
2771
|
+
}
|
|
2611
2772
|
const ID_COERCIONS = /* @__PURE__ */ new Set(["keep", "canonical", "string", "number"]);
|
|
2612
2773
|
const NODE_ID_SOURCES = /* @__PURE__ */ new Set(["id", "label", "index"]);
|
|
2613
2774
|
const DUPLICATE_POLICIES = /* @__PURE__ */ new Set(["keep", "error", "first", "last", "sum", "min", "max"]);
|
|
@@ -4368,4 +4529,4 @@ export {
|
|
|
4368
4529
|
INVALID_ENCODING_CODE as y,
|
|
4369
4530
|
INVALID_UTF8_CODE as z
|
|
4370
4531
|
};
|
|
4371
|
-
//# sourceMappingURL=export-
|
|
4532
|
+
//# sourceMappingURL=export-rKLeGQMo.js.map
|