@dpuse/dpuse-tool-rust-csv-core-parser 1.0.106 → 1.0.115

package/dist/dpuse-tool-rust-csv-core-parser.es.js

@@ -1,4 +1,4 @@
-//#region node_modules/@dpuse/dpuse-shared/dist/dpuse-shared.es.js
+// node_modules/@dpuse/dpuse-shared/dist/dpuse-shared.es.js
 function e(e) {
 	return new Map(Object.entries(e));
 }
@@ -300,7 +300,7 @@ var T = {
 	status: /* @__PURE__ */ d(C),
 	statusId: /* @__PURE__ */ d(S)
 };
-({ ...T }), e({ en: "Data Positioning Events" }), e({ en: "Delimited Text" }), e({ en: "JSON" }), e({ en: "SPSS" }), e({ en: "XLSX" }), e({ en: "XML" }), e({ en: "Newline" }), e({ en: "Carriage Return" }), e({ en: "Carriage Return/Newline" }), e({ en: "Colon" }), e({ en: "Comma" }), e({ en: "Exclamation Mark" }), e({ en: "Record Separator" }), e({ en: "Semicolon" }), e({ en: "Space" }), e({ en: "Tab" }), e({ en: "Underscore" }), e({ en: "Unit Separator" }), e({ en: "Vertical Bar" }), y([
+({ ...T }), e({ en: "Arrow" }), e({ en: "Avro" }), e({ en: "Delimited text (CSV, TSV…)" }), e({ en: "JSON" }), e({ en: "JSON Lines (JSONL)" }), e({ en: "Markdown (MD)" }), e({ en: "OpenDocument spreadsheet (ODS)" }), e({ en: "Parquet" }), e({ en: "SPSS (SAV)" }), e({ en: "SQLite" }), e({ en: "Excel (XLSX, XLSM)" }), e({ en: "XML" }), e({ en: "Newline" }), e({ en: "Carriage Return" }), e({ en: "Carriage Return/Newline" }), e({ en: "Colon" }), e({ en: "Comma" }), e({ en: "Exclamation Mark" }), e({ en: "Record Separator" }), e({ en: "Semicolon" }), e({ en: "Space" }), e({ en: "Tab" }), e({ en: "Underscore" }), e({ en: "Unit Separator" }), e({ en: "Vertical Bar" }), y([
 	"app",
 	"engine",
 	"connector",
@@ -318,6 +318,7 @@ var E = {
 ({ ...E }), y([
 	"abortOperation",
 	"auditObjectContent",
+	"closeStore",
 	"createObject",
 	"describeConnection",
 	"dropObject",
@@ -326,7 +327,10 @@ var E = {
 	"getReadableStream",
 	"getRecord",
 	"listNodes",
+	"loadObjectRecords",
+	"openStore",
 	"previewObject",
+	"queryRecords",
 	"removeRecords",
 	"retrieveChunks",
 	"retrieveRecords",
@@ -336,6 +340,10 @@ var E = {
 	"curatedDataset",
 	"database",
 	"fileStore"
+]), y([
+	"apiEndpoint",
+	"file",
+	"table"
 ]), y([
 	"bidirectional",
 	"destination",
@@ -346,10 +354,6 @@ var E = {
 	"disabled",
 	"oAuth2",
 	"none"
-]), { ...E }, y(["list"]), { ...E }, y([
-	"list",
-	"render",
-	"setColorMode"
 ]), { ...E };
 var D = class extends Error {
 	data;
@@ -361,14 +365,582 @@ var D = class extends Error {
 	constructor(e, t, n, r) {
 		super(e, t, n, r), this.name = "ConnectorError";
 	}
-}, k = {
+};
+function k(e) {
+	try {
+		e();
+	} catch {}
+}
+y([
+	"arrow",
+	"avro",
+	"dtv",
+	"json",
+	"jsonl",
+	"markdown",
+	"ods",
+	"parquet",
+	"sav",
+	"sqlite",
+	"xlsx",
+	"xml"
+]), { ...E };
+async function A(e, t, n) {
+	let r = j(e), { issues: i, parsedRecords: a, valueDelimiterId: o } = await M(e, t.valueDelimiterIds, t.isComplete, n);
+	return {
+		issues: i,
+		parsedRecords: a,
+		recordDelimiterId: r,
+		valueDelimiterId: o
+	};
+}
+function j(e) {
+	let t = (e.match(/\r\n/g) ?? []).length, n = (e.match(/(?<!\r)\n/g) ?? []).length, r = (e.match(/\r(?!\n)/g) ?? []).length;
+	return t >= n && t >= r ? "\r\n" : n >= t && n >= r ? "\n" : r >= t && r >= n ? "\r" : "\n";
+}
+async function M(e, t, n, r) {
+	let i, a, o, s = [], c, l = !1;
+	for (let u of t) {
+		let { issues: t, records: d } = await r(e, u, n);
+		if (t.length > 0) {
+			c = N(c, {
+				delimiter: u,
+				issues: t,
+				records: d
+			});
+			continue;
+		}
+		if (l = !0, d.length === 0) continue;
+		let { averageValueCount: f, sumOfValueCountDiffs: p } = P(d);
+		(!o || p <= o) && (!a || f > a) && (i = u, a = f, o = p, s = d);
+	}
+	if (!l && c != null) {
+		let { delimiter: e, issues: t, records: n } = c;
+		return {
+			issues: t,
+			parsedRecords: n,
+			valueDelimiterId: F(n) ? e : void 0
+		};
+	}
+	return {
+		issues: [],
+		parsedRecords: s,
+		valueDelimiterId: i != null && F(s) ? i : void 0
+	};
+}
+function N(e, t) {
+	return e == null || t.issues.length < e.issues.length ? t : e;
+}
+function P(e) {
+	let t = 0, n = 0, r;
+	for (let i of e) r != null && (n += Math.abs(i.length - r)), r = i.length, t += i.length;
+	return {
+		averageValueCount: t / e.length,
+		sumOfValueCountDiffs: n
+	};
+}
+function F(e) {
+	let t = /* @__PURE__ */ new Map();
+	for (let n of e) t.set(n.length, (t.get(n.length) ?? 0) + 1);
+	let [n = 0, r = 0] = [...t].toSorted((e, t) => t[1] - e[1])[0] ?? [];
+	return n > 1 && r * 2 > e.length;
+}
+y(["list"]), { ...E }, y([
+	"list",
+	"render",
+	"setColorMode"
+]), { ...E };
+var I = {
+	ascii: {
+		id: "ascii",
+		groupId: null,
+		decoderId: "windows-1252"
+	},
+	big5: {
+		id: "big5",
+		groupId: "chineseTraditional",
+		decoderId: "big5"
+	},
+	cp273: {
+		id: "cp273",
+		groupId: "western",
+		decoderId: null
+	},
+	cp424: {
+		id: "cp424",
+		groupId: "hebrew",
+		decoderId: null
+	},
+	cp437: {
+		id: "cp437",
+		groupId: "western",
+		decoderId: null
+	},
+	cp500: {
+		id: "cp500",
+		groupId: "western",
+		decoderId: null
+	},
+	cp720: {
+		id: "cp720",
+		groupId: "arabic",
+		decoderId: null
+	},
+	cp737: {
+		id: "cp737",
+		groupId: "greek",
+		decoderId: null
+	},
+	cp775: {
+		id: "cp775",
+		groupId: "baltic",
+		decoderId: null
+	},
+	cp850: {
+		id: "cp850",
+		groupId: "western",
+		decoderId: null
+	},
+	cp852: {
+		id: "cp852",
+		groupId: "centralEuropean",
+		decoderId: null
+	},
+	cp856: {
+		id: "cp856",
+		groupId: "hebrew",
+		decoderId: null
+	},
+	cp857: {
+		id: "cp857",
+		groupId: "turkish",
+		decoderId: null
+	},
+	cp858: {
+		id: "cp858",
+		groupId: "western",
+		decoderId: null
+	},
+	cp860: {
+		id: "cp860",
+		groupId: "western",
+		decoderId: null
+	},
+	cp861: {
+		id: "cp861",
+		groupId: "nordic",
+		decoderId: null
+	},
+	cp862: {
+		id: "cp862",
+		groupId: "hebrew",
+		decoderId: null
+	},
+	cp863: {
+		id: "cp863",
+		groupId: "western",
+		decoderId: null
+	},
+	cp864: {
+		id: "cp864",
+		groupId: "arabic",
+		decoderId: null
+	},
+	cp865: {
+		id: "cp865",
+		groupId: "nordic",
+		decoderId: null
+	},
+	cp869: {
+		id: "cp869",
+		groupId: "greek",
+		decoderId: null
+	},
+	cp874: {
+		id: "cp874",
+		groupId: "thai",
+		decoderId: "windows-874"
+	},
+	cp875: {
+		id: "cp875",
+		groupId: "greek",
+		decoderId: null
+	},
+	cp932: {
+		id: "cp932",
+		groupId: "japanese",
+		decoderId: "shift_jis"
+	},
+	cp949: {
+		id: "cp949",
+		groupId: "korean",
+		decoderId: "euc-kr"
+	},
+	cp1006: {
+		id: "cp1006",
+		groupId: "arabic",
+		decoderId: null
+	},
+	cp1026: {
+		id: "cp1026",
+		groupId: "turkish",
+		decoderId: null
+	},
+	cp1125: {
+		id: "cp1125",
+		groupId: "cyrillic",
+		decoderId: null
+	},
+	cp1140: {
+		id: "cp1140",
+		groupId: "western",
+		decoderId: null
+	},
+	cp1258: {
+		id: "cp1258",
+		groupId: "vietnamese",
+		decoderId: "windows-1258"
+	},
+	"euc-jp": {
+		id: "euc-jp",
+		groupId: "japanese",
+		decoderId: "euc-jp"
+	},
+	"euc-kr": {
+		id: "euc-kr",
+		groupId: "korean",
+		decoderId: "euc-kr"
+	},
+	gb18030: {
+		id: "gb18030",
+		groupId: "chineseSimplified",
+		decoderId: "gb18030"
+	},
+	"hp-roman8": {
+		id: "hp-roman8",
+		groupId: "western",
+		decoderId: null
+	},
+	"hz-gb-2312": {
+		id: "hz-gb-2312",
+		groupId: "chineseSimplified",
+		decoderId: null
+	},
+	ibm855: {
+		id: "ibm855",
+		groupId: "cyrillic",
+		decoderId: null
+	},
+	ibm866: {
+		id: "ibm866",
+		groupId: "cyrillic",
+		decoderId: "ibm866"
+	},
+	"iso-2022-jp": {
+		id: "iso-2022-jp",
+		groupId: "japanese",
+		decoderId: "iso-2022-jp"
+	},
+	"iso-2022-kr": {
+		id: "iso-2022-kr",
+		groupId: "korean",
+		decoderId: null
+	},
+	"iso-8859-1": {
+		id: "iso-8859-1",
+		groupId: "western",
+		decoderId: "windows-1252"
+	},
+	"iso-8859-2": {
+		id: "iso-8859-2",
+		groupId: "centralEuropean",
+		decoderId: "iso-8859-2"
+	},
+	"iso-8859-5": {
+		id: "iso-8859-5",
+		groupId: "cyrillic",
+		decoderId: "iso-8859-5"
+	},
+	"iso-8859-6": {
+		id: "iso-8859-6",
+		groupId: "arabic",
+		decoderId: "iso-8859-6"
+	},
+	"iso-8859-7": {
+		id: "iso-8859-7",
+		groupId: "greek",
+		decoderId: "iso-8859-7"
+	},
+	"iso-8859-8": {
+		id: "iso-8859-8",
+		groupId: "hebrew",
+		decoderId: "iso-8859-8-i"
+	},
+	"iso-8859-9": {
+		id: "iso-8859-9",
+		groupId: "turkish",
+		decoderId: "windows-1254"
+	},
+	"iso-8859-13": {
+		id: "iso-8859-13",
+		groupId: "baltic",
+		decoderId: "iso-8859-13"
+	},
+	iso2022_jp_2004: {
+		id: "iso2022_jp_2004",
+		groupId: "japanese",
+		decoderId: null
+	},
+	iso2022_jp_ext: {
+		id: "iso2022_jp_ext",
+		groupId: "japanese",
+		decoderId: null
+	},
+	"iso8859-3": {
+		id: "iso8859-3",
+		groupId: "southernEuropean",
+		decoderId: "iso-8859-3"
+	},
+	"iso8859-4": {
+		id: "iso8859-4",
+		groupId: "baltic",
+		decoderId: "iso-8859-4"
+	},
+	"iso8859-10": {
+		id: "iso8859-10",
+		groupId: "nordic",
+		decoderId: "iso-8859-10"
+	},
+	"iso8859-14": {
+		id: "iso8859-14",
+		groupId: "celtic",
+		decoderId: "iso-8859-14"
+	},
+	"iso8859-15": {
+		id: "iso8859-15",
+		groupId: "western",
+		decoderId: "iso-8859-15"
+	},
+	"iso8859-16": {
+		id: "iso8859-16",
+		groupId: "romanian",
+		decoderId: "iso-8859-16"
+	},
+	johab: {
+		id: "johab",
+		groupId: "korean",
+		decoderId: null
+	},
+	"koi8-r": {
+		id: "koi8-r",
+		groupId: "cyrillic",
+		decoderId: "koi8-r"
+	},
+	"koi8-t": {
+		id: "koi8-t",
+		groupId: "cyrillic",
+		decoderId: null
+	},
+	"koi8-u": {
+		id: "koi8-u",
+		groupId: "cyrillic",
+		decoderId: "koi8-u"
+	},
+	kz1048: {
+		id: "kz1048",
+		groupId: "cyrillic",
+		decoderId: null
+	},
+	maccyrillic: {
+		id: "maccyrillic",
+		groupId: "cyrillic",
+		decoderId: "x-mac-cyrillic"
+	},
+	macgreek: {
+		id: "macgreek",
+		groupId: "greek",
+		decoderId: null
+	},
+	maciceland: {
+		id: "maciceland",
+		groupId: "nordic",
+		decoderId: null
+	},
+	maclatin2: {
+		id: "maclatin2",
+		groupId: "centralEuropean",
+		decoderId: null
+	},
+	macroman: {
+		id: "macroman",
+		groupId: "western",
+		decoderId: "macintosh"
+	},
+	macturkish: {
+		id: "macturkish",
+		groupId: "turkish",
+		decoderId: null
+	},
+	ptcp154: {
+		id: "ptcp154",
+		groupId: "cyrillic",
+		decoderId: null
+	},
+	shift_jis: {
+		id: "shift_jis",
+		groupId: "japanese",
+		decoderId: "shift_jis"
+	},
+	"tis-620": {
+		id: "tis-620",
+		groupId: "thai",
+		decoderId: "windows-874"
+	},
+	"utf-7": {
+		id: "utf-7",
+		groupId: "other",
+		decoderId: null
+	},
+	"utf-8": {
+		id: "utf-8",
+		groupId: null,
+		decoderId: "utf-8"
+	},
+	"utf-8-sig": {
+		id: "utf-8-sig",
+		groupId: null,
+		decoderId: "utf-8"
+	},
+	"utf-16": {
+		id: "utf-16",
+		groupId: "unicode16",
+		decoderId: null
+	},
+	"utf-16-be": {
+		id: "utf-16-be",
+		groupId: "unicode16",
+		decoderId: "utf-16be"
+	},
+	"utf-16-le": {
+		id: "utf-16-le",
+		groupId: "unicode16",
+		decoderId: "utf-16le"
+	},
+	"utf-32": {
+		id: "utf-32",
+		groupId: "unicode32",
+		decoderId: null
+	},
+	"utf-32-be": {
+		id: "utf-32-be",
+		groupId: "unicode32",
+		decoderId: null
+	},
+	"utf-32-le": {
+		id: "utf-32-le",
+		groupId: "unicode32",
+		decoderId: null
+	},
+	"windows-1250": {
+		id: "windows-1250",
+		groupId: "centralEuropean",
+		decoderId: "windows-1250"
+	},
+	"windows-1251": {
+		id: "windows-1251",
+		groupId: "cyrillic",
+		decoderId: "windows-1251"
+	},
+	"windows-1252": {
+		id: "windows-1252",
+		groupId: "western",
+		decoderId: "windows-1252"
+	},
+	"windows-1253": {
+		id: "windows-1253",
+		groupId: "greek",
+		decoderId: "windows-1253"
+	},
+	"windows-1254": {
+		id: "windows-1254",
+		groupId: "turkish",
+		decoderId: "windows-1254"
+	},
+	"windows-1255": {
+		id: "windows-1255",
+		groupId: "hebrew",
+		decoderId: "windows-1255"
+	},
+	"windows-1256": {
+		id: "windows-1256",
+		groupId: "arabic",
+		decoderId: "windows-1256"
+	},
+	"windows-1257": {
+		id: "windows-1257",
+		groupId: "baltic",
+		decoderId: "windows-1257"
+	}
+};
+function L(e) {
+	return Object.hasOwn(I, e);
+}
+function R(e) {
+	return L(e) ? I[e].decoderId ?? e : e;
+}
+// src/index.ts
+var z = {
 	id: "rust-csv-core",
 	name: "Rust CSV Core",
 	version: "0.1.0"
-}, A = { rustBindingsPromise: void 0 }, j = class {
-	config = k;
+}, B = 1e4, V = { rustBindingsPromise: void 0 }, H = class {
+	config = z;
+	async parseStream(e, t, n) {
+		let { RecordReader: r } = await K();
+		t.abortSignal.throwIfAborted();
+		let i = new r(U(t.valueDelimiterId)), a = W(t.encodingId), o = Math.max(1, Math.floor(t.chunkSize ?? B)), s = e.getReader(), c = () => {
+			k(() => void s.cancel(t.abortSignal.reason));
+		};
+		t.abortSignal.addEventListener("abort", c, { once: !0 });
+		let l = [], u = async (e) => {
+			for (let t of G(e)) {
+				if (l.push(t), l.length < o) continue;
+				let e = l;
+				l = [], await n("parsingRecordArray", e);
+			}
+		};
+		try {
+			let e = await s.read();
+			for (; !e.done;) t.abortSignal.throwIfAborted(), await u(i.pushChunk(a == null ? e.value : a.transcode(e.value))), e = await s.read();
+			return t.abortSignal.throwIfAborted(), a != null && await u(i.pushChunk(a.finish())), await u(i.finish()), l.length > 0 && await n("parsingRecordArray", l), {
+				byteCount: i.byteCount,
+				commentLineCount: 0,
+				emptyLineCount: i.emptyLineCount,
+				lineCount: i.lineCount,
+				nonUniformRecordCount: i.nonUniformRecordCount,
+				recordCount: i.recordCount
+			};
+		} finally {
+			t.abortSignal.removeEventListener("abort", c), i.free(), s.releaseLock();
+		}
+	}
+	async parseText(e, t) {
+		let { RecordReader: n } = await K();
+		return await A(e, t, (e, t) => {
+			let r = new n(U(t));
+			try {
+				let t = G(r.pushChunk(new TextEncoder().encode(e)));
+				return t.push(...G(r.finish())), Promise.resolve({
+					issues: [],
+					records: t
+				});
+			} finally {
+				r.free();
+			}
+		});
+	}
 	async processWithTransferableStream(e, t = {}, n) {
-		let r = await M(), i = t.delimiter?.codePointAt(0) ?? 44, a = t.hasHeaders ?? !0, o = performance.now(), s = 0, c = (e) => {
+		let r = await K(), i = t.delimiter?.codePointAt(0) ?? 44, a = t.hasHeaders ?? !0, o = performance.now(), s = 0, c = (e) => {
 			s += e, n && n(e);
 		};
 		try {
@@ -382,7 +954,7 @@ var D = class extends Error {
 		}
 	}
 	async processWithChunks(e, t = {}, n) {
-		let { CsvSession: r } = await M(), i = t.delimiter?.codePointAt(0) ?? 44, a = t.hasHeaders ?? !0, o = performance.now(), s = 0, c = new r(i, a);
+		let { CsvSession: r } = await K(), i = t.delimiter?.codePointAt(0) ?? 44, a = t.hasHeaders ?? !0, o = performance.now(), s = 0, c = new r(i, a);
 		try {
 			let t = e.getReader();
 			try {
@@ -406,11 +978,43 @@ var D = class extends Error {
 		}
 	}
 };
-async function M() {
-	return A.rustBindingsPromise ??= (async () => {
-		let e = await import("./dpuse_tool_rust_csv_core_parser-DxibblRh.js");
+function U(e) {
+	if (e === "0x1E") return 30;
+	if (e === "0x1F") return 31;
+	let t = e.codePointAt(0);
+	if (t == null || t > 127) throw Error(`Delimiter '${e}' is not a single ASCII character.`);
+	return t;
+}
+function W(e) {
+	let t = new TextDecoder(R(e));
+	if (t.encoding === "utf-8") return;
+	let n = new TextEncoder();
+	return {
+		finish: () => n.encode(t.decode()),
+		transcode: (e) => n.encode(t.decode(e, { stream: !0 }))
+	};
+}
+function G(e) {
+	let { recordValueCounts: t, text: n, valueEnds: r, valueQuoted: i } = e;
+	e.free();
+	let a = [], o = 0, s = 0;
+	for (let e of t) {
+		let t = [];
+		for (let a = 0; a < e; a++) {
+			let e = r[o] ?? s;
+			t.push({
+				value: n.slice(s, e),
+				valueWasQuoted: i[o] === 1
+			}), s = e, o++;
+		}
+		a.push(t);
+	}
+	return a;
+}
+async function K() {
+	return V.rustBindingsPromise ??= (async () => {
+		let e = await import("./dpuse_tool_rust_csv_core_parser-BZJW07sd.js");
 		return await e.default(), e;
-	})(), A.rustBindingsPromise;
+	})(), V.rustBindingsPromise;
 }
-//#endregion
-export { j as Tool, k as config };
+export { H as Tool, z as config };