@frictionless-ts/table 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +9 -0
- package/README.md +3 -0
- package/build/assets/geojson.json +216 -0
- package/build/assets/topojson.json +259 -0
- package/build/data/index.d.ts +2 -0
- package/build/data/index.js +2 -0
- package/build/data/record.d.ts +4 -0
- package/build/data/record.js +84 -0
- package/build/data/record.spec.d.ts +1 -0
- package/build/data/record.spec.js +269 -0
- package/build/dialect/Options.d.ts +3 -0
- package/build/dialect/Options.js +3 -0
- package/build/dialect/index.d.ts +2 -0
- package/build/dialect/index.js +2 -0
- package/build/dialect/infer.d.ts +4 -0
- package/build/dialect/infer.js +2 -0
- package/build/field/Field.d.ts +5 -0
- package/build/field/Field.js +2 -0
- package/build/field/Mapping.d.ts +11 -0
- package/build/field/Mapping.js +2 -0
- package/build/field/checks/enum.d.ts +8 -0
- package/build/field/checks/enum.js +71 -0
- package/build/field/checks/enum.spec.d.ts +1 -0
- package/build/field/checks/enum.spec.js +327 -0
- package/build/field/checks/maxLength.d.ts +7 -0
- package/build/field/checks/maxLength.js +17 -0
- package/build/field/checks/maxLength.spec.d.ts +1 -0
- package/build/field/checks/maxLength.spec.js +49 -0
- package/build/field/checks/maximum.d.ts +11 -0
- package/build/field/checks/maximum.js +73 -0
- package/build/field/checks/maximum.spec.d.ts +1 -0
- package/build/field/checks/maximum.spec.js +476 -0
- package/build/field/checks/minLength.d.ts +7 -0
- package/build/field/checks/minLength.js +17 -0
- package/build/field/checks/minLength.spec.d.ts +1 -0
- package/build/field/checks/minLength.spec.js +56 -0
- package/build/field/checks/minimum.d.ts +11 -0
- package/build/field/checks/minimum.js +73 -0
- package/build/field/checks/minimum.spec.d.ts +1 -0
- package/build/field/checks/minimum.spec.js +476 -0
- package/build/field/checks/pattern.d.ts +7 -0
- package/build/field/checks/pattern.js +17 -0
- package/build/field/checks/pattern.spec.d.ts +1 -0
- package/build/field/checks/pattern.spec.js +66 -0
- package/build/field/checks/required.d.ts +7 -0
- package/build/field/checks/required.js +14 -0
- package/build/field/checks/required.spec.d.ts +1 -0
- package/build/field/checks/required.spec.js +24 -0
- package/build/field/checks/type.d.ts +7 -0
- package/build/field/checks/type.js +14 -0
- package/build/field/checks/type.spec.d.ts +1 -0
- package/build/field/checks/type.spec.js +255 -0
- package/build/field/checks/unique.d.ts +7 -0
- package/build/field/checks/unique.js +17 -0
- package/build/field/checks/unique.spec.d.ts +1 -0
- package/build/field/checks/unique.spec.js +97 -0
- package/build/field/denormalize.d.ts +6 -0
- package/build/field/denormalize.js +13 -0
- package/build/field/desubstitute.d.ts +3 -0
- package/build/field/desubstitute.js +13 -0
- package/build/field/index.d.ts +8 -0
- package/build/field/index.js +6 -0
- package/build/field/inspect.d.ts +6 -0
- package/build/field/inspect.js +131 -0
- package/build/field/inspect.spec.d.ts +1 -0
- package/build/field/inspect.spec.js +385 -0
- package/build/field/narrow.d.ts +3 -0
- package/build/field/narrow.js +14 -0
- package/build/field/narrow.spec.d.ts +1 -0
- package/build/field/narrow.spec.js +52 -0
- package/build/field/normalize.d.ts +5 -0
- package/build/field/normalize.js +14 -0
- package/build/field/parse.d.ts +3 -0
- package/build/field/parse.js +47 -0
- package/build/field/parse.spec.d.ts +1 -0
- package/build/field/parse.spec.js +33 -0
- package/build/field/stringify.d.ts +3 -0
- package/build/field/stringify.js +43 -0
- package/build/field/stringify.spec.d.ts +1 -0
- package/build/field/stringify.spec.js +41 -0
- package/build/field/substitute.d.ts +3 -0
- package/build/field/substitute.js +16 -0
- package/build/field/types/array.d.ts +3 -0
- package/build/field/types/array.js +5 -0
- package/build/field/types/array.spec.d.ts +1 -0
- package/build/field/types/array.spec.js +358 -0
- package/build/field/types/boolean.d.ts +4 -0
- package/build/field/types/boolean.js +31 -0
- package/build/field/types/boolean.spec.d.ts +1 -0
- package/build/field/types/boolean.spec.js +76 -0
- package/build/field/types/date.d.ts +4 -0
- package/build/field/types/date.js +14 -0
- package/build/field/types/date.spec.d.ts +1 -0
- package/build/field/types/date.spec.js +52 -0
- package/build/field/types/datetime.d.ts +4 -0
- package/build/field/types/datetime.js +15 -0
- package/build/field/types/datetime.spec.d.ts +1 -0
- package/build/field/types/datetime.spec.js +62 -0
- package/build/field/types/duration.d.ts +4 -0
- package/build/field/types/duration.js +9 -0
- package/build/field/types/duration.spec.d.ts +1 -0
- package/build/field/types/duration.spec.js +37 -0
- package/build/field/types/geojson.d.ts +3 -0
- package/build/field/types/geojson.js +9 -0
- package/build/field/types/geojson.spec.d.ts +1 -0
- package/build/field/types/geojson.spec.js +522 -0
- package/build/field/types/geopoint.d.ts +4 -0
- package/build/field/types/geopoint.js +59 -0
- package/build/field/types/geopoint.spec.d.ts +1 -0
- package/build/field/types/geopoint.spec.js +173 -0
- package/build/field/types/integer.d.ts +4 -0
- package/build/field/types/integer.js +41 -0
- package/build/field/types/integer.spec.d.ts +1 -0
- package/build/field/types/integer.spec.js +102 -0
- package/build/field/types/json.d.ts +6 -0
- package/build/field/types/json.js +68 -0
- package/build/field/types/list.d.ts +4 -0
- package/build/field/types/list.js +30 -0
- package/build/field/types/list.spec.d.ts +1 -0
- package/build/field/types/list.spec.js +230 -0
- package/build/field/types/number.d.ts +4 -0
- package/build/field/types/number.js +50 -0
- package/build/field/types/number.spec.d.ts +1 -0
- package/build/field/types/number.spec.js +101 -0
- package/build/field/types/object.d.ts +3 -0
- package/build/field/types/object.js +5 -0
- package/build/field/types/object.spec.d.ts +1 -0
- package/build/field/types/object.spec.js +393 -0
- package/build/field/types/string.d.ts +4 -0
- package/build/field/types/string.js +32 -0
- package/build/field/types/string.spec.d.ts +1 -0
- package/build/field/types/string.spec.js +162 -0
- package/build/field/types/time.d.ts +4 -0
- package/build/field/types/time.js +18 -0
- package/build/field/types/time.spec.d.ts +1 -0
- package/build/field/types/time.spec.js +53 -0
- package/build/field/types/year.d.ts +4 -0
- package/build/field/types/year.js +16 -0
- package/build/field/types/year.spec.d.ts +1 -0
- package/build/field/types/year.spec.js +50 -0
- package/build/field/types/yearmonth.d.ts +4 -0
- package/build/field/types/yearmonth.js +14 -0
- package/build/field/types/yearmonth.spec.d.ts +1 -0
- package/build/field/types/yearmonth.spec.js +36 -0
- package/build/helpers.d.ts +4 -0
- package/build/helpers.js +12 -0
- package/build/index.d.ts +40 -0
- package/build/index.js +27 -0
- package/build/plugin.d.ts +27 -0
- package/build/plugin.js +2 -0
- package/build/plugins/arrow/index.d.ts +2 -0
- package/build/plugins/arrow/index.js +3 -0
- package/build/plugins/arrow/plugin.d.ts +7 -0
- package/build/plugins/arrow/plugin.js +22 -0
- package/build/plugins/arrow/plugin.spec.d.ts +1 -0
- package/build/plugins/arrow/plugin.spec.js +161 -0
- package/build/plugins/arrow/table/index.d.ts +2 -0
- package/build/plugins/arrow/table/index.js +3 -0
- package/build/plugins/arrow/table/load.d.ts +4 -0
- package/build/plugins/arrow/table/load.js +23 -0
- package/build/plugins/arrow/table/load.spec.d.ts +1 -0
- package/build/plugins/arrow/table/load.spec.js +56 -0
- package/build/plugins/arrow/table/save.d.ts +3 -0
- package/build/plugins/arrow/table/save.js +31 -0
- package/build/plugins/arrow/table/save.spec.d.ts +1 -0
- package/build/plugins/arrow/table/save.spec.js +81 -0
- package/build/plugins/csv/dialect/index.d.ts +1 -0
- package/build/plugins/csv/dialect/index.js +2 -0
- package/build/plugins/csv/dialect/infer.d.ts +4 -0
- package/build/plugins/csv/dialect/infer.js +44 -0
- package/build/plugins/csv/dialect/infer.spec.d.ts +1 -0
- package/build/plugins/csv/dialect/infer.spec.js +54 -0
- package/build/plugins/csv/index.d.ts +2 -0
- package/build/plugins/csv/index.js +3 -0
- package/build/plugins/csv/plugin.d.ts +8 -0
- package/build/plugins/csv/plugin.js +22 -0
- package/build/plugins/csv/plugin.spec.d.ts +1 -0
- package/build/plugins/csv/plugin.spec.js +161 -0
- package/build/plugins/csv/table/index.d.ts +2 -0
- package/build/plugins/csv/table/index.js +3 -0
- package/build/plugins/csv/table/load.d.ts +6 -0
- package/build/plugins/csv/table/load.js +86 -0
- package/build/plugins/csv/table/load.spec.d.ts +1 -0
- package/build/plugins/csv/table/load.spec.js +293 -0
- package/build/plugins/csv/table/save.d.ts +5 -0
- package/build/plugins/csv/table/save.js +29 -0
- package/build/plugins/csv/table/save.spec.d.ts +1 -0
- package/build/plugins/csv/table/save.spec.js +137 -0
- package/build/plugins/inline/index.d.ts +2 -0
- package/build/plugins/inline/index.js +3 -0
- package/build/plugins/inline/plugin.d.ts +7 -0
- package/build/plugins/inline/plugin.js +14 -0
- package/build/plugins/inline/table/index.d.ts +1 -0
- package/build/plugins/inline/table/index.js +2 -0
- package/build/plugins/inline/table/load.d.ts +6 -0
- package/build/plugins/inline/table/load.js +24 -0
- package/build/plugins/inline/table/load.spec.d.ts +1 -0
- package/build/plugins/inline/table/load.spec.js +160 -0
- package/build/plugins/json/buffer/decode.d.ts +4 -0
- package/build/plugins/json/buffer/decode.js +10 -0
- package/build/plugins/json/buffer/encode.d.ts +4 -0
- package/build/plugins/json/buffer/encode.js +8 -0
- package/build/plugins/json/buffer/index.d.ts +2 -0
- package/build/plugins/json/buffer/index.js +3 -0
- package/build/plugins/json/index.d.ts +2 -0
- package/build/plugins/json/index.js +3 -0
- package/build/plugins/json/plugin.d.ts +7 -0
- package/build/plugins/json/plugin.js +25 -0
- package/build/plugins/json/plugin.spec.d.ts +1 -0
- package/build/plugins/json/plugin.spec.js +163 -0
- package/build/plugins/json/table/index.d.ts +2 -0
- package/build/plugins/json/table/index.js +3 -0
- package/build/plugins/json/table/load.d.ts +6 -0
- package/build/plugins/json/table/load.js +55 -0
- package/build/plugins/json/table/load.spec.d.ts +1 -0
- package/build/plugins/json/table/load.spec.js +200 -0
- package/build/plugins/json/table/parse.d.ts +3 -0
- package/build/plugins/json/table/parse.js +6 -0
- package/build/plugins/json/table/save.d.ts +5 -0
- package/build/plugins/json/table/save.js +45 -0
- package/build/plugins/json/table/save.spec.d.ts +1 -0
- package/build/plugins/json/table/save.spec.js +147 -0
- package/build/plugins/ods/index.d.ts +2 -0
- package/build/plugins/ods/index.js +3 -0
- package/build/plugins/ods/plugin.d.ts +7 -0
- package/build/plugins/ods/plugin.js +23 -0
- package/build/plugins/ods/plugin.spec.d.ts +1 -0
- package/build/plugins/ods/plugin.spec.js +142 -0
- package/build/plugins/ods/table/index.d.ts +2 -0
- package/build/plugins/ods/table/index.js +3 -0
- package/build/plugins/ods/table/load.d.ts +4 -0
- package/build/plugins/ods/table/load.js +41 -0
- package/build/plugins/ods/table/load.spec.d.ts +1 -0
- package/build/plugins/ods/table/load.spec.js +167 -0
- package/build/plugins/ods/table/save.d.ts +3 -0
- package/build/plugins/ods/table/save.js +26 -0
- package/build/plugins/ods/table/save.spec.d.ts +1 -0
- package/build/plugins/ods/table/save.spec.js +75 -0
- package/build/plugins/ods/table/test.d.ts +5 -0
- package/build/plugins/ods/table/test.js +23 -0
- package/build/plugins/parquet/index.d.ts +2 -0
- package/build/plugins/parquet/index.js +3 -0
- package/build/plugins/parquet/plugin.d.ts +7 -0
- package/build/plugins/parquet/plugin.js +23 -0
- package/build/plugins/parquet/plugin.spec.d.ts +1 -0
- package/build/plugins/parquet/plugin.spec.js +142 -0
- package/build/plugins/parquet/table/index.d.ts +2 -0
- package/build/plugins/parquet/table/index.js +3 -0
- package/build/plugins/parquet/table/load.d.ts +4 -0
- package/build/plugins/parquet/table/load.js +23 -0
- package/build/plugins/parquet/table/load.spec.d.ts +1 -0
- package/build/plugins/parquet/table/load.spec.js +56 -0
- package/build/plugins/parquet/table/save.d.ts +3 -0
- package/build/plugins/parquet/table/save.js +32 -0
- package/build/plugins/parquet/table/save.spec.d.ts +1 -0
- package/build/plugins/parquet/table/save.spec.js +81 -0
- package/build/plugins/xlxs/index.d.ts +2 -0
- package/build/plugins/xlxs/index.js +3 -0
- package/build/plugins/xlxs/plugin.d.ts +7 -0
- package/build/plugins/xlxs/plugin.js +23 -0
- package/build/plugins/xlxs/plugin.spec.d.ts +1 -0
- package/build/plugins/xlxs/plugin.spec.js +142 -0
- package/build/plugins/xlxs/table/index.d.ts +2 -0
- package/build/plugins/xlxs/table/index.js +3 -0
- package/build/plugins/xlxs/table/load.d.ts +4 -0
- package/build/plugins/xlxs/table/load.js +43 -0
- package/build/plugins/xlxs/table/load.spec.d.ts +1 -0
- package/build/plugins/xlxs/table/load.spec.js +167 -0
- package/build/plugins/xlxs/table/save.d.ts +3 -0
- package/build/plugins/xlxs/table/save.js +28 -0
- package/build/plugins/xlxs/table/save.spec.d.ts +1 -0
- package/build/plugins/xlxs/table/save.spec.js +75 -0
- package/build/plugins/xlxs/table/test.d.ts +5 -0
- package/build/plugins/xlxs/table/test.js +23 -0
- package/build/schema/Mapping.d.ts +6 -0
- package/build/schema/Mapping.js +2 -0
- package/build/schema/Options.d.ts +22 -0
- package/build/schema/Options.js +2 -0
- package/build/schema/Schema.d.ts +4 -0
- package/build/schema/Schema.js +2 -0
- package/build/schema/helpers.d.ts +3 -0
- package/build/schema/helpers.js +6 -0
- package/build/schema/index.d.ts +8 -0
- package/build/schema/index.js +5 -0
- package/build/schema/infer.d.ts +13 -0
- package/build/schema/infer.js +199 -0
- package/build/schema/infer.spec.d.ts +1 -0
- package/build/schema/infer.spec.js +304 -0
- package/build/schema/match.d.ts +6 -0
- package/build/schema/match.js +8 -0
- package/build/table/Frame.d.ts +2 -0
- package/build/table/Frame.js +2 -0
- package/build/table/Table.d.ts +2 -0
- package/build/table/Table.js +2 -0
- package/build/table/checks/unique.d.ts +7 -0
- package/build/table/checks/unique.js +23 -0
- package/build/table/checks/unique.spec.d.ts +1 -0
- package/build/table/checks/unique.spec.js +187 -0
- package/build/table/denormalize.d.ts +6 -0
- package/build/table/denormalize.js +15 -0
- package/build/table/helpers.d.ts +19 -0
- package/build/table/helpers.js +62 -0
- package/build/table/helpers.spec.d.ts +1 -0
- package/build/table/helpers.spec.js +352 -0
- package/build/table/index.d.ts +9 -0
- package/build/table/index.js +8 -0
- package/build/table/inspect.d.ts +8 -0
- package/build/table/inspect.js +165 -0
- package/build/table/inspect.spec.d.ts +1 -0
- package/build/table/inspect.spec.js +335 -0
- package/build/table/normalize.d.ts +6 -0
- package/build/table/normalize.js +27 -0
- package/build/table/normalize.spec.d.ts +1 -0
- package/build/table/normalize.spec.js +222 -0
- package/build/table/query.d.ts +3 -0
- package/build/table/query.js +6 -0
- package/package.json +45 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { inferFormat } from "@frictionless-ts/metadata";
|
|
2
|
+
import { loadArrowTable, saveArrowTable } from "./table/index.js";
|
|
3
|
+
export class ArrowPlugin {
|
|
4
|
+
async loadTable(resource, options) {
|
|
5
|
+
const isArrow = getIsArrow(resource);
|
|
6
|
+
if (!isArrow)
|
|
7
|
+
return undefined;
|
|
8
|
+
return await loadArrowTable(resource, options);
|
|
9
|
+
}
|
|
10
|
+
async saveTable(table, options) {
|
|
11
|
+
const { path, format } = options;
|
|
12
|
+
const isArrow = getIsArrow({ path, format });
|
|
13
|
+
if (!isArrow)
|
|
14
|
+
return undefined;
|
|
15
|
+
return await saveArrowTable(table, options);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
function getIsArrow(resource) {
|
|
19
|
+
const format = inferFormat(resource);
|
|
20
|
+
return format === "arrow" || format === "feather";
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGx1Z2luLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vcGx1Z2lucy9hcnJvdy9wbHVnaW4udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLDJCQUEyQixDQUFBO0FBT3ZELE9BQU8sRUFBRSxjQUFjLEVBQUUsY0FBYyxFQUFFLE1BQU0sa0JBQWtCLENBQUE7QUFFakUsTUFBTSxPQUFPLFdBQVc7SUFDdEIsS0FBSyxDQUFDLFNBQVMsQ0FBQyxRQUEyQixFQUFFLE9BQTBCO1FBQ3JFLE1BQU0sT0FBTyxHQUFHLFVBQVUsQ0FBQyxRQUFRLENBQUMsQ0FBQTtRQUNwQyxJQUFJLENBQUMsT0FBTztZQUFFLE9BQU8sU0FBUyxDQUFBO1FBRTlCLE9BQU8sTUFBTSxjQUFjLENBQUMsUUFBUSxFQUFFLE9BQU8sQ0FBQyxDQUFBO0lBQ2hELENBQUM7SUFFRCxLQUFLLENBQUMsU0FBUyxDQUFDLEtBQVksRUFBRSxPQUF5QjtRQUNyRCxNQUFNLEVBQUUsSUFBSSxFQUFFLE1BQU0sRUFBRSxHQUFHLE9BQU8sQ0FBQTtRQUVoQyxNQUFNLE9BQU8sR0FBRyxVQUFVLENBQUMsRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLENBQUMsQ0FBQTtRQUM1QyxJQUFJLENBQUMsT0FBTztZQUFFLE9BQU8sU0FBUyxDQUFBO1FBRTlCLE9BQU8sTUFBTSxjQUFjLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFBO0lBQzdDLENBQUM7Q0FDRjtBQUVELFNBQVMsVUFBVSxDQUFDLFFBQTJCO0lBQzdDLE1BQU0sTUFBTSxHQUFHLFdBQVcsQ0FBQyxRQUFRLENBQUMsQ0FBQTtJQUNwQyxPQUFPLE1BQU0sS0FBSyxPQUFPLElBQUksTUFBTSxLQUFLLFNBQVMsQ0FBQTtBQUNuRCxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IHR5cGUgeyBSZXNvdXJjZSB9IGZyb20gXCJAZnJpY3Rpb25sZXNzLXRzL21ldGFkYXRhXCJcbmltcG9ydCB7IGluZmVyRm9ybWF0IH0gZnJvbSBcIkBmcmljdGlvbmxlc3MtdHMvbWV0YWRhdGFcIlxuaW1wb3J0IHR5cGUge1xuICBMb2FkVGFibGVPcHRpb25zLFxuICBTYXZlVGFibGVPcHRpb25zLFxuICBUYWJsZVBsdWdpbixcbn0gZnJvbSBcIi4uLy4uL3BsdWdpbi50c1wiXG5pbXBvcnQgdHlwZSB7IFRhYmxlIH0gZnJvbSBcIi4uLy4uL3RhYmxlL2luZGV4LnRzXCJcbmltcG9ydCB7IGxvYWRBcnJvd1RhYmxlLCBzYXZlQXJyb3dUYWJsZSB9IGZyb20gXCIuL3RhYmxlL2luZGV4LnRzXCJcblxuZXhwb3J0IGNsYXNzIEFycm93UGx1Z2luIGltcGxlbWVudHMgVGFibGVQbHVnaW4ge1xuICBhc3luYyBsb2FkVGFibGUocmVzb3VyY2U6IFBhcnRpYWw8UmVzb3VyY2U+LCBvcHRpb25zPzogTG9hZFRhYmxlT3B0aW9ucykge1xuICAgIGNvbnN0IGlzQXJyb3cgPSBnZXRJc0Fycm93KHJlc291cmNlKVxuICAgIGlmICghaXNBcnJvdykgcmV0dXJuIHVuZGVmaW5lZFxuXG4gICAgcmV0dXJuIGF3YWl0IGxvYWRBcnJvd1RhYmxlKHJlc291cmNlLCBvcHRpb25zKVxuICB9XG5cbiAgYXN5bmMgc2F2ZVRhYmxlKHRhYmxlOiBUYWJsZSwgb3B0aW9uczogU2F2ZVRhYmxlT3B0aW9ucykge1xuICAgIGNvbnN0IHsgcGF0aCwgZm9ybWF0IH0gPSBvcHRpb25zXG5cbiAgICBjb25zdCBpc0Fycm93ID0gZ2V0SXNBcnJvdyh7IHBhdGgsIGZvcm1hdCB9KVxuICAgIGlmICghaXNBcnJvdykgcmV0dXJuIHVuZGVmaW5lZFxuXG4gICAgcmV0dXJuIGF3YWl0IHNhdmVBcnJvd1RhYmxlKHRhYmxlLCBvcHRpb25zKVxuICB9XG59XG5cbmZ1bmN0aW9uIGdldElzQXJyb3cocmVzb3VyY2U6IFBhcnRpYWw8UmVzb3VyY2U+KSB7XG4gIGNvbnN0IGZvcm1hdCA9IGluZmVyRm9ybWF0KHJlc291cmNlKVxuICByZXR1cm4gZm9ybWF0ID09PSBcImFycm93XCIgfHwgZm9ybWF0ID09PSBcImZlYXRoZXJcIlxufVxuIl19
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import * as pl from "nodejs-polars";
|
|
2
|
+
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
3
|
+
import { ArrowPlugin } from "./plugin.js";
|
|
4
|
+
import * as tableModule from "./table/index.js";
|
|
5
|
+
vi.mock("./table/index.ts", () => ({
|
|
6
|
+
loadArrowTable: vi.fn(),
|
|
7
|
+
saveArrowTable: vi.fn(),
|
|
8
|
+
}));
|
|
9
|
+
describe("ArrowPlugin", () => {
|
|
10
|
+
let plugin;
|
|
11
|
+
let mockLoadArrowTable;
|
|
12
|
+
let mockSaveArrowTable;
|
|
13
|
+
beforeEach(() => {
|
|
14
|
+
plugin = new ArrowPlugin();
|
|
15
|
+
mockLoadArrowTable = vi.mocked(tableModule.loadArrowTable);
|
|
16
|
+
mockSaveArrowTable = vi.mocked(tableModule.saveArrowTable);
|
|
17
|
+
vi.clearAllMocks();
|
|
18
|
+
});
|
|
19
|
+
describe("loadTable", () => {
|
|
20
|
+
it("should load table from arrow file", async () => {
|
|
21
|
+
const resource = {
|
|
22
|
+
path: "test.arrow",
|
|
23
|
+
};
|
|
24
|
+
const mockTable = pl.DataFrame().lazy();
|
|
25
|
+
mockLoadArrowTable.mockResolvedValue(mockTable);
|
|
26
|
+
const result = await plugin.loadTable(resource);
|
|
27
|
+
expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, undefined);
|
|
28
|
+
expect(result).toEqual(mockTable);
|
|
29
|
+
});
|
|
30
|
+
it("should load table from feather file", async () => {
|
|
31
|
+
const resource = {
|
|
32
|
+
path: "test.feather",
|
|
33
|
+
};
|
|
34
|
+
const mockTable = pl.DataFrame().lazy();
|
|
35
|
+
mockLoadArrowTable.mockResolvedValue(mockTable);
|
|
36
|
+
const result = await plugin.loadTable(resource);
|
|
37
|
+
expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, undefined);
|
|
38
|
+
expect(result).toEqual(mockTable);
|
|
39
|
+
});
|
|
40
|
+
it("should return undefined for non-arrow files", async () => {
|
|
41
|
+
const resource = {
|
|
42
|
+
path: "test.csv",
|
|
43
|
+
};
|
|
44
|
+
const result = await plugin.loadTable(resource);
|
|
45
|
+
expect(mockLoadArrowTable).not.toHaveBeenCalled();
|
|
46
|
+
expect(result).toBeUndefined();
|
|
47
|
+
});
|
|
48
|
+
it("should handle explicit arrow format specification", async () => {
|
|
49
|
+
const resource = {
|
|
50
|
+
path: "test.txt",
|
|
51
|
+
format: "arrow",
|
|
52
|
+
};
|
|
53
|
+
const mockTable = pl.DataFrame().lazy();
|
|
54
|
+
mockLoadArrowTable.mockResolvedValue(mockTable);
|
|
55
|
+
const result = await plugin.loadTable(resource);
|
|
56
|
+
expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, undefined);
|
|
57
|
+
expect(result).toEqual(mockTable);
|
|
58
|
+
});
|
|
59
|
+
it("should handle explicit feather format specification", async () => {
|
|
60
|
+
const resource = {
|
|
61
|
+
path: "test.txt",
|
|
62
|
+
format: "feather",
|
|
63
|
+
};
|
|
64
|
+
const mockTable = pl.DataFrame().lazy();
|
|
65
|
+
mockLoadArrowTable.mockResolvedValue(mockTable);
|
|
66
|
+
const result = await plugin.loadTable(resource);
|
|
67
|
+
expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, undefined);
|
|
68
|
+
expect(result).toEqual(mockTable);
|
|
69
|
+
});
|
|
70
|
+
it("should pass through load options", async () => {
|
|
71
|
+
const resource = {
|
|
72
|
+
path: "test.arrow",
|
|
73
|
+
};
|
|
74
|
+
const options = { denormalized: true };
|
|
75
|
+
const mockTable = pl.DataFrame().lazy();
|
|
76
|
+
mockLoadArrowTable.mockResolvedValue(mockTable);
|
|
77
|
+
await plugin.loadTable(resource, options);
|
|
78
|
+
expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, options);
|
|
79
|
+
});
|
|
80
|
+
it("should handle paths with directories", async () => {
|
|
81
|
+
const resource = {
|
|
82
|
+
path: "/path/to/data.arrow",
|
|
83
|
+
};
|
|
84
|
+
const mockTable = pl.DataFrame().lazy();
|
|
85
|
+
mockLoadArrowTable.mockResolvedValue(mockTable);
|
|
86
|
+
await plugin.loadTable(resource);
|
|
87
|
+
expect(mockLoadArrowTable).toHaveBeenCalledWith(resource, undefined);
|
|
88
|
+
});
|
|
89
|
+
it("should return undefined for parquet files", async () => {
|
|
90
|
+
const resource = {
|
|
91
|
+
path: "test.parquet",
|
|
92
|
+
};
|
|
93
|
+
const result = await plugin.loadTable(resource);
|
|
94
|
+
expect(mockLoadArrowTable).not.toHaveBeenCalled();
|
|
95
|
+
expect(result).toBeUndefined();
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
describe("saveTable", () => {
|
|
99
|
+
it("should save table to arrow file", async () => {
|
|
100
|
+
const table = pl.DataFrame().lazy();
|
|
101
|
+
const options = { path: "output.arrow" };
|
|
102
|
+
mockSaveArrowTable.mockResolvedValue("output.arrow");
|
|
103
|
+
const result = await plugin.saveTable(table, options);
|
|
104
|
+
expect(mockSaveArrowTable).toHaveBeenCalledWith(table, options);
|
|
105
|
+
expect(result).toBe("output.arrow");
|
|
106
|
+
});
|
|
107
|
+
it("should save table to feather file", async () => {
|
|
108
|
+
const table = pl.DataFrame().lazy();
|
|
109
|
+
const options = { path: "output.feather" };
|
|
110
|
+
mockSaveArrowTable.mockResolvedValue("output.feather");
|
|
111
|
+
const result = await plugin.saveTable(table, options);
|
|
112
|
+
expect(mockSaveArrowTable).toHaveBeenCalledWith(table, options);
|
|
113
|
+
expect(result).toBe("output.feather");
|
|
114
|
+
});
|
|
115
|
+
it("should return undefined for non-arrow files", async () => {
|
|
116
|
+
const table = pl.DataFrame().lazy();
|
|
117
|
+
const options = { path: "output.csv" };
|
|
118
|
+
const result = await plugin.saveTable(table, options);
|
|
119
|
+
expect(mockSaveArrowTable).not.toHaveBeenCalled();
|
|
120
|
+
expect(result).toBeUndefined();
|
|
121
|
+
});
|
|
122
|
+
it("should handle explicit arrow format specification", async () => {
|
|
123
|
+
const table = pl.DataFrame().lazy();
|
|
124
|
+
const options = { path: "output.txt", format: "arrow" };
|
|
125
|
+
mockSaveArrowTable.mockResolvedValue("output.txt");
|
|
126
|
+
const result = await plugin.saveTable(table, options);
|
|
127
|
+
expect(mockSaveArrowTable).toHaveBeenCalledWith(table, options);
|
|
128
|
+
expect(result).toBe("output.txt");
|
|
129
|
+
});
|
|
130
|
+
it("should handle explicit feather format specification", async () => {
|
|
131
|
+
const table = pl.DataFrame().lazy();
|
|
132
|
+
const options = { path: "output.txt", format: "feather" };
|
|
133
|
+
mockSaveArrowTable.mockResolvedValue("output.txt");
|
|
134
|
+
const result = await plugin.saveTable(table, options);
|
|
135
|
+
expect(mockSaveArrowTable).toHaveBeenCalledWith(table, options);
|
|
136
|
+
expect(result).toBe("output.txt");
|
|
137
|
+
});
|
|
138
|
+
it("should handle paths with directories", async () => {
|
|
139
|
+
const table = pl.DataFrame().lazy();
|
|
140
|
+
const options = { path: "/path/to/output.arrow" };
|
|
141
|
+
mockSaveArrowTable.mockResolvedValue("/path/to/output.arrow");
|
|
142
|
+
await plugin.saveTable(table, options);
|
|
143
|
+
expect(mockSaveArrowTable).toHaveBeenCalledWith(table, options);
|
|
144
|
+
});
|
|
145
|
+
it("should return undefined for files without extension", async () => {
|
|
146
|
+
const table = pl.DataFrame().lazy();
|
|
147
|
+
const options = { path: "output" };
|
|
148
|
+
const result = await plugin.saveTable(table, options);
|
|
149
|
+
expect(mockSaveArrowTable).not.toHaveBeenCalled();
|
|
150
|
+
expect(result).toBeUndefined();
|
|
151
|
+
});
|
|
152
|
+
it("should return undefined for parquet files", async () => {
|
|
153
|
+
const table = pl.DataFrame().lazy();
|
|
154
|
+
const options = { path: "output.parquet" };
|
|
155
|
+
const result = await plugin.saveTable(table, options);
|
|
156
|
+
expect(mockSaveArrowTable).not.toHaveBeenCalled();
|
|
157
|
+
expect(result).toBeUndefined();
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGx1Z2luLnNwZWMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9wbHVnaW5zL2Fycm93L3BsdWdpbi5zcGVjLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sS0FBSyxFQUFFLE1BQU0sZUFBZSxDQUFBO0FBQ25DLE9BQU8sRUFBRSxVQUFVLEVBQUUsUUFBUSxFQUFFLE1BQU0sRUFBRSxFQUFFLEVBQUUsRUFBRSxFQUFFLE1BQU0sUUFBUSxDQUFBO0FBQzdELE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxhQUFhLENBQUE7QUFDekMsT0FBTyxLQUFLLFdBQVcsTUFBTSxrQkFBa0IsQ0FBQTtBQUUvQyxFQUFFLENBQUMsSUFBSSxDQUFDLGtCQUFrQixFQUFFLEdBQUcsRUFBRSxDQUFDLENBQUM7SUFDakMsY0FBYyxFQUFFLEVBQUUsQ0FBQyxFQUFFLEVBQUU7SUFDdkIsY0FBYyxFQUFFLEVBQUUsQ0FBQyxFQUFFLEVBQUU7Q0FDeEIsQ0FBQyxDQUFDLENBQUE7QUFFSCxRQUFRLENBQUMsYUFBYSxFQUFFLEdBQUcsRUFBRTtJQUMzQixJQUFJLE1BQW1CLENBQUE7SUFDdkIsSUFBSSxrQkFBNEMsQ0FBQTtJQUNoRCxJQUFJLGtCQUE0QyxDQUFBO0lBRWhELFVBQVUsQ0FBQyxHQUFHLEVBQUU7UUFDZCxNQUFNLEdBQUcsSUFBSSxXQUFXLEVBQUUsQ0FBQTtRQUMxQixrQkFBa0IsR0FBRyxFQUFFLENBQUMsTUFBTSxDQUFDLFdBQVcsQ0FBQyxjQUFjLENBQUMsQ0FBQTtRQUMxRCxrQkFBa0IsR0FBRyxFQUFFLENBQUMsTUFBTSxDQUFDLFdBQVcsQ0FBQyxjQUFjLENBQUMsQ0FBQTtRQUMxRCxFQUFFLENBQUMsYUFBYSxFQUFFLENBQUE7SUFDcEIsQ0FBQyxDQUFDLENBQUE7SUFFRixRQUFRLENBQUMsV0FBVyxFQUFFLEdBQUcsRUFBRTtRQUN6QixFQUFFLENBQUMsbUNBQW1DLEVBQUUsS0FBSyxJQUFJLEVBQUU7WUFDakQsTUFBTSxRQUFRLEdBQXNCO2dCQUNsQyxJQUFJLEVBQUUsWUFBWTthQUNuQixDQUFBO1lBQ0QsTUFBTSxTQUFTLEdBQUcsRUFBRSxDQUFDLFNBQVMsRUFBRSxDQUFDLElBQUksRUFBRSxDQUFBO1lBQ3ZDLGtCQUFrQixDQUFDLGlCQUFpQixDQUFDLFNBQVMsQ0FBQyxDQUFBO1lBRS9DLE1BQU0sTUFBTSxHQUFHLE1BQU0sTUFBTSxDQUFDLFNBQVMsQ0FBQyxRQUFRLENBQUMsQ0FBQTtZQUUvQyxNQUFNLENBQUMsa0JBQWtCLENBQUMsQ0FBQyxvQkFBb0IsQ0FBQyxRQUFRLEVBQUUsU0FBUyxDQUFDLENBQUE7WUFDcEUsTUFBTSxDQUFDLE1BQU0sQ0FBQyxDQUFDLE9BQU8sQ0FBQyxTQUFTLENBQUMsQ0FBQTtRQUNuQyxDQUFDLENBQUMsQ0FBQTtRQUVGLEVBQUUsQ0FBQyxxQ0FBcUMsRUFBRSxLQUFLLElBQUksRUFBRTtZQUNuRCxNQUFNLFFBQVEsR0FBc0I7Z0JBQ2xDLElBQUksRUFBRSxjQUFjO2FBQ3JCLENBQUE7WUFDRCxNQUFNLFNBQVMsR0FBRyxFQUFFLENBQUMsU0FBUyxFQUFFLENBQUMsSUFBSSxFQUFFLENBQUE7WUFDdkMsa0JBQWtCLENBQUMsaUJBQWlCLENBQUMsU0FBUyxDQUFDLENBQUE7WUFFL0MsTUFBTSxNQUFNLEdBQUcsTUFBTSxNQUFNLENBQUMsU0FBUyxDQUFDLFFBQVEsQ0FBQyxDQUFBO1lBRS9DLE1BQU0sQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDLG9CQUFvQixDQUFDLFFBQVEsRUFBRSxTQUFTLENBQUMsQ0FBQTtZQUNwRSxNQUFNLENBQUMsTUFBTSxDQUFDLENBQUMsT0FBTyxDQUFDLFNBQVMsQ0FBQyxDQUFBO1FBQ25DLENBQUMsQ0FBQyxDQUFBO1FBRUYsRUFBRSxDQUFDLDZDQUE2QyxFQUFFLEtBQUssSUFBSSxFQUFFO1lBQzNELE1BQU0sUUFBUSxHQUFzQjtnQkFDbEMsSUFBSSxFQUFFLFVBQVU7YUFDakIsQ0FBQTtZQUVELE1BQU0sTUFBTSxHQUFHLE1BQU0sTUFBTSxDQUFDLFNBQVMsQ0FBQyxRQUFRLENBQUMsQ0FBQTtZQUUvQyxNQUFNLENBQUMsa0JBQWtCLENBQUMsQ0FBQyxHQUFHLENBQUMsZ0JBQWdCLEVBQUUsQ0FBQTtZQUNqRCxNQUFNLENBQUMsTUFBTSxDQUFDLENBQUMsYUFBYSxFQUFFLENBQUE7UUFDaEMsQ0FBQyxDQUFDLENBQUE7UUFFRixFQUFFLENBQUMsbURBQW1ELEVBQUUsS0FBSyxJQUFJLEVBQUU7WUFDakUsTUFBTSxRQUFRLEdBQXNCO2dCQUNsQyxJQUFJLEVBQUUsVUFBVTtnQkFDaEIsTUFBTSxFQUFFLE9BQU87YUFDaEIsQ0FBQTtZQUNELE1BQU0sU0FBUyxHQUFHLEVBQUUsQ0FBQyxTQUFTLEVBQUUsQ0FBQyxJQUFJLEVBQUUsQ0FBQTtZQUN2QyxrQkFBa0IsQ0FBQyxpQkFBaUIsQ0FBQyxTQUFTLENBQUMsQ0FBQTtZQUUvQyxNQUFNLE1BQU0sR0FBRyxNQUFNLE1BQU0sQ0FBQyxTQUFTLENBQUMsUUFBUSxDQUFDLENBQUE7WUFFL0MsTUFBTSxDQUFDLGtCQUFrQixDQUFDLENBQUMsb0JBQW9CLENBQUMsUUFBUSxFQUFFLFNBQVMsQ0FBQyxDQUFBO1lBQ3BFLE1BQU0sQ0FBQyxNQUFNLENBQUMsQ0FBQyxPQUFPLENBQUMsU0FBUyxDQUFDLENBQUE7UUFDbkMsQ0FBQyxDQUFDLENBQUE7UUFFRixFQUFFLENBQUMscURBQXFELEVBQUUsS0FBSyxJQUFJLEVBQUU7WUFDbkUsTUFBTSxRQUFRLEdBQXNCO2dCQUNsQyxJQUFJLEVBQUUsVUFBVTtnQkFDaEIsTUFBTSxFQUFFLFNBQVM7YUFDbEIsQ0FBQTtZQUNELE1BQU0sU0FBUyxHQUFHLEVBQUUsQ0FBQyxTQUFTLEVBQUUsQ0FBQyxJQUFJLEVBQUUsQ0FBQTtZQUN2QyxrQkFBa0IsQ0FBQyxpQkFBaUIsQ0FBQyxTQUFTLENBQUMsQ0FBQTtZQUUvQyxNQUFNLE1BQU0sR0FBRyxNQUFNLE1BQU0sQ0FBQyxTQUFTLENBQUMsUUFBUSxDQUFDLENBQUE7WUFFL0MsTUFBTSxDQUFDLGtCQUFrQixDQUFDLENBQUMsb0JBQW9CLENBQUMsUUFBUSxFQUFFLFNBQVMsQ0FBQyxDQUFBO1lBQ3BFLE1BQU0sQ0FBQyxNQUFNLENBQUMsQ0FBQyxPQUFPLENBQUMsU0FBUyxDQUFDLENBQUE7UUFDbkMsQ0FBQyxDQUFDLENBQUE7UUFFRixFQUFFLENBQUMsa0NBQWtDLEVBQUUsS0FBSyxJQUFJLEVBQUU7WUFDaEQsTUFBTSxRQUFRLEdBQXNCO2dCQUNsQyxJQUFJLEVBQUUsWUFBWTthQUNuQixDQUFBO1lBQ0QsTUFBTSxPQUFPLEdBQUcsRUFBRSxZQUFZLEVBQUUsSUFBSSxFQUFFLENBQUE7WUFDdEMsTUFBTSxTQUFTLEdBQUcsRUFBRSxDQUFDLFNBQVMsRUFBRSxDQUFDLElBQUksRUFBRSxDQUFBO1lBQ3ZDLGtCQUFrQixDQUFDLGlCQUFpQixDQUFDLFNBQVMsQ0FBQyxDQUFBO1lBRS9DLE1BQU0sTUFBTSxDQUFDLFNBQVMsQ0FBQyxRQUFRLEVBQUUsT0FBTyxDQUFDLENBQUE7WUFFekMsTUFBTSxDQUFDLGtCQUFrQixDQUFDLENBQUMsb0JBQW9CLENBQUMsUUFBUSxFQUFFLE9BQU8sQ0FBQyxDQUFBO1FBQ3BFLENBQUMsQ0FBQyxDQUFBO1FBRUYsRUFBRSxDQUFDLHNDQUFzQyxFQUFFLEtBQUssSUFBSSxFQUFFO1lBQ3BELE1BQU0sUUFBUSxHQUFzQjtnQkFDbEMsSUFBSSxFQUFFLHFCQUFxQjthQUM1QixDQUFBO1lBQ0QsTUFBTSxTQUFTLEdBQUcsRUFBRSxDQUFDLFNBQVMsRUFBRSxDQUFDLElBQUksRUFBRSxDQUFBO1lBQ3ZDLGtCQUFrQixDQUFDLGlCQUFpQixDQUFDLFNBQVMsQ0FBQyxDQUFBO1lBRS9DLE1BQU0sTUFBTSxDQUFDLFNBQVMsQ0FBQyxRQUFRLENBQUMsQ0FBQTtZQUVoQyxNQUFNLENBQUMsa0JBQWtCLENBQUMsQ0FBQyxvQkFBb0IsQ0FBQyxRQUFRLEVBQUUsU0FBUyxDQUFDLENBQUE7UUFDdEUsQ0FBQyxDQUFDLENBQUE7UUFFRixFQUFFLENBQUMsMkNBQTJDLEVBQUUsS0FBSyxJQUFJLEVBQUU7WUFDekQsTUFBTSxRQUFRLEdBQXNCO2dCQUNsQyxJQUFJLEVBQUUsY0FBYzthQUNyQixDQUFBO1lBRUQsTUFBTSxNQUFNLEdBQUcsTUFBTSxNQUFNLENBQUMsU0FBUyxDQUFDLFFBQVEsQ0FBQyxDQUFBO1lBRS9DLE1BQU0sQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxnQkFBZ0IsRUFBRSxDQUFBO1lBQ2pELE1BQU0sQ0FBQyxNQUFNLENBQUMsQ0FBQyxhQUFhLEVBQUUsQ0FBQTtRQUNoQyxDQUFDLENBQUMsQ0FBQTtJQUNKLENBQUMsQ0FBQyxDQUFBO0lBRUYsUUFBUSxDQUFDLFdBQVcsRUFBRSxHQUFHLEVBQUU7UUFDekIsRUFBRSxDQUFDLGlDQUFpQyxFQUFFLEtBQUssSUFBSSxFQUFFO1lBQy9DLE1BQU0sS0FBSyxHQUFHLEVBQUUsQ0FBQyxTQUFTLEVBQUUsQ0FBQyxJQUFJLEVBQUUsQ0FBQTtZQUNuQyxNQUFNLE9BQU8sR0FBRyxFQUFFLElBQUksRUFBRSxjQUFjLEVBQUUsQ0FBQTtZQUN4QyxrQkFBa0IsQ0FBQyxpQkFBaUIsQ0FBQyxjQUFjLENBQUMsQ0FBQTtZQUVwRCxNQUFNLE1BQU0sR0FBRyxNQUFNLE1BQU0sQ0FBQyxTQUFTLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFBO1lBRXJELE1BQU0sQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDLG9CQUFvQixDQUFDLEtBQUssRUFBRSxPQUFPLENBQUMsQ0FBQTtZQUMvRCxNQUFNLENBQUMsTUFBTSxDQUFDLENBQUMsSUFBSSxDQUFDLGNBQWMsQ0FBQyxDQUFBO1FBQ3JDLENBQUMsQ0FBQyxDQUFBO1FBRUYsRUFBRSxDQUFDLG1DQUFtQyxFQUFFLEtBQUssSUFBSSxFQUFFO1lBQ2pELE1BQU0sS0FBSyxHQUFHLEVBQUUsQ0FBQyxTQUFTLEVBQUUsQ0FBQyxJQUFJLEVBQUUsQ0FBQTtZQUNuQyxNQUFNLE9BQU8sR0FBRyxFQUFFLElBQUksRUFBRSxnQkFBZ0IsRUFBRSxDQUFBO1lBQzFDLGtCQUFrQixDQUFDLGlCQUFpQixDQUFDLGdCQUFnQixDQUFDLENBQUE7WUFFdEQsTUFBTSxNQUFNLEdBQUcsTUFBTSxNQUFNLENBQUMsU0FBUyxDQUFDLEtBQUssRUFBRSxPQUFPLENBQUMsQ0FBQTtZQUVyRCxNQUFNLENBQUMsa0JBQWtCLENBQUMsQ0FBQyxvQkFBb0IsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLENBQUE7WUFDL0QsTUFBTSxDQUFDLE1BQU0sQ0FBQyxDQUFDLElBQUksQ0FBQyxnQkFBZ0IsQ0FBQyxDQUFBO1FBQ3ZDLENBQUMsQ0FBQyxDQUFBO1FBRUYsRUFBRSxDQUFDLDZDQUE2QyxFQUFFLEtBQUssSUFBSSxFQUFFO1lBQzNELE1BQU0sS0FBSyxHQUFHLEVBQUUsQ0FBQyxTQUFTLEVBQUUsQ0FBQyxJQUFJLEVBQUUsQ0FBQTtZQUNuQyxNQUFNLE9BQU8sR0FBRyxFQUFFLElBQUksRUFBRSxZQUFZLEVBQUUsQ0FBQTtZQUV0QyxNQUFNLE1BQU0sR0FBRyxNQUFNLE1BQU0sQ0FBQyxTQUFTLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFBO1lBRXJELE1BQU0sQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxnQkFBZ0IsRUFBRSxDQUFBO1lBQ2pELE1BQU0sQ0FBQyxNQUFNLENBQUMsQ0FBQyxhQUFhLEVBQUUsQ0FBQTtRQUNoQyxDQUFDLENBQUMsQ0FBQTtRQUVGLEVBQUUsQ0FBQyxtREFBbUQsRUFBRSxLQUFLLElBQUksRUFBRTtZQUNqRSxNQUFNLEtBQUssR0FBRyxFQUFFLENBQUMsU0FBUyxFQUFFLENBQUMsSUFBSSxFQUFFLENBQUE7WUFDbkMsTUFBTSxPQUFPLEdBQUcsRUFBRSxJQUFJLEVBQUUsWUFBWSxFQUFFLE1BQU0sRUFBRSxPQUFnQixFQUFFLENBQUE7WUFDaEUsa0JBQWtCLENBQUMsaUJBQWlCLENBQUMsWUFBWSxDQUFDLENBQUE7WUFFbEQsTUFBTSxNQUFNLEdBQUcsTUFBTSxNQUFNLENBQUMsU0FBUyxDQUFDLEtBQUssRUFBRSxPQUFPLENBQUMsQ0FBQTtZQUVyRCxNQUFNLENBQUMsa0JBQWtCLENBQUMsQ0FBQyxvQkFBb0IsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLENBQUE7WUFDL0QsTUFBTSxDQUFDLE1BQU0sQ0FBQyxDQUFDLElBQUksQ0FBQyxZQUFZLENBQUMsQ0FBQTtRQUNuQyxDQUFDLENBQUMsQ0FBQTtRQUVGLEVBQUUsQ0FBQyxxREFBcUQsRUFBRSxLQUFLLElBQUksRUFBRTtZQUNuRSxNQUFNLEtBQUssR0FBRyxFQUFFLENBQUMsU0FBUyxFQUFFLENBQUMsSUFBSSxFQUFFLENBQUE7WUFDbkMsTUFBTSxPQUFPLEdBQUcsRUFBRSxJQUFJLEVBQUUsWUFBWSxFQUFFLE1BQU0sRUFBRSxTQUFrQixFQUFFLENBQUE7WUFDbEUsa0JBQWtCLENBQUMsaUJBQWlCLENBQUMsWUFBWSxDQUFDLENBQUE7WUFFbEQsTUFBTSxNQUFNLEdBQUcsTUFBTSxNQUFNLENBQUMsU0FBUyxDQUFDLEtBQUssRUFBRSxPQUFPLENBQUMsQ0FBQTtZQUVyRCxNQUFNLENBQUMsa0JBQWtCLENBQUMsQ0FBQyxvQkFBb0IsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLENBQUE7WUFDL0QsTUFBTSxDQUFDLE1BQU0sQ0FBQyxDQUFDLElBQUksQ0FBQyxZQUFZLENBQUMsQ0FBQTtRQUNuQyxDQUFDLENBQUMsQ0FBQTtRQUVGLEVBQUUsQ0FBQyxzQ0FBc0MsRUFBRSxLQUFLLElBQUksRUFBRTtZQUNwRCxNQUFNLEtBQUssR0FBRyxFQUFFLENBQUMsU0FBUyxFQUFFLENBQUMsSUFBSSxFQUFFLENBQUE7WUFDbkMsTUFBTSxPQUFPLEdBQUcsRUFBRSxJQUFJLEVBQUUsdUJBQXVCLEVBQUUsQ0FBQTtZQUNqRCxrQkFBa0IsQ0FBQyxpQkFBaUIsQ0FBQyx1QkFBdUIsQ0FBQyxDQUFBO1lBRTdELE1BQU0sTUFBTSxDQUFDLFNBQVMsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLENBQUE7WUFFdEMsTUFBTSxDQUFDLGtCQUFrQixDQUFDLENBQUMsb0JBQW9CLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFBO1FBQ2pFLENBQUMsQ0FBQyxDQUFBO1FBRUYsRUFBRSxDQUFDLHFEQUFxRCxFQUFFLEtBQUssSUFBSSxFQUFFO1lBQ25FLE1BQU0sS0FBSyxHQUFHLEVBQUUsQ0FBQyxTQUFTLEVBQUUsQ0FBQyxJQUFJLEVBQUUsQ0FBQTtZQUNuQyxNQUFNLE9BQU8sR0FBRyxFQUFFLElBQUksRUFBRSxRQUFRLEVBQUUsQ0FBQTtZQUVsQyxNQUFNLE1BQU0sR0FBRyxNQUFNLE1BQU0sQ0FBQyxTQUFTLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFBO1lBRXJELE1BQU0sQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxnQkFBZ0IsRUFBRSxDQUFBO1lBQ2pELE1BQU0sQ0FBQyxNQUFNLENBQUMsQ0FBQyxhQUFhLEVBQUUsQ0FBQTtRQUNoQyxDQUFDLENBQUMsQ0FBQTtRQUVGLEVBQUUsQ0FBQywyQ0FBMkMsRUFBRSxLQUFLLElBQUksRUFBRTtZQUN6RCxNQUFNLEtBQUssR0FBRyxFQUFFLENBQUMsU0FBUyxFQUFFLENBQUMsSUFBSSxFQUFFLENBQUE7WUFDbkMsTUFBTSxPQUFPLEdBQUcsRUFBRSxJQUFJLEVBQUUsZ0JBQWdCLEVBQUUsQ0FBQTtZQUUxQyxNQUFNLE1BQU0sR0FBRyxNQUFNLE1BQU0sQ0FBQyxTQUFTLENBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQyxDQUFBO1lBRXJELE1BQU0sQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxnQkFBZ0IsRUFBRSxDQUFBO1lBQ2pELE1BQU0sQ0FBQyxNQUFNLENBQUMsQ0FBQyxhQUFhLEVBQUUsQ0FBQTtRQUNoQyxDQUFDLENBQUMsQ0FBQTtJQUNKLENBQUMsQ0FBQyxDQUFBO0FBQ0osQ0FBQyxDQUFDLENBQUEiLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgdHlwZSB7IFJlc291cmNlIH0gZnJvbSBcIkBmcmljdGlvbmxlc3MtdHMvbWV0YWRhdGFcIlxuaW1wb3J0ICogYXMgcGwgZnJvbSBcIm5vZGVqcy1wb2xhcnNcIlxuaW1wb3J0IHsgYmVmb3JlRWFjaCwgZGVzY3JpYmUsIGV4cGVjdCwgaXQsIHZpIH0gZnJvbSBcInZpdGVzdFwiXG5pbXBvcnQgeyBBcnJvd1BsdWdpbiB9IGZyb20gXCIuL3BsdWdpbi50c1wiXG5pbXBvcnQgKiBhcyB0YWJsZU1vZHVsZSBmcm9tIFwiLi90YWJsZS9pbmRleC50c1wiXG5cbnZpLm1vY2soXCIuL3RhYmxlL2luZGV4LnRzXCIsICgpID0+ICh7XG4gIGxvYWRBcnJvd1RhYmxlOiB2aS5mbigpLFxuICBzYXZlQXJyb3dUYWJsZTogdmkuZm4oKSxcbn0pKVxuXG5kZXNjcmliZShcIkFycm93UGx1Z2luXCIsICgpID0+IHtcbiAgbGV0IHBsdWdpbjogQXJyb3dQbHVnaW5cbiAgbGV0IG1vY2tMb2FkQXJyb3dUYWJsZTogUmV0dXJuVHlwZTx0eXBlb2YgdmkuZm4+XG4gIGxldCBtb2NrU2F2ZUFycm93VGFibGU6IFJldHVyblR5cGU8dHlwZW9mIHZpLmZuPlxuXG4gIGJlZm9yZUVhY2goKCkgPT4ge1xuICAgIHBsdWdpbiA9IG5ldyBBcnJvd1BsdWdpbigpXG4gICAgbW9ja0xvYWRBcnJvd1RhYmxlID0gdmkubW9ja2VkKHRhYmxlTW9kdWxlLmxvYWRBcnJvd1RhYmxlKVxuICAgIG1vY2tTYXZlQXJyb3dUYWJsZSA9IHZpLm1vY2tlZCh0YWJsZU1vZHVsZS5zYXZlQXJyb3dUYWJsZSlcbiAgICB2aS5jbGVhckFsbE1vY2tzKClcbiAgfSlcblxuICBkZXNjcmliZShcImxvYWRUYWJsZVwiLCAoKSA9PiB7XG4gICAgaXQoXCJzaG91bGQgbG9hZCB0YWJsZSBmcm9tIGFycm93IGZpbGVcIiwgYXN5bmMgKCkgPT4ge1xuICAgICAgY29uc3QgcmVzb3VyY2U6IFBhcnRpYWw8UmVzb3VyY2U+ID0ge1xuICAgICAgICBwYXRoOiBcInRlc3QuYXJyb3dcIixcbiAgICAgIH1cbiAgICAgIGNvbnN0IG1vY2tUYWJsZSA9IHBsLkRhdGFGcmFtZSgpLmxhenkoKVxuICAgICAgbW9ja0xvYWRBcnJvd1RhYmxlLm1vY2tSZXNvbHZlZFZhbHVlKG1vY2tUYWJsZSlcblxuICAgICAgY29uc3QgcmVzdWx0ID0gYXdhaXQgcGx1Z2luLmxvYWRUYWJsZShyZXNvdXJjZSlcblxuICAgICAgZXhwZWN0KG1vY2tMb2FkQXJyb3dUYWJsZSkudG9IYXZlQmVlbkNhbGxlZFdpdGgocmVzb3VyY2UsIHVuZGVmaW5lZClcbiAgICAgIGV4cGVjdChyZXN1bHQpLnRvRXF1YWwobW9ja1RhYmxlKVxuICAgIH0pXG5cbiAgICBpdChcInNob3VsZCBsb2FkIHRhYmxlIGZyb20gZmVhdGhlciBmaWxlXCIsIGFzeW5jICgpID0+IHtcbiAgICAgIGNvbnN0IHJlc291cmNlOiBQYXJ0aWFsPFJlc291cmNlPiA9IHtcbiAgICAgICAgcGF0aDogXCJ0ZXN0LmZlYXRoZXJcIixcbiAgICAgIH1cbiAgICAgIGNvbnN0IG1vY2tUYWJsZSA9IHBsLkRhdGFGcmFtZSgpLmxhenkoKVxuICAgICAgbW9ja0xvYWRBcnJvd1RhYmxlLm1vY2tSZXNvbHZlZFZhbHVlKG1vY2tUYWJsZSlcblxuICAgICAgY29uc3QgcmVzdWx0ID0gYXdhaXQgcGx1Z2luLmxvYWRUYWJsZShyZXNvdXJjZSlcblxuICAgICAgZXhwZWN0KG1vY2tMb2FkQXJyb3dUYWJsZSkudG9IYXZlQmVlbkNhbGxlZFdpdGgocmVzb3VyY2UsIHVuZGVmaW5lZClcbiAgICAgIGV4cGVjdChyZXN1bHQpLnRvRXF1YWwobW9ja1RhYmxlKVxuICAgIH0pXG5cbiAgICBpdChcInNob3VsZCByZXR1cm4gdW5kZWZpbmVkIGZvciBub24tYXJyb3cgZmlsZXNcIiwgYXN5bmMgKCkgPT4ge1xuICAgICAgY29uc3QgcmVzb3VyY2U6IFBhcnRpYWw8UmVzb3VyY2U+ID0ge1xuICAgICAgICBwYXRoOiBcInRlc3QuY3N2XCIsXG4gICAgICB9XG5cbiAgICAgIGNvbnN0IHJlc3VsdCA9IGF3YWl0IHBsdWdpbi5sb2FkVGFibGUocmVzb3VyY2UpXG5cbiAgICAgIGV4cGVjdChtb2NrTG9hZEFycm93VGFibGUpLm5vdC50b0hhdmVCZWVuQ2FsbGVkKClcbiAgICAgIGV4cGVjdChyZXN1bHQpLnRvQmVVbmRlZmluZWQoKVxuICAgIH0pXG5cbiAgICBpdChcInNob3VsZCBoYW5kbGUgZXhwbGljaXQgYXJyb3cgZm9ybWF0IHNwZWNpZmljYXRpb25cIiwgYXN5bmMgKCkgPT4ge1xuICAgICAgY29uc3QgcmVzb3VyY2U6IFBhcnRpYWw8UmVzb3VyY2U+ID0ge1xuICAgICAgICBwYXRoOiBcInRlc3QudHh0XCIsXG4gICAgICAgIGZvcm1hdDogXCJhcnJvd1wiLFxuICAgICAgfVxuICAgICAgY29uc3QgbW9ja1RhYmxlID0gcGwuRGF0YUZyYW1lKCkubGF6eSgpXG4gICAgICBtb2NrTG9hZEFycm93VGFibGUubW9ja1Jlc29sdmVkVmFsdWUobW9ja1RhYmxlKVxuXG4gICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCBwbHVnaW4ubG9hZFRhYmxlKHJlc291cmNlKVxuXG4gICAgICBleHBlY3QobW9ja0xvYWRBcnJvd1RhYmxlKS50b0hhdmVCZWVuQ2FsbGVkV2l0aChyZXNvdXJjZSwgdW5kZWZpbmVkKVxuICAgICAgZXhwZWN0KHJlc3VsdCkudG9FcXVhbChtb2NrVGFibGUpXG4gICAgfSlcblxuICAgIGl0KFwic2hvdWxkIGhhbmRsZSBleHBsaWNpdCBmZWF0aGVyIGZvcm1hdCBzcGVjaWZpY2F0aW9uXCIsIGFzeW5jICgpID0+IHtcbiAgICAgIGNvbnN0IHJlc291cmNlOiBQYXJ0aWFsPFJlc291cmNlPiA9IHtcbiAgICAgICAgcGF0aDogXCJ0ZXN0LnR4dFwiLFxuICAgICAgICBmb3JtYXQ6IFwiZmVhdGhlclwiLFxuICAgICAgfVxuICAgICAgY29uc3QgbW9ja1RhYmxlID0gcGwuRGF0YUZyYW1lKCkubGF6eSgpXG4gICAgICBtb2NrTG9hZEFycm93VGFibGUubW9ja1Jlc29sdmVkVmFsdWUobW9ja1RhYmxlKVxuXG4gICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCBwbHVnaW4ubG9hZFRhYmxlKHJlc291cmNlKVxuXG4gICAgICBleHBlY3QobW9ja0xvYWRBcnJvd1RhYmxlKS50b0hhdmVCZWVuQ2FsbGVkV2l0aChyZXNvdXJjZSwgdW5kZWZpbmVkKVxuICAgICAgZXhwZWN0KHJlc3VsdCkudG9FcXVhbChtb2NrVGFibGUpXG4gICAgfSlcblxuICAgIGl0KFwic2hvdWxkIHBhc3MgdGhyb3VnaCBsb2FkIG9wdGlvbnNcIiwgYXN5bmMgKCkgPT4ge1xuICAgICAgY29uc3QgcmVzb3VyY2U6IFBhcnRpYWw8UmVzb3VyY2U+ID0ge1xuICAgICAgICBwYXRoOiBcInRlc3QuYXJyb3dcIixcbiAgICAgIH1cbiAgICAgIGNvbnN0IG9wdGlvbnMgPSB7IGRlbm9ybWFsaXplZDogdHJ1ZSB9XG4gICAgICBjb25zdCBtb2NrVGFibGUgPSBwbC5EYXRhRnJhbWUoKS5sYXp5KClcbiAgICAgIG1vY2tMb2FkQXJyb3dUYWJsZS5tb2NrUmVzb2x2ZWRWYWx1ZShtb2NrVGFibGUpXG5cbiAgICAgIGF3YWl0IHBsdWdpbi5sb2FkVGFibGUocmVzb3VyY2UsIG9wdGlvbnMpXG5cbiAgICAgIGV4cGVjdChtb2NrTG9hZEFycm93VGFibGUpLnRvSGF2ZUJlZW5DYWxsZWRXaXRoKHJlc291cmNlLCBvcHRpb25zKVxuICAgIH0pXG5cbiAgICBpdChcInNob3VsZCBoYW5kbGUgcGF0aHMgd2l0aCBkaXJlY3Rvcmllc1wiLCBhc3luYyAoKSA9PiB7XG4gICAgICBjb25zdCByZXNvdXJjZTogUGFydGlhbDxSZXNvdXJjZT4gPSB7XG4gICAgICAgIHBhdGg6IFwiL3BhdGgvdG8vZGF0YS5hcnJvd1wiLFxuICAgICAgfVxuICAgICAgY29uc3QgbW9ja1RhYmxlID0gcGwuRGF0YUZyYW1lKCkubGF6eSgpXG4gICAgICBtb2NrTG9hZEFycm93VGFibGUubW9ja1Jlc29sdmVkVmFsdWUobW9ja1RhYmxlKVxuXG4gICAgICBhd2FpdCBwbHVnaW4ubG9hZFRhYmxlKHJlc291cmNlKVxuXG4gICAgICBleHBlY3QobW9ja0xvYWRBcnJvd1RhYmxlKS50b0hhdmVCZWVuQ2FsbGVkV2l0aChyZXNvdXJjZSwgdW5kZWZpbmVkKVxuICAgIH0pXG5cbiAgICBpdChcInNob3VsZCByZXR1cm4gdW5kZWZpbmVkIGZvciBwYXJxdWV0IGZpbGVzXCIsIGFzeW5jICgpID0+IHtcbiAgICAgIGNvbnN0IHJlc291cmNlOiBQYXJ0aWFsPFJlc291cmNlPiA9IHtcbiAgICAgICAgcGF0aDogXCJ0ZXN0LnBhcnF1ZXRcIixcbiAgICAgIH1cblxuICAgICAgY29uc3QgcmVzdWx0ID0gYXdhaXQgcGx1Z2luLmxvYWRUYWJsZShyZXNvdXJjZSlcblxuICAgICAgZXhwZWN0KG1vY2tMb2FkQXJyb3dUYWJsZSkubm90LnRvSGF2ZUJlZW5DYWxsZWQoKVxuICAgICAgZXhwZWN0KHJlc3VsdCkudG9CZVVuZGVmaW5lZCgpXG4gICAgfSlcbiAgfSlcblxuICBkZXNjcmliZShcInNhdmVUYWJsZVwiLCAoKSA9PiB7XG4gICAgaXQoXCJzaG91bGQgc2F2ZSB0YWJsZSB0byBhcnJvdyBmaWxlXCIsIGFzeW5jICgpID0+IHtcbiAgICAgIGNvbnN0IHRhYmxlID0gcGwuRGF0YUZyYW1lKCkubGF6eSgpXG4gICAgICBjb25zdCBvcHRpb25zID0geyBwYXRoOiBcIm91dHB1dC5hcnJvd1wiIH1cbiAgICAgIG1vY2tTYXZlQXJyb3dUYWJsZS5tb2NrUmVzb2x2ZWRWYWx1ZShcIm91dHB1dC5hcnJvd1wiKVxuXG4gICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCBwbHVnaW4uc2F2ZVRhYmxlKHRhYmxlLCBvcHRpb25zKVxuXG4gICAgICBleHBlY3QobW9ja1NhdmVBcnJvd1RhYmxlKS50b0hhdmVCZWVuQ2FsbGVkV2l0aCh0YWJsZSwgb3B0aW9ucylcbiAgICAgIGV4cGVjdChyZXN1bHQpLnRvQmUoXCJvdXRwdXQuYXJyb3dcIilcbiAgICB9KVxuXG4gICAgaXQoXCJzaG91bGQgc2F2ZSB0YWJsZSB0byBmZWF0aGVyIGZpbGVcIiwgYXN5bmMgKCkgPT4ge1xuICAgICAgY29uc3QgdGFibGUgPSBwbC5EYXRhRnJhbWUoKS5sYXp5KClcbiAgICAgIGNvbnN0IG9wdGlvbnMgPSB7IHBhdGg6IFwib3V0cHV0LmZlYXRoZXJcIiB9XG4gICAgICBtb2NrU2F2ZUFycm93VGFibGUubW9ja1Jlc29sdmVkVmFsdWUoXCJvdXRwdXQuZmVhdGhlclwiKVxuXG4gICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCBwbHVnaW4uc2F2ZVRhYmxlKHRhYmxlLCBvcHRpb25zKVxuXG4gICAgICBleHBlY3QobW9ja1NhdmVBcnJvd1RhYmxlKS50b0hhdmVCZWVuQ2FsbGVkV2l0aCh0YWJsZSwgb3B0aW9ucylcbiAgICAgIGV4cGVjdChyZXN1bHQpLnRvQmUoXCJvdXRwdXQuZmVhdGhlclwiKVxuICAgIH0pXG5cbiAgICBpdChcInNob3VsZCByZXR1cm4gdW5kZWZpbmVkIGZvciBub24tYXJyb3cgZmlsZXNcIiwgYXN5bmMgKCkgPT4ge1xuICAgICAgY29uc3QgdGFibGUgPSBwbC5EYXRhRnJhbWUoKS5sYXp5KClcbiAgICAgIGNvbnN0IG9wdGlvbnMgPSB7IHBhdGg6IFwib3V0cHV0LmNzdlwiIH1cblxuICAgICAgY29uc3QgcmVzdWx0ID0gYXdhaXQgcGx1Z2luLnNhdmVUYWJsZSh0YWJsZSwgb3B0aW9ucylcblxuICAgICAgZXhwZWN0KG1vY2tTYXZlQXJyb3dUYWJsZSkubm90LnRvSGF2ZUJlZW5DYWxsZWQoKVxuICAgICAgZXhwZWN0KHJlc3VsdCkudG9CZVVuZGVmaW5lZCgpXG4gICAgfSlcblxuICAgIGl0KFwic2hvdWxkIGhhbmRsZSBleHBsaWNpdCBhcnJvdyBmb3JtYXQgc3BlY2lmaWNhdGlvblwiLCBhc3luYyAoKSA9PiB7XG4gICAgICBjb25zdCB0YWJsZSA9IHBsLkRhdGFGcmFtZSgpLmxhenkoKVxuICAgICAgY29uc3Qgb3B0aW9ucyA9IHsgcGF0aDogXCJvdXRwdXQudHh0XCIsIGZvcm1hdDogXCJhcnJvd1wiIGFzIGNvbnN0IH1cbiAgICAgIG1vY2tTYXZlQXJyb3dUYWJsZS5tb2NrUmVzb2x2ZWRWYWx1ZShcIm91dHB1dC50eHRcIilcblxuICAgICAgY29uc3QgcmVzdWx0ID0gYXdhaXQgcGx1Z2luLnNhdmVUYWJsZSh0YWJsZSwgb3B0aW9ucylcblxuICAgICAgZXhwZWN0KG1vY2tTYXZlQXJyb3dUYWJsZSkudG9IYXZlQmVlbkNhbGxlZFdpdGgodGFibGUsIG9wdGlvbnMpXG4gICAgICBleHBlY3QocmVzdWx0KS50b0JlKFwib3V0cHV0LnR4dFwiKVxuICAgIH0pXG5cbiAgICBpdChcInNob3VsZCBoYW5kbGUgZXhwbGljaXQgZmVhdGhlciBmb3JtYXQgc3BlY2lmaWNhdGlvblwiLCBhc3luYyAoKSA9PiB7XG4gICAgICBjb25zdCB0YWJsZSA9IHBsLkRhdGFGcmFtZSgpLmxhenkoKVxuICAgICAgY29uc3Qgb3B0aW9ucyA9IHsgcGF0aDogXCJvdXRwdXQudHh0XCIsIGZvcm1hdDogXCJmZWF0aGVyXCIgYXMgY29uc3QgfVxuICAgICAgbW9ja1NhdmVBcnJvd1RhYmxlLm1vY2tSZXNvbHZlZFZhbHVlKFwib3V0cHV0LnR4dFwiKVxuXG4gICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCBwbHVnaW4uc2F2ZVRhYmxlKHRhYmxlLCBvcHRpb25zKVxuXG4gICAgICBleHBlY3QobW9ja1NhdmVBcnJvd1RhYmxlKS50b0hhdmVCZWVuQ2FsbGVkV2l0aCh0YWJsZSwgb3B0aW9ucylcbiAgICAgIGV4cGVjdChyZXN1bHQpLnRvQmUoXCJvdXRwdXQudHh0XCIpXG4gICAgfSlcblxuICAgIGl0KFwic2hvdWxkIGhhbmRsZSBwYXRocyB3aXRoIGRpcmVjdG9yaWVzXCIsIGFzeW5jICgpID0+IHtcbiAgICAgIGNvbnN0IHRhYmxlID0gcGwuRGF0YUZyYW1lKCkubGF6eSgpXG4gICAgICBjb25zdCBvcHRpb25zID0geyBwYXRoOiBcIi9wYXRoL3RvL291dHB1dC5hcnJvd1wiIH1cbiAgICAgIG1vY2tTYXZlQXJyb3dUYWJsZS5tb2NrUmVzb2x2ZWRWYWx1ZShcIi9wYXRoL3RvL291dHB1dC5hcnJvd1wiKVxuXG4gICAgICBhd2FpdCBwbHVnaW4uc2F2ZVRhYmxlKHRhYmxlLCBvcHRpb25zKVxuXG4gICAgICBleHBlY3QobW9ja1NhdmVBcnJvd1RhYmxlKS50b0hhdmVCZWVuQ2FsbGVkV2l0aCh0YWJsZSwgb3B0aW9ucylcbiAgICB9KVxuXG4gICAgaXQoXCJzaG91bGQgcmV0dXJuIHVuZGVmaW5lZCBmb3IgZmlsZXMgd2l0aG91dCBleHRlbnNpb25cIiwgYXN5bmMgKCkgPT4ge1xuICAgICAgY29uc3QgdGFibGUgPSBwbC5EYXRhRnJhbWUoKS5sYXp5KClcbiAgICAgIGNvbnN0IG9wdGlvbnMgPSB7IHBhdGg6IFwib3V0cHV0XCIgfVxuXG4gICAgICBjb25zdCByZXN1bHQgPSBhd2FpdCBwbHVnaW4uc2F2ZVRhYmxlKHRhYmxlLCBvcHRpb25zKVxuXG4gICAgICBleHBlY3QobW9ja1NhdmVBcnJvd1RhYmxlKS5ub3QudG9IYXZlQmVlbkNhbGxlZCgpXG4gICAgICBleHBlY3QocmVzdWx0KS50b0JlVW5kZWZpbmVkKClcbiAgICB9KVxuXG4gICAgaXQoXCJzaG91bGQgcmV0dXJuIHVuZGVmaW5lZCBmb3IgcGFycXVldCBmaWxlc1wiLCBhc3luYyAoKSA9PiB7XG4gICAgICBjb25zdCB0YWJsZSA9IHBsLkRhdGFGcmFtZSgpLmxhenkoKVxuICAgICAgY29uc3Qgb3B0aW9ucyA9IHsgcGF0aDogXCJvdXRwdXQucGFycXVldFwiIH1cblxuICAgICAgY29uc3QgcmVzdWx0ID0gYXdhaXQgcGx1Z2luLnNhdmVUYWJsZSh0YWJsZSwgb3B0aW9ucylcblxuICAgICAgZXhwZWN0KG1vY2tTYXZlQXJyb3dUYWJsZSkubm90LnRvSGF2ZUJlZW5DYWxsZWQoKVxuICAgICAgZXhwZWN0KHJlc3VsdCkudG9CZVVuZGVmaW5lZCgpXG4gICAgfSlcbiAgfSlcbn0pXG4iXX0=
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
export { loadArrowTable } from "./load.js";
|
|
2
|
+
export { saveArrowTable } from "./save.js";
|
|
3
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi9wbHVnaW5zL2Fycm93L3RhYmxlL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxjQUFjLEVBQUUsTUFBTSxXQUFXLENBQUE7QUFDMUMsT0FBTyxFQUFFLGNBQWMsRUFBRSxNQUFNLFdBQVcsQ0FBQSIsInNvdXJjZXNDb250ZW50IjpbImV4cG9ydCB7IGxvYWRBcnJvd1RhYmxlIH0gZnJvbSBcIi4vbG9hZC50c1wiXG5leHBvcnQgeyBzYXZlQXJyb3dUYWJsZSB9IGZyb20gXCIuL3NhdmUudHNcIlxuIl19
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Resource } from "@frictionless-ts/metadata";
|
|
2
|
+
import type { LoadTableOptions } from "../../../plugin.ts";
|
|
3
|
+
import * as pl from "nodejs-polars";
|
|
4
|
+
export declare function loadArrowTable(resource: Partial<Resource>, options?: LoadTableOptions): Promise<pl.LazyDataFrame<any>>;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { resolveSchema } from "@frictionless-ts/metadata";
|
|
2
|
+
import { prefetchFiles } from "@frictionless-ts/dataset";
|
|
3
|
+
import { inferSchemaFromTable } from "../../../schema/index.js";
|
|
4
|
+
import { normalizeTable } from "../../../table/index.js";
|
|
5
|
+
import * as pl from "nodejs-polars";
|
|
6
|
+
export async function loadArrowTable(resource, options) {
|
|
7
|
+
const [firstPath, ...restPaths] = await prefetchFiles(resource.path);
|
|
8
|
+
if (!firstPath) {
|
|
9
|
+
throw new Error("Resource path is not defined");
|
|
10
|
+
}
|
|
11
|
+
let table = pl.scanIPC(firstPath);
|
|
12
|
+
if (restPaths.length) {
|
|
13
|
+
table = pl.concat([table, ...restPaths.map(path => pl.scanIPC(path))]);
|
|
14
|
+
}
|
|
15
|
+
if (!options?.denormalized) {
|
|
16
|
+
let schema = await resolveSchema(resource.schema);
|
|
17
|
+
if (!schema)
|
|
18
|
+
schema = await inferSchemaFromTable(table, options);
|
|
19
|
+
table = await normalizeTable(table, schema);
|
|
20
|
+
}
|
|
21
|
+
return table;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibG9hZC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uL3BsdWdpbnMvYXJyb3cvdGFibGUvbG9hZC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxPQUFPLEVBQUUsYUFBYSxFQUFFLE1BQU0sMkJBQTJCLENBQUE7QUFDekQsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLDBCQUEwQixDQUFBO0FBRXhELE9BQU8sRUFBRSxvQkFBb0IsRUFBRSxNQUFNLDBCQUEwQixDQUFBO0FBQy9ELE9BQU8sRUFBRSxjQUFjLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQTtBQUN4RCxPQUFPLEtBQUssRUFBRSxNQUFNLGVBQWUsQ0FBQTtBQUVuQyxNQUFNLENBQUMsS0FBSyxVQUFVLGNBQWMsQ0FDbEMsUUFBMkIsRUFDM0IsT0FBMEI7SUFFMUIsTUFBTSxDQUFDLFNBQVMsRUFBRSxHQUFHLFNBQVMsQ0FBQyxHQUFHLE1BQU0sYUFBYSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQTtJQUNwRSxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7UUFDZixNQUFNLElBQUksS0FBSyxDQUFDLDhCQUE4QixDQUFDLENBQUE7SUFDakQsQ0FBQztJQUVELElBQUksS0FBSyxHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQUMsU0FBUyxDQUFDLENBQUE7SUFDakMsSUFBSSxTQUFTLENBQUMsTUFBTSxFQUFFLENBQUM7UUFDckIsS0FBSyxHQUFHLEVBQUUsQ0FBQyxNQUFNLENBQUMsQ0FBQyxLQUFLLEVBQUUsR0FBRyxTQUFTLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsRUFBRSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQTtJQUN4RSxDQUFDO0lBRUQsSUFBSSxDQUFDLE9BQU8sRUFBRSxZQUFZLEVBQUUsQ0FBQztRQUMzQixJQUFJLE1BQU0sR0FBRyxNQUFNLGFBQWEsQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLENBQUE7UUFDakQsSUFBSSxDQUFDLE1BQU07WUFBRSxNQUFNLEdBQUcsTUFBTSxvQkFBb0IsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLENBQUE7UUFDaEUsS0FBSyxHQUFHLE1BQU0sY0FBYyxDQUFDLEtBQUssRUFBRSxNQUFNLENBQUMsQ0FBQTtJQUM3QyxDQUFDO0lBRUQsT0FBTyxLQUFLLENBQUE7QUFDZCxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IHR5cGUgeyBSZXNvdXJjZSB9IGZyb20gXCJAZnJpY3Rpb25sZXNzLXRzL21ldGFkYXRhXCJcbmltcG9ydCB7IHJlc29sdmVTY2hlbWEgfSBmcm9tIFwiQGZyaWN0aW9ubGVzcy10cy9tZXRhZGF0YVwiXG5pbXBvcnQgeyBwcmVmZXRjaEZpbGVzIH0gZnJvbSBcIkBmcmljdGlvbmxlc3MtdHMvZGF0YXNldFwiXG5pbXBvcnQgdHlwZSB7IExvYWRUYWJsZU9wdGlvbnMgfSBmcm9tIFwiLi4vLi4vLi4vcGx1Z2luLnRzXCJcbmltcG9ydCB7IGluZmVyU2NoZW1hRnJvbVRhYmxlIH0gZnJvbSBcIi4uLy4uLy4uL3NjaGVtYS9pbmRleC50c1wiXG5pbXBvcnQgeyBub3JtYWxpemVUYWJsZSB9IGZyb20gXCIuLi8uLi8uLi90YWJsZS9pbmRleC50c1wiXG5pbXBvcnQgKiBhcyBwbCBmcm9tIFwibm9kZWpzLXBvbGFyc1wiXG5cbmV4cG9ydCBhc3luYyBmdW5jdGlvbiBsb2FkQXJyb3dUYWJsZShcbiAgcmVzb3VyY2U6IFBhcnRpYWw8UmVzb3VyY2U+LFxuICBvcHRpb25zPzogTG9hZFRhYmxlT3B0aW9ucyxcbikge1xuICBjb25zdCBbZmlyc3RQYXRoLCAuLi5yZXN0UGF0aHNdID0gYXdhaXQgcHJlZmV0Y2hGaWxlcyhyZXNvdXJjZS5wYXRoKVxuICBpZiAoIWZpcnN0UGF0aCkge1xuICAgIHRocm93IG5ldyBFcnJvcihcIlJlc291cmNlIHBhdGggaXMgbm90IGRlZmluZWRcIilcbiAgfVxuXG4gIGxldCB0YWJsZSA9IHBsLnNjYW5JUEMoZmlyc3RQYXRoKVxuICBpZiAocmVzdFBhdGhzLmxlbmd0aCkge1xuICAgIHRhYmxlID0gcGwuY29uY2F0KFt0YWJsZSwgLi4ucmVzdFBhdGhzLm1hcChwYXRoID0+IHBsLnNjYW5JUEMocGF0aCkpXSlcbiAgfVxuXG4gIGlmICghb3B0aW9ucz8uZGVub3JtYWxpemVkKSB7XG4gICAgbGV0IHNjaGVtYSA9IGF3YWl0IHJlc29sdmVTY2hlbWEocmVzb3VyY2Uuc2NoZW1hKVxuICAgIGlmICghc2NoZW1hKSBzY2hlbWEgPSBhd2FpdCBpbmZlclNjaGVtYUZyb21UYWJsZSh0YWJsZSwgb3B0aW9ucylcbiAgICB0YWJsZSA9IGF3YWl0IG5vcm1hbGl6ZVRhYmxlKHRhYmxlLCBzY2hlbWEpXG4gIH1cblxuICByZXR1cm4gdGFibGVcbn1cbiJdfQ==
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { getTempFilePath } from "@frictionless-ts/dataset";
|
|
2
|
+
import * as pl from "nodejs-polars";
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
import { useRecording } from "vitest-polly";
|
|
5
|
+
import { loadArrowTable } from "./load.js";
|
|
6
|
+
useRecording();
|
|
7
|
+
describe("loadArrowTable", () => {
|
|
8
|
+
describe("file variations", () => {
|
|
9
|
+
it("should load local file", async () => {
|
|
10
|
+
const path = getTempFilePath();
|
|
11
|
+
pl.DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeIPC(path);
|
|
12
|
+
const table = await loadArrowTable({ path });
|
|
13
|
+
expect((await table.collect()).toRecords()).toEqual([
|
|
14
|
+
{ id: 1, name: "english" },
|
|
15
|
+
{ id: 2, name: "中文" },
|
|
16
|
+
]);
|
|
17
|
+
});
|
|
18
|
+
it("should load local file (multipart)", async () => {
|
|
19
|
+
const path1 = getTempFilePath();
|
|
20
|
+
const path2 = getTempFilePath();
|
|
21
|
+
pl.DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeIPC(path1);
|
|
22
|
+
pl.DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeIPC(path2);
|
|
23
|
+
const table = await loadArrowTable({ path: [path1, path2] });
|
|
24
|
+
expect((await table.collect()).toRecords()).toEqual([
|
|
25
|
+
{ id: 1, name: "english" },
|
|
26
|
+
{ id: 2, name: "中文" },
|
|
27
|
+
{ id: 1, name: "english" },
|
|
28
|
+
{ id: 2, name: "中文" },
|
|
29
|
+
]);
|
|
30
|
+
});
|
|
31
|
+
it("should load remote file", async () => {
|
|
32
|
+
const table = await loadArrowTable({
|
|
33
|
+
path: "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/arrow/table/fixtures/table.arrow",
|
|
34
|
+
});
|
|
35
|
+
expect((await table.collect()).toRecords()).toEqual([
|
|
36
|
+
{ id: 1, name: "english" },
|
|
37
|
+
{ id: 2, name: "中文" },
|
|
38
|
+
]);
|
|
39
|
+
});
|
|
40
|
+
it("should load remote file (multipart)", async () => {
|
|
41
|
+
const table = await loadArrowTable({
|
|
42
|
+
path: [
|
|
43
|
+
"https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/arrow/table/fixtures/table.arrow",
|
|
44
|
+
"https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/arrow/table/fixtures/table.arrow",
|
|
45
|
+
],
|
|
46
|
+
});
|
|
47
|
+
expect((await table.collect()).toRecords()).toEqual([
|
|
48
|
+
{ id: 1, name: "english" },
|
|
49
|
+
{ id: 2, name: "中文" },
|
|
50
|
+
{ id: 1, name: "english" },
|
|
51
|
+
{ id: 2, name: "中文" },
|
|
52
|
+
]);
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibG9hZC5zcGVjLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vLi4vcGx1Z2lucy9hcnJvdy90YWJsZS9sb2FkLnNwZWMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLGVBQWUsRUFBRSxNQUFNLDBCQUEwQixDQUFBO0FBQzFELE9BQU8sS0FBSyxFQUFFLE1BQU0sZUFBZSxDQUFBO0FBQ25DLE9BQU8sRUFBRSxRQUFRLEVBQUUsTUFBTSxFQUFFLEVBQUUsRUFBRSxNQUFNLFFBQVEsQ0FBQTtBQUM3QyxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sY0FBYyxDQUFBO0FBQzNDLE9BQU8sRUFBRSxjQUFjLEVBQUUsTUFBTSxXQUFXLENBQUE7QUFFMUMsWUFBWSxFQUFFLENBQUE7QUFFZCxRQUFRLENBQUMsZ0JBQWdCLEVBQUUsR0FBRyxFQUFFO0lBQzlCLFFBQVEsQ0FBQyxpQkFBaUIsRUFBRSxHQUFHLEVBQUU7UUFDL0IsRUFBRSxDQUFDLHdCQUF3QixFQUFFLEtBQUssSUFBSSxFQUFFO1lBQ3RDLE1BQU0sSUFBSSxHQUFHLGVBQWUsRUFBRSxDQUFBO1lBQzlCLEVBQUUsQ0FBQyxTQUFTLENBQUMsRUFBRSxFQUFFLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsSUFBSSxFQUFFLENBQUMsU0FBUyxFQUFFLElBQUksQ0FBQyxFQUFFLENBQUMsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLENBQUE7WUFFcEUsTUFBTSxLQUFLLEdBQUcsTUFBTSxjQUFjLENBQUMsRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFBO1lBQzVDLE1BQU0sQ0FBQyxDQUFDLE1BQU0sS0FBSyxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUMsU0FBUyxFQUFFLENBQUMsQ0FBQyxPQUFPLENBQUM7Z0JBQ2xELEVBQUUsRUFBRSxFQUFFLENBQUMsRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFO2dCQUMxQixFQUFFLEVBQUUsRUFBRSxDQUFDLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRTthQUN0QixDQUFDLENBQUE7UUFDSixDQUFDLENBQUMsQ0FBQTtRQUVGLEVBQUUsQ0FBQyxvQ0FBb0MsRUFBRSxLQUFLLElBQUksRUFBRTtZQUNsRCxNQUFNLEtBQUssR0FBRyxlQUFlLEVBQUUsQ0FBQTtZQUMvQixNQUFNLEtBQUssR0FBRyxlQUFlLEVBQUUsQ0FBQTtZQUMvQixFQUFFLENBQUMsU0FBUyxDQUFDLEVBQUUsRUFBRSxFQUFFLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLElBQUksRUFBRSxDQUFDLFNBQVMsRUFBRSxJQUFJLENBQUMsRUFBRSxDQUFDLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFBO1lBQ3JFLEVBQUUsQ0FBQyxTQUFTLENBQUMsRUFBRSxFQUFFLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsSUFBSSxFQUFFLENBQUMsU0FBUyxFQUFFLElBQUksQ0FBQyxFQUFFLENBQUMsQ0FBQyxRQUFRLENBQUMsS0FBSyxDQUFDLENBQUE7WUFFckUsTUFBTSxLQUFLLEdBQUcsTUFBTSxjQUFjLENBQUMsRUFBRSxJQUFJLEVBQUUsQ0FBQyxLQUFLLEVBQUUsS0FBSyxDQUFDLEVBQUUsQ0FBQyxDQUFBO1lBQzVELE1BQU0sQ0FBQyxDQUFDLE1BQU0sS0FBSyxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUMsU0FBUyxFQUFFLENBQUMsQ0FBQyxPQUFPLENBQUM7Z0JBQ2xELEVBQUUsRUFBRSxFQUFFLENBQUMsRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFO2dCQUMxQixFQUFFLEVBQUUsRUFBRSxDQUFDLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRTtnQkFDckIsRUFBRSxFQUFFLEVBQUUsQ0FBQyxFQUFFLElBQUksRUFBRSxTQUFTLEVBQUU7Z0JBQzFCLEVBQUUsRUFBRSxFQUFFLENBQUMsRUFBRSxJQUFJLEVBQUUsSUFBSSxFQUFFO2FBQ3RCLENBQUMsQ0FBQTtRQUNKLENBQUMsQ0FBQyxDQUFBO1FBRUYsRUFBRSxDQUFDLHlCQUF5QixFQUFFLEtBQUssSUFBSSxFQUFFO1lBQ3ZDLE1BQU0sS0FBSyxHQUFHLE1BQU0sY0FBYyxDQUFDO2dCQUNqQyxJQUFJLEVBQUUsbUdBQW1HO2FBQzFHLENBQUMsQ0FBQTtZQUVGLE1BQU0sQ0FBQyxDQUFDLE1BQU0sS0FBSyxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUMsU0FBUyxFQUFFLENBQUMsQ0FBQyxPQUFPLENBQUM7Z0JBQ2xELEVBQUUsRUFBRSxFQUFFLENBQUMsRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFO2dCQUMxQixFQUFFLEVBQUUsRUFBRSxDQUFDLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRTthQUN0QixDQUFDLENBQUE7UUFDSixDQUFDLENBQUMsQ0FBQTtRQUVGLEVBQUUsQ0FBQyxxQ0FBcUMsRUFBRSxLQUFLLElBQUksRUFBRTtZQUNuRCxNQUFNLEtBQUssR0FBRyxNQUFNLGNBQWMsQ0FBQztnQkFDakMsSUFBSSxFQUFFO29CQUNKLG1HQUFtRztvQkFDbkcsbUdBQW1HO2lCQUNwRzthQUNGLENBQUMsQ0FBQTtZQUVGLE1BQU0sQ0FBQyxDQUFDLE1BQU0sS0FBSyxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUMsU0FBUyxFQUFFLENBQUMsQ0FBQyxPQUFPLENBQUM7Z0JBQ2xELEVBQUUsRUFBRSxFQUFFLENBQUMsRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFO2dCQUMxQixFQUFFLEVBQUUsRUFBRSxDQUFDLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRTtnQkFDckIsRUFBRSxFQUFFLEVBQUUsQ0FBQyxFQUFFLElBQUksRUFBRSxTQUFTLEVBQUU7Z0JBQzFCLEVBQUUsRUFBRSxFQUFFLENBQUMsRUFBRSxJQUFJLEVBQUUsSUFBSSxFQUFFO2FBQ3RCLENBQUMsQ0FBQTtRQUNKLENBQUMsQ0FBQyxDQUFBO0lBQ0osQ0FBQyxDQUFDLENBQUE7QUFDSixDQUFDLENBQUMsQ0FBQSIsInNvdXJjZXNDb250ZW50IjpbImltcG9ydCB7IGdldFRlbXBGaWxlUGF0aCB9IGZyb20gXCJAZnJpY3Rpb25sZXNzLXRzL2RhdGFzZXRcIlxuaW1wb3J0ICogYXMgcGwgZnJvbSBcIm5vZGVqcy1wb2xhcnNcIlxuaW1wb3J0IHsgZGVzY3JpYmUsIGV4cGVjdCwgaXQgfSBmcm9tIFwidml0ZXN0XCJcbmltcG9ydCB7IHVzZVJlY29yZGluZyB9IGZyb20gXCJ2aXRlc3QtcG9sbHlcIlxuaW1wb3J0IHsgbG9hZEFycm93VGFibGUgfSBmcm9tIFwiLi9sb2FkLnRzXCJcblxudXNlUmVjb3JkaW5nKClcblxuZGVzY3JpYmUoXCJsb2FkQXJyb3dUYWJsZVwiLCAoKSA9PiB7XG4gIGRlc2NyaWJlKFwiZmlsZSB2YXJpYXRpb25zXCIsICgpID0+IHtcbiAgICBpdChcInNob3VsZCBsb2FkIGxvY2FsIGZpbGVcIiwgYXN5bmMgKCkgPT4ge1xuICAgICAgY29uc3QgcGF0aCA9IGdldFRlbXBGaWxlUGF0aCgpXG4gICAgICBwbC5EYXRhRnJhbWUoeyBpZDogWzEsIDJdLCBuYW1lOiBbXCJlbmdsaXNoXCIsIFwi5Lit5paHXCJdIH0pLndyaXRlSVBDKHBhdGgpXG5cbiAgICAgIGNvbnN0IHRhYmxlID0gYXdhaXQgbG9hZEFycm93VGFibGUoeyBwYXRoIH0pXG4gICAgICBleHBlY3QoKGF3YWl0IHRhYmxlLmNvbGxlY3QoKSkudG9SZWNvcmRzKCkpLnRvRXF1YWwoW1xuICAgICAgICB7IGlkOiAxLCBuYW1lOiBcImVuZ2xpc2hcIiB9LFxuICAgICAgICB7IGlkOiAyLCBuYW1lOiBcIuS4reaWh1wiIH0sXG4gICAgICBdKVxuICAgIH0pXG5cbiAgICBpdChcInNob3VsZCBsb2FkIGxvY2FsIGZpbGUgKG11bHRpcGFydClcIiwgYXN5bmMgKCkgPT4ge1xuICAgICAgY29uc3QgcGF0aDEgPSBnZXRUZW1wRmlsZVBhdGgoKVxuICAgICAgY29uc3QgcGF0aDIgPSBnZXRUZW1wRmlsZVBhdGgoKVxuICAgICAgcGwuRGF0YUZyYW1lKHsgaWQ6IFsxLCAyXSwgbmFtZTogW1wiZW5nbGlzaFwiLCBcIuS4reaWh1wiXSB9KS53cml0ZUlQQyhwYXRoMSlcbiAgICAgIHBsLkRhdGFGcmFtZSh7IGlkOiBbMSwgMl0sIG5hbWU6IFtcImVuZ2xpc2hcIiwgXCLkuK3mlodcIl0gfSkud3JpdGVJUEMocGF0aDIpXG5cbiAgICAgIGNvbnN0IHRhYmxlID0gYXdhaXQgbG9hZEFycm93VGFibGUoeyBwYXRoOiBbcGF0aDEsIHBhdGgyXSB9KVxuICAgICAgZXhwZWN0KChhd2FpdCB0YWJsZS5jb2xsZWN0KCkpLnRvUmVjb3JkcygpKS50b0VxdWFsKFtcbiAgICAgICAgeyBpZDogMSwgbmFtZTogXCJlbmdsaXNoXCIgfSxcbiAgICAgICAgeyBpZDogMiwgbmFtZTogXCLkuK3mlodcIiB9LFxuICAgICAgICB7IGlkOiAxLCBuYW1lOiBcImVuZ2xpc2hcIiB9LFxuICAgICAgICB7IGlkOiAyLCBuYW1lOiBcIuS4reaWh1wiIH0sXG4gICAgICBdKVxuICAgIH0pXG5cbiAgICBpdChcInNob3VsZCBsb2FkIHJlbW90ZSBmaWxlXCIsIGFzeW5jICgpID0+IHtcbiAgICAgIGNvbnN0IHRhYmxlID0gYXdhaXQgbG9hZEFycm93VGFibGUoe1xuICAgICAgICBwYXRoOiBcImh0dHBzOi8vcmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbS9kYXRpc3RocS9kcGtpdC9yZWZzL2hlYWRzL21haW4vYXJyb3cvdGFibGUvZml4dHVyZXMvdGFibGUuYXJyb3dcIixcbiAgICAgIH0pXG5cbiAgICAgIGV4cGVjdCgoYXdhaXQgdGFibGUuY29sbGVjdCgpKS50b1JlY29yZHMoKSkudG9FcXVhbChbXG4gICAgICAgIHsgaWQ6IDEsIG5hbWU6IFwiZW5nbGlzaFwiIH0sXG4gICAgICAgIHsgaWQ6IDIsIG5hbWU6IFwi5Lit5paHXCIgfSxcbiAgICAgIF0pXG4gICAgfSlcblxuICAgIGl0KFwic2hvdWxkIGxvYWQgcmVtb3RlIGZpbGUgKG11bHRpcGFydClcIiwgYXN5bmMgKCkgPT4ge1xuICAgICAgY29uc3QgdGFibGUgPSBhd2FpdCBsb2FkQXJyb3dUYWJsZSh7XG4gICAgICAgIHBhdGg6IFtcbiAgICAgICAgICBcImh0dHBzOi8vcmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbS9kYXRpc3RocS9kcGtpdC9yZWZzL2hlYWRzL21haW4vYXJyb3cvdGFibGUvZml4dHVyZXMvdGFibGUuYXJyb3dcIixcbiAgICAgICAgICBcImh0dHBzOi8vcmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbS9kYXRpc3RocS9kcGtpdC9yZWZzL2hlYWRzL21haW4vYXJyb3cvdGFibGUvZml4dHVyZXMvdGFibGUuYXJyb3dcIixcbiAgICAgICAgXSxcbiAgICAgIH0pXG5cbiAgICAgIGV4cGVjdCgoYXdhaXQgdGFibGUuY29sbGVjdCgpKS50b1JlY29yZHMoKSkudG9FcXVhbChbXG4gICAgICAgIHsgaWQ6IDEsIG5hbWU6IFwiZW5nbGlzaFwiIH0sXG4gICAgICAgIHsgaWQ6IDIsIG5hbWU6IFwi5Lit5paHXCIgfSxcbiAgICAgICAgeyBpZDogMSwgbmFtZTogXCJlbmdsaXNoXCIgfSxcbiAgICAgICAgeyBpZDogMiwgbmFtZTogXCLkuK3mlodcIiB9LFxuICAgICAgXSlcbiAgICB9KVxuICB9KVxufSlcbiJdfQ==
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { assertLocalPathVacant } from "@frictionless-ts/dataset";
|
|
2
|
+
import { inferSchemaFromTable } from "../../../schema/index.js";
|
|
3
|
+
import { denormalizeTable } from "../../../table/index.js";
|
|
4
|
+
// TODO: rebase on sinkIPC when it is available
|
|
5
|
+
// https://github.com/pola-rs/nodejs-polars/issues/353
|
|
6
|
+
export async function saveArrowTable(table, options) {
|
|
7
|
+
const { path, overwrite } = options;
|
|
8
|
+
if (!overwrite) {
|
|
9
|
+
await assertLocalPathVacant(path);
|
|
10
|
+
}
|
|
11
|
+
const schema = options.schema ??
|
|
12
|
+
(await inferSchemaFromTable(table, {
|
|
13
|
+
...options,
|
|
14
|
+
keepStrings: true,
|
|
15
|
+
}));
|
|
16
|
+
table = await denormalizeTable(table, schema, {
|
|
17
|
+
nativeTypes: [
|
|
18
|
+
"boolean",
|
|
19
|
+
"datetime",
|
|
20
|
+
"integer",
|
|
21
|
+
"list",
|
|
22
|
+
"number",
|
|
23
|
+
"string",
|
|
24
|
+
"year",
|
|
25
|
+
],
|
|
26
|
+
});
|
|
27
|
+
const frame = await table.collect();
|
|
28
|
+
frame.writeIPC(path);
|
|
29
|
+
return path;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2F2ZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uL3BsdWdpbnMvYXJyb3cvdGFibGUvc2F2ZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUscUJBQXFCLEVBQUUsTUFBTSwwQkFBMEIsQ0FBQTtBQUVoRSxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSwwQkFBMEIsQ0FBQTtBQUMvRCxPQUFPLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQTtBQUcxRCwrQ0FBK0M7QUFDL0Msc0RBQXNEO0FBRXRELE1BQU0sQ0FBQyxLQUFLLFVBQVUsY0FBYyxDQUFDLEtBQVksRUFBRSxPQUF5QjtJQUMxRSxNQUFNLEVBQUUsSUFBSSxFQUFFLFNBQVMsRUFBRSxHQUFHLE9BQU8sQ0FBQTtJQUVuQyxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7UUFDZixNQUFNLHFCQUFxQixDQUFDLElBQUksQ0FBQyxDQUFBO0lBQ25DLENBQUM7SUFFRCxNQUFNLE1BQU0sR0FDVixPQUFPLENBQUMsTUFBTTtRQUNkLENBQUMsTUFBTSxvQkFBb0IsQ0FBQyxLQUFLLEVBQUU7WUFDakMsR0FBRyxPQUFPO1lBQ1YsV0FBVyxFQUFFLElBQUk7U0FDbEIsQ0FBQyxDQUFDLENBQUE7SUFFTCxLQUFLLEdBQUcsTUFBTSxnQkFBZ0IsQ0FBQyxLQUFLLEVBQUUsTUFBTSxFQUFFO1FBQzVDLFdBQVcsRUFBRTtZQUNYLFNBQVM7WUFDVCxVQUFVO1lBQ1YsU0FBUztZQUNULE1BQU07WUFDTixRQUFRO1lBQ1IsUUFBUTtZQUNSLE1BQU07U0FDUDtLQUNGLENBQUMsQ0FBQTtJQUVGLE1BQU0sS0FBSyxHQUFHLE1BQU0sS0FBSyxDQUFDLE9BQU8sRUFBRSxDQUFBO0lBQ25DLEtBQUssQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLENBQUE7SUFFcEIsT0FBTyxJQUFJLENBQUE7QUFDYixDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IHsgYXNzZXJ0TG9jYWxQYXRoVmFjYW50IH0gZnJvbSBcIkBmcmljdGlvbmxlc3MtdHMvZGF0YXNldFwiXG5pbXBvcnQgdHlwZSB7IFNhdmVUYWJsZU9wdGlvbnMgfSBmcm9tIFwiLi4vLi4vLi4vcGx1Z2luLnRzXCJcbmltcG9ydCB7IGluZmVyU2NoZW1hRnJvbVRhYmxlIH0gZnJvbSBcIi4uLy4uLy4uL3NjaGVtYS9pbmRleC50c1wiXG5pbXBvcnQgeyBkZW5vcm1hbGl6ZVRhYmxlIH0gZnJvbSBcIi4uLy4uLy4uL3RhYmxlL2luZGV4LnRzXCJcbmltcG9ydCB0eXBlIHsgVGFibGUgfSBmcm9tIFwiLi4vLi4vLi4vdGFibGUvaW5kZXgudHNcIlxuXG4vLyBUT0RPOiByZWJhc2Ugb24gc2lua0lQQyB3aGVuIGl0IGlzIGF2YWlsYWJsZVxuLy8gaHR0cHM6Ly9naXRodWIuY29tL3BvbGEtcnMvbm9kZWpzLXBvbGFycy9pc3N1ZXMvMzUzXG5cbmV4cG9ydCBhc3luYyBmdW5jdGlvbiBzYXZlQXJyb3dUYWJsZSh0YWJsZTogVGFibGUsIG9wdGlvbnM6IFNhdmVUYWJsZU9wdGlvbnMpIHtcbiAgY29uc3QgeyBwYXRoLCBvdmVyd3JpdGUgfSA9IG9wdGlvbnNcblxuICBpZiAoIW92ZXJ3cml0ZSkge1xuICAgIGF3YWl0IGFzc2VydExvY2FsUGF0aFZhY2FudChwYXRoKVxuICB9XG5cbiAgY29uc3Qgc2NoZW1hID1cbiAgICBvcHRpb25zLnNjaGVtYSA/P1xuICAgIChhd2FpdCBpbmZlclNjaGVtYUZyb21UYWJsZSh0YWJsZSwge1xuICAgICAgLi4ub3B0aW9ucyxcbiAgICAgIGtlZXBTdHJpbmdzOiB0cnVlLFxuICAgIH0pKVxuXG4gIHRhYmxlID0gYXdhaXQgZGVub3JtYWxpemVUYWJsZSh0YWJsZSwgc2NoZW1hLCB7XG4gICAgbmF0aXZlVHlwZXM6IFtcbiAgICAgIFwiYm9vbGVhblwiLFxuICAgICAgXCJkYXRldGltZVwiLFxuICAgICAgXCJpbnRlZ2VyXCIsXG4gICAgICBcImxpc3RcIixcbiAgICAgIFwibnVtYmVyXCIsXG4gICAgICBcInN0cmluZ1wiLFxuICAgICAgXCJ5ZWFyXCIsXG4gICAgXSxcbiAgfSlcblxuICBjb25zdCBmcmFtZSA9IGF3YWl0IHRhYmxlLmNvbGxlY3QoKVxuICBmcmFtZS53cml0ZUlQQyhwYXRoKVxuXG4gIHJldHVybiBwYXRoXG59XG4iXX0=
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { getTempFilePath } from "@frictionless-ts/dataset";
|
|
2
|
+
import * as pl from "nodejs-polars";
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
import { loadArrowTable } from "./load.js";
|
|
5
|
+
import { saveArrowTable } from "./save.js";
|
|
6
|
+
describe("saveArrowTable", () => {
|
|
7
|
+
it("should save table to Arrow file", async () => {
|
|
8
|
+
const path = getTempFilePath();
|
|
9
|
+
const source = pl
|
|
10
|
+
.DataFrame({
|
|
11
|
+
id: [1.0, 2.0, 3.0],
|
|
12
|
+
name: ["Alice", "Bob", "Charlie"],
|
|
13
|
+
})
|
|
14
|
+
.lazy();
|
|
15
|
+
await saveArrowTable(source, { path });
|
|
16
|
+
const table = await loadArrowTable({ path });
|
|
17
|
+
expect((await table.collect()).toRecords()).toEqual([
|
|
18
|
+
{ id: 1.0, name: "Alice" },
|
|
19
|
+
{ id: 2.0, name: "Bob" },
|
|
20
|
+
{ id: 3.0, name: "Charlie" },
|
|
21
|
+
]);
|
|
22
|
+
});
|
|
23
|
+
it("should save and load various data types", async () => {
|
|
24
|
+
const path = getTempFilePath();
|
|
25
|
+
const source = pl
|
|
26
|
+
.DataFrame([
|
|
27
|
+
pl.Series("array", ["[1, 2, 3]"], pl.String),
|
|
28
|
+
pl.Series("boolean", [true], pl.Bool),
|
|
29
|
+
pl.Series("date", [new Date(Date.UTC(2025, 0, 1))], pl.Date),
|
|
30
|
+
pl.Series("datetime", [new Date(Date.UTC(2025, 0, 1))], pl.Datetime),
|
|
31
|
+
pl.Series("duration", ["P23DT23H"], pl.String),
|
|
32
|
+
pl.Series("geojson", ['{"value": 1}'], pl.String),
|
|
33
|
+
pl.Series("geopoint", [[40.0, 50.0]], pl.List(pl.Float32)),
|
|
34
|
+
pl.Series("integer", [1], pl.Int32),
|
|
35
|
+
pl.Series("list", [[1.0, 2.0, 3.0]], pl.List(pl.Float32)),
|
|
36
|
+
pl.Series("number", [1.1], pl.Float64),
|
|
37
|
+
pl.Series("object", ['{"value": 1}']),
|
|
38
|
+
pl.Series("string", ["string"], pl.String),
|
|
39
|
+
pl.Series("time", [new Date(Date.UTC(2025, 0, 1))], pl.Time),
|
|
40
|
+
pl.Series("year", [2025], pl.Int32),
|
|
41
|
+
pl.Series("yearmonth", [[2025, 1]], pl.List(pl.Int16)),
|
|
42
|
+
])
|
|
43
|
+
.lazy();
|
|
44
|
+
await saveArrowTable(source, {
|
|
45
|
+
path,
|
|
46
|
+
fieldTypes: {
|
|
47
|
+
array: "array",
|
|
48
|
+
geojson: "geojson",
|
|
49
|
+
geopoint: "geopoint",
|
|
50
|
+
list: "list",
|
|
51
|
+
object: "object",
|
|
52
|
+
// TODO: Remove time after:
|
|
53
|
+
// https://github.com/pola-rs/nodejs-polars/issues/364
|
|
54
|
+
time: "time",
|
|
55
|
+
year: "year",
|
|
56
|
+
yearmonth: "yearmonth",
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
const target = await loadArrowTable({ path }, { denormalized: true });
|
|
60
|
+
expect((await target.collect()).toRecords()).toEqual([
|
|
61
|
+
{
|
|
62
|
+
array: "[1, 2, 3]",
|
|
63
|
+
boolean: true,
|
|
64
|
+
date: "2025-01-01",
|
|
65
|
+
datetime: new Date(Date.UTC(2025, 0, 1)),
|
|
66
|
+
duration: "P23DT23H",
|
|
67
|
+
geojson: '{"value": 1}',
|
|
68
|
+
geopoint: "40.0,50.0",
|
|
69
|
+
integer: 1,
|
|
70
|
+
list: [1.0, 2.0, 3.0],
|
|
71
|
+
number: 1.1,
|
|
72
|
+
object: '{"value": 1}',
|
|
73
|
+
string: "string",
|
|
74
|
+
time: "00:00:00",
|
|
75
|
+
year: 2025,
|
|
76
|
+
yearmonth: "2025-01",
|
|
77
|
+
},
|
|
78
|
+
]);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2F2ZS5zcGVjLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vLi4vcGx1Z2lucy9hcnJvdy90YWJsZS9zYXZlLnNwZWMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLGVBQWUsRUFBRSxNQUFNLDBCQUEwQixDQUFBO0FBQzFELE9BQU8sS0FBSyxFQUFFLE1BQU0sZUFBZSxDQUFBO0FBQ25DLE9BQU8sRUFBRSxRQUFRLEVBQUUsTUFBTSxFQUFFLEVBQUUsRUFBRSxNQUFNLFFBQVEsQ0FBQTtBQUM3QyxPQUFPLEVBQUUsY0FBYyxFQUFFLE1BQU0sV0FBVyxDQUFBO0FBQzFDLE9BQU8sRUFBRSxjQUFjLEVBQUUsTUFBTSxXQUFXLENBQUE7QUFFMUMsUUFBUSxDQUFDLGdCQUFnQixFQUFFLEdBQUcsRUFBRTtJQUM5QixFQUFFLENBQUMsaUNBQWlDLEVBQUUsS0FBSyxJQUFJLEVBQUU7UUFDL0MsTUFBTSxJQUFJLEdBQUcsZUFBZSxFQUFFLENBQUE7UUFDOUIsTUFBTSxNQUFNLEdBQUcsRUFBRTthQUNkLFNBQVMsQ0FBQztZQUNULEVBQUUsRUFBRSxDQUFDLEdBQUcsRUFBRSxHQUFHLEVBQUUsR0FBRyxDQUFDO1lBQ25CLElBQUksRUFBRSxDQUFDLE9BQU8sRUFBRSxLQUFLLEVBQUUsU0FBUyxDQUFDO1NBQ2xDLENBQUM7YUFDRCxJQUFJLEVBQUUsQ0FBQTtRQUVULE1BQU0sY0FBYyxDQUFDLE1BQU0sRUFBRSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUE7UUFFdEMsTUFBTSxLQUFLLEdBQUcsTUFBTSxjQUFjLENBQUMsRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFBO1FBQzVDLE1BQU0sQ0FBQyxDQUFDLE1BQU0sS0FBSyxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUMsU0FBUyxFQUFFLENBQUMsQ0FBQyxPQUFPLENBQUM7WUFDbEQsRUFBRSxFQUFFLEVBQUUsR0FBRyxFQUFFLElBQUksRUFBRSxPQUFPLEVBQUU7WUFDMUIsRUFBRSxFQUFFLEVBQUUsR0FBRyxFQUFFLElBQUksRUFBRSxLQUFLLEVBQUU7WUFDeEIsRUFBRSxFQUFFLEVBQUUsR0FBRyxFQUFFLElBQUksRUFBRSxTQUFTLEVBQUU7U0FDN0IsQ0FBQyxDQUFBO0lBQ0osQ0FBQyxDQUFDLENBQUE7SUFFRixFQUFFLENBQUMseUNBQXlDLEVBQUUsS0FBSyxJQUFJLEVBQUU7UUFDdkQsTUFBTSxJQUFJLEdBQUcsZUFBZSxFQUFFLENBQUE7UUFFOUIsTUFBTSxNQUFNLEdBQUcsRUFBRTthQUNkLFNBQVMsQ0FBQztZQUNULEVBQUUsQ0FBQyxNQUFNLENBQUMsT0FBTyxFQUFFLENBQUMsV0FBVyxDQUFDLEVBQUUsRUFBRSxDQUFDLE1BQU0sQ0FBQztZQUM1QyxFQUFFLENBQUMsTUFBTSxDQUFDLFNBQVMsRUFBRSxDQUFDLElBQUksQ0FBQyxFQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUM7WUFDckMsRUFBRSxDQUFDLE1BQU0sQ0FBQyxNQUFNLEVBQUUsQ0FBQyxJQUFJLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLElBQUksRUFBRSxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUM7WUFDNUQsRUFBRSxDQUFDLE1BQU0sQ0FBQyxVQUFVLEVBQUUsQ0FBQyxJQUFJLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLElBQUksRUFBRSxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxRQUFRLENBQUM7WUFDcEUsRUFBRSxDQUFDLE1BQU0sQ0FBQyxVQUFVLEVBQUUsQ0FBQyxVQUFVLENBQUMsRUFBRSxFQUFFLENBQUMsTUFBTSxDQUFDO1lBQzlDLEVBQUUsQ0FBQyxNQUFNLENBQUMsU0FBUyxFQUFFLENBQUMsY0FBYyxDQUFDLEVBQUUsRUFBRSxDQUFDLE1BQU0sQ0FBQztZQUNqRCxFQUFFLENBQUMsTUFBTSxDQUFDLFVBQVUsRUFBRSxDQUFDLENBQUMsSUFBSSxFQUFFLElBQUksQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsT0FBTyxDQUFDLENBQUM7WUFDMUQsRUFBRSxDQUFDLE1BQU0sQ0FBQyxTQUFTLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxFQUFFLENBQUMsS0FBSyxDQUFDO1lBQ25DLEVBQUUsQ0FBQyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQyxHQUFHLEVBQUUsR0FBRyxFQUFFLEdBQUcsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsT0FBTyxDQUFDLENBQUM7WUFDekQsRUFBRSxDQUFDLE1BQU0sQ0FBQyxRQUFRLEVBQUUsQ0FBQyxHQUFHLENBQUMsRUFBRSxFQUFFLENBQUMsT0FBTyxDQUFDO1lBQ3RDLEVBQUUsQ0FBQyxNQUFNLENBQUMsUUFBUSxFQUFFLENBQUMsY0FBYyxDQUFDLENBQUM7WUFDckMsRUFBRSxDQUFDLE1BQU0sQ0FBQyxRQUFRLEVBQUUsQ0FBQyxRQUFRLENBQUMsRUFBRSxFQUFFLENBQUMsTUFBTSxDQUFDO1lBQzFDLEVBQUUsQ0FBQyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsSUFBSSxJQUFJLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUMsRUFBRSxFQUFFLENBQUMsSUFBSSxDQUFDO1lBQzVELEVBQUUsQ0FBQyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsSUFBSSxDQUFDLEVBQUUsRUFBRSxDQUFDLEtBQUssQ0FBQztZQUNuQyxFQUFFLENBQUMsTUFBTSxDQUFDLFdBQVcsRUFBRSxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsS0FBSyxDQUFDLENBQUM7U0FDdkQsQ0FBQzthQUNELElBQUksRUFBRSxDQUFBO1FBRVQsTUFBTSxjQUFjLENBQUMsTUFBTSxFQUFFO1lBQzNCLElBQUk7WUFDSixVQUFVLEVBQUU7Z0JBQ1YsS0FBSyxFQUFFLE9BQU87Z0JBQ2QsT0FBTyxFQUFFLFNBQVM7Z0JBQ2xCLFFBQVEsRUFBRSxVQUFVO2dCQUNwQixJQUFJLEVBQUUsTUFBTTtnQkFDWixNQUFNLEVBQUUsUUFBUTtnQkFDaEIsMkJBQTJCO2dCQUMzQixzREFBc0Q7Z0JBQ3RELElBQUksRUFBRSxNQUFNO2dCQUNaLElBQUksRUFBRSxNQUFNO2dCQUNaLFNBQVMsRUFBRSxXQUFXO2FBQ3ZCO1NBQ0YsQ0FBQyxDQUFBO1FBRUYsTUFBTSxNQUFNLEdBQUcsTUFBTSxjQUFjLENBQUMsRUFBRSxJQUFJLEVBQUUsRUFBRSxFQUFFLFlBQVksRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFBO1FBQ3JFLE1BQU0sQ0FBQyxDQUFDLE1BQU0sTUFBTSxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUMsU0FBUyxFQUFFLENBQUMsQ0FBQyxPQUFPLENBQUM7WUFDbkQ7Z0JBQ0UsS0FBSyxFQUFFLFdBQVc7Z0JBQ2xCLE9BQU8sRUFBRSxJQUFJO2dCQUNiLElBQUksRUFBRSxZQUFZO2dCQUNsQixRQUFRLEVBQUUsSUFBSSxJQUFJLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDO2dCQUN4QyxRQUFRLEVBQUUsVUFBVTtnQkFDcEIsT0FBTyxFQUFFLGNBQWM7Z0JBQ3ZCLFFBQVEsRUFBRSxXQUFXO2dCQUNyQixPQUFPLEVBQUUsQ0FBQztnQkFDVixJQUFJLEVBQUUsQ0FBQyxHQUFHLEVBQUUsR0FBRyxFQUFFLEdBQUcsQ0FBQztnQkFDckIsTUFBTSxFQUFFLEdBQUc7Z0JBQ1gsTUFBTSxFQUFFLGNBQWM7Z0JBQ3RCLE1BQU0sRUFBRSxRQUFRO2dCQUNoQixJQUFJLEVBQUUsVUFBVTtnQkFDaEIsSUFBSSxFQUFFLElBQUk7Z0JBQ1YsU0FBUyxFQUFFLFNBQVM7YUFDckI7U0FDRixDQUFDLENBQUE7SUFDSixDQUFDLENBQUMsQ0FBQTtBQUNKLENBQUMsQ0FBQyxDQUFBIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IHsgZ2V0VGVtcEZpbGVQYXRoIH0gZnJvbSBcIkBmcmljdGlvbmxlc3MtdHMvZGF0YXNldFwiXG5pbXBvcnQgKiBhcyBwbCBmcm9tIFwibm9kZWpzLXBvbGFyc1wiXG5pbXBvcnQgeyBkZXNjcmliZSwgZXhwZWN0LCBpdCB9IGZyb20gXCJ2aXRlc3RcIlxuaW1wb3J0IHsgbG9hZEFycm93VGFibGUgfSBmcm9tIFwiLi9sb2FkLnRzXCJcbmltcG9ydCB7IHNhdmVBcnJvd1RhYmxlIH0gZnJvbSBcIi4vc2F2ZS50c1wiXG5cbmRlc2NyaWJlKFwic2F2ZUFycm93VGFibGVcIiwgKCkgPT4ge1xuICBpdChcInNob3VsZCBzYXZlIHRhYmxlIHRvIEFycm93IGZpbGVcIiwgYXN5bmMgKCkgPT4ge1xuICAgIGNvbnN0IHBhdGggPSBnZXRUZW1wRmlsZVBhdGgoKVxuICAgIGNvbnN0IHNvdXJjZSA9IHBsXG4gICAgICAuRGF0YUZyYW1lKHtcbiAgICAgICAgaWQ6IFsxLjAsIDIuMCwgMy4wXSxcbiAgICAgICAgbmFtZTogW1wiQWxpY2VcIiwgXCJCb2JcIiwgXCJDaGFybGllXCJdLFxuICAgICAgfSlcbiAgICAgIC5sYXp5KClcblxuICAgIGF3YWl0IHNhdmVBcnJvd1RhYmxlKHNvdXJjZSwgeyBwYXRoIH0pXG5cbiAgICBjb25zdCB0YWJsZSA9IGF3YWl0IGxvYWRBcnJvd1RhYmxlKHsgcGF0aCB9KVxuICAgIGV4cGVjdCgoYXdhaXQgdGFibGUuY29sbGVjdCgpKS50b1JlY29yZHMoKSkudG9FcXVhbChbXG4gICAgICB7IGlkOiAxLjAsIG5hbWU6IFwiQWxpY2VcIiB9LFxuICAgICAgeyBpZDogMi4wLCBuYW1lOiBcIkJvYlwiIH0sXG4gICAgICB7IGlkOiAzLjAsIG5hbWU6IFwiQ2hhcmxpZVwiIH0sXG4gICAgXSlcbiAgfSlcblxuICBpdChcInNob3VsZCBzYXZlIGFuZCBsb2FkIHZhcmlvdXMgZGF0YSB0eXBlc1wiLCBhc3luYyAoKSA9PiB7XG4gICAgY29uc3QgcGF0aCA9IGdldFRlbXBGaWxlUGF0aCgpXG5cbiAgICBjb25zdCBzb3VyY2UgPSBwbFxuICAgICAgLkRhdGFGcmFtZShbXG4gICAgICAgIHBsLlNlcmllcyhcImFycmF5XCIsIFtcIlsxLCAyLCAzXVwiXSwgcGwuU3RyaW5nKSxcbiAgICAgICAgcGwuU2VyaWVzKFwiYm9vbGVhblwiLCBbdHJ1ZV0sIHBsLkJvb2wpLFxuICAgICAgICBwbC5TZXJpZXMoXCJkYXRlXCIsIFtuZXcgRGF0ZShEYXRlLlVUQygyMDI1LCAwLCAxKSldLCBwbC5EYXRlKSxcbiAgICAgICAgcGwuU2VyaWVzKFwiZGF0ZXRpbWVcIiwgW25ldyBEYXRlKERhdGUuVVRDKDIwMjUsIDAsIDEpKV0sIHBsLkRhdGV0aW1lKSxcbiAgICAgICAgcGwuU2VyaWVzKFwiZHVyYXRpb25cIiwgW1wiUDIzRFQyM0hcIl0sIHBsLlN0cmluZyksXG4gICAgICAgIHBsLlNlcmllcyhcImdlb2pzb25cIiwgWyd7XCJ2YWx1ZVwiOiAxfSddLCBwbC5TdHJpbmcpLFxuICAgICAgICBwbC5TZXJpZXMoXCJnZW9wb2ludFwiLCBbWzQwLjAsIDUwLjBdXSwgcGwuTGlzdChwbC5GbG9hdDMyKSksXG4gICAgICAgIHBsLlNlcmllcyhcImludGVnZXJcIiwgWzFdLCBwbC5JbnQzMiksXG4gICAgICAgIHBsLlNlcmllcyhcImxpc3RcIiwgW1sxLjAsIDIuMCwgMy4wXV0sIHBsLkxpc3QocGwuRmxvYXQzMikpLFxuICAgICAgICBwbC5TZXJpZXMoXCJudW1iZXJcIiwgWzEuMV0sIHBsLkZsb2F0NjQpLFxuICAgICAgICBwbC5TZXJpZXMoXCJvYmplY3RcIiwgWyd7XCJ2YWx1ZVwiOiAxfSddKSxcbiAgICAgICAgcGwuU2VyaWVzKFwic3RyaW5nXCIsIFtcInN0cmluZ1wiXSwgcGwuU3RyaW5nKSxcbiAgICAgICAgcGwuU2VyaWVzKFwidGltZVwiLCBbbmV3IERhdGUoRGF0ZS5VVEMoMjAyNSwgMCwgMSkpXSwgcGwuVGltZSksXG4gICAgICAgIHBsLlNlcmllcyhcInllYXJcIiwgWzIwMjVdLCBwbC5JbnQzMiksXG4gICAgICAgIHBsLlNlcmllcyhcInllYXJtb250aFwiLCBbWzIwMjUsIDFdXSwgcGwuTGlzdChwbC5JbnQxNikpLFxuICAgICAgXSlcbiAgICAgIC5sYXp5KClcblxuICAgIGF3YWl0IHNhdmVBcnJvd1RhYmxlKHNvdXJjZSwge1xuICAgICAgcGF0aCxcbiAgICAgIGZpZWxkVHlwZXM6IHtcbiAgICAgICAgYXJyYXk6IFwiYXJyYXlcIixcbiAgICAgICAgZ2VvanNvbjogXCJnZW9qc29uXCIsXG4gICAgICAgIGdlb3BvaW50OiBcImdlb3BvaW50XCIsXG4gICAgICAgIGxpc3Q6IFwibGlzdFwiLFxuICAgICAgICBvYmplY3Q6IFwib2JqZWN0XCIsXG4gICAgICAgIC8vIFRPRE86IFJlbW92ZSB0aW1lIGFmdGVyOlxuICAgICAgICAvLyBodHRwczovL2dpdGh1Yi5jb20vcG9sYS1ycy9ub2RlanMtcG9sYXJzL2lzc3Vlcy8zNjRcbiAgICAgICAgdGltZTogXCJ0aW1lXCIsXG4gICAgICAgIHllYXI6IFwieWVhclwiLFxuICAgICAgICB5ZWFybW9udGg6IFwieWVhcm1vbnRoXCIsXG4gICAgICB9LFxuICAgIH0pXG5cbiAgICBjb25zdCB0YXJnZXQgPSBhd2FpdCBsb2FkQXJyb3dUYWJsZSh7IHBhdGggfSwgeyBkZW5vcm1hbGl6ZWQ6IHRydWUgfSlcbiAgICBleHBlY3QoKGF3YWl0IHRhcmdldC5jb2xsZWN0KCkpLnRvUmVjb3JkcygpKS50b0VxdWFsKFtcbiAgICAgIHtcbiAgICAgICAgYXJyYXk6IFwiWzEsIDIsIDNdXCIsXG4gICAgICAgIGJvb2xlYW46IHRydWUsXG4gICAgICAgIGRhdGU6IFwiMjAyNS0wMS0wMVwiLFxuICAgICAgICBkYXRldGltZTogbmV3IERhdGUoRGF0ZS5VVEMoMjAyNSwgMCwgMSkpLFxuICAgICAgICBkdXJhdGlvbjogXCJQMjNEVDIzSFwiLFxuICAgICAgICBnZW9qc29uOiAne1widmFsdWVcIjogMX0nLFxuICAgICAgICBnZW9wb2ludDogXCI0MC4wLDUwLjBcIixcbiAgICAgICAgaW50ZWdlcjogMSxcbiAgICAgICAgbGlzdDogWzEuMCwgMi4wLCAzLjBdLFxuICAgICAgICBudW1iZXI6IDEuMSxcbiAgICAgICAgb2JqZWN0OiAne1widmFsdWVcIjogMX0nLFxuICAgICAgICBzdHJpbmc6IFwic3RyaW5nXCIsXG4gICAgICAgIHRpbWU6IFwiMDA6MDA6MDBcIixcbiAgICAgICAgeWVhcjogMjAyNSxcbiAgICAgICAgeWVhcm1vbnRoOiBcIjIwMjUtMDFcIixcbiAgICAgIH0sXG4gICAgXSlcbiAgfSlcbn0pXG4iXX0=
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { inferCsvDialect } from "./infer.ts";
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
export { inferCsvDialect } from "./infer.js";
|
|
2
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi9wbHVnaW5zL2Nzdi9kaWFsZWN0L2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxlQUFlLEVBQUUsTUFBTSxZQUFZLENBQUEiLCJzb3VyY2VzQ29udGVudCI6WyJleHBvcnQgeyBpbmZlckNzdkRpYWxlY3QgfSBmcm9tIFwiLi9pbmZlci50c1wiXG4iXX0=
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { text } from "node:stream/consumers";
|
|
2
|
+
import { loadFileStream } from "@frictionless-ts/dataset";
|
|
3
|
+
import { default as CsvSnifferFactory } from "csv-sniffer";
|
|
4
|
+
const CSV_DELIMITERS = [",", ";", ":", "|", "\t", "^", "*", "&"];
|
|
5
|
+
const TSV_DELIMITERS = ["\t"];
|
|
6
|
+
export async function inferCsvDialect(resource, options) {
|
|
7
|
+
const { sampleBytes = 10_000 } = options ?? {};
|
|
8
|
+
const isTabs = resource.format === "tsv";
|
|
9
|
+
const dialect = {};
|
|
10
|
+
if (resource.path) {
|
|
11
|
+
const stream = await loadFileStream(resource.path, {
|
|
12
|
+
maxBytes: sampleBytes,
|
|
13
|
+
});
|
|
14
|
+
const sample = await text(stream);
|
|
15
|
+
const result = sniffSample(sample, isTabs ? TSV_DELIMITERS : CSV_DELIMITERS);
|
|
16
|
+
if (result?.delimiter) {
|
|
17
|
+
dialect.delimiter = result.delimiter;
|
|
18
|
+
}
|
|
19
|
+
if (result?.quoteChar) {
|
|
20
|
+
dialect.quoteChar = result.quoteChar;
|
|
21
|
+
}
|
|
22
|
+
//if (result.lineTerminator) {
|
|
23
|
+
// dialect.lineTerminator = result.lineTerminator
|
|
24
|
+
//}
|
|
25
|
+
// TODO: it gives false positives
|
|
26
|
+
//if (!result.hasHeader) {
|
|
27
|
+
// dialect.header = false
|
|
28
|
+
//}
|
|
29
|
+
}
|
|
30
|
+
return dialect;
|
|
31
|
+
}
|
|
32
|
+
// Sniffer can fail for some reasons
|
|
33
|
+
function sniffSample(sample, delimiters) {
|
|
34
|
+
try {
|
|
35
|
+
const CsvSniffer = CsvSnifferFactory();
|
|
36
|
+
const sniffer = new CsvSniffer(delimiters);
|
|
37
|
+
const result = sniffer.sniff(sample);
|
|
38
|
+
return result;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5mZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi9wbHVnaW5zL2Nzdi9kaWFsZWN0L2luZmVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxJQUFJLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQTtBQUM1QyxPQUFPLEVBQUUsY0FBYyxFQUFFLE1BQU0sMEJBQTBCLENBQUE7QUFFekQsT0FBTyxFQUFFLE9BQU8sSUFBSSxpQkFBaUIsRUFBRSxNQUFNLGFBQWEsQ0FBQTtBQUUxRCxNQUFNLGNBQWMsR0FBRyxDQUFDLEdBQUcsRUFBRSxHQUFHLEVBQUUsR0FBRyxFQUFFLEdBQUcsRUFBRSxJQUFJLEVBQUUsR0FBRyxFQUFFLEdBQUcsRUFBRSxHQUFHLENBQUMsQ0FBQTtBQUNoRSxNQUFNLGNBQWMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFBO0FBRTdCLE1BQU0sQ0FBQyxLQUFLLFVBQVUsZUFBZSxDQUNuQyxRQUEyQixFQUMzQixPQUVDO0lBRUQsTUFBTSxFQUFFLFdBQVcsR0FBRyxNQUFNLEVBQUUsR0FBRyxPQUFPLElBQUksRUFBRSxDQUFBO0lBQzlDLE1BQU0sTUFBTSxHQUFHLFFBQVEsQ0FBQyxNQUFNLEtBQUssS0FBSyxDQUFBO0lBRXhDLE1BQU0sT0FBTyxHQUFZLEVBQUUsQ0FBQTtJQUUzQixJQUFJLFFBQVEsQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUNsQixNQUFNLE1BQU0sR0FBRyxNQUFNLGNBQWMsQ0FBQyxRQUFRLENBQUMsSUFBSSxFQUFFO1lBQ2pELFFBQVEsRUFBRSxXQUFXO1NBQ3RCLENBQUMsQ0FBQTtRQUVGLE1BQU0sTUFBTSxHQUFHLE1BQU0sSUFBSSxDQUFDLE1BQU0sQ0FBQyxDQUFBO1FBQ2pDLE1BQU0sTUFBTSxHQUFHLFdBQVcsQ0FBQyxNQUFNLEVBQUUsTUFBTSxDQUFDLENBQUMsQ0FBQyxjQUFjLENBQUMsQ0FBQyxDQUFDLGNBQWMsQ0FBQyxDQUFBO1FBRTVFLElBQUksTUFBTSxFQUFFLFNBQVMsRUFBRSxDQUFDO1lBQ3RCLE9BQU8sQ0FBQyxTQUFTLEdBQUcsTUFBTSxDQUFDLFNBQVMsQ0FBQTtRQUN0QyxDQUFDO1FBRUQsSUFBSSxNQUFNLEVBQUUsU0FBUyxFQUFFLENBQUM7WUFDdEIsT0FBTyxDQUFDLFNBQVMsR0FBRyxNQUFNLENBQUMsU0FBUyxDQUFBO1FBQ3RDLENBQUM7UUFFRCw4QkFBOEI7UUFDOUIsa0RBQWtEO1FBQ2xELEdBQUc7UUFFSCxpQ0FBaUM7UUFDakMsMEJBQTBCO1FBQzFCLDBCQUEwQjtRQUMxQixHQUFHO0lBQ0wsQ0FBQztJQUVELE9BQU8sT0FBTyxDQUFBO0FBQ2hCLENBQUM7QUFFRCxvQ0FBb0M7QUFDcEMsU0FBUyxXQUFXLENBQUMsTUFBYyxFQUFFLFVBQW9CO0lBQ3ZELElBQUksQ0FBQztRQUNILE1BQU0sVUFBVSxHQUFHLGlCQUFpQixFQUFFLENBQUE7UUFDdEMsTUFBTSxPQUFPLEdBQUcsSUFBSSxVQUFVLENBQUMsVUFBVSxDQUFDLENBQUE7UUFDMUMsTUFBTSxNQUFNLEdBQUcsT0FBTyxDQUFDLEtBQUssQ0FBQyxNQUFNLENBQUMsQ0FBQTtRQUNwQyxPQUFPLE1BQU0sQ0FBQTtJQUNmLENBQUM7SUFBQyxNQUFNLENBQUM7UUFDUCxPQUFPLFNBQVMsQ0FBQTtJQUNsQixDQUFDO0FBQ0gsQ0FBQyIsInNvdXJjZXNDb250ZW50IjpbImltcG9ydCB7IHRleHQgfSBmcm9tIFwibm9kZTpzdHJlYW0vY29uc3VtZXJzXCJcbmltcG9ydCB7IGxvYWRGaWxlU3RyZWFtIH0gZnJvbSBcIkBmcmljdGlvbmxlc3MtdHMvZGF0YXNldFwiXG5pbXBvcnQgdHlwZSB7IERpYWxlY3QsIFJlc291cmNlIH0gZnJvbSBcIkBmcmljdGlvbmxlc3MtdHMvbWV0YWRhdGFcIlxuaW1wb3J0IHsgZGVmYXVsdCBhcyBDc3ZTbmlmZmVyRmFjdG9yeSB9IGZyb20gXCJjc3Ytc25pZmZlclwiXG5cbmNvbnN0IENTVl9ERUxJTUlURVJTID0gW1wiLFwiLCBcIjtcIiwgXCI6XCIsIFwifFwiLCBcIlxcdFwiLCBcIl5cIiwgXCIqXCIsIFwiJlwiXVxuY29uc3QgVFNWX0RFTElNSVRFUlMgPSBbXCJcXHRcIl1cblxuZXhwb3J0IGFzeW5jIGZ1bmN0aW9uIGluZmVyQ3N2RGlhbGVjdChcbiAgcmVzb3VyY2U6IFBhcnRpYWw8UmVzb3VyY2U+LFxuICBvcHRpb25zPzoge1xuICAgIHNhbXBsZUJ5dGVzPzogbnVtYmVyXG4gIH0sXG4pIHtcbiAgY29uc3QgeyBzYW1wbGVCeXRlcyA9IDEwXzAwMCB9ID0gb3B0aW9ucyA/PyB7fVxuICBjb25zdCBpc1RhYnMgPSByZXNvdXJjZS5mb3JtYXQgPT09IFwidHN2XCJcblxuICBjb25zdCBkaWFsZWN0OiBEaWFsZWN0ID0ge31cblxuICBpZiAocmVzb3VyY2UucGF0aCkge1xuICAgIGNvbnN0IHN0cmVhbSA9IGF3YWl0IGxvYWRGaWxlU3RyZWFtKHJlc291cmNlLnBhdGgsIHtcbiAgICAgIG1heEJ5dGVzOiBzYW1wbGVCeXRlcyxcbiAgICB9KVxuXG4gICAgY29uc3Qgc2FtcGxlID0gYXdhaXQgdGV4dChzdHJlYW0pXG4gICAgY29uc3QgcmVzdWx0ID0gc25pZmZTYW1wbGUoc2FtcGxlLCBpc1RhYnMgPyBUU1ZfREVMSU1JVEVSUyA6IENTVl9ERUxJTUlURVJTKVxuXG4gICAgaWYgKHJlc3VsdD8uZGVsaW1pdGVyKSB7XG4gICAgICBkaWFsZWN0LmRlbGltaXRlciA9IHJlc3VsdC5kZWxpbWl0ZXJcbiAgICB9XG5cbiAgICBpZiAocmVzdWx0Py5xdW90ZUNoYXIpIHtcbiAgICAgIGRpYWxlY3QucXVvdGVDaGFyID0gcmVzdWx0LnF1b3RlQ2hhclxuICAgIH1cblxuICAgIC8vaWYgKHJlc3VsdC5saW5lVGVybWluYXRvcikge1xuICAgIC8vICBkaWFsZWN0LmxpbmVUZXJtaW5hdG9yID0gcmVzdWx0LmxpbmVUZXJtaW5hdG9yXG4gICAgLy99XG5cbiAgICAvLyBUT0RPOiBpdCBnaXZlcyBmYWxzZSBwb3NpdGl2ZXNcbiAgICAvL2lmICghcmVzdWx0Lmhhc0hlYWRlcikge1xuICAgIC8vICBkaWFsZWN0LmhlYWRlciA9IGZhbHNlXG4gICAgLy99XG4gIH1cblxuICByZXR1cm4gZGlhbGVjdFxufVxuXG4vLyBTbmlmZmVyIGNhbiBmYWlsIGZvciBzb21lIHJlYXNvbnNcbmZ1bmN0aW9uIHNuaWZmU2FtcGxlKHNhbXBsZTogc3RyaW5nLCBkZWxpbWl0ZXJzOiBzdHJpbmdbXSkge1xuICB0cnkge1xuICAgIGNvbnN0IENzdlNuaWZmZXIgPSBDc3ZTbmlmZmVyRmFjdG9yeSgpXG4gICAgY29uc3Qgc25pZmZlciA9IG5ldyBDc3ZTbmlmZmVyKGRlbGltaXRlcnMpXG4gICAgY29uc3QgcmVzdWx0ID0gc25pZmZlci5zbmlmZihzYW1wbGUpXG4gICAgcmV0dXJuIHJlc3VsdFxuICB9IGNhdGNoIHtcbiAgICByZXR1cm4gdW5kZWZpbmVkXG4gIH1cbn1cbiJdfQ==
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { writeTempFile } from "@frictionless-ts/dataset";
|
|
2
|
+
import { describe, expect, it } from "vitest";
|
|
3
|
+
import { inferCsvDialect } from "./infer.js";
|
|
4
|
+
describe("inferCsvDialect", () => {
|
|
5
|
+
it("should infer a simple CSV file", async () => {
|
|
6
|
+
const path = await writeTempFile("id,name\n1,english\n2,中文");
|
|
7
|
+
const dialect = await inferCsvDialect({ path });
|
|
8
|
+
expect(dialect).toEqual({
|
|
9
|
+
delimiter: ",",
|
|
10
|
+
});
|
|
11
|
+
});
|
|
12
|
+
it("should infer quoteChar", async () => {
|
|
13
|
+
const path = await writeTempFile('id,name\n1,"John Doe"\n2,"Jane Smith"');
|
|
14
|
+
const dialect = await inferCsvDialect({ path });
|
|
15
|
+
expect(dialect).toEqual({
|
|
16
|
+
delimiter: ",",
|
|
17
|
+
quoteChar: '"',
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
it("should infer quoteChar with single quotes", async () => {
|
|
21
|
+
const path = await writeTempFile("id,name\n1,'John Doe'\n2,'Jane Smith'");
|
|
22
|
+
const dialect = await inferCsvDialect({ path });
|
|
23
|
+
expect(dialect).toEqual({
|
|
24
|
+
delimiter: ",",
|
|
25
|
+
quoteChar: "'",
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
// TODO: it gives false positives
|
|
29
|
+
it.skip("should infer header false when no header present", async () => {
|
|
30
|
+
const path = await writeTempFile("1,english\n2,中文\n3,español");
|
|
31
|
+
const dialect = await inferCsvDialect({ path });
|
|
32
|
+
expect(dialect).toEqual({
|
|
33
|
+
delimiter: ",",
|
|
34
|
+
header: false,
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
it("should not set header when header is present", async () => {
|
|
38
|
+
const path = await writeTempFile("id,name\n1,english\n2,中文");
|
|
39
|
+
const dialect = await inferCsvDialect({ path });
|
|
40
|
+
expect(dialect).toEqual({
|
|
41
|
+
delimiter: ",",
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
// TODO: recover if possible with csv-sniffer
|
|
45
|
+
it.skip("should infer complex CSV with quotes and header", async () => {
|
|
46
|
+
const path = await writeTempFile('name,description\n"Product A","A great product with, commas"\n"Product B","Another product"');
|
|
47
|
+
const dialect = await inferCsvDialect({ path });
|
|
48
|
+
expect(dialect).toEqual({
|
|
49
|
+
delimiter: ",",
|
|
50
|
+
quoteChar: '"',
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5mZXIuc3BlYy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uL3BsdWdpbnMvY3N2L2RpYWxlY3QvaW5mZXIuc3BlYy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsYUFBYSxFQUFFLE1BQU0sMEJBQTBCLENBQUE7QUFDeEQsT0FBTyxFQUFFLFFBQVEsRUFBRSxNQUFNLEVBQUUsRUFBRSxFQUFFLE1BQU0sUUFBUSxDQUFBO0FBQzdDLE9BQU8sRUFBRSxlQUFlLEVBQUUsTUFBTSxZQUFZLENBQUE7QUFFNUMsUUFBUSxDQUFDLGlCQUFpQixFQUFFLEdBQUcsRUFBRTtJQUMvQixFQUFFLENBQUMsZ0NBQWdDLEVBQUUsS0FBSyxJQUFJLEVBQUU7UUFDOUMsTUFBTSxJQUFJLEdBQUcsTUFBTSxhQUFhLENBQUMsMEJBQTBCLENBQUMsQ0FBQTtRQUM1RCxNQUFNLE9BQU8sR0FBRyxNQUFNLGVBQWUsQ0FBQyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUE7UUFFL0MsTUFBTSxDQUFDLE9BQU8sQ0FBQyxDQUFDLE9BQU8sQ0FBQztZQUN0QixTQUFTLEVBQUUsR0FBRztTQUNmLENBQUMsQ0FBQTtJQUNKLENBQUMsQ0FBQyxDQUFBO0lBRUYsRUFBRSxDQUFDLHdCQUF3QixFQUFFLEtBQUssSUFBSSxFQUFFO1FBQ3RDLE1BQU0sSUFBSSxHQUFHLE1BQU0sYUFBYSxDQUFDLHVDQUF1QyxDQUFDLENBQUE7UUFDekUsTUFBTSxPQUFPLEdBQUcsTUFBTSxlQUFlLENBQUMsRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFBO1FBRS9DLE1BQU0sQ0FBQyxPQUFPLENBQUMsQ0FBQyxPQUFPLENBQUM7WUFDdEIsU0FBUyxFQUFFLEdBQUc7WUFDZCxTQUFTLEVBQUUsR0FBRztTQUNmLENBQUMsQ0FBQTtJQUNKLENBQUMsQ0FBQyxDQUFBO0lBRUYsRUFBRSxDQUFDLDJDQUEyQyxFQUFFLEtBQUssSUFBSSxFQUFFO1FBQ3pELE1BQU0sSUFBSSxHQUFHLE1BQU0sYUFBYSxDQUFDLHVDQUF1QyxDQUFDLENBQUE7UUFDekUsTUFBTSxPQUFPLEdBQUcsTUFBTSxlQUFlLENBQUMsRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFBO1FBRS9DLE1BQU0sQ0FBQyxPQUFPLENBQUMsQ0FBQyxPQUFPLENBQUM7WUFDdEIsU0FBUyxFQUFFLEdBQUc7WUFDZCxTQUFTLEVBQUUsR0FBRztTQUNmLENBQUMsQ0FBQTtJQUNKLENBQUMsQ0FBQyxDQUFBO0lBRUYsaUNBQWlDO0lBQ2pDLEVBQUUsQ0FBQyxJQUFJLENBQUMsa0RBQWtELEVBQUUsS0FBSyxJQUFJLEVBQUU7UUFDckUsTUFBTSxJQUFJLEdBQUcsTUFBTSxhQUFhLENBQUMsNEJBQTRCLENBQUMsQ0FBQTtRQUM5RCxNQUFNLE9BQU8sR0FBRyxNQUFNLGVBQWUsQ0FBQyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUE7UUFFL0MsTUFBTSxDQUFDLE9BQU8sQ0FBQyxDQUFDLE9BQU8sQ0FBQztZQUN0QixTQUFTLEVBQUUsR0FBRztZQUNkLE1BQU0sRUFBRSxLQUFLO1NBQ2QsQ0FBQyxDQUFBO0lBQ0osQ0FBQyxDQUFDLENBQUE7SUFFRixFQUFFLENBQUMsOENBQThDLEVBQUUsS0FBSyxJQUFJLEVBQUU7UUFDNUQsTUFBTSxJQUFJLEdBQUcsTUFBTSxhQUFhLENBQUMsMEJBQTBCLENBQUMsQ0FBQTtRQUM1RCxNQUFNLE9BQU8sR0FBRyxNQUFNLGVBQWUsQ0FBQyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUE7UUFFL0MsTUFBTSxDQUFDLE9BQU8sQ0FBQyxDQUFDLE9BQU8sQ0FBQztZQUN0QixTQUFTLEVBQUUsR0FBRztTQUNmLENBQUMsQ0FBQTtJQUNKLENBQUMsQ0FBQyxDQUFBO0lBRUYsNkNBQTZDO0lBQzdDLEVBQUUsQ0FBQyxJQUFJLENBQUMsaURBQWlELEVBQUUsS0FBSyxJQUFJLEVBQUU7UUFDcEUsTUFBTSxJQUFJLEdBQUcsTUFBTSxhQUFhLENBQzlCLDZGQUE2RixDQUM5RixDQUFBO1FBRUQsTUFBTSxPQUFPLEdBQUcsTUFBTSxlQUFlLENBQUMsRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFBO1FBQy9DLE1BQU0sQ0FBQyxPQUFPLENBQUMsQ0FBQyxPQUFPLENBQUM7WUFDdEIsU0FBUyxFQUFFLEdBQUc7WUFDZCxTQUFTLEVBQUUsR0FBRztTQUNmLENBQUMsQ0FBQTtJQUNKLENBQUMsQ0FBQyxDQUFBO0FBQ0osQ0FBQyxDQUFDLENBQUEiLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgeyB3cml0ZVRlbXBGaWxlIH0gZnJvbSBcIkBmcmljdGlvbmxlc3MtdHMvZGF0YXNldFwiXG5pbXBvcnQgeyBkZXNjcmliZSwgZXhwZWN0LCBpdCB9IGZyb20gXCJ2aXRlc3RcIlxuaW1wb3J0IHsgaW5mZXJDc3ZEaWFsZWN0IH0gZnJvbSBcIi4vaW5mZXIudHNcIlxuXG5kZXNjcmliZShcImluZmVyQ3N2RGlhbGVjdFwiLCAoKSA9PiB7XG4gIGl0KFwic2hvdWxkIGluZmVyIGEgc2ltcGxlIENTViBmaWxlXCIsIGFzeW5jICgpID0+IHtcbiAgICBjb25zdCBwYXRoID0gYXdhaXQgd3JpdGVUZW1wRmlsZShcImlkLG5hbWVcXG4xLGVuZ2xpc2hcXG4yLOS4reaWh1wiKVxuICAgIGNvbnN0IGRpYWxlY3QgPSBhd2FpdCBpbmZlckNzdkRpYWxlY3QoeyBwYXRoIH0pXG5cbiAgICBleHBlY3QoZGlhbGVjdCkudG9FcXVhbCh7XG4gICAgICBkZWxpbWl0ZXI6IFwiLFwiLFxuICAgIH0pXG4gIH0pXG5cbiAgaXQoXCJzaG91bGQgaW5mZXIgcXVvdGVDaGFyXCIsIGFzeW5jICgpID0+IHtcbiAgICBjb25zdCBwYXRoID0gYXdhaXQgd3JpdGVUZW1wRmlsZSgnaWQsbmFtZVxcbjEsXCJKb2huIERvZVwiXFxuMixcIkphbmUgU21pdGhcIicpXG4gICAgY29uc3QgZGlhbGVjdCA9IGF3YWl0IGluZmVyQ3N2RGlhbGVjdCh7IHBhdGggfSlcblxuICAgIGV4cGVjdChkaWFsZWN0KS50b0VxdWFsKHtcbiAgICAgIGRlbGltaXRlcjogXCIsXCIsXG4gICAgICBxdW90ZUNoYXI6ICdcIicsXG4gICAgfSlcbiAgfSlcblxuICBpdChcInNob3VsZCBpbmZlciBxdW90ZUNoYXIgd2l0aCBzaW5nbGUgcXVvdGVzXCIsIGFzeW5jICgpID0+IHtcbiAgICBjb25zdCBwYXRoID0gYXdhaXQgd3JpdGVUZW1wRmlsZShcImlkLG5hbWVcXG4xLCdKb2huIERvZSdcXG4yLCdKYW5lIFNtaXRoJ1wiKVxuICAgIGNvbnN0IGRpYWxlY3QgPSBhd2FpdCBpbmZlckNzdkRpYWxlY3QoeyBwYXRoIH0pXG5cbiAgICBleHBlY3QoZGlhbGVjdCkudG9FcXVhbCh7XG4gICAgICBkZWxpbWl0ZXI6IFwiLFwiLFxuICAgICAgcXVvdGVDaGFyOiBcIidcIixcbiAgICB9KVxuICB9KVxuXG4gIC8vIFRPRE86IGl0IGdpdmVzIGZhbHNlIHBvc2l0aXZlc1xuICBpdC5za2lwKFwic2hvdWxkIGluZmVyIGhlYWRlciBmYWxzZSB3aGVuIG5vIGhlYWRlciBwcmVzZW50XCIsIGFzeW5jICgpID0+IHtcbiAgICBjb25zdCBwYXRoID0gYXdhaXQgd3JpdGVUZW1wRmlsZShcIjEsZW5nbGlzaFxcbjIs5Lit5paHXFxuMyxlc3Bhw7FvbFwiKVxuICAgIGNvbnN0IGRpYWxlY3QgPSBhd2FpdCBpbmZlckNzdkRpYWxlY3QoeyBwYXRoIH0pXG5cbiAgICBleHBlY3QoZGlhbGVjdCkudG9FcXVhbCh7XG4gICAgICBkZWxpbWl0ZXI6IFwiLFwiLFxuICAgICAgaGVhZGVyOiBmYWxzZSxcbiAgICB9KVxuICB9KVxuXG4gIGl0KFwic2hvdWxkIG5vdCBzZXQgaGVhZGVyIHdoZW4gaGVhZGVyIGlzIHByZXNlbnRcIiwgYXN5bmMgKCkgPT4ge1xuICAgIGNvbnN0IHBhdGggPSBhd2FpdCB3cml0ZVRlbXBGaWxlKFwiaWQsbmFtZVxcbjEsZW5nbGlzaFxcbjIs5Lit5paHXCIpXG4gICAgY29uc3QgZGlhbGVjdCA9IGF3YWl0IGluZmVyQ3N2RGlhbGVjdCh7IHBhdGggfSlcblxuICAgIGV4cGVjdChkaWFsZWN0KS50b0VxdWFsKHtcbiAgICAgIGRlbGltaXRlcjogXCIsXCIsXG4gICAgfSlcbiAgfSlcblxuICAvLyBUT0RPOiByZWNvdmVyIGlmIHBvc3NpYmxlIHdpdGggY3N2LXNuaWZmZXJcbiAgaXQuc2tpcChcInNob3VsZCBpbmZlciBjb21wbGV4IENTViB3aXRoIHF1b3RlcyBhbmQgaGVhZGVyXCIsIGFzeW5jICgpID0+IHtcbiAgICBjb25zdCBwYXRoID0gYXdhaXQgd3JpdGVUZW1wRmlsZShcbiAgICAgICduYW1lLGRlc2NyaXB0aW9uXFxuXCJQcm9kdWN0IEFcIixcIkEgZ3JlYXQgcHJvZHVjdCB3aXRoLCBjb21tYXNcIlxcblwiUHJvZHVjdCBCXCIsXCJBbm90aGVyIHByb2R1Y3RcIicsXG4gICAgKVxuXG4gICAgY29uc3QgZGlhbGVjdCA9IGF3YWl0IGluZmVyQ3N2RGlhbGVjdCh7IHBhdGggfSlcbiAgICBleHBlY3QoZGlhbGVjdCkudG9FcXVhbCh7XG4gICAgICBkZWxpbWl0ZXI6IFwiLFwiLFxuICAgICAgcXVvdGVDaGFyOiAnXCInLFxuICAgIH0pXG4gIH0pXG59KVxuIl19
|