@loaders.gl/arrow 4.0.4 → 4.1.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-loader.d.ts +1 -0
- package/dist/arrow-loader.d.ts.map +1 -1
- package/dist/arrow-loader.js +1 -1
- package/dist/arrow-loader.js.map +1 -1
- package/dist/arrow-worker.js +241 -3
- package/dist/arrow-writer.js +1 -1
- package/dist/arrow-writer.js.map +1 -1
- package/dist/dist.dev.js +1496 -641
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts +8 -4
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts.map +1 -1
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +75 -54
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +1 -1
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.d.ts +13 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.d.ts.map +1 -0
- package/dist/geoarrow/{convert-geoarrow-to-geojson.js → convert-geoarrow-to-geojson-geometry.js} +34 -27
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js.map +1 -0
- package/dist/geoarrow/get-arrow-bounds.d.ts.map +1 -1
- package/dist/geoarrow/get-arrow-bounds.js.map +1 -1
- package/dist/geoarrow-loader.d.ts.map +1 -1
- package/dist/geoarrow-loader.js +0 -1
- package/dist/geoarrow-loader.js.map +1 -1
- package/dist/geoarrow-writer.js +1 -1
- package/dist/geoarrow-writer.js.map +1 -1
- package/dist/index.cjs +436 -347
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -3
- package/dist/index.js.map +1 -1
- package/dist/lib/arrow-table-batch.d.ts.map +1 -1
- package/dist/lib/arrow-table-batch.js.map +1 -1
- package/dist/lib/arrow-table.d.ts.map +1 -1
- package/dist/lib/arrow-table.js.map +1 -1
- package/dist/lib/encode-arrow.d.ts.map +1 -1
- package/dist/lib/encode-arrow.js.map +1 -1
- package/dist/lib/encode-geoarrow.d.ts.map +1 -1
- package/dist/lib/encode-geoarrow.js.map +1 -1
- package/dist/parsers/parse-arrow-in-batches.d.ts +2 -1
- package/dist/parsers/parse-arrow-in-batches.d.ts.map +1 -1
- package/dist/parsers/parse-arrow-in-batches.js +8 -1
- package/dist/parsers/parse-arrow-in-batches.js.map +1 -1
- package/dist/parsers/parse-arrow-sync.d.ts.map +1 -1
- package/dist/parsers/parse-arrow-sync.js +2 -0
- package/dist/parsers/parse-arrow-sync.js.map +1 -1
- package/dist/parsers/parse-geoarrow-in-batches.d.ts.map +1 -1
- package/dist/parsers/parse-geoarrow-in-batches.js.map +1 -1
- package/dist/parsers/parse-geoarrow-sync.d.ts.map +1 -1
- package/dist/parsers/parse-geoarrow-sync.js.map +1 -1
- package/dist/schema/arrow-type-utils.d.ts.map +1 -1
- package/dist/schema/arrow-type-utils.js.map +1 -1
- package/dist/schema/convert-arrow-schema.d.ts.map +1 -1
- package/dist/schema/convert-arrow-schema.js.map +1 -1
- package/dist/tables/convert-arrow-to-columnar-table.d.ts.map +1 -1
- package/dist/tables/convert-arrow-to-columnar-table.js +1 -0
- package/dist/tables/convert-arrow-to-columnar-table.js.map +1 -1
- package/dist/tables/convert-arrow-to-geojson-table.d.ts +1 -1
- package/dist/tables/convert-arrow-to-geojson-table.d.ts.map +1 -1
- package/dist/tables/convert-arrow-to-geojson-table.js +14 -8
- package/dist/tables/convert-arrow-to-geojson-table.js.map +1 -1
- package/dist/tables/convert-columnar-to-row-table.d.ts.map +1 -1
- package/dist/tables/convert-columnar-to-row-table.js +1 -0
- package/dist/tables/convert-columnar-to-row-table.js.map +1 -1
- package/dist/tables/convert-table-to-arrow.d.ts.map +1 -1
- package/dist/tables/convert-table-to-arrow.js.map +1 -1
- package/dist/triangulate-on-worker.d.ts +40 -6
- package/dist/triangulate-on-worker.d.ts.map +1 -1
- package/dist/triangulate-on-worker.js +12 -2
- package/dist/triangulate-on-worker.js.map +1 -1
- package/dist/triangulation-worker.js +11703 -34
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/workers/arrow-worker.js.map +1 -1
- package/dist/workers/hard-clone.d.ts +23 -0
- package/dist/workers/hard-clone.d.ts.map +1 -0
- package/dist/workers/hard-clone.js +57 -0
- package/dist/workers/hard-clone.js.map +1 -0
- package/dist/workers/triangulation-worker-node.d.ts.map +1 -1
- package/dist/workers/triangulation-worker-node.js.map +1 -1
- package/dist/workers/triangulation-worker.js +37 -2
- package/dist/workers/triangulation-worker.js.map +1 -1
- package/package.json +18 -12
- package/src/arrow-loader.ts +3 -1
- package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts +105 -62
- package/src/geoarrow/{convert-geoarrow-to-geojson.ts → convert-geoarrow-to-geojson-geometry.ts} +58 -47
- package/src/geoarrow/get-arrow-bounds.ts +2 -1
- package/src/geoarrow-loader.ts +2 -5
- package/src/index.ts +11 -4
- package/src/lib/arrow-table-batch.ts +2 -1
- package/src/lib/arrow-table.ts +2 -1
- package/src/lib/encode-arrow.ts +2 -1
- package/src/lib/encode-geoarrow.ts +2 -1
- package/src/parsers/parse-arrow-in-batches.ts +9 -2
- package/src/parsers/parse-arrow-sync.ts +8 -2
- package/src/parsers/parse-geoarrow-in-batches.ts +2 -1
- package/src/parsers/parse-geoarrow-sync.ts +2 -1
- package/src/schema/arrow-type-utils.ts +2 -1
- package/src/schema/convert-arrow-schema.ts +2 -1
- package/src/tables/convert-arrow-to-columnar-table.ts +3 -1
- package/src/tables/convert-arrow-to-geojson-table.ts +20 -8
- package/src/tables/convert-columnar-to-row-table.ts +3 -1
- package/src/tables/convert-table-to-arrow.ts +2 -1
- package/src/triangulate-on-worker.ts +53 -9
- package/src/types.ts +2 -1
- package/src/workers/arrow-worker.ts +2 -1
- package/src/workers/hard-clone.ts +162 -0
- package/src/workers/triangulation-worker-node.ts +2 -1
- package/src/workers/triangulation-worker.ts +59 -4
- package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts +0 -20
- package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts.map +0 -1
- package/dist/geoarrow/convert-geoarrow-to-geojson.js.map +0 -1
package/dist/arrow-loader.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ import type { ArrowTable } from './lib/arrow-table';
|
|
|
4
4
|
export type ArrowLoaderOptions = LoaderOptions & {
|
|
5
5
|
arrow?: {
|
|
6
6
|
shape: 'arrow-table' | 'columnar-table' | 'array-row-table' | 'object-row-table';
|
|
7
|
+
batchDebounceMs?: number;
|
|
7
8
|
};
|
|
8
9
|
};
|
|
9
10
|
/** ArrowJS table loader */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"arrow-loader.d.ts","sourceRoot":"","sources":["../src/arrow-loader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"arrow-loader.d.ts","sourceRoot":"","sources":["../src/arrow-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EACV,aAAa,EACb,eAAe,EACf,aAAa,EACb,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAQlD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG;IAC/C,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,aAAa,GAAG,gBAAgB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;QACjF,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH,CAAC;AAEF,2BAA2B;AAC3B,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,kBAAkB,CAoB3E,CAAC;AAEF,2BAA2B;AAC3B,eAAO,MAAM,WAAW,EAAE,gBAAgB,CACxC,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,aAAa,EAC3D,eAAe,EACf,kBAAkB,CAQnB,CAAC"}
|
package/dist/arrow-loader.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { parseArrowSync } from "./parsers/parse-arrow-sync.js";
|
|
2
2
|
import { parseArrowInBatches } from "./parsers/parse-arrow-in-batches.js";
|
|
3
|
-
const VERSION = typeof
|
|
3
|
+
const VERSION = typeof "4.1.0-alpha.10" !== 'undefined' ? "4.1.0-alpha.10" : 'latest';
|
|
4
4
|
export const ArrowWorkerLoader = {
|
|
5
5
|
name: 'Apache Arrow',
|
|
6
6
|
id: 'arrow',
|
package/dist/arrow-loader.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"arrow-loader.js","names":["parseArrowSync","parseArrowInBatches","VERSION","
|
|
1
|
+
{"version":3,"file":"arrow-loader.js","names":["parseArrowSync","parseArrowInBatches","VERSION","ArrowWorkerLoader","name","id","module","version","category","extensions","mimeTypes","binary","tests","options","arrow","shape","ArrowLoader","parse","arraybuffer","parseSync","parseInBatches"],"sources":["../src/arrow-loader.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {\n ArrayRowTable,\n ArrowTableBatch,\n ColumnarTable,\n ObjectRowTable\n} from '@loaders.gl/schema';\nimport type {ArrowTable} from './lib/arrow-table';\nimport {parseArrowSync} from './parsers/parse-arrow-sync';\nimport {parseArrowInBatches} from './parsers/parse-arrow-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ArrowLoaderOptions = LoaderOptions & {\n arrow?: {\n shape: 'arrow-table' | 'columnar-table' | 'array-row-table' | 'object-row-table';\n batchDebounceMs?: number;\n };\n};\n\n/** ArrowJS table loader */\nexport const ArrowWorkerLoader: Loader<ArrowTable, never, ArrowLoaderOptions> = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n // worker: true,\n category: 'table',\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n tests: ['ARROW'],\n options: {\n arrow: {\n shape: 'columnar-table'\n }\n }\n};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser<\n ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable,\n ArrowTableBatch,\n ArrowLoaderOptions\n> = {\n ...ArrowWorkerLoader,\n parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n parseArrowSync(arraybuffer, options?.arrow),\n parseSync: (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n parseArrowSync(arraybuffer, options?.arrow),\n parseInBatches: parseArrowInBatches\n};\n"],"mappings":"SAYQA,cAAc;AAAA,SACdC,mBAAmB;AAI3B,MAAMC,OAAO,GAAG,uBAAkB,KAAK,WAAW,sBAAiB,QAAQ;AAU3E,OAAO,MAAMC,iBAAgE,GAAG;EAC9EC,IAAI,EAAE,cAAc;EACpBC,EAAE,EAAE,OAAO;EACXC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAEL,OAAO;EAEhBM,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;EAChCC,SAAS,EAAE,CACT,mCAAmC,EACnC,qCAAqC,EACrC,0BAA0B,CAC3B;EACDC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,OAAO,CAAC;EAChBC,OAAO,EAAE;IACPC,KAAK,EAAE;MACLC,KAAK,EAAE;IACT;EACF;AACF,CAAC;AAGD,OAAO,MAAMC,WAIZ,GAAG;EACF,GAAGb,iBAAiB;EACpBc,KAAK,EAAE,MAAAA,CAAOC,WAAwB,EAAEL,OAA4B,KAClEb,cAAc,CAACkB,WAAW,EAAEL,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEC,KAAK,CAAC;EAC7CK,SAAS,EAAEA,CAACD,WAAwB,EAAEL,OAA4B,KAChEb,cAAc,CAACkB,WAAW,EAAEL,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEC,KAAK,CAAC;EAC7CM,cAAc,EAAEnB;AAClB,CAAC"}
|
package/dist/arrow-worker.js
CHANGED
|
@@ -1879,6 +1879,16 @@
|
|
|
1879
1879
|
proto.ArrayType = Int32Array;
|
|
1880
1880
|
return proto[Symbol.toStringTag] = "Date";
|
|
1881
1881
|
})(Date_.prototype);
|
|
1882
|
+
var DateDay = class extends Date_ {
|
|
1883
|
+
constructor() {
|
|
1884
|
+
super(DateUnit.DAY);
|
|
1885
|
+
}
|
|
1886
|
+
};
|
|
1887
|
+
var DateMillisecond = class extends Date_ {
|
|
1888
|
+
constructor() {
|
|
1889
|
+
super(DateUnit.MILLISECOND);
|
|
1890
|
+
}
|
|
1891
|
+
};
|
|
1882
1892
|
var Time_ = class extends DataType {
|
|
1883
1893
|
constructor(unit, bitWidth) {
|
|
1884
1894
|
super();
|
|
@@ -1907,6 +1917,26 @@
|
|
|
1907
1917
|
proto.bitWidth = null;
|
|
1908
1918
|
return proto[Symbol.toStringTag] = "Time";
|
|
1909
1919
|
})(Time_.prototype);
|
|
1920
|
+
var TimeSecond = class extends Time_ {
|
|
1921
|
+
constructor() {
|
|
1922
|
+
super(TimeUnit.SECOND, 32);
|
|
1923
|
+
}
|
|
1924
|
+
};
|
|
1925
|
+
var TimeMillisecond = class extends Time_ {
|
|
1926
|
+
constructor() {
|
|
1927
|
+
super(TimeUnit.MILLISECOND, 32);
|
|
1928
|
+
}
|
|
1929
|
+
};
|
|
1930
|
+
var TimeMicrosecond = class extends Time_ {
|
|
1931
|
+
constructor() {
|
|
1932
|
+
super(TimeUnit.MICROSECOND, 64);
|
|
1933
|
+
}
|
|
1934
|
+
};
|
|
1935
|
+
var TimeNanosecond = class extends Time_ {
|
|
1936
|
+
constructor() {
|
|
1937
|
+
super(TimeUnit.NANOSECOND, 64);
|
|
1938
|
+
}
|
|
1939
|
+
};
|
|
1910
1940
|
var Timestamp_ = class extends DataType {
|
|
1911
1941
|
constructor(unit, timezone) {
|
|
1912
1942
|
super();
|
|
@@ -1927,6 +1957,26 @@
|
|
|
1927
1957
|
proto.ArrayType = Int32Array;
|
|
1928
1958
|
return proto[Symbol.toStringTag] = "Timestamp";
|
|
1929
1959
|
})(Timestamp_.prototype);
|
|
1960
|
+
var TimestampSecond = class extends Timestamp_ {
|
|
1961
|
+
constructor(timezone) {
|
|
1962
|
+
super(TimeUnit.SECOND, timezone);
|
|
1963
|
+
}
|
|
1964
|
+
};
|
|
1965
|
+
var TimestampMillisecond = class extends Timestamp_ {
|
|
1966
|
+
constructor(timezone) {
|
|
1967
|
+
super(TimeUnit.MILLISECOND, timezone);
|
|
1968
|
+
}
|
|
1969
|
+
};
|
|
1970
|
+
var TimestampMicrosecond = class extends Timestamp_ {
|
|
1971
|
+
constructor(timezone) {
|
|
1972
|
+
super(TimeUnit.MICROSECOND, timezone);
|
|
1973
|
+
}
|
|
1974
|
+
};
|
|
1975
|
+
var TimestampNanosecond = class extends Timestamp_ {
|
|
1976
|
+
constructor(timezone) {
|
|
1977
|
+
super(TimeUnit.NANOSECOND, timezone);
|
|
1978
|
+
}
|
|
1979
|
+
};
|
|
1930
1980
|
var Interval_ = class extends DataType {
|
|
1931
1981
|
constructor(unit) {
|
|
1932
1982
|
super();
|
|
@@ -1945,6 +1995,16 @@
|
|
|
1945
1995
|
proto.ArrayType = Int32Array;
|
|
1946
1996
|
return proto[Symbol.toStringTag] = "Interval";
|
|
1947
1997
|
})(Interval_.prototype);
|
|
1998
|
+
var IntervalDayTime = class extends Interval_ {
|
|
1999
|
+
constructor() {
|
|
2000
|
+
super(IntervalUnit.DAY_TIME);
|
|
2001
|
+
}
|
|
2002
|
+
};
|
|
2003
|
+
var IntervalYearMonth = class extends Interval_ {
|
|
2004
|
+
constructor() {
|
|
2005
|
+
super(IntervalUnit.YEAR_MONTH);
|
|
2006
|
+
}
|
|
2007
|
+
};
|
|
1948
2008
|
var List = class extends DataType {
|
|
1949
2009
|
constructor(child) {
|
|
1950
2010
|
super();
|
|
@@ -11893,14 +11953,189 @@ return true;`);
|
|
|
11893
11953
|
}
|
|
11894
11954
|
return {
|
|
11895
11955
|
shape: "columnar-table",
|
|
11956
|
+
schema: table.schema,
|
|
11896
11957
|
data: columnarTable
|
|
11897
11958
|
};
|
|
11898
11959
|
}
|
|
11899
11960
|
|
|
11961
|
+
// src/schema/convert-arrow-schema.ts
|
|
11962
|
+
function serializeArrowSchema(arrowSchema) {
|
|
11963
|
+
return {
|
|
11964
|
+
fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
|
|
11965
|
+
metadata: serializeArrowMetadata(arrowSchema.metadata)
|
|
11966
|
+
};
|
|
11967
|
+
}
|
|
11968
|
+
function serializeArrowMetadata(arrowMetadata) {
|
|
11969
|
+
return Object.fromEntries(arrowMetadata);
|
|
11970
|
+
}
|
|
11971
|
+
function serializeArrowField(field) {
|
|
11972
|
+
return {
|
|
11973
|
+
name: field.name,
|
|
11974
|
+
type: serializeArrowType(field.type),
|
|
11975
|
+
nullable: field.nullable,
|
|
11976
|
+
metadata: serializeArrowMetadata(field.metadata)
|
|
11977
|
+
};
|
|
11978
|
+
}
|
|
11979
|
+
function serializeArrowType(arrowType) {
|
|
11980
|
+
switch (arrowType.constructor) {
|
|
11981
|
+
case Null:
|
|
11982
|
+
return "null";
|
|
11983
|
+
case Binary:
|
|
11984
|
+
return "binary";
|
|
11985
|
+
case Bool:
|
|
11986
|
+
return "bool";
|
|
11987
|
+
case Int_:
|
|
11988
|
+
const intType = arrowType;
|
|
11989
|
+
return `${intType.isSigned ? "u" : ""}int${intType.bitWidth}`;
|
|
11990
|
+
case Int8:
|
|
11991
|
+
return "int8";
|
|
11992
|
+
case Int16:
|
|
11993
|
+
return "int16";
|
|
11994
|
+
case Int32:
|
|
11995
|
+
return "int32";
|
|
11996
|
+
case Int64:
|
|
11997
|
+
return "int64";
|
|
11998
|
+
case Uint8:
|
|
11999
|
+
return "uint8";
|
|
12000
|
+
case Uint16:
|
|
12001
|
+
return "uint16";
|
|
12002
|
+
case Uint32:
|
|
12003
|
+
return "uint32";
|
|
12004
|
+
case Uint64:
|
|
12005
|
+
return "uint64";
|
|
12006
|
+
case Float:
|
|
12007
|
+
const precision = arrowType.precision;
|
|
12008
|
+
switch (precision) {
|
|
12009
|
+
case Precision.HALF:
|
|
12010
|
+
return "float16";
|
|
12011
|
+
case Precision.SINGLE:
|
|
12012
|
+
return "float32";
|
|
12013
|
+
case Precision.DOUBLE:
|
|
12014
|
+
return "float64";
|
|
12015
|
+
default:
|
|
12016
|
+
return "float16";
|
|
12017
|
+
}
|
|
12018
|
+
case Float16:
|
|
12019
|
+
return "float16";
|
|
12020
|
+
case Float32:
|
|
12021
|
+
return "float32";
|
|
12022
|
+
case Float64:
|
|
12023
|
+
return "float64";
|
|
12024
|
+
case Utf8:
|
|
12025
|
+
return "utf8";
|
|
12026
|
+
case Decimal:
|
|
12027
|
+
const decimal = arrowType;
|
|
12028
|
+
return {
|
|
12029
|
+
type: "decimal",
|
|
12030
|
+
bitWidth: decimal.bitWidth,
|
|
12031
|
+
precision: decimal.precision,
|
|
12032
|
+
scale: decimal.scale
|
|
12033
|
+
};
|
|
12034
|
+
case Date_:
|
|
12035
|
+
const dateUnit = arrowType.unit;
|
|
12036
|
+
return dateUnit === DateUnit.DAY ? "date-day" : "date-millisecond";
|
|
12037
|
+
case DateDay:
|
|
12038
|
+
return "date-day";
|
|
12039
|
+
case DateMillisecond:
|
|
12040
|
+
return "date-millisecond";
|
|
12041
|
+
case Time_:
|
|
12042
|
+
const timeUnit = arrowType.unit;
|
|
12043
|
+
switch (timeUnit) {
|
|
12044
|
+
case TimeUnit.SECOND:
|
|
12045
|
+
return "time-second";
|
|
12046
|
+
case TimeUnit.MILLISECOND:
|
|
12047
|
+
return "time-millisecond";
|
|
12048
|
+
case TimeUnit.MICROSECOND:
|
|
12049
|
+
return "time-microsecond";
|
|
12050
|
+
case TimeUnit.NANOSECOND:
|
|
12051
|
+
return "time-nanosecond";
|
|
12052
|
+
default:
|
|
12053
|
+
return "time-second";
|
|
12054
|
+
}
|
|
12055
|
+
case TimeMillisecond:
|
|
12056
|
+
return "time-millisecond";
|
|
12057
|
+
case TimeSecond:
|
|
12058
|
+
return "time-second";
|
|
12059
|
+
case TimeMicrosecond:
|
|
12060
|
+
return "time-microsecond";
|
|
12061
|
+
case TimeNanosecond:
|
|
12062
|
+
return "time-nanosecond";
|
|
12063
|
+
case Timestamp_:
|
|
12064
|
+
const timeStampUnit = arrowType.unit;
|
|
12065
|
+
switch (timeStampUnit) {
|
|
12066
|
+
case TimeUnit.SECOND:
|
|
12067
|
+
return "timestamp-second";
|
|
12068
|
+
case TimeUnit.MILLISECOND:
|
|
12069
|
+
return "timestamp-millisecond";
|
|
12070
|
+
case TimeUnit.MICROSECOND:
|
|
12071
|
+
return "timestamp-microsecond";
|
|
12072
|
+
case TimeUnit.NANOSECOND:
|
|
12073
|
+
return "timestamp-nanosecond";
|
|
12074
|
+
default:
|
|
12075
|
+
return "timestamp-second";
|
|
12076
|
+
}
|
|
12077
|
+
case TimestampSecond:
|
|
12078
|
+
return "timestamp-second";
|
|
12079
|
+
case TimestampMillisecond:
|
|
12080
|
+
return "timestamp-millisecond";
|
|
12081
|
+
case TimestampMicrosecond:
|
|
12082
|
+
return "timestamp-microsecond";
|
|
12083
|
+
case TimestampNanosecond:
|
|
12084
|
+
return "timestamp-nanosecond";
|
|
12085
|
+
case Interval_:
|
|
12086
|
+
const intervalUnit = arrowType.unit;
|
|
12087
|
+
switch (intervalUnit) {
|
|
12088
|
+
case IntervalUnit.DAY_TIME:
|
|
12089
|
+
return "interval-daytime";
|
|
12090
|
+
case IntervalUnit.YEAR_MONTH:
|
|
12091
|
+
return "interval-yearmonth";
|
|
12092
|
+
default:
|
|
12093
|
+
return "interval-daytime";
|
|
12094
|
+
}
|
|
12095
|
+
case IntervalDayTime:
|
|
12096
|
+
return "interval-daytime";
|
|
12097
|
+
case IntervalYearMonth:
|
|
12098
|
+
return "interval-yearmonth";
|
|
12099
|
+
case Map_:
|
|
12100
|
+
const mapType = arrowType;
|
|
12101
|
+
return {
|
|
12102
|
+
type: "map",
|
|
12103
|
+
keysSorted: mapType.keysSorted,
|
|
12104
|
+
children: mapType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
12105
|
+
};
|
|
12106
|
+
case List:
|
|
12107
|
+
const listType = arrowType;
|
|
12108
|
+
const listField = listType.valueField;
|
|
12109
|
+
return {
|
|
12110
|
+
type: "list",
|
|
12111
|
+
children: [serializeArrowField(listField)]
|
|
12112
|
+
};
|
|
12113
|
+
case FixedSizeList:
|
|
12114
|
+
const fixedSizeList = arrowType;
|
|
12115
|
+
return {
|
|
12116
|
+
type: "fixed-size-list",
|
|
12117
|
+
listSize: fixedSizeList.listSize,
|
|
12118
|
+
children: [serializeArrowField(fixedSizeList.children[0])]
|
|
12119
|
+
};
|
|
12120
|
+
case Struct:
|
|
12121
|
+
const structType = arrowType;
|
|
12122
|
+
return {
|
|
12123
|
+
type: "struct",
|
|
12124
|
+
children: structType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
12125
|
+
};
|
|
12126
|
+
default:
|
|
12127
|
+
throw new Error(`arrow type not supported: ${arrowType.constructor.name}`);
|
|
12128
|
+
}
|
|
12129
|
+
}
|
|
12130
|
+
|
|
11900
12131
|
// src/parsers/parse-arrow-sync.ts
|
|
11901
12132
|
function parseArrowSync(arrayBuffer, options) {
|
|
11902
12133
|
const apacheArrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);
|
|
11903
|
-
const arrowTable = {
|
|
12134
|
+
const arrowTable = {
|
|
12135
|
+
shape: "arrow-table",
|
|
12136
|
+
schema: serializeArrowSchema(apacheArrowTable.schema),
|
|
12137
|
+
data: apacheArrowTable
|
|
12138
|
+
};
|
|
11904
12139
|
const shape = options?.shape || "arrow-table";
|
|
11905
12140
|
switch (shape) {
|
|
11906
12141
|
case "arrow-table":
|
|
@@ -11919,11 +12154,14 @@ return true;`);
|
|
|
11919
12154
|
}
|
|
11920
12155
|
|
|
11921
12156
|
// src/parsers/parse-arrow-in-batches.ts
|
|
11922
|
-
function parseArrowInBatches(asyncIterator) {
|
|
12157
|
+
function parseArrowInBatches(asyncIterator, options) {
|
|
11923
12158
|
async function* makeArrowAsyncIterator() {
|
|
11924
12159
|
const readers = RecordBatchReader.readAll(asyncIterator);
|
|
11925
12160
|
for await (const reader of readers) {
|
|
11926
12161
|
for await (const recordBatch of reader) {
|
|
12162
|
+
if (options?.arrow?.batchDebounceMs !== void 0 && options?.arrow?.batchDebounceMs > 0) {
|
|
12163
|
+
await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));
|
|
12164
|
+
}
|
|
11927
12165
|
const arrowTabledBatch = {
|
|
11928
12166
|
shape: "arrow-table",
|
|
11929
12167
|
batchType: "data",
|
|
@@ -11939,7 +12177,7 @@ return true;`);
|
|
|
11939
12177
|
}
|
|
11940
12178
|
|
|
11941
12179
|
// src/arrow-loader.ts
|
|
11942
|
-
var VERSION = true ? "4.0.
|
|
12180
|
+
var VERSION = true ? "4.1.0-alpha.10" : "latest";
|
|
11943
12181
|
var ArrowWorkerLoader = {
|
|
11944
12182
|
name: "Apache Arrow",
|
|
11945
12183
|
id: "arrow",
|
package/dist/arrow-writer.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { encodeArrowSync } from "./lib/encode-arrow.js";
|
|
2
|
-
const VERSION = typeof
|
|
2
|
+
const VERSION = typeof "4.1.0-alpha.10" !== 'undefined' ? "4.1.0-alpha.10" : 'latest';
|
|
3
3
|
export const ArrowWriter = {
|
|
4
4
|
name: 'Apache Arrow',
|
|
5
5
|
id: 'arrow',
|
package/dist/arrow-writer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"arrow-writer.js","names":["encodeArrowSync","VERSION","
|
|
1
|
+
{"version":3,"file":"arrow-writer.js","names":["encodeArrowSync","VERSION","ArrowWriter","name","id","module","version","extensions","mimeTypes","binary","options","encode","encodeArrow","data","encodeSync"],"sources":["../src/arrow-writer.ts"],"sourcesContent":["// import type {} from '@loaders.gl/loader-utils';\n\nimport type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';\nimport {ColumnarTable} from './lib/encode-arrow';\nimport {encodeArrowSync} from './lib/encode-arrow';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\ntype ArrowWriterOptions = WriterOptions & {\n arrow?: {};\n};\n\n/** Apache Arrow writer */\nexport const ArrowWriter: WriterWithEncoder<ColumnarTable, never, ArrowWriterOptions> = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n options: {},\n encode: async function encodeArrow(data, options?): Promise<ArrayBuffer> {\n return encodeArrowSync(data);\n },\n encodeSync(data, options?) {\n return encodeArrowSync(data);\n }\n};\n"],"mappings":"SAIQA,eAAe;AAIvB,MAAMC,OAAO,GAAG,uBAAkB,KAAK,WAAW,sBAAiB,QAAQ;AAO3E,OAAO,MAAMC,WAAwE,GAAG;EACtFC,IAAI,EAAE,cAAc;EACpBC,EAAE,EAAE,OAAO;EACXC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAEL,OAAO;EAChBM,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;EAChCC,SAAS,EAAE,CACT,mCAAmC,EACnC,qCAAqC,EACrC,0BAA0B,CAC3B;EACDC,MAAM,EAAE,IAAI;EACZC,OAAO,EAAE,CAAC,CAAC;EACXC,MAAM,EAAE,eAAeC,WAAWA,CAACC,IAAI,EAAEH,OAAQ,EAAwB;IACvE,OAAOV,eAAe,CAACa,IAAI,CAAC;EAC9B,CAAC;EACDC,UAAUA,CAACD,IAAI,EAAEH,OAAQ,EAAE;IACzB,OAAOV,eAAe,CAACa,IAAI,CAAC;EAC9B;AACF,CAAC"}
|