@loaders.gl/json 4.0.0-alpha.23 → 4.0.0-alpha.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +87 -51
- package/dist/es5/geojson-loader.js +14 -5
- package/dist/es5/geojson-loader.js.map +1 -1
- package/dist/es5/geojson-writer.js.map +1 -1
- package/dist/es5/json-loader.js +1 -1
- package/dist/es5/json-writer.js +1 -1
- package/dist/es5/json-writer.js.map +1 -1
- package/dist/es5/lib/encoder-utils/encode-table-row.js +54 -0
- package/dist/es5/lib/encoder-utils/encode-table-row.js.map +1 -0
- package/dist/es5/lib/encoder-utils/encode-utils.js.map +1 -0
- package/dist/es5/lib/encoder-utils/utf8-encoder.js.map +1 -0
- package/dist/es5/lib/encoders/geojson-encoder.js +47 -84
- package/dist/es5/lib/encoders/geojson-encoder.js.map +1 -1
- package/dist/es5/lib/encoders/json-encoder.js +3 -3
- package/dist/es5/lib/encoders/json-encoder.js.map +1 -1
- package/dist/es5/ndgeoson-loader.js +1 -1
- package/dist/es5/ndjson-loader.js +1 -1
- package/dist/esm/geojson-loader.js +14 -5
- package/dist/esm/geojson-loader.js.map +1 -1
- package/dist/esm/geojson-writer.js.map +1 -1
- package/dist/esm/json-loader.js +1 -1
- package/dist/esm/json-writer.js +1 -1
- package/dist/esm/json-writer.js.map +1 -1
- package/dist/esm/lib/encoder-utils/encode-table-row.js +44 -0
- package/dist/esm/lib/encoder-utils/encode-table-row.js.map +1 -0
- package/dist/esm/lib/encoder-utils/encode-utils.js.map +1 -0
- package/dist/esm/lib/encoder-utils/utf8-encoder.js.map +1 -0
- package/dist/esm/lib/encoders/geojson-encoder.js +9 -52
- package/dist/esm/lib/encoders/geojson-encoder.js.map +1 -1
- package/dist/esm/lib/encoders/json-encoder.js +3 -3
- package/dist/esm/lib/encoders/json-encoder.js.map +1 -1
- package/dist/esm/ndgeoson-loader.js +1 -1
- package/dist/esm/ndjson-loader.js +1 -1
- package/dist/geojson-loader.d.ts.map +1 -1
- package/dist/geojson-worker.js +14 -183
- package/dist/geojson-writer.d.ts +8 -3
- package/dist/geojson-writer.d.ts.map +1 -1
- package/dist/json-writer.d.ts +13 -3
- package/dist/json-writer.d.ts.map +1 -1
- package/dist/lib/encoder-utils/encode-table-row.d.ts +7 -0
- package/dist/lib/encoder-utils/encode-table-row.d.ts.map +1 -0
- package/dist/lib/encoder-utils/encode-utils.d.ts.map +1 -0
- package/dist/lib/encoder-utils/utf8-encoder.d.ts.map +1 -0
- package/dist/lib/encoders/geojson-encoder.d.ts +2 -8
- package/dist/lib/encoders/geojson-encoder.d.ts.map +1 -1
- package/dist/lib/encoders/json-encoder.d.ts +1 -10
- package/dist/lib/encoders/json-encoder.d.ts.map +1 -1
- package/package.json +5 -5
- package/src/geojson-loader.ts +19 -5
- package/src/geojson-writer.ts +8 -3
- package/src/json-writer.ts +12 -4
- package/src/lib/encoder-utils/encode-table-row.ts +69 -0
- package/src/lib/encoders/geojson-encoder.ts +15 -78
- package/src/lib/encoders/json-encoder.ts +3 -11
- package/dist/bundle.js +0 -5
- package/dist/es5/lib/encoders/encode-utils.js.map +0 -1
- package/dist/es5/lib/encoders/utf8-encoder.js.map +0 -1
- package/dist/esm/lib/encoders/encode-utils.js.map +0 -1
- package/dist/esm/lib/encoders/utf8-encoder.js.map +0 -1
- package/dist/geojson-loader.js +0 -79
- package/dist/geojson-writer.js +0 -22
- package/dist/index.js +0 -24
- package/dist/json-loader.js +0 -42
- package/dist/json-writer.js +0 -18
- package/dist/lib/clarinet/clarinet.js +0 -535
- package/dist/lib/encoders/encode-utils.d.ts.map +0 -1
- package/dist/lib/encoders/encode-utils.js +0 -47
- package/dist/lib/encoders/geojson-encoder.js +0 -104
- package/dist/lib/encoders/json-encoder.js +0 -22
- package/dist/lib/encoders/utf8-encoder.d.ts.map +0 -1
- package/dist/lib/encoders/utf8-encoder.js +0 -32
- package/dist/lib/json-parser/json-parser.js +0 -98
- package/dist/lib/json-parser/streaming-json-parser.js +0 -100
- package/dist/lib/jsonpath/jsonpath.js +0 -89
- package/dist/lib/parsers/parse-json-in-batches.js +0 -101
- package/dist/lib/parsers/parse-json.js +0 -32
- package/dist/lib/parsers/parse-ndjson-in-batches.js +0 -36
- package/dist/lib/parsers/parse-ndjson.js +0 -17
- package/dist/ndgeoson-loader.js +0 -36
- package/dist/ndjson-loader.js +0 -27
- package/dist/workers/geojson-worker.js +0 -5
- /package/dist/es5/lib/{encoders → encoder-utils}/encode-utils.js +0 -0
- /package/dist/es5/lib/{encoders → encoder-utils}/utf8-encoder.js +0 -0
- /package/dist/esm/lib/{encoders → encoder-utils}/encode-utils.js +0 -0
- /package/dist/esm/lib/{encoders → encoder-utils}/utf8-encoder.js +0 -0
- /package/dist/lib/{encoders → encoder-utils}/encode-utils.d.ts +0 -0
- /package/dist/lib/{encoders → encoder-utils}/utf8-encoder.d.ts +0 -0
- /package/src/lib/{encoders → encoder-utils}/encode-utils.ts +0 -0
- /package/src/lib/{encoders → encoder-utils}/utf8-encoder.ts +0 -0
|
@@ -1,20 +1,12 @@
|
|
|
1
1
|
// loaders.gl, MIT license
|
|
2
2
|
// Copyright 2022 Foursquare Labs, Inc.
|
|
3
3
|
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {detectGeometryColumnIndex
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
type
|
|
10
|
-
|
|
11
|
-
export type GeoJSONWriterOptions = {
|
|
12
|
-
geojson?: {
|
|
13
|
-
featureArray?: boolean;
|
|
14
|
-
geometryColumn?: number | null;
|
|
15
|
-
};
|
|
16
|
-
chunkSize?: number;
|
|
17
|
-
};
|
|
4
|
+
import type {TableBatch} from '@loaders.gl/schema';
|
|
5
|
+
import {getTableLength} from '@loaders.gl/schema';
|
|
6
|
+
import {detectGeometryColumnIndex} from '../encoder-utils/encode-utils';
|
|
7
|
+
import {encodeTableRow} from '../encoder-utils/encode-table-row';
|
|
8
|
+
import {Utf8ArrayBufferEncoder} from '../encoder-utils/utf8-encoder';
|
|
9
|
+
import type {GeoJSONWriterOptions} from '../../geojson-writer';
|
|
18
10
|
|
|
19
11
|
/**
|
|
20
12
|
* Encode a table as GeoJSON
|
|
@@ -24,6 +16,7 @@ export async function* encodeTableAsGeojsonInBatches(
|
|
|
24
16
|
batchIterator: AsyncIterable<TableBatch>, // | Iterable<TableBatch>,
|
|
25
17
|
inputOpts: GeoJSONWriterOptions = {}
|
|
26
18
|
): AsyncIterable<ArrayBuffer> {
|
|
19
|
+
// @ts-expect-error
|
|
27
20
|
const options: Required<GeoJSONWriterOptions> = {geojson: {}, chunkSize: 10000, ...inputOpts};
|
|
28
21
|
|
|
29
22
|
const utf8Encoder = new Utf8ArrayBufferEncoder(options.chunkSize);
|
|
@@ -37,12 +30,13 @@ export async function* encodeTableAsGeojsonInBatches(
|
|
|
37
30
|
|
|
38
31
|
let isFirstLine = true;
|
|
39
32
|
|
|
40
|
-
|
|
41
|
-
|
|
33
|
+
let start = 0;
|
|
34
|
+
for await (const tableBatch of batchIterator) {
|
|
35
|
+
const end = start + getTableLength(tableBatch);
|
|
42
36
|
|
|
43
37
|
// Deduce geometry column if not already done
|
|
44
38
|
if (!geometryColumn) {
|
|
45
|
-
geometryColumn = geometryColumn || detectGeometryColumnIndex(
|
|
39
|
+
geometryColumn = geometryColumn || detectGeometryColumnIndex(tableBatch);
|
|
46
40
|
}
|
|
47
41
|
|
|
48
42
|
for (let rowIndex = start; rowIndex < end; ++rowIndex) {
|
|
@@ -53,13 +47,16 @@ export async function* encodeTableAsGeojsonInBatches(
|
|
|
53
47
|
utf8Encoder.push('\n');
|
|
54
48
|
isFirstLine = false;
|
|
55
49
|
|
|
56
|
-
|
|
50
|
+
encodeTableRow(tableBatch, rowIndex, geometryColumn, utf8Encoder);
|
|
57
51
|
|
|
58
52
|
// eslint-disable-next-line max-depth
|
|
59
53
|
if (utf8Encoder.isFull()) {
|
|
60
54
|
yield utf8Encoder.getArrayBufferBatch();
|
|
61
55
|
}
|
|
56
|
+
|
|
57
|
+
start = end;
|
|
62
58
|
}
|
|
59
|
+
|
|
63
60
|
const arrayBufferBatch = utf8Encoder.getArrayBufferBatch();
|
|
64
61
|
if (arrayBufferBatch.byteLength > 0) {
|
|
65
62
|
yield arrayBufferBatch;
|
|
@@ -77,63 +74,3 @@ export async function* encodeTableAsGeojsonInBatches(
|
|
|
77
74
|
// Note: Since we pushed a few final lines, the last batch will always exist, no need to check first
|
|
78
75
|
yield utf8Encoder.getArrayBufferBatch();
|
|
79
76
|
}
|
|
80
|
-
|
|
81
|
-
// Helpers
|
|
82
|
-
|
|
83
|
-
/**
|
|
84
|
-
* Encode a row. Currently this ignores properties in the geometry column.
|
|
85
|
-
*/
|
|
86
|
-
function encodeRow(
|
|
87
|
-
table: Table,
|
|
88
|
-
rowIndex: number,
|
|
89
|
-
geometryColumnIndex: number,
|
|
90
|
-
utf8Encoder: Utf8ArrayBufferEncoder
|
|
91
|
-
): void {
|
|
92
|
-
const row = getTableRowAsObject(table, rowIndex);
|
|
93
|
-
if (!row) return;
|
|
94
|
-
const featureWithProperties = getFeatureFromRow(table, row, geometryColumnIndex);
|
|
95
|
-
const featureString = JSON.stringify(featureWithProperties);
|
|
96
|
-
utf8Encoder.push(featureString);
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
/**
|
|
100
|
-
* Encode a row as a Feature. Currently this ignores properties objects in the geometry column.
|
|
101
|
-
*/
|
|
102
|
-
function getFeatureFromRow(table: Table, row: Row, geometryColumnIndex: number): Feature {
|
|
103
|
-
// Extract non-feature/geometry properties
|
|
104
|
-
const properties = getRowPropertyObject(table, row, [geometryColumnIndex]);
|
|
105
|
-
|
|
106
|
-
// Extract geometry feature
|
|
107
|
-
const columnName = table.schema?.fields[geometryColumnIndex].name;
|
|
108
|
-
let featureOrGeometry =
|
|
109
|
-
columnName && (row[columnName] as {[key: string]: unknown} | string | null | undefined);
|
|
110
|
-
|
|
111
|
-
// GeoJSON support null geometries
|
|
112
|
-
if (!featureOrGeometry) {
|
|
113
|
-
// @ts-ignore Feature type does not support null geometries
|
|
114
|
-
return {type: 'Feature', geometry: null, properties};
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
// Support string geometries?
|
|
118
|
-
// TODO: This assumes GeoJSON strings, which may not be the correct format
|
|
119
|
-
// (could be WKT, encoded WKB...)
|
|
120
|
-
if (typeof featureOrGeometry === 'string') {
|
|
121
|
-
try {
|
|
122
|
-
featureOrGeometry = JSON.parse(featureOrGeometry);
|
|
123
|
-
} catch (err) {
|
|
124
|
-
throw new Error('Invalid string geometry');
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
if (typeof featureOrGeometry !== 'object' || typeof featureOrGeometry?.type !== 'string') {
|
|
129
|
-
throw new Error('invalid geometry column value');
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
if (featureOrGeometry?.type === 'Feature') {
|
|
133
|
-
// @ts-ignore Feature type does not support null geometries
|
|
134
|
-
return {...featureOrGeometry, properties};
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
// @ts-ignore Feature type does not support null geometries
|
|
138
|
-
return {type: 'Feature', geometry: featureOrGeometry, properties};
|
|
139
|
-
}
|
|
@@ -2,21 +2,13 @@
|
|
|
2
2
|
// Copyright 2022 Foursquare Labs, Inc.
|
|
3
3
|
|
|
4
4
|
import {Table, makeRowIterator} from '@loaders.gl/schema';
|
|
5
|
-
|
|
6
|
-
type RowArray = unknown[];
|
|
7
|
-
type RowObject = {[key: string]: unknown};
|
|
8
|
-
type TableJSON = RowArray[] | RowObject[];
|
|
9
|
-
|
|
10
|
-
export type JSONWriterOptions = {
|
|
11
|
-
shape?: 'object-row-table' | 'array-row-table';
|
|
12
|
-
wrapper?: (table: TableJSON) => unknown;
|
|
13
|
-
};
|
|
5
|
+
import type {JSONWriterOptions} from '../../json-writer';
|
|
14
6
|
|
|
15
7
|
/**
|
|
16
8
|
* Encode a table as a JSON string
|
|
17
9
|
*/
|
|
18
|
-
export function encodeTableAsJSON(table: Table, options
|
|
19
|
-
const shape = options
|
|
10
|
+
export function encodeTableAsJSON(table: Table, options?: JSONWriterOptions): string {
|
|
11
|
+
const shape = options?.json?.shape || 'object-row-table';
|
|
20
12
|
|
|
21
13
|
const strings: string[] = [];
|
|
22
14
|
const rowIterator = makeRowIterator(table, shape);
|
package/dist/bundle.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"encode-utils.js","names":["_schema","require","detectGeometryColumnIndex","table","_table$schema$fields$","_table$schema","geometryIndex","schema","fields","findIndex","field","name","getTableLength","row","getTableRowAsArray","columnIndex","getTableNumCols","value","_typeof2","default","Error","getRowPropertyObject","excludeColumnIndices","arguments","length","undefined","properties","_table$schema2","columnName","includes"],"sources":["../../../../src/lib/encoders/encode-utils.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright 2022 Foursquare Labs, Inc.\n\nimport {Table, getTableLength, getTableNumCols, getTableRowAsArray} from '@loaders.gl/schema';\n\ntype Row = {[key: string]: unknown};\n\n/**\n * Attempts to identify which column contains geometry\n * Currently just returns name (key) of first object-valued column\n * @todo look for hints in schema metadata\n * @todo look for WKB\n */\nexport function detectGeometryColumnIndex(table: Table): number {\n // TODO - look for hints in schema metadata\n\n // look for a column named geometry\n const geometryIndex = table.schema?.fields.findIndex((field) => field.name === 'geometry') ?? -1;\n if (geometryIndex > -1) {\n return geometryIndex;\n }\n\n // look at the data\n // TODO - this drags in the indices\n if (getTableLength(table) > 0) {\n const row = getTableRowAsArray(table, 0);\n for (let columnIndex = 0; columnIndex < getTableNumCols(table); columnIndex++) {\n const value = row?.[columnIndex];\n if (value && typeof value === 'object') {\n return columnIndex;\n }\n }\n }\n\n throw new Error('Failed to detect geometry column');\n}\n\n/**\n * Return a row as a property (key/value) object, excluding selected columns\n */\nexport function getRowPropertyObject(\n table: Table,\n row: Row,\n excludeColumnIndices: number[] = []\n): {[columnName: string]: unknown} {\n const properties = {};\n for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {\n const columnName = table.schema?.fields[columnIndex].name;\n if (columnName && !excludeColumnIndices.includes(columnIndex)) {\n properties[columnName] = row[columnName];\n }\n }\n return properties;\n}\n"],"mappings":";;;;;;;;;AAGA,IAAAA,OAAA,GAAAC,OAAA;AAUO,SAASC,yBAAyBA,CAACC,KAAY,EAAU;EAAA,IAAAC,qBAAA,EAAAC,aAAA;EAI9D,IAAMC,aAAa,IAAAF,qBAAA,IAAAC,aAAA,GAAGF,KAAK,CAACI,MAAM,cAAAF,aAAA,uBAAZA,aAAA,CAAcG,MAAM,CAACC,SAAS,CAAC,UAACC,KAAK;IAAA,OAAKA,KAAK,CAACC,IAAI,KAAK,UAAU;EAAA,EAAC,cAAAP,qBAAA,cAAAA,qBAAA,GAAI,CAAC,CAAC;EAChG,IAAIE,aAAa,GAAG,CAAC,CAAC,EAAE;IACtB,OAAOA,aAAa;EACtB;EAIA,IAAI,IAAAM,sBAAc,EAACT,KAAK,CAAC,GAAG,CAAC,EAAE;IAC7B,IAAMU,GAAG,GAAG,IAAAC,0BAAkB,EAACX,KAAK,EAAE,CAAC,CAAC;IACxC,KAAK,IAAIY,WAAW,GAAG,CAAC,EAAEA,WAAW,GAAG,IAAAC,uBAAe,EAACb,KAAK,CAAC,EAAEY,WAAW,EAAE,EAAE;MAC7E,IAAME,KAAK,GAAGJ,GAAG,aAAHA,GAAG,uBAAHA,GAAG,CAAGE,WAAW,CAAC;MAChC,IAAIE,KAAK,IAAI,IAAAC,QAAA,CAAAC,OAAA,EAAOF,KAAK,MAAK,QAAQ,EAAE;QACtC,OAAOF,WAAW;MACpB;IACF;EACF;EAEA,MAAM,IAAIK,KAAK,CAAC,kCAAkC,CAAC;AACrD;AAKO,SAASC,oBAAoBA,CAClClB,KAAY,EACZU,GAAQ,EAEyB;EAAA,IADjCS,oBAA8B,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,EAAE;EAEnC,IAAMG,UAAU,GAAG,CAAC,CAAC;EACrB,KAAK,IAAIX,WAAW,GAAG,CAAC,EAAEA,WAAW,GAAG,IAAAC,uBAAe,EAACb,KAAK,CAAC,EAAE,EAAEY,WAAW,EAAE;IAAA,IAAAY,cAAA;IAC7E,IAAMC,WAAU,IAAAD,cAAA,GAAGxB,KAAK,CAACI,MAAM,cAAAoB,cAAA,uBAAZA,cAAA,CAAcnB,MAAM,CAACO,WAAW,CAAC,CAACJ,IAAI;IACzD,IAAIiB,WAAU,IAAI,CAACN,oBAAoB,CAACO,QAAQ,CAACd,WAAW,CAAC,EAAE;MAC7DW,UAAU,CAACE,WAAU,CAAC,GAAGf,GAAG,CAACe,WAAU,CAAC;IAC1C;EACF;EACA,OAAOF,UAAU;AACnB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"utf8-encoder.js","names":["Utf8ArrayBufferEncoder","chunkSize","_classCallCheck2","default","_defineProperty2","TextEncoder","_createClass2","key","value","push","_len","arguments","length","strings","Array","_key","_i","_strings","string","totalLength","isFull","getArrayBufferBatch","textEncoder","encode","getStringBatch","buffer","stringChunk","join","exports"],"sources":["../../../../src/lib/encoders/utf8-encoder.ts"],"sourcesContent":["// loaders.gl, MIT License\n\n/* global TextEncoder */\nexport class Utf8ArrayBufferEncoder {\n private readonly chunkSize: number;\n private strings: string[] = [];\n private totalLength = 0;\n private textEncoder: TextEncoder = new TextEncoder();\n\n constructor(chunkSize: number) {\n this.chunkSize = chunkSize;\n }\n\n push(...strings: string[]): void {\n for (const string of strings) {\n this.strings.push(string);\n this.totalLength += string.length;\n }\n }\n\n isFull(): boolean {\n return this.totalLength >= this.chunkSize;\n }\n\n getArrayBufferBatch(): ArrayBufferLike {\n return this.textEncoder.encode(this.getStringBatch()).buffer;\n }\n\n getStringBatch(): string {\n const stringChunk = this.strings.join('');\n this.strings = [];\n this.totalLength = 0;\n return stringChunk;\n }\n}\n"],"mappings":";;;;;;;;;;IAGaA,sBAAsB;EAMjC,SAAAA,uBAAYC,SAAiB,EAAE;IAAA,IAAAC,gBAAA,CAAAC,OAAA,QAAAH,sBAAA;IAAA,IAAAI,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA,mBAJH,EAAE;IAAA,IAAAC,gBAAA,CAAAD,OAAA,uBACR,CAAC;IAAA,IAAAC,gBAAA,CAAAD,OAAA,uBACY,IAAIE,WAAW,CAAC,CAAC;IAGlD,IAAI,CAACJ,SAAS,GAAGA,SAAS;EAC5B;EAAC,IAAAK,aAAA,CAAAH,OAAA,EAAAH,sBAAA;IAAAO,GAAA;IAAAC,KAAA,EAED,SAAAC,KAAA,EAAiC;MAAA,SAAAC,IAAA,GAAAC,SAAA,CAAAC,MAAA,EAAzBC,OAAO,OAAAC,KAAA,CAAAJ,IAAA,GAAAK,IAAA,MAAAA,IAAA,GAAAL,IAAA,EAAAK,IAAA;QAAPF,OAAO,CAAAE,IAAA,IAAAJ,SAAA,CAAAI,IAAA;MAAA;MACb,SAAAC,EAAA,MAAAC,QAAA,GAAqBJ,OAAO,EAAAG,EAAA,GAAAC,QAAA,CAAAL,MAAA,EAAAI,EAAA,IAAE;QAAzB,IAAME,MAAM,GAAAD,QAAA,CAAAD,EAAA;QACf,IAAI,CAACH,OAAO,CAACJ,IAAI,CAACS,MAAM,CAAC;QACzB,IAAI,CAACC,WAAW,IAAID,MAAM,CAACN,MAAM;MACnC;IACF;EAAC;IAAAL,GAAA;IAAAC,KAAA,EAED,SAAAY,OAAA,EAAkB;MAChB,OAAO,IAAI,CAACD,WAAW,IAAI,IAAI,CAAClB,SAAS;IAC3C;EAAC;IAAAM,GAAA;IAAAC,KAAA,EAED,SAAAa,oBAAA,EAAuC;MACrC,OAAO,IAAI,CAACC,WAAW,CAACC,MAAM,CAAC,IAAI,CAACC,cAAc,CAAC,CAAC,CAAC,CAACC,MAAM;IAC9D;EAAC;IAAAlB,GAAA;IAAAC,KAAA,EAED,SAAAgB,eAAA,EAAyB;MACvB,IAAME,WAAW,GAAG,IAAI,CAACb,OAAO,CAACc,IAAI,CAAC,EAAE,CAAC;MACzC,IAAI,CAACd,OAAO,GAAG,EAAE;MACjB,IAAI,CAACM,WAAW,GAAG,CAAC;MACpB,OAAOO,WAAW;IACpB;EAAC;EAAA,OAAA1B,sBAAA;AAAA;AAAA4B,OAAA,CAAA5B,sBAAA,GAAAA,sBAAA"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"encode-utils.js","names":["getTableLength","getTableNumCols","getTableRowAsArray","detectGeometryColumnIndex","table","_table$schema$fields$","_table$schema","geometryIndex","schema","fields","findIndex","field","name","row","columnIndex","value","Error","getRowPropertyObject","excludeColumnIndices","arguments","length","undefined","properties","_table$schema2","columnName","includes"],"sources":["../../../../src/lib/encoders/encode-utils.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright 2022 Foursquare Labs, Inc.\n\nimport {Table, getTableLength, getTableNumCols, getTableRowAsArray} from '@loaders.gl/schema';\n\ntype Row = {[key: string]: unknown};\n\n/**\n * Attempts to identify which column contains geometry\n * Currently just returns name (key) of first object-valued column\n * @todo look for hints in schema metadata\n * @todo look for WKB\n */\nexport function detectGeometryColumnIndex(table: Table): number {\n // TODO - look for hints in schema metadata\n\n // look for a column named geometry\n const geometryIndex = table.schema?.fields.findIndex((field) => field.name === 'geometry') ?? -1;\n if (geometryIndex > -1) {\n return geometryIndex;\n }\n\n // look at the data\n // TODO - this drags in the indices\n if (getTableLength(table) > 0) {\n const row = getTableRowAsArray(table, 0);\n for (let columnIndex = 0; columnIndex < getTableNumCols(table); columnIndex++) {\n const value = row?.[columnIndex];\n if (value && typeof value === 'object') {\n return columnIndex;\n }\n }\n }\n\n throw new Error('Failed to detect geometry column');\n}\n\n/**\n * Return a row as a property (key/value) object, excluding selected columns\n */\nexport function getRowPropertyObject(\n table: Table,\n row: Row,\n excludeColumnIndices: number[] = []\n): {[columnName: string]: unknown} {\n const properties = {};\n for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {\n const columnName = table.schema?.fields[columnIndex].name;\n if (columnName && !excludeColumnIndices.includes(columnIndex)) {\n properties[columnName] = row[columnName];\n }\n }\n return properties;\n}\n"],"mappings":"AAGA,SAAeA,cAAc,EAAEC,eAAe,EAAEC,kBAAkB,QAAO,oBAAoB;AAU7F,OAAO,SAASC,yBAAyBA,CAACC,KAAY,EAAU;EAAA,IAAAC,qBAAA,EAAAC,aAAA;EAI9D,MAAMC,aAAa,IAAAF,qBAAA,IAAAC,aAAA,GAAGF,KAAK,CAACI,MAAM,cAAAF,aAAA,uBAAZA,aAAA,CAAcG,MAAM,CAACC,SAAS,CAAEC,KAAK,IAAKA,KAAK,CAACC,IAAI,KAAK,UAAU,CAAC,cAAAP,qBAAA,cAAAA,qBAAA,GAAI,CAAC,CAAC;EAChG,IAAIE,aAAa,GAAG,CAAC,CAAC,EAAE;IACtB,OAAOA,aAAa;EACtB;EAIA,IAAIP,cAAc,CAACI,KAAK,CAAC,GAAG,CAAC,EAAE;IAC7B,MAAMS,GAAG,GAAGX,kBAAkB,CAACE,KAAK,EAAE,CAAC,CAAC;IACxC,KAAK,IAAIU,WAAW,GAAG,CAAC,EAAEA,WAAW,GAAGb,eAAe,CAACG,KAAK,CAAC,EAAEU,WAAW,EAAE,EAAE;MAC7E,MAAMC,KAAK,GAAGF,GAAG,aAAHA,GAAG,uBAAHA,GAAG,CAAGC,WAAW,CAAC;MAChC,IAAIC,KAAK,IAAI,OAAOA,KAAK,KAAK,QAAQ,EAAE;QACtC,OAAOD,WAAW;MACpB;IACF;EACF;EAEA,MAAM,IAAIE,KAAK,CAAC,kCAAkC,CAAC;AACrD;AAKA,OAAO,SAASC,oBAAoBA,CAClCb,KAAY,EACZS,GAAQ,EAEyB;EAAA,IADjCK,oBAA8B,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,EAAE;EAEnC,MAAMG,UAAU,GAAG,CAAC,CAAC;EACrB,KAAK,IAAIR,WAAW,GAAG,CAAC,EAAEA,WAAW,GAAGb,eAAe,CAACG,KAAK,CAAC,EAAE,EAAEU,WAAW,EAAE;IAAA,IAAAS,cAAA;IAC7E,MAAMC,UAAU,IAAAD,cAAA,GAAGnB,KAAK,CAACI,MAAM,cAAAe,cAAA,uBAAZA,cAAA,CAAcd,MAAM,CAACK,WAAW,CAAC,CAACF,IAAI;IACzD,IAAIY,UAAU,IAAI,CAACN,oBAAoB,CAACO,QAAQ,CAACX,WAAW,CAAC,EAAE;MAC7DQ,UAAU,CAACE,UAAU,CAAC,GAAGX,GAAG,CAACW,UAAU,CAAC;IAC1C;EACF;EACA,OAAOF,UAAU;AACnB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"utf8-encoder.js","names":["Utf8ArrayBufferEncoder","constructor","chunkSize","_defineProperty","TextEncoder","push","_len","arguments","length","strings","Array","_key","string","totalLength","isFull","getArrayBufferBatch","textEncoder","encode","getStringBatch","buffer","stringChunk","join"],"sources":["../../../../src/lib/encoders/utf8-encoder.ts"],"sourcesContent":["// loaders.gl, MIT License\n\n/* global TextEncoder */\nexport class Utf8ArrayBufferEncoder {\n private readonly chunkSize: number;\n private strings: string[] = [];\n private totalLength = 0;\n private textEncoder: TextEncoder = new TextEncoder();\n\n constructor(chunkSize: number) {\n this.chunkSize = chunkSize;\n }\n\n push(...strings: string[]): void {\n for (const string of strings) {\n this.strings.push(string);\n this.totalLength += string.length;\n }\n }\n\n isFull(): boolean {\n return this.totalLength >= this.chunkSize;\n }\n\n getArrayBufferBatch(): ArrayBufferLike {\n return this.textEncoder.encode(this.getStringBatch()).buffer;\n }\n\n getStringBatch(): string {\n const stringChunk = this.strings.join('');\n this.strings = [];\n this.totalLength = 0;\n return stringChunk;\n }\n}\n"],"mappings":";AAGA,OAAO,MAAMA,sBAAsB,CAAC;EAMlCC,WAAWA,CAACC,SAAiB,EAAE;IAAAC,eAAA;IAAAA,eAAA,kBAJH,EAAE;IAAAA,eAAA,sBACR,CAAC;IAAAA,eAAA,sBACY,IAAIC,WAAW,CAAC,CAAC;IAGlD,IAAI,CAACF,SAAS,GAAGA,SAAS;EAC5B;EAEAG,IAAIA,CAAA,EAA6B;IAAA,SAAAC,IAAA,GAAAC,SAAA,CAAAC,MAAA,EAAzBC,OAAO,OAAAC,KAAA,CAAAJ,IAAA,GAAAK,IAAA,MAAAA,IAAA,GAAAL,IAAA,EAAAK,IAAA;MAAPF,OAAO,CAAAE,IAAA,IAAAJ,SAAA,CAAAI,IAAA;IAAA;IACb,KAAK,MAAMC,MAAM,IAAIH,OAAO,EAAE;MAC5B,IAAI,CAACA,OAAO,CAACJ,IAAI,CAACO,MAAM,CAAC;MACzB,IAAI,CAACC,WAAW,IAAID,MAAM,CAACJ,MAAM;IACnC;EACF;EAEAM,MAAMA,CAAA,EAAY;IAChB,OAAO,IAAI,CAACD,WAAW,IAAI,IAAI,CAACX,SAAS;EAC3C;EAEAa,mBAAmBA,CAAA,EAAoB;IACrC,OAAO,IAAI,CAACC,WAAW,CAACC,MAAM,CAAC,IAAI,CAACC,cAAc,CAAC,CAAC,CAAC,CAACC,MAAM;EAC9D;EAEAD,cAAcA,CAAA,EAAW;IACvB,MAAME,WAAW,GAAG,IAAI,CAACX,OAAO,CAACY,IAAI,CAAC,EAAE,CAAC;IACzC,IAAI,CAACZ,OAAO,GAAG,EAAE;IACjB,IAAI,CAACI,WAAW,GAAG,CAAC;IACpB,OAAOO,WAAW;EACpB;AACF"}
|
package/dist/geojson-loader.js
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT license
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.GeoJSONLoader = exports.GeoJSONWorkerLoader = void 0;
|
|
5
|
-
const gis_1 = require("@loaders.gl/gis");
|
|
6
|
-
const parse_json_1 = require("./lib/parsers/parse-json");
|
|
7
|
-
const parse_json_in_batches_1 = require("./lib/parsers/parse-json-in-batches");
|
|
8
|
-
// __VERSION__ is injected by babel-plugin-version-inline
|
|
9
|
-
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
10
|
-
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
11
|
-
/**
|
|
12
|
-
* GeoJSON loader
|
|
13
|
-
*/
|
|
14
|
-
exports.GeoJSONWorkerLoader = {
|
|
15
|
-
name: 'GeoJSON',
|
|
16
|
-
id: 'geojson',
|
|
17
|
-
module: 'geojson',
|
|
18
|
-
version: VERSION,
|
|
19
|
-
worker: true,
|
|
20
|
-
extensions: ['geojson'],
|
|
21
|
-
mimeTypes: ['application/geo+json'],
|
|
22
|
-
category: 'geometry',
|
|
23
|
-
text: true,
|
|
24
|
-
options: {
|
|
25
|
-
geojson: {
|
|
26
|
-
shape: 'object-row-table'
|
|
27
|
-
},
|
|
28
|
-
json: {
|
|
29
|
-
shape: 'object-row-table',
|
|
30
|
-
jsonpaths: ['$', '$.features']
|
|
31
|
-
},
|
|
32
|
-
gis: {
|
|
33
|
-
format: 'geojson'
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
};
|
|
37
|
-
exports.GeoJSONLoader = {
|
|
38
|
-
...exports.GeoJSONWorkerLoader,
|
|
39
|
-
// @ts-expect-error
|
|
40
|
-
parse,
|
|
41
|
-
// @ts-expect-error
|
|
42
|
-
parseTextSync,
|
|
43
|
-
parseInBatches
|
|
44
|
-
};
|
|
45
|
-
async function parse(arrayBuffer, options) {
|
|
46
|
-
return parseTextSync(new TextDecoder().decode(arrayBuffer), options);
|
|
47
|
-
}
|
|
48
|
-
function parseTextSync(text, options) {
|
|
49
|
-
// Apps can call the parse method directly, we so apply default options here
|
|
50
|
-
options = { ...exports.GeoJSONLoader.options, ...options };
|
|
51
|
-
options.geojson = { ...exports.GeoJSONLoader.options.geojson, ...options.geojson };
|
|
52
|
-
options.gis = options.gis || {};
|
|
53
|
-
const table = (0, parse_json_1.parseJSONSync)(text, options);
|
|
54
|
-
table.shape = 'geojson-row-table';
|
|
55
|
-
switch (options.gis.format) {
|
|
56
|
-
case 'binary':
|
|
57
|
-
return (0, gis_1.geojsonToBinary)(table.data);
|
|
58
|
-
default:
|
|
59
|
-
return table;
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
function parseInBatches(asyncIterator, options) {
|
|
63
|
-
// Apps can call the parse method directly, we so apply default options here
|
|
64
|
-
options = { ...exports.GeoJSONLoader.options, ...options };
|
|
65
|
-
options.json = { ...exports.GeoJSONLoader.options.geojson, ...options.geojson };
|
|
66
|
-
const geojsonIterator = (0, parse_json_in_batches_1.parseJSONInBatches)(asyncIterator, options);
|
|
67
|
-
switch (options.gis.format) {
|
|
68
|
-
case 'binary':
|
|
69
|
-
return makeBinaryGeometryIterator(geojsonIterator);
|
|
70
|
-
default:
|
|
71
|
-
return geojsonIterator;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
async function* makeBinaryGeometryIterator(geojsonIterator) {
|
|
75
|
-
for await (const batch of geojsonIterator) {
|
|
76
|
-
batch.data = (0, gis_1.geojsonToBinary)(batch.data);
|
|
77
|
-
yield batch;
|
|
78
|
-
}
|
|
79
|
-
}
|
package/dist/geojson-writer.js
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT license
|
|
3
|
-
// Copyright Foursquare, Inc 20222
|
|
4
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.GeoJSONWriter = void 0;
|
|
6
|
-
const geojson_encoder_1 = require("./lib/encoders/geojson-encoder");
|
|
7
|
-
exports.GeoJSONWriter = {
|
|
8
|
-
id: 'geojson',
|
|
9
|
-
version: 'latest',
|
|
10
|
-
module: 'geojson',
|
|
11
|
-
name: 'GeoJSON',
|
|
12
|
-
extensions: ['geojson'],
|
|
13
|
-
mimeTypes: ['application/geo+json'],
|
|
14
|
-
options: {
|
|
15
|
-
geojson: {
|
|
16
|
-
featureArray: false,
|
|
17
|
-
geometryColumn: null
|
|
18
|
-
}
|
|
19
|
-
},
|
|
20
|
-
text: true,
|
|
21
|
-
encodeInBatches: (tableIterator, options) => (0, geojson_encoder_1.encodeTableAsGeojsonInBatches)(tableIterator, options)
|
|
22
|
-
};
|
package/dist/index.js
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT license
|
|
3
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
4
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
5
|
-
};
|
|
6
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports._rebuildJsonObject = exports._ClarinetParser = exports._JSONPath = exports._GeoJSONWriter = exports._GeoJSONWorkerLoader = exports._GeoJSONLoader = exports.JSONWriter = exports.NDJSONLoader = exports.JSONLoader = void 0;
|
|
8
|
-
var json_loader_1 = require("./json-loader");
|
|
9
|
-
Object.defineProperty(exports, "JSONLoader", { enumerable: true, get: function () { return json_loader_1.JSONLoader; } });
|
|
10
|
-
var ndjson_loader_1 = require("./ndjson-loader");
|
|
11
|
-
Object.defineProperty(exports, "NDJSONLoader", { enumerable: true, get: function () { return ndjson_loader_1.NDJSONLoader; } });
|
|
12
|
-
var json_writer_1 = require("./json-writer");
|
|
13
|
-
Object.defineProperty(exports, "JSONWriter", { enumerable: true, get: function () { return json_writer_1.JSONWriter; } });
|
|
14
|
-
var geojson_loader_1 = require("./geojson-loader");
|
|
15
|
-
Object.defineProperty(exports, "_GeoJSONLoader", { enumerable: true, get: function () { return geojson_loader_1.GeoJSONLoader; } });
|
|
16
|
-
Object.defineProperty(exports, "_GeoJSONWorkerLoader", { enumerable: true, get: function () { return geojson_loader_1.GeoJSONWorkerLoader; } });
|
|
17
|
-
var geojson_writer_1 = require("./geojson-writer");
|
|
18
|
-
Object.defineProperty(exports, "_GeoJSONWriter", { enumerable: true, get: function () { return geojson_writer_1.GeoJSONWriter; } });
|
|
19
|
-
var jsonpath_1 = require("./lib/jsonpath/jsonpath");
|
|
20
|
-
Object.defineProperty(exports, "_JSONPath", { enumerable: true, get: function () { return __importDefault(jsonpath_1).default; } });
|
|
21
|
-
var clarinet_1 = require("./lib/clarinet/clarinet");
|
|
22
|
-
Object.defineProperty(exports, "_ClarinetParser", { enumerable: true, get: function () { return __importDefault(clarinet_1).default; } });
|
|
23
|
-
var parse_json_in_batches_1 = require("./lib/parsers/parse-json-in-batches");
|
|
24
|
-
Object.defineProperty(exports, "_rebuildJsonObject", { enumerable: true, get: function () { return parse_json_in_batches_1.rebuildJsonObject; } });
|
package/dist/json-loader.js
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT license
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.JSONLoader = void 0;
|
|
5
|
-
const parse_json_1 = require("./lib/parsers/parse-json");
|
|
6
|
-
const parse_json_in_batches_1 = require("./lib/parsers/parse-json-in-batches");
|
|
7
|
-
// __VERSION__ is injected by babel-plugin-version-inline
|
|
8
|
-
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
9
|
-
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
10
|
-
const DEFAULT_JSON_LOADER_OPTIONS = {
|
|
11
|
-
json: {
|
|
12
|
-
shape: 'object-row-table',
|
|
13
|
-
table: false,
|
|
14
|
-
jsonpaths: []
|
|
15
|
-
// batchSize: 'auto'
|
|
16
|
-
}
|
|
17
|
-
};
|
|
18
|
-
exports.JSONLoader = {
|
|
19
|
-
name: 'JSON',
|
|
20
|
-
id: 'json',
|
|
21
|
-
module: 'json',
|
|
22
|
-
version: VERSION,
|
|
23
|
-
extensions: ['json', 'geojson'],
|
|
24
|
-
mimeTypes: ['application/json'],
|
|
25
|
-
category: 'table',
|
|
26
|
-
text: true,
|
|
27
|
-
parse,
|
|
28
|
-
parseTextSync,
|
|
29
|
-
parseInBatches,
|
|
30
|
-
options: DEFAULT_JSON_LOADER_OPTIONS
|
|
31
|
-
};
|
|
32
|
-
async function parse(arrayBuffer, options) {
|
|
33
|
-
return parseTextSync(new TextDecoder().decode(arrayBuffer), options);
|
|
34
|
-
}
|
|
35
|
-
function parseTextSync(text, options) {
|
|
36
|
-
const jsonOptions = { ...options, json: { ...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json } };
|
|
37
|
-
return (0, parse_json_1.parseJSONSync)(text, jsonOptions);
|
|
38
|
-
}
|
|
39
|
-
function parseInBatches(asyncIterator, options) {
|
|
40
|
-
const jsonOptions = { ...options, json: { ...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json } };
|
|
41
|
-
return (0, parse_json_in_batches_1.parseJSONInBatches)(asyncIterator, jsonOptions);
|
|
42
|
-
}
|
package/dist/json-writer.js
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT license
|
|
3
|
-
// Copyright 2022 Foursquare Labs, Inc.
|
|
4
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.JSONWriter = void 0;
|
|
6
|
-
const json_encoder_1 = require("./lib/encoders/json-encoder");
|
|
7
|
-
exports.JSONWriter = {
|
|
8
|
-
id: 'json',
|
|
9
|
-
version: 'latest',
|
|
10
|
-
module: 'json',
|
|
11
|
-
name: 'JSON',
|
|
12
|
-
extensions: ['json'],
|
|
13
|
-
mimeTypes: ['application/json'],
|
|
14
|
-
options: {},
|
|
15
|
-
text: true,
|
|
16
|
-
encode: async (table, options) => new TextEncoder().encode((0, json_encoder_1.encodeTableAsJSON)(table, options)).buffer,
|
|
17
|
-
encodeText: (table, options) => (0, json_encoder_1.encodeTableAsJSON)(table, options)
|
|
18
|
-
};
|