@loaders.gl/shapefile 3.1.3 → 4.0.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.js +2 -2
- package/dist/bundle.js.map +1 -0
- package/dist/dbf-loader.js +20 -29
- package/dist/dbf-loader.js.map +1 -0
- package/dist/dbf-worker.js +1 -1
- package/dist/index.js +4 -11
- package/dist/index.js.map +1 -0
- package/dist/lib/parsers/parse-dbf.js +264 -300
- package/dist/lib/parsers/parse-dbf.js.map +1 -0
- package/dist/lib/parsers/parse-shapefile.js +209 -231
- package/dist/lib/parsers/parse-shapefile.js.map +1 -0
- package/dist/lib/parsers/parse-shp-geometry.js +212 -265
- package/dist/lib/parsers/parse-shp-geometry.js.map +1 -0
- package/dist/lib/parsers/parse-shp-header.js +27 -38
- package/dist/lib/parsers/parse-shp-header.js.map +1 -0
- package/dist/lib/parsers/parse-shp.js +136 -152
- package/dist/lib/parsers/parse-shp.js.map +1 -0
- package/dist/lib/parsers/parse-shx.js +19 -25
- package/dist/lib/parsers/parse-shx.js.map +1 -0
- package/dist/lib/streaming/binary-chunk-reader.js +128 -152
- package/dist/lib/streaming/binary-chunk-reader.js.map +1 -0
- package/dist/lib/streaming/binary-reader.js +33 -50
- package/dist/lib/streaming/binary-reader.js.map +1 -0
- package/dist/lib/streaming/zip-batch-iterators.js +48 -57
- package/dist/lib/streaming/zip-batch-iterators.js.map +1 -0
- package/dist/shapefile-loader.js +22 -30
- package/dist/shapefile-loader.js.map +1 -0
- package/dist/shp-loader.js +22 -32
- package/dist/shp-loader.js.map +1 -0
- package/dist/shp-worker.js +1 -1
- package/dist/workers/dbf-worker.js +4 -5
- package/dist/workers/dbf-worker.js.map +1 -0
- package/dist/workers/shp-worker.js +4 -5
- package/dist/workers/shp-worker.js.map +1 -0
- package/package.json +9 -9
- package/dist/es5/bundle.js +0 -7
- package/dist/es5/bundle.js.map +0 -1
- package/dist/es5/dbf-loader.js +0 -68
- package/dist/es5/dbf-loader.js.map +0 -1
- package/dist/es5/index.js +0 -42
- package/dist/es5/index.js.map +0 -1
- package/dist/es5/lib/parsers/parse-dbf.js +0 -454
- package/dist/es5/lib/parsers/parse-dbf.js.map +0 -1
- package/dist/es5/lib/parsers/parse-shapefile.js +0 -497
- package/dist/es5/lib/parsers/parse-shapefile.js.map +0 -1
- package/dist/es5/lib/parsers/parse-shp-geometry.js +0 -288
- package/dist/es5/lib/parsers/parse-shp-geometry.js.map +0 -1
- package/dist/es5/lib/parsers/parse-shp-header.js +0 -39
- package/dist/es5/lib/parsers/parse-shp-header.js.map +0 -1
- package/dist/es5/lib/parsers/parse-shp.js +0 -283
- package/dist/es5/lib/parsers/parse-shp.js.map +0 -1
- package/dist/es5/lib/parsers/parse-shx.js +0 -31
- package/dist/es5/lib/parsers/parse-shx.js.map +0 -1
- package/dist/es5/lib/streaming/binary-chunk-reader.js +0 -218
- package/dist/es5/lib/streaming/binary-chunk-reader.js.map +0 -1
- package/dist/es5/lib/streaming/binary-reader.js +0 -56
- package/dist/es5/lib/streaming/binary-reader.js.map +0 -1
- package/dist/es5/lib/streaming/zip-batch-iterators.js +0 -118
- package/dist/es5/lib/streaming/zip-batch-iterators.js.map +0 -1
- package/dist/es5/shapefile-loader.js +0 -34
- package/dist/es5/shapefile-loader.js.map +0 -1
- package/dist/es5/shp-loader.js +0 -71
- package/dist/es5/shp-loader.js.map +0 -1
- package/dist/es5/workers/dbf-worker.js +0 -8
- package/dist/es5/workers/dbf-worker.js.map +0 -1
- package/dist/es5/workers/shp-worker.js +0 -8
- package/dist/es5/workers/shp-worker.js.map +0 -1
- package/dist/esm/bundle.js +0 -5
- package/dist/esm/bundle.js.map +0 -1
- package/dist/esm/dbf-loader.js +0 -23
- package/dist/esm/dbf-loader.js.map +0 -1
- package/dist/esm/index.js +0 -4
- package/dist/esm/index.js.map +0 -1
- package/dist/esm/lib/parsers/parse-dbf.js +0 -299
- package/dist/esm/lib/parsers/parse-dbf.js.map +0 -1
- package/dist/esm/lib/parsers/parse-shapefile.js +0 -226
- package/dist/esm/lib/parsers/parse-shapefile.js.map +0 -1
- package/dist/esm/lib/parsers/parse-shp-geometry.js +0 -234
- package/dist/esm/lib/parsers/parse-shp-geometry.js.map +0 -1
- package/dist/esm/lib/parsers/parse-shp-header.js +0 -32
- package/dist/esm/lib/parsers/parse-shp-header.js.map +0 -1
- package/dist/esm/lib/parsers/parse-shp.js +0 -154
- package/dist/esm/lib/parsers/parse-shp.js.map +0 -1
- package/dist/esm/lib/parsers/parse-shx.js +0 -22
- package/dist/esm/lib/parsers/parse-shx.js.map +0 -1
- package/dist/esm/lib/streaming/binary-chunk-reader.js +0 -137
- package/dist/esm/lib/streaming/binary-chunk-reader.js.map +0 -1
- package/dist/esm/lib/streaming/binary-reader.js +0 -35
- package/dist/esm/lib/streaming/binary-reader.js.map +0 -1
- package/dist/esm/lib/streaming/zip-batch-iterators.js +0 -52
- package/dist/esm/lib/streaming/zip-batch-iterators.js.map +0 -1
- package/dist/esm/shapefile-loader.js +0 -23
- package/dist/esm/shapefile-loader.js.map +0 -1
- package/dist/esm/shp-loader.js +0 -25
- package/dist/esm/shp-loader.js.map +0 -1
- package/dist/esm/workers/dbf-worker.js +0 -4
- package/dist/esm/workers/dbf-worker.js.map +0 -1
- package/dist/esm/workers/shp-worker.js +0 -4
- package/dist/esm/workers/shp-worker.js.map +0 -1
package/dist/esm/bundle.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}
|
package/dist/esm/dbf-loader.js
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { parseDBF, parseDBFInBatches } from './lib/parsers/parse-dbf';
|
|
2
|
-
const VERSION = typeof "3.1.3" !== 'undefined' ? "3.1.3" : 'latest';
|
|
3
|
-
export const DBFWorkerLoader = {
|
|
4
|
-
name: 'DBF',
|
|
5
|
-
id: 'dbf',
|
|
6
|
-
module: 'shapefile',
|
|
7
|
-
version: VERSION,
|
|
8
|
-
worker: true,
|
|
9
|
-
category: 'table',
|
|
10
|
-
extensions: ['dbf'],
|
|
11
|
-
mimeTypes: ['application/x-dbf'],
|
|
12
|
-
options: {
|
|
13
|
-
dbf: {
|
|
14
|
-
encoding: 'latin1'
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
};
|
|
18
|
-
export const DBFLoader = { ...DBFWorkerLoader,
|
|
19
|
-
parse: async (arrayBuffer, options) => parseDBF(arrayBuffer, options),
|
|
20
|
-
parseSync: parseDBF,
|
|
21
|
-
parseInBatches: parseDBFInBatches
|
|
22
|
-
};
|
|
23
|
-
//# sourceMappingURL=dbf-loader.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/dbf-loader.ts"],"names":["parseDBF","parseDBFInBatches","VERSION","DBFWorkerLoader","name","id","module","version","worker","category","extensions","mimeTypes","options","dbf","encoding","DBFLoader","parse","arrayBuffer","parseSync","parseInBatches"],"mappings":"AACA,SAAQA,QAAR,EAAkBC,iBAAlB,QAA0C,yBAA1C;AAIA,MAAMC,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAKA,OAAO,MAAMC,eAAuB,GAAG;AACrCC,EAAAA,IAAI,EAAE,KAD+B;AAErCC,EAAAA,EAAE,EAAE,KAFiC;AAGrCC,EAAAA,MAAM,EAAE,WAH6B;AAIrCC,EAAAA,OAAO,EAAEL,OAJ4B;AAKrCM,EAAAA,MAAM,EAAE,IAL6B;AAMrCC,EAAAA,QAAQ,EAAE,OAN2B;AAOrCC,EAAAA,UAAU,EAAE,CAAC,KAAD,CAPyB;AAQrCC,EAAAA,SAAS,EAAE,CAAC,mBAAD,CAR0B;AASrCC,EAAAA,OAAO,EAAE;AACPC,IAAAA,GAAG,EAAE;AACHC,MAAAA,QAAQ,EAAE;AADP;AADE;AAT4B,CAAhC;AAiBP,OAAO,MAAMC,SAA2B,GAAG,EACzC,GAAGZ,eADsC;AAEzCa,EAAAA,KAAK,EAAE,OAAOC,WAAP,EAAoBL,OAApB,KAAgCZ,QAAQ,CAACiB,WAAD,EAAcL,OAAd,CAFN;AAGzCM,EAAAA,SAAS,EAAElB,QAH8B;AAIzCmB,EAAAA,cAAc,EAAElB;AAJyB,CAApC","sourcesContent":["import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils';\nimport {parseDBF, parseDBFInBatches} from './lib/parsers/parse-dbf';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\n/**\n * DBFLoader - DBF files are used to contain non-geometry columns in Shapefiles\n */\nexport const DBFWorkerLoader: Loader = {\n name: 'DBF',\n id: 'dbf',\n module: 'shapefile',\n version: VERSION,\n worker: true,\n category: 'table',\n extensions: ['dbf'],\n mimeTypes: ['application/x-dbf'],\n options: {\n dbf: {\n encoding: 'latin1'\n }\n }\n};\n\n/** DBF file loader */\nexport const DBFLoader: LoaderWithParser = {\n ...DBFWorkerLoader,\n parse: async (arrayBuffer, options) => parseDBF(arrayBuffer, options),\n parseSync: parseDBF,\n parseInBatches: parseDBFInBatches\n};\n"],"file":"dbf-loader.js"}
|
package/dist/esm/index.js
DELETED
package/dist/esm/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/index.ts"],"names":["ShapefileLoader","DBFLoader","DBFWorkerLoader","SHPLoader","SHPWorkerLoader"],"mappings":"AAAA,SAAQA,eAAR,QAA8B,oBAA9B;AACA,SAAQC,SAAR,EAAmBC,eAAnB,QAAyC,cAAzC;AACA,SAAQC,SAAR,EAAmBC,eAAnB,QAAyC,cAAzC","sourcesContent":["export {ShapefileLoader} from './shapefile-loader';\nexport {DBFLoader, DBFWorkerLoader} from './dbf-loader';\nexport {SHPLoader, SHPWorkerLoader} from './shp-loader';\n"],"file":"index.js"}
|
|
@@ -1,299 +0,0 @@
|
|
|
1
|
-
import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
|
|
2
|
-
import { Schema, Field, Bool, Utf8, Float64, TimestampMillisecond } from '@loaders.gl/schema';
|
|
3
|
-
import BinaryChunkReader from '../streaming/binary-chunk-reader';
|
|
4
|
-
const LITTLE_ENDIAN = true;
|
|
5
|
-
const DBF_HEADER_SIZE = 32;
|
|
6
|
-
var STATE;
|
|
7
|
-
|
|
8
|
-
(function (STATE) {
|
|
9
|
-
STATE[STATE["START"] = 0] = "START";
|
|
10
|
-
STATE[STATE["FIELD_DESCRIPTORS"] = 1] = "FIELD_DESCRIPTORS";
|
|
11
|
-
STATE[STATE["FIELD_PROPERTIES"] = 2] = "FIELD_PROPERTIES";
|
|
12
|
-
STATE[STATE["END"] = 3] = "END";
|
|
13
|
-
STATE[STATE["ERROR"] = 4] = "ERROR";
|
|
14
|
-
})(STATE || (STATE = {}));
|
|
15
|
-
|
|
16
|
-
class DBFParser {
|
|
17
|
-
constructor(options) {
|
|
18
|
-
_defineProperty(this, "binaryReader", new BinaryChunkReader());
|
|
19
|
-
|
|
20
|
-
_defineProperty(this, "textDecoder", void 0);
|
|
21
|
-
|
|
22
|
-
_defineProperty(this, "state", STATE.START);
|
|
23
|
-
|
|
24
|
-
_defineProperty(this, "result", {
|
|
25
|
-
data: []
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
this.textDecoder = new TextDecoder(options.encoding);
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
write(arrayBuffer) {
|
|
32
|
-
this.binaryReader.write(arrayBuffer);
|
|
33
|
-
this.state = parseState(this.state, this.result, this.binaryReader, this.textDecoder);
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
end() {
|
|
37
|
-
this.binaryReader.end();
|
|
38
|
-
this.state = parseState(this.state, this.result, this.binaryReader, this.textDecoder);
|
|
39
|
-
|
|
40
|
-
if (this.state !== STATE.END) {
|
|
41
|
-
this.state = STATE.ERROR;
|
|
42
|
-
this.result.error = 'DBF incomplete file';
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export function parseDBF(arrayBuffer, options = {}) {
|
|
49
|
-
const loaderOptions = options.dbf || {};
|
|
50
|
-
const {
|
|
51
|
-
encoding
|
|
52
|
-
} = loaderOptions;
|
|
53
|
-
const dbfParser = new DBFParser({
|
|
54
|
-
encoding
|
|
55
|
-
});
|
|
56
|
-
dbfParser.write(arrayBuffer);
|
|
57
|
-
dbfParser.end();
|
|
58
|
-
const {
|
|
59
|
-
data,
|
|
60
|
-
schema
|
|
61
|
-
} = dbfParser.result;
|
|
62
|
-
|
|
63
|
-
switch (options.tables && options.tables.format) {
|
|
64
|
-
case 'table':
|
|
65
|
-
return {
|
|
66
|
-
schema,
|
|
67
|
-
rows: data
|
|
68
|
-
};
|
|
69
|
-
|
|
70
|
-
case 'rows':
|
|
71
|
-
default:
|
|
72
|
-
return data;
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
export async function* parseDBFInBatches(asyncIterator, options = {}) {
|
|
76
|
-
const loaderOptions = options.dbf || {};
|
|
77
|
-
const {
|
|
78
|
-
encoding
|
|
79
|
-
} = loaderOptions;
|
|
80
|
-
const parser = new DBFParser({
|
|
81
|
-
encoding
|
|
82
|
-
});
|
|
83
|
-
let headerReturned = false;
|
|
84
|
-
|
|
85
|
-
for await (const arrayBuffer of asyncIterator) {
|
|
86
|
-
parser.write(arrayBuffer);
|
|
87
|
-
|
|
88
|
-
if (!headerReturned && parser.result.dbfHeader) {
|
|
89
|
-
headerReturned = true;
|
|
90
|
-
yield parser.result.dbfHeader;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
if (parser.result.data.length > 0) {
|
|
94
|
-
yield parser.result.data;
|
|
95
|
-
parser.result.data = [];
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
parser.end();
|
|
100
|
-
|
|
101
|
-
if (parser.result.data.length > 0) {
|
|
102
|
-
yield parser.result.data;
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
function parseState(state, result, binaryReader, textDecoder) {
|
|
107
|
-
while (true) {
|
|
108
|
-
try {
|
|
109
|
-
switch (state) {
|
|
110
|
-
case STATE.ERROR:
|
|
111
|
-
case STATE.END:
|
|
112
|
-
return state;
|
|
113
|
-
|
|
114
|
-
case STATE.START:
|
|
115
|
-
const dataView = binaryReader.getDataView(DBF_HEADER_SIZE, 'DBF header');
|
|
116
|
-
|
|
117
|
-
if (!dataView) {
|
|
118
|
-
return state;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
result.dbfHeader = parseDBFHeader(dataView);
|
|
122
|
-
result.progress = {
|
|
123
|
-
bytesUsed: 0,
|
|
124
|
-
rowsTotal: result.dbfHeader.nRecords,
|
|
125
|
-
rows: 0
|
|
126
|
-
};
|
|
127
|
-
state = STATE.FIELD_DESCRIPTORS;
|
|
128
|
-
break;
|
|
129
|
-
|
|
130
|
-
case STATE.FIELD_DESCRIPTORS:
|
|
131
|
-
const fieldDescriptorView = binaryReader.getDataView(result.dbfHeader.headerLength - DBF_HEADER_SIZE, 'DBF field descriptors');
|
|
132
|
-
|
|
133
|
-
if (!fieldDescriptorView) {
|
|
134
|
-
return state;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
result.dbfFields = parseFieldDescriptors(fieldDescriptorView, textDecoder);
|
|
138
|
-
result.schema = new Schema(result.dbfFields.map(dbfField => makeField(dbfField)));
|
|
139
|
-
state = STATE.FIELD_PROPERTIES;
|
|
140
|
-
binaryReader.skip(1);
|
|
141
|
-
break;
|
|
142
|
-
|
|
143
|
-
case STATE.FIELD_PROPERTIES:
|
|
144
|
-
const {
|
|
145
|
-
recordLength = 0,
|
|
146
|
-
nRecords = 0
|
|
147
|
-
} = (result === null || result === void 0 ? void 0 : result.dbfHeader) || {};
|
|
148
|
-
|
|
149
|
-
while (result.data.length < nRecords) {
|
|
150
|
-
const recordView = binaryReader.getDataView(recordLength - 1);
|
|
151
|
-
|
|
152
|
-
if (!recordView) {
|
|
153
|
-
return state;
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
binaryReader.skip(1);
|
|
157
|
-
const row = parseRow(recordView, result.dbfFields, textDecoder);
|
|
158
|
-
result.data.push(row);
|
|
159
|
-
result.progress.rows = result.data.length;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
state = STATE.END;
|
|
163
|
-
break;
|
|
164
|
-
|
|
165
|
-
default:
|
|
166
|
-
state = STATE.ERROR;
|
|
167
|
-
result.error = "illegal parser state ".concat(state);
|
|
168
|
-
return state;
|
|
169
|
-
}
|
|
170
|
-
} catch (error) {
|
|
171
|
-
state = STATE.ERROR;
|
|
172
|
-
result.error = "DBF parsing failed: ".concat(error.message);
|
|
173
|
-
return state;
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
function parseDBFHeader(headerView) {
|
|
179
|
-
return {
|
|
180
|
-
year: headerView.getUint8(1) + 1900,
|
|
181
|
-
month: headerView.getUint8(2),
|
|
182
|
-
day: headerView.getUint8(3),
|
|
183
|
-
nRecords: headerView.getUint32(4, LITTLE_ENDIAN),
|
|
184
|
-
headerLength: headerView.getUint16(8, LITTLE_ENDIAN),
|
|
185
|
-
recordLength: headerView.getUint16(10, LITTLE_ENDIAN),
|
|
186
|
-
languageDriver: headerView.getUint8(29)
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
function parseFieldDescriptors(view, textDecoder) {
|
|
191
|
-
const nFields = (view.byteLength - 1) / 32;
|
|
192
|
-
const fields = [];
|
|
193
|
-
let offset = 0;
|
|
194
|
-
|
|
195
|
-
for (let i = 0; i < nFields; i++) {
|
|
196
|
-
const name = textDecoder.decode(new Uint8Array(view.buffer, view.byteOffset + offset, 11)).replace(/\u0000/g, '');
|
|
197
|
-
fields.push({
|
|
198
|
-
name,
|
|
199
|
-
dataType: String.fromCharCode(view.getUint8(offset + 11)),
|
|
200
|
-
fieldLength: view.getUint8(offset + 16),
|
|
201
|
-
decimal: view.getUint8(offset + 17)
|
|
202
|
-
});
|
|
203
|
-
offset += 32;
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
return fields;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
function parseRow(view, fields, textDecoder) {
|
|
210
|
-
const out = {};
|
|
211
|
-
let offset = 0;
|
|
212
|
-
|
|
213
|
-
for (const field of fields) {
|
|
214
|
-
const text = textDecoder.decode(new Uint8Array(view.buffer, view.byteOffset + offset, field.fieldLength));
|
|
215
|
-
out[field.name] = parseField(text, field.dataType);
|
|
216
|
-
offset += field.fieldLength;
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
return out;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
function parseField(text, dataType) {
|
|
223
|
-
switch (dataType) {
|
|
224
|
-
case 'B':
|
|
225
|
-
return parseNumber(text);
|
|
226
|
-
|
|
227
|
-
case 'C':
|
|
228
|
-
return parseCharacter(text);
|
|
229
|
-
|
|
230
|
-
case 'F':
|
|
231
|
-
return parseNumber(text);
|
|
232
|
-
|
|
233
|
-
case 'N':
|
|
234
|
-
return parseNumber(text);
|
|
235
|
-
|
|
236
|
-
case 'O':
|
|
237
|
-
return parseNumber(text);
|
|
238
|
-
|
|
239
|
-
case 'D':
|
|
240
|
-
return parseDate(text);
|
|
241
|
-
|
|
242
|
-
case 'L':
|
|
243
|
-
return parseBoolean(text);
|
|
244
|
-
|
|
245
|
-
default:
|
|
246
|
-
throw new Error('Unsupported data type');
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
function parseDate(str) {
|
|
251
|
-
return Date.UTC(str.slice(0, 4), parseInt(str.slice(4, 6), 10) - 1, str.slice(6, 8));
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
function parseBoolean(value) {
|
|
255
|
-
return /^[nf]$/i.test(value) ? false : /^[yt]$/i.test(value) ? true : null;
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
function parseNumber(text) {
|
|
259
|
-
const number = parseFloat(text);
|
|
260
|
-
return isNaN(number) ? null : number;
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
function parseCharacter(text) {
|
|
264
|
-
return text.trim() || null;
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
function makeField({
|
|
268
|
-
name,
|
|
269
|
-
dataType,
|
|
270
|
-
fieldLength,
|
|
271
|
-
decimal
|
|
272
|
-
}) {
|
|
273
|
-
switch (dataType) {
|
|
274
|
-
case 'B':
|
|
275
|
-
return new Field(name, new Float64(), true);
|
|
276
|
-
|
|
277
|
-
case 'C':
|
|
278
|
-
return new Field(name, new Utf8(), true);
|
|
279
|
-
|
|
280
|
-
case 'F':
|
|
281
|
-
return new Field(name, new Float64(), true);
|
|
282
|
-
|
|
283
|
-
case 'N':
|
|
284
|
-
return new Field(name, new Float64(), true);
|
|
285
|
-
|
|
286
|
-
case 'O':
|
|
287
|
-
return new Field(name, new Float64(), true);
|
|
288
|
-
|
|
289
|
-
case 'D':
|
|
290
|
-
return new Field(name, new TimestampMillisecond(), true);
|
|
291
|
-
|
|
292
|
-
case 'L':
|
|
293
|
-
return new Field(name, new Bool(), true);
|
|
294
|
-
|
|
295
|
-
default:
|
|
296
|
-
throw new Error('Unsupported data type');
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
//# sourceMappingURL=parse-dbf.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/lib/parsers/parse-dbf.ts"],"names":["Schema","Field","Bool","Utf8","Float64","TimestampMillisecond","BinaryChunkReader","LITTLE_ENDIAN","DBF_HEADER_SIZE","STATE","DBFParser","constructor","options","START","data","textDecoder","TextDecoder","encoding","write","arrayBuffer","binaryReader","state","parseState","result","end","END","ERROR","error","parseDBF","loaderOptions","dbf","dbfParser","schema","tables","format","rows","parseDBFInBatches","asyncIterator","parser","headerReturned","dbfHeader","length","dataView","getDataView","parseDBFHeader","progress","bytesUsed","rowsTotal","nRecords","FIELD_DESCRIPTORS","fieldDescriptorView","headerLength","dbfFields","parseFieldDescriptors","map","dbfField","makeField","FIELD_PROPERTIES","skip","recordLength","recordView","row","parseRow","push","message","headerView","year","getUint8","month","day","getUint32","getUint16","languageDriver","view","nFields","byteLength","fields","offset","i","name","decode","Uint8Array","buffer","byteOffset","replace","dataType","String","fromCharCode","fieldLength","decimal","out","field","text","parseField","parseNumber","parseCharacter","parseDate","parseBoolean","Error","str","Date","UTC","slice","parseInt","value","test","number","parseFloat","isNaN","trim"],"mappings":";AAAA,SAAQA,MAAR,EAAgBC,KAAhB,EAAuBC,IAAvB,EAA6BC,IAA7B,EAAmCC,OAAnC,EAA4CC,oBAA5C,QAAuE,oBAAvE;AACA,OAAOC,iBAAP,MAA8B,kCAA9B;AA4CA,MAAMC,aAAa,GAAG,IAAtB;AACA,MAAMC,eAAe,GAAG,EAAxB;IAEKC,K;;WAAAA,K;AAAAA,EAAAA,K,CAAAA,K;AAAAA,EAAAA,K,CAAAA,K;AAAAA,EAAAA,K,CAAAA,K;AAAAA,EAAAA,K,CAAAA,K;AAAAA,EAAAA,K,CAAAA,K;GAAAA,K,KAAAA,K;;AAQL,MAAMC,SAAN,CAAgB;AAQdC,EAAAA,WAAW,CAACC,OAAD,EAA8B;AAAA,0CAP1B,IAAIN,iBAAJ,EAO0B;;AAAA;;AAAA,mCALjCG,KAAK,CAACI,KAK2B;;AAAA,oCAJrB;AAClBC,MAAAA,IAAI,EAAE;AADY,KAIqB;;AACvC,SAAKC,WAAL,GAAmB,IAAIC,WAAJ,CAAgBJ,OAAO,CAACK,QAAxB,CAAnB;AACD;;AAKDC,EAAAA,KAAK,CAACC,WAAD,EAAiC;AACpC,SAAKC,YAAL,CAAkBF,KAAlB,CAAwBC,WAAxB;AACA,SAAKE,KAAL,GAAaC,UAAU,CAAC,KAAKD,KAAN,EAAa,KAAKE,MAAlB,EAA0B,KAAKH,YAA/B,EAA6C,KAAKL,WAAlD,CAAvB;AAOD;;AAEDS,EAAAA,GAAG,GAAS;AACV,SAAKJ,YAAL,CAAkBI,GAAlB;AACA,SAAKH,KAAL,GAAaC,UAAU,CAAC,KAAKD,KAAN,EAAa,KAAKE,MAAlB,EAA0B,KAAKH,YAA/B,EAA6C,KAAKL,WAAlD,CAAvB;;AAEA,QAAI,KAAKM,KAAL,KAAeZ,KAAK,CAACgB,GAAzB,EAA8B;AAC5B,WAAKJ,KAAL,GAAaZ,KAAK,CAACiB,KAAnB;AACA,WAAKH,MAAL,CAAYI,KAAZ,GAAoB,qBAApB;AACD;AACF;;AAlCa;;AA0ChB,OAAO,SAASC,QAAT,CACLT,WADK,EAELP,OAAY,GAAG,EAFV,EAG2B;AAChC,QAAMiB,aAAa,GAAGjB,OAAO,CAACkB,GAAR,IAAe,EAArC;AACA,QAAM;AAACb,IAAAA;AAAD,MAAaY,aAAnB;AAEA,QAAME,SAAS,GAAG,IAAIrB,SAAJ,CAAc;AAACO,IAAAA;AAAD,GAAd,CAAlB;AACAc,EAAAA,SAAS,CAACb,KAAV,CAAgBC,WAAhB;AACAY,EAAAA,SAAS,CAACP,GAAV;AAEA,QAAM;AAACV,IAAAA,IAAD;AAAOkB,IAAAA;AAAP,MAAiBD,SAAS,CAACR,MAAjC;;AACA,UAAQX,OAAO,CAACqB,MAAR,IAAkBrB,OAAO,CAACqB,MAAR,CAAeC,MAAzC;AACE,SAAK,OAAL;AAEE,aAAO;AAACF,QAAAA,MAAD;AAASG,QAAAA,IAAI,EAAErB;AAAf,OAAP;;AAEF,SAAK,MAAL;AACA;AACE,aAAOA,IAAP;AAPJ;AASD;AAKD,OAAO,gBAAgBsB,iBAAhB,CACLC,aADK,EAELzB,OAAY,GAAG,EAFV,EAGsD;AAC3D,QAAMiB,aAAa,GAAGjB,OAAO,CAACkB,GAAR,IAAe,EAArC;AACA,QAAM;AAACb,IAAAA;AAAD,MAAaY,aAAnB;AAEA,QAAMS,MAAM,GAAG,IAAI5B,SAAJ,CAAc;AAACO,IAAAA;AAAD,GAAd,CAAf;AACA,MAAIsB,cAAc,GAAG,KAArB;;AACA,aAAW,MAAMpB,WAAjB,IAAgCkB,aAAhC,EAA+C;AAC7CC,IAAAA,MAAM,CAACpB,KAAP,CAAaC,WAAb;;AACA,QAAI,CAACoB,cAAD,IAAmBD,MAAM,CAACf,MAAP,CAAciB,SAArC,EAAgD;AAC9CD,MAAAA,cAAc,GAAG,IAAjB;AACA,YAAMD,MAAM,CAACf,MAAP,CAAciB,SAApB;AACD;;AAED,QAAIF,MAAM,CAACf,MAAP,CAAcT,IAAd,CAAmB2B,MAAnB,GAA4B,CAAhC,EAAmC;AACjC,YAAMH,MAAM,CAACf,MAAP,CAAcT,IAApB;AACAwB,MAAAA,MAAM,CAACf,MAAP,CAAcT,IAAd,GAAqB,EAArB;AACD;AACF;;AACDwB,EAAAA,MAAM,CAACd,GAAP;;AACA,MAAIc,MAAM,CAACf,MAAP,CAAcT,IAAd,CAAmB2B,MAAnB,GAA4B,CAAhC,EAAmC;AACjC,UAAMH,MAAM,CAACf,MAAP,CAAcT,IAApB;AACD;AACF;;AAUD,SAASQ,UAAT,CACED,KADF,EAEEE,MAFF,EAGEH,YAHF,EAIEL,WAJF,EAKS;AAEP,SAAO,IAAP,EAAa;AACX,QAAI;AACF,cAAQM,KAAR;AACE,aAAKZ,KAAK,CAACiB,KAAX;AACA,aAAKjB,KAAK,CAACgB,GAAX;AACE,iBAAOJ,KAAP;;AAEF,aAAKZ,KAAK,CAACI,KAAX;AAEE,gBAAM6B,QAAQ,GAAGtB,YAAY,CAACuB,WAAb,CAAyBnC,eAAzB,EAA0C,YAA1C,CAAjB;;AACA,cAAI,CAACkC,QAAL,EAAe;AACb,mBAAOrB,KAAP;AACD;;AACDE,UAAAA,MAAM,CAACiB,SAAP,GAAmBI,cAAc,CAACF,QAAD,CAAjC;AACAnB,UAAAA,MAAM,CAACsB,QAAP,GAAkB;AAChBC,YAAAA,SAAS,EAAE,CADK;AAEhBC,YAAAA,SAAS,EAAExB,MAAM,CAACiB,SAAP,CAAiBQ,QAFZ;AAGhBb,YAAAA,IAAI,EAAE;AAHU,WAAlB;AAKAd,UAAAA,KAAK,GAAGZ,KAAK,CAACwC,iBAAd;AACA;;AAEF,aAAKxC,KAAK,CAACwC,iBAAX;AAEE,gBAAMC,mBAAmB,GAAG9B,YAAY,CAACuB,WAAb,CAE1BpB,MAAM,CAACiB,SAAP,CAAiBW,YAAjB,GAAgC3C,eAFN,EAG1B,uBAH0B,CAA5B;;AAKA,cAAI,CAAC0C,mBAAL,EAA0B;AACxB,mBAAO7B,KAAP;AACD;;AAEDE,UAAAA,MAAM,CAAC6B,SAAP,GAAmBC,qBAAqB,CAACH,mBAAD,EAAsBnC,WAAtB,CAAxC;AACAQ,UAAAA,MAAM,CAACS,MAAP,GAAgB,IAAIhC,MAAJ,CAAWuB,MAAM,CAAC6B,SAAP,CAAiBE,GAAjB,CAAsBC,QAAD,IAAcC,SAAS,CAACD,QAAD,CAA5C,CAAX,CAAhB;AAEAlC,UAAAA,KAAK,GAAGZ,KAAK,CAACgD,gBAAd;AAIArC,UAAAA,YAAY,CAACsC,IAAb,CAAkB,CAAlB;AACA;;AAEF,aAAKjD,KAAK,CAACgD,gBAAX;AACE,gBAAM;AAACE,YAAAA,YAAY,GAAG,CAAhB;AAAmBX,YAAAA,QAAQ,GAAG;AAA9B,cAAmC,CAAAzB,MAAM,SAAN,IAAAA,MAAM,WAAN,YAAAA,MAAM,CAAEiB,SAAR,KAAqB,EAA9D;;AACA,iBAAOjB,MAAM,CAACT,IAAP,CAAY2B,MAAZ,GAAqBO,QAA5B,EAAsC;AACpC,kBAAMY,UAAU,GAAGxC,YAAY,CAACuB,WAAb,CAAyBgB,YAAY,GAAG,CAAxC,CAAnB;;AACA,gBAAI,CAACC,UAAL,EAAiB;AACf,qBAAOvC,KAAP;AACD;;AAEDD,YAAAA,YAAY,CAACsC,IAAb,CAAkB,CAAlB;AAGA,kBAAMG,GAAG,GAAGC,QAAQ,CAACF,UAAD,EAAarC,MAAM,CAAC6B,SAApB,EAA+BrC,WAA/B,CAApB;AACAQ,YAAAA,MAAM,CAACT,IAAP,CAAYiD,IAAZ,CAAiBF,GAAjB;AAEAtC,YAAAA,MAAM,CAACsB,QAAP,CAAgBV,IAAhB,GAAuBZ,MAAM,CAACT,IAAP,CAAY2B,MAAnC;AACD;;AACDpB,UAAAA,KAAK,GAAGZ,KAAK,CAACgB,GAAd;AACA;;AAEF;AACEJ,UAAAA,KAAK,GAAGZ,KAAK,CAACiB,KAAd;AACAH,UAAAA,MAAM,CAACI,KAAP,kCAAuCN,KAAvC;AACA,iBAAOA,KAAP;AA/DJ;AAiED,KAlED,CAkEE,OAAOM,KAAP,EAAc;AACdN,MAAAA,KAAK,GAAGZ,KAAK,CAACiB,KAAd;AACAH,MAAAA,MAAM,CAACI,KAAP,iCAAuCA,KAAD,CAAiBqC,OAAvD;AACA,aAAO3C,KAAP;AACD;AACF;AACF;;AAKD,SAASuB,cAAT,CAAwBqB,UAAxB,EAAyD;AACvD,SAAO;AAELC,IAAAA,IAAI,EAAED,UAAU,CAACE,QAAX,CAAoB,CAApB,IAAyB,IAF1B;AAGLC,IAAAA,KAAK,EAAEH,UAAU,CAACE,QAAX,CAAoB,CAApB,CAHF;AAILE,IAAAA,GAAG,EAAEJ,UAAU,CAACE,QAAX,CAAoB,CAApB,CAJA;AAMLnB,IAAAA,QAAQ,EAAEiB,UAAU,CAACK,SAAX,CAAqB,CAArB,EAAwB/D,aAAxB,CANL;AAQL4C,IAAAA,YAAY,EAAEc,UAAU,CAACM,SAAX,CAAqB,CAArB,EAAwBhE,aAAxB,CART;AAULoD,IAAAA,YAAY,EAAEM,UAAU,CAACM,SAAX,CAAqB,EAArB,EAAyBhE,aAAzB,CAVT;AAYLiE,IAAAA,cAAc,EAAEP,UAAU,CAACE,QAAX,CAAoB,EAApB;AAZX,GAAP;AAcD;;AAKD,SAASd,qBAAT,CAA+BoB,IAA/B,EAA+C1D,WAA/C,EAAqF;AAGnF,QAAM2D,OAAO,GAAG,CAACD,IAAI,CAACE,UAAL,GAAkB,CAAnB,IAAwB,EAAxC;AACA,QAAMC,MAAkB,GAAG,EAA3B;AACA,MAAIC,MAAM,GAAG,CAAb;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGJ,OAApB,EAA6BI,CAAC,EAA9B,EAAkC;AAChC,UAAMC,IAAI,GAAGhE,WAAW,CACrBiE,MADU,CACH,IAAIC,UAAJ,CAAeR,IAAI,CAACS,MAApB,EAA4BT,IAAI,CAACU,UAAL,GAAkBN,MAA9C,EAAsD,EAAtD,CADG,EAGVO,OAHU,CAGF,SAHE,EAGS,EAHT,CAAb;AAKAR,IAAAA,MAAM,CAACb,IAAP,CAAY;AACVgB,MAAAA,IADU;AAEVM,MAAAA,QAAQ,EAAEC,MAAM,CAACC,YAAP,CAAoBd,IAAI,CAACN,QAAL,CAAcU,MAAM,GAAG,EAAvB,CAApB,CAFA;AAGVW,MAAAA,WAAW,EAAEf,IAAI,CAACN,QAAL,CAAcU,MAAM,GAAG,EAAvB,CAHH;AAIVY,MAAAA,OAAO,EAAEhB,IAAI,CAACN,QAAL,CAAcU,MAAM,GAAG,EAAvB;AAJC,KAAZ;AAMAA,IAAAA,MAAM,IAAI,EAAV;AACD;;AACD,SAAOD,MAAP;AACD;;AAuBD,SAASd,QAAT,CACEW,IADF,EAEEG,MAFF,EAGE7D,WAHF,EAIwB;AACtB,QAAM2E,GAAG,GAAG,EAAZ;AACA,MAAIb,MAAM,GAAG,CAAb;;AACA,OAAK,MAAMc,KAAX,IAAoBf,MAApB,EAA4B;AAC1B,UAAMgB,IAAI,GAAG7E,WAAW,CAACiE,MAAZ,CACX,IAAIC,UAAJ,CAAeR,IAAI,CAACS,MAApB,EAA4BT,IAAI,CAACU,UAAL,GAAkBN,MAA9C,EAAsDc,KAAK,CAACH,WAA5D,CADW,CAAb;AAGAE,IAAAA,GAAG,CAACC,KAAK,CAACZ,IAAP,CAAH,GAAkBc,UAAU,CAACD,IAAD,EAAOD,KAAK,CAACN,QAAb,CAA5B;AACAR,IAAAA,MAAM,IAAIc,KAAK,CAACH,WAAhB;AACD;;AAED,SAAOE,GAAP;AACD;;AAQD,SAASG,UAAT,CAAoBD,IAApB,EAAkCP,QAAlC,EAAsF;AACpF,UAAQA,QAAR;AACE,SAAK,GAAL;AACE,aAAOS,WAAW,CAACF,IAAD,CAAlB;;AACF,SAAK,GAAL;AACE,aAAOG,cAAc,CAACH,IAAD,CAArB;;AACF,SAAK,GAAL;AACE,aAAOE,WAAW,CAACF,IAAD,CAAlB;;AACF,SAAK,GAAL;AACE,aAAOE,WAAW,CAACF,IAAD,CAAlB;;AACF,SAAK,GAAL;AACE,aAAOE,WAAW,CAACF,IAAD,CAAlB;;AACF,SAAK,GAAL;AACE,aAAOI,SAAS,CAACJ,IAAD,CAAhB;;AACF,SAAK,GAAL;AACE,aAAOK,YAAY,CAACL,IAAD,CAAnB;;AACF;AACE,YAAM,IAAIM,KAAJ,CAAU,uBAAV,CAAN;AAhBJ;AAkBD;;AAOD,SAASF,SAAT,CAAmBG,GAAnB,EAAqC;AACnC,SAAOC,IAAI,CAACC,GAAL,CAASF,GAAG,CAACG,KAAJ,CAAU,CAAV,EAAa,CAAb,CAAT,EAA0BC,QAAQ,CAACJ,GAAG,CAACG,KAAJ,CAAU,CAAV,EAAa,CAAb,CAAD,EAAkB,EAAlB,CAAR,GAAgC,CAA1D,EAA6DH,GAAG,CAACG,KAAJ,CAAU,CAAV,EAAa,CAAb,CAA7D,CAAP;AACD;;AAUD,SAASL,YAAT,CAAsBO,KAAtB,EAAqD;AACnD,SAAO,UAAUC,IAAV,CAAeD,KAAf,IAAwB,KAAxB,GAAgC,UAAUC,IAAV,CAAeD,KAAf,IAAwB,IAAxB,GAA+B,IAAtE;AACD;;AAOD,SAASV,WAAT,CAAqBF,IAArB,EAAkD;AAChD,QAAMc,MAAM,GAAGC,UAAU,CAACf,IAAD,CAAzB;AACA,SAAOgB,KAAK,CAACF,MAAD,CAAL,GAAgB,IAAhB,GAAuBA,MAA9B;AACD;;AAOD,SAASX,cAAT,CAAwBH,IAAxB,EAAqD;AACnD,SAAOA,IAAI,CAACiB,IAAL,MAAe,IAAtB;AACD;;AASD,SAASrD,SAAT,CAAmB;AAACuB,EAAAA,IAAD;AAAOM,EAAAA,QAAP;AAAiBG,EAAAA,WAAjB;AAA8BC,EAAAA;AAA9B,CAAnB,EAAkE;AAChE,UAAQJ,QAAR;AACE,SAAK,GAAL;AACE,aAAO,IAAIpF,KAAJ,CAAU8E,IAAV,EAAgB,IAAI3E,OAAJ,EAAhB,EAA+B,IAA/B,CAAP;;AACF,SAAK,GAAL;AACE,aAAO,IAAIH,KAAJ,CAAU8E,IAAV,EAAgB,IAAI5E,IAAJ,EAAhB,EAA4B,IAA5B,CAAP;;AACF,SAAK,GAAL;AACE,aAAO,IAAIF,KAAJ,CAAU8E,IAAV,EAAgB,IAAI3E,OAAJ,EAAhB,EAA+B,IAA/B,CAAP;;AACF,SAAK,GAAL;AACE,aAAO,IAAIH,KAAJ,CAAU8E,IAAV,EAAgB,IAAI3E,OAAJ,EAAhB,EAA+B,IAA/B,CAAP;;AACF,SAAK,GAAL;AACE,aAAO,IAAIH,KAAJ,CAAU8E,IAAV,EAAgB,IAAI3E,OAAJ,EAAhB,EAA+B,IAA/B,CAAP;;AACF,SAAK,GAAL;AACE,aAAO,IAAIH,KAAJ,CAAU8E,IAAV,EAAgB,IAAI1E,oBAAJ,EAAhB,EAA4C,IAA5C,CAAP;;AACF,SAAK,GAAL;AACE,aAAO,IAAIJ,KAAJ,CAAU8E,IAAV,EAAgB,IAAI7E,IAAJ,EAAhB,EAA4B,IAA5B,CAAP;;AACF;AACE,YAAM,IAAIgG,KAAJ,CAAU,uBAAV,CAAN;AAhBJ;AAkBD","sourcesContent":["import {Schema, Field, Bool, Utf8, Float64, TimestampMillisecond} from '@loaders.gl/schema';\nimport BinaryChunkReader from '../streaming/binary-chunk-reader';\n\ntype DBFRowsOutput = object[];\n\ninterface DBFTableOutput {\n schema?: Schema;\n rows: DBFRowsOutput;\n}\n\ntype DBFHeader = {\n // Last updated date\n year: number;\n month: number;\n day: number;\n // Number of records in data file\n nRecords: number;\n // Length of header in bytes\n headerLength: number;\n // Length of each record\n recordLength: number;\n // Not sure if this is usually set\n languageDriver: number;\n};\n\ntype DBFField = {\n name: string;\n dataType: string;\n fieldLength: number;\n decimal: number;\n};\n\ntype DBFResult = {\n data: {[key: string]: any}[];\n schema?: Schema;\n error?: string;\n dbfHeader?: DBFHeader;\n dbfFields?: DBFField[];\n progress?: {\n bytesUsed: number;\n rowsTotal: number;\n rows: number;\n };\n};\n\nconst LITTLE_ENDIAN = true;\nconst DBF_HEADER_SIZE = 32;\n\nenum STATE {\n START = 0, // Expecting header\n FIELD_DESCRIPTORS = 1,\n FIELD_PROPERTIES = 2,\n END = 3,\n ERROR = 4\n}\n\nclass DBFParser {\n binaryReader = new BinaryChunkReader();\n textDecoder: TextDecoder;\n state = STATE.START;\n result: DBFResult = {\n data: []\n };\n\n constructor(options: {encoding: string}) {\n this.textDecoder = new TextDecoder(options.encoding);\n }\n\n /**\n * @param arrayBuffer\n */\n write(arrayBuffer: ArrayBuffer): void {\n this.binaryReader.write(arrayBuffer);\n this.state = parseState(this.state, this.result, this.binaryReader, this.textDecoder);\n // this.result.progress.bytesUsed = this.binaryReader.bytesUsed();\n\n // important events:\n // - schema available\n // - first rows available\n // - all rows available\n }\n\n end(): void {\n this.binaryReader.end();\n this.state = parseState(this.state, this.result, this.binaryReader, this.textDecoder);\n // this.result.progress.bytesUsed = this.binaryReader.bytesUsed();\n if (this.state !== STATE.END) {\n this.state = STATE.ERROR;\n this.result.error = 'DBF incomplete file';\n }\n }\n}\n\n/**\n * @param arrayBuffer\n * @param options\n * @returns DBFTable or rows\n */\nexport function parseDBF(\n arrayBuffer: ArrayBuffer,\n options: any = {}\n): DBFRowsOutput | DBFTableOutput {\n const loaderOptions = options.dbf || {};\n const {encoding} = loaderOptions;\n\n const dbfParser = new DBFParser({encoding});\n dbfParser.write(arrayBuffer);\n dbfParser.end();\n\n const {data, schema} = dbfParser.result;\n switch (options.tables && options.tables.format) {\n case 'table':\n // TODO - parse columns\n return {schema, rows: data};\n\n case 'rows':\n default:\n return data;\n }\n}\n/**\n * @param asyncIterator\n * @param options\n */\nexport async function* parseDBFInBatches(\n asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n options: any = {}\n): AsyncIterable<DBFHeader | DBFRowsOutput | DBFTableOutput> {\n const loaderOptions = options.dbf || {};\n const {encoding} = loaderOptions;\n\n const parser = new DBFParser({encoding});\n let headerReturned = false;\n for await (const arrayBuffer of asyncIterator) {\n parser.write(arrayBuffer);\n if (!headerReturned && parser.result.dbfHeader) {\n headerReturned = true;\n yield parser.result.dbfHeader;\n }\n\n if (parser.result.data.length > 0) {\n yield parser.result.data;\n parser.result.data = [];\n }\n }\n parser.end();\n if (parser.result.data.length > 0) {\n yield parser.result.data;\n }\n}\n/**\n * https://www.dbase.com/Knowledgebase/INT/db7_file_fmt.htm\n * @param state\n * @param result\n * @param binaryReader\n * @param textDecoder\n * @returns\n */\n/* eslint-disable complexity, max-depth */\nfunction parseState(\n state: STATE,\n result: DBFResult,\n binaryReader: {[key: string]: any},\n textDecoder: TextDecoder\n): STATE {\n // eslint-disable-next-line no-constant-condition\n while (true) {\n try {\n switch (state) {\n case STATE.ERROR:\n case STATE.END:\n return state;\n\n case STATE.START:\n // Parse initial file header\n const dataView = binaryReader.getDataView(DBF_HEADER_SIZE, 'DBF header');\n if (!dataView) {\n return state;\n }\n result.dbfHeader = parseDBFHeader(dataView);\n result.progress = {\n bytesUsed: 0,\n rowsTotal: result.dbfHeader.nRecords,\n rows: 0\n };\n state = STATE.FIELD_DESCRIPTORS;\n break;\n\n case STATE.FIELD_DESCRIPTORS:\n // Parse DBF field descriptors (schema)\n const fieldDescriptorView = binaryReader.getDataView(\n // @ts-ignore\n result.dbfHeader.headerLength - DBF_HEADER_SIZE,\n 'DBF field descriptors'\n );\n if (!fieldDescriptorView) {\n return state;\n }\n\n result.dbfFields = parseFieldDescriptors(fieldDescriptorView, textDecoder);\n result.schema = new Schema(result.dbfFields.map((dbfField) => makeField(dbfField)));\n\n state = STATE.FIELD_PROPERTIES;\n\n // TODO(kyle) Not exactly sure why start offset needs to be headerLength + 1?\n // parsedbf uses ((fields.length + 1) << 5) + 2;\n binaryReader.skip(1);\n break;\n\n case STATE.FIELD_PROPERTIES:\n const {recordLength = 0, nRecords = 0} = result?.dbfHeader || {};\n while (result.data.length < nRecords) {\n const recordView = binaryReader.getDataView(recordLength - 1);\n if (!recordView) {\n return state;\n }\n // Note: Avoid actually reading the last byte, which may not be present\n binaryReader.skip(1);\n\n // @ts-ignore\n const row = parseRow(recordView, result.dbfFields, textDecoder);\n result.data.push(row);\n // @ts-ignore\n result.progress.rows = result.data.length;\n }\n state = STATE.END;\n break;\n\n default:\n state = STATE.ERROR;\n result.error = `illegal parser state ${state}`;\n return state;\n }\n } catch (error) {\n state = STATE.ERROR;\n result.error = `DBF parsing failed: ${(error as Error).message}`;\n return state;\n }\n }\n}\n\n/**\n * @param headerView\n */\nfunction parseDBFHeader(headerView: DataView): DBFHeader {\n return {\n // Last updated date\n year: headerView.getUint8(1) + 1900,\n month: headerView.getUint8(2),\n day: headerView.getUint8(3),\n // Number of records in data file\n nRecords: headerView.getUint32(4, LITTLE_ENDIAN),\n // Length of header in bytes\n headerLength: headerView.getUint16(8, LITTLE_ENDIAN),\n // Length of each record\n recordLength: headerView.getUint16(10, LITTLE_ENDIAN),\n // Not sure if this is usually set\n languageDriver: headerView.getUint8(29)\n };\n}\n\n/**\n * @param view\n */\nfunction parseFieldDescriptors(view: DataView, textDecoder: TextDecoder): DBFField[] {\n // NOTE: this might overestimate the number of fields if the \"Database\n // Container\" container exists and is included in the headerLength\n const nFields = (view.byteLength - 1) / 32;\n const fields: DBFField[] = [];\n let offset = 0;\n for (let i = 0; i < nFields; i++) {\n const name = textDecoder\n .decode(new Uint8Array(view.buffer, view.byteOffset + offset, 11))\n // eslint-disable-next-line no-control-regex\n .replace(/\\u0000/g, '');\n\n fields.push({\n name,\n dataType: String.fromCharCode(view.getUint8(offset + 11)),\n fieldLength: view.getUint8(offset + 16),\n decimal: view.getUint8(offset + 17)\n });\n offset += 32;\n }\n return fields;\n}\n\n/*\n * @param {BinaryChunkReader} binaryReader\nfunction parseRows(binaryReader, fields, nRecords, recordLength, textDecoder) {\n const rows = [];\n for (let i = 0; i < nRecords; i++) {\n const recordView = binaryReader.getDataView(recordLength - 1);\n binaryReader.skip(1);\n // @ts-ignore\n rows.push(parseRow(recordView, fields, textDecoder));\n }\n return rows;\n}\n */\n\n/**\n *\n * @param view\n * @param fields\n * @param textDecoder\n * @returns\n */\nfunction parseRow(\n view: DataView,\n fields: DBFField[],\n textDecoder: TextDecoder\n): {[key: string]: any} {\n const out = {};\n let offset = 0;\n for (const field of fields) {\n const text = textDecoder.decode(\n new Uint8Array(view.buffer, view.byteOffset + offset, field.fieldLength)\n );\n out[field.name] = parseField(text, field.dataType);\n offset += field.fieldLength;\n }\n\n return out;\n}\n\n/**\n * Should NaN be coerced to null?\n * @param text\n * @param dataType\n * @returns Field depends on a type of the data\n */\nfunction parseField(text: string, dataType: string): string | number | boolean | null {\n switch (dataType) {\n case 'B':\n return parseNumber(text);\n case 'C':\n return parseCharacter(text);\n case 'F':\n return parseNumber(text);\n case 'N':\n return parseNumber(text);\n case 'O':\n return parseNumber(text);\n case 'D':\n return parseDate(text);\n case 'L':\n return parseBoolean(text);\n default:\n throw new Error('Unsupported data type');\n }\n}\n\n/**\n * Parse YYYYMMDD to date in milliseconds\n * @param str YYYYMMDD\n * @returns new Date as a number\n */\nfunction parseDate(str: any): number {\n return Date.UTC(str.slice(0, 4), parseInt(str.slice(4, 6), 10) - 1, str.slice(6, 8));\n}\n\n/**\n * Read boolean value\n * any of Y, y, T, t coerce to true\n * any of N, n, F, f coerce to false\n * otherwise null\n * @param value\n * @returns boolean | null\n */\nfunction parseBoolean(value: string): boolean | null {\n return /^[nf]$/i.test(value) ? false : /^[yt]$/i.test(value) ? true : null;\n}\n\n/**\n * Return null instead of NaN\n * @param text\n * @returns number | null\n */\nfunction parseNumber(text: string): number | null {\n const number = parseFloat(text);\n return isNaN(number) ? null : number;\n}\n\n/**\n *\n * @param text\n * @returns string | null\n */\nfunction parseCharacter(text: string): string | null {\n return text.trim() || null;\n}\n\n/**\n * Create a standard Arrow-style `Field` from field descriptor.\n * TODO - use `fieldLength` and `decimal` to generate smaller types?\n * @param param0\n * @returns Field\n */\n// eslint-disable\nfunction makeField({name, dataType, fieldLength, decimal}): Field {\n switch (dataType) {\n case 'B':\n return new Field(name, new Float64(), true);\n case 'C':\n return new Field(name, new Utf8(), true);\n case 'F':\n return new Field(name, new Float64(), true);\n case 'N':\n return new Field(name, new Float64(), true);\n case 'O':\n return new Field(name, new Float64(), true);\n case 'D':\n return new Field(name, new TimestampMillisecond(), true);\n case 'L':\n return new Field(name, new Bool(), true);\n default:\n throw new Error('Unsupported data type');\n }\n}\n"],"file":"parse-dbf.js"}
|
|
@@ -1,226 +0,0 @@
|
|
|
1
|
-
import { binaryToGeometry, transformGeoJsonCoords } from '@loaders.gl/gis';
|
|
2
|
-
import { Proj4Projection } from '@math.gl/proj4';
|
|
3
|
-
import { parseShx } from './parse-shx';
|
|
4
|
-
import { zipBatchIterators } from '../streaming/zip-batch-iterators';
|
|
5
|
-
import { SHPLoader } from '../../shp-loader';
|
|
6
|
-
import { DBFLoader } from '../../dbf-loader';
|
|
7
|
-
export async function* parseShapefileInBatches(asyncIterator, options, context) {
|
|
8
|
-
const {
|
|
9
|
-
reproject = false,
|
|
10
|
-
_targetCrs = 'WGS84'
|
|
11
|
-
} = (options === null || options === void 0 ? void 0 : options.gis) || {};
|
|
12
|
-
const {
|
|
13
|
-
shx,
|
|
14
|
-
cpg,
|
|
15
|
-
prj
|
|
16
|
-
} = await loadShapefileSidecarFiles(options, context);
|
|
17
|
-
const shapeIterable = await context.parseInBatches(asyncIterator, SHPLoader, options);
|
|
18
|
-
let propertyIterable;
|
|
19
|
-
const dbfResponse = await context.fetch(replaceExtension((context === null || context === void 0 ? void 0 : context.url) || '', 'dbf'));
|
|
20
|
-
|
|
21
|
-
if (dbfResponse.ok) {
|
|
22
|
-
propertyIterable = await context.parseInBatches(dbfResponse, DBFLoader, { ...options,
|
|
23
|
-
dbf: {
|
|
24
|
-
encoding: cpg || 'latin1'
|
|
25
|
-
}
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
let shapeHeader = (await shapeIterable.next()).value;
|
|
30
|
-
|
|
31
|
-
if (shapeHeader && shapeHeader.batchType === 'metadata') {
|
|
32
|
-
shapeHeader = (await shapeIterable.next()).value;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
let dbfHeader = {};
|
|
36
|
-
|
|
37
|
-
if (propertyIterable) {
|
|
38
|
-
dbfHeader = (await propertyIterable.next()).value;
|
|
39
|
-
|
|
40
|
-
if (dbfHeader && dbfHeader.batchType === 'metadata') {
|
|
41
|
-
dbfHeader = (await propertyIterable.next()).value;
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
let iterator;
|
|
46
|
-
|
|
47
|
-
if (propertyIterable) {
|
|
48
|
-
iterator = zipBatchIterators(shapeIterable, propertyIterable);
|
|
49
|
-
} else {
|
|
50
|
-
iterator = shapeIterable;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
for await (const item of iterator) {
|
|
54
|
-
let geometries;
|
|
55
|
-
let properties;
|
|
56
|
-
|
|
57
|
-
if (!propertyIterable) {
|
|
58
|
-
geometries = item;
|
|
59
|
-
} else {
|
|
60
|
-
[geometries, properties] = item;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
const geojsonGeometries = parseGeometries(geometries);
|
|
64
|
-
let features = joinProperties(geojsonGeometries, properties);
|
|
65
|
-
|
|
66
|
-
if (reproject) {
|
|
67
|
-
features = reprojectFeatures(features, prj, _targetCrs);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
yield {
|
|
71
|
-
encoding: cpg,
|
|
72
|
-
prj,
|
|
73
|
-
shx,
|
|
74
|
-
header: shapeHeader,
|
|
75
|
-
data: features
|
|
76
|
-
};
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
export async function parseShapefile(arrayBuffer, options, context) {
|
|
80
|
-
const {
|
|
81
|
-
reproject = false,
|
|
82
|
-
_targetCrs = 'WGS84'
|
|
83
|
-
} = (options === null || options === void 0 ? void 0 : options.gis) || {};
|
|
84
|
-
const {
|
|
85
|
-
shx,
|
|
86
|
-
cpg,
|
|
87
|
-
prj
|
|
88
|
-
} = await loadShapefileSidecarFiles(options, context);
|
|
89
|
-
const {
|
|
90
|
-
header,
|
|
91
|
-
geometries
|
|
92
|
-
} = await context.parse(arrayBuffer, SHPLoader, options);
|
|
93
|
-
const geojsonGeometries = parseGeometries(geometries);
|
|
94
|
-
let properties = [];
|
|
95
|
-
const dbfResponse = await context.fetch(replaceExtension(context.url, 'dbf'));
|
|
96
|
-
|
|
97
|
-
if (dbfResponse.ok) {
|
|
98
|
-
properties = await context.parse(dbfResponse, DBFLoader, {
|
|
99
|
-
dbf: {
|
|
100
|
-
encoding: cpg || 'latin1'
|
|
101
|
-
}
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
let features = joinProperties(geojsonGeometries, properties);
|
|
106
|
-
|
|
107
|
-
if (reproject) {
|
|
108
|
-
features = reprojectFeatures(features, prj, _targetCrs);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
return {
|
|
112
|
-
encoding: cpg,
|
|
113
|
-
prj,
|
|
114
|
-
shx,
|
|
115
|
-
header,
|
|
116
|
-
data: features
|
|
117
|
-
};
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
function parseGeometries(geometries) {
|
|
121
|
-
const geojsonGeometries = [];
|
|
122
|
-
|
|
123
|
-
for (const geom of geometries) {
|
|
124
|
-
geojsonGeometries.push(binaryToGeometry(geom));
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
return geojsonGeometries;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
function joinProperties(geometries, properties) {
|
|
131
|
-
const features = [];
|
|
132
|
-
|
|
133
|
-
for (let i = 0; i < geometries.length; i++) {
|
|
134
|
-
const geometry = geometries[i];
|
|
135
|
-
const feature = {
|
|
136
|
-
type: 'Feature',
|
|
137
|
-
geometry,
|
|
138
|
-
properties: properties && properties[i] || {}
|
|
139
|
-
};
|
|
140
|
-
features.push(feature);
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
return features;
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
function reprojectFeatures(features, sourceCrs, targetCrs) {
|
|
147
|
-
if (!sourceCrs && !targetCrs) {
|
|
148
|
-
return features;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
const projection = new Proj4Projection({
|
|
152
|
-
from: sourceCrs || 'WGS84',
|
|
153
|
-
to: targetCrs || 'WGS84'
|
|
154
|
-
});
|
|
155
|
-
return transformGeoJsonCoords(features, coord => projection.project(coord));
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
export async function loadShapefileSidecarFiles(options, context) {
|
|
159
|
-
const {
|
|
160
|
-
url,
|
|
161
|
-
fetch
|
|
162
|
-
} = context;
|
|
163
|
-
const shxPromise = fetch(replaceExtension(url, 'shx'));
|
|
164
|
-
const cpgPromise = fetch(replaceExtension(url, 'cpg'));
|
|
165
|
-
const prjPromise = fetch(replaceExtension(url, 'prj'));
|
|
166
|
-
await Promise.all([shxPromise, cpgPromise, prjPromise]);
|
|
167
|
-
let shx;
|
|
168
|
-
let cpg;
|
|
169
|
-
let prj;
|
|
170
|
-
const shxResponse = await shxPromise;
|
|
171
|
-
|
|
172
|
-
if (shxResponse.ok) {
|
|
173
|
-
const arrayBuffer = await shxResponse.arrayBuffer();
|
|
174
|
-
shx = parseShx(arrayBuffer);
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
const cpgResponse = await cpgPromise;
|
|
178
|
-
|
|
179
|
-
if (cpgResponse.ok) {
|
|
180
|
-
cpg = await cpgResponse.text();
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
const prjResponse = await prjPromise;
|
|
184
|
-
|
|
185
|
-
if (prjResponse.ok) {
|
|
186
|
-
prj = await prjResponse.text();
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
return {
|
|
190
|
-
shx,
|
|
191
|
-
cpg,
|
|
192
|
-
prj
|
|
193
|
-
};
|
|
194
|
-
}
|
|
195
|
-
export function replaceExtension(url, newExtension) {
|
|
196
|
-
const baseName = basename(url);
|
|
197
|
-
const extension = extname(url);
|
|
198
|
-
const isUpperCase = extension === extension.toUpperCase();
|
|
199
|
-
|
|
200
|
-
if (isUpperCase) {
|
|
201
|
-
newExtension = newExtension.toUpperCase();
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
return "".concat(baseName, ".").concat(newExtension);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
function basename(url) {
|
|
208
|
-
const extIndex = url && url.lastIndexOf('.');
|
|
209
|
-
|
|
210
|
-
if (typeof extIndex === 'number') {
|
|
211
|
-
return extIndex >= 0 ? url.substr(0, extIndex) : '';
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
return extIndex;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
function extname(url) {
|
|
218
|
-
const extIndex = url && url.lastIndexOf('.');
|
|
219
|
-
|
|
220
|
-
if (typeof extIndex === 'number') {
|
|
221
|
-
return extIndex >= 0 ? url.substr(extIndex + 1) : '';
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
return extIndex;
|
|
225
|
-
}
|
|
226
|
-
//# sourceMappingURL=parse-shapefile.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/lib/parsers/parse-shapefile.ts"],"names":["binaryToGeometry","transformGeoJsonCoords","Proj4Projection","parseShx","zipBatchIterators","SHPLoader","DBFLoader","parseShapefileInBatches","asyncIterator","options","context","reproject","_targetCrs","gis","shx","cpg","prj","loadShapefileSidecarFiles","shapeIterable","parseInBatches","propertyIterable","dbfResponse","fetch","replaceExtension","url","ok","dbf","encoding","shapeHeader","next","value","batchType","dbfHeader","iterator","item","geometries","properties","geojsonGeometries","parseGeometries","features","joinProperties","reprojectFeatures","header","data","parseShapefile","arrayBuffer","parse","geom","push","i","length","geometry","feature","type","sourceCrs","targetCrs","projection","from","to","coord","project","shxPromise","cpgPromise","prjPromise","Promise","all","shxResponse","cpgResponse","text","prjResponse","newExtension","baseName","basename","extension","extname","isUpperCase","toUpperCase","extIndex","lastIndexOf","substr"],"mappings":"AAKA,SAAQA,gBAAR,EAA0BC,sBAA1B,QAAuD,iBAAvD;AACA,SAAQC,eAAR,QAA8B,gBAA9B;AACA,SAAQC,QAAR,QAAuB,aAAvB;AACA,SAAQC,iBAAR,QAAgC,kCAAhC;AACA,SAAQC,SAAR,QAAwB,kBAAxB;AACA,SAAQC,SAAR,QAAwB,kBAAxB;AAkBA,OAAO,gBAAgBC,uBAAhB,CACLC,aADK,EAELC,OAFK,EAGLC,OAHK,EAI2B;AAChC,QAAM;AAACC,IAAAA,SAAS,GAAG,KAAb;AAAoBC,IAAAA,UAAU,GAAG;AAAjC,MAA4C,CAAAH,OAAO,SAAP,IAAAA,OAAO,WAAP,YAAAA,OAAO,CAAEI,GAAT,KAAgB,EAAlE;AACA,QAAM;AAACC,IAAAA,GAAD;AAAMC,IAAAA,GAAN;AAAWC,IAAAA;AAAX,MAAkB,MAAMC,yBAAyB,CAACR,OAAD,EAAUC,OAAV,CAAvD;AAIA,QAAMQ,aAAkB,GAAG,MAAMR,OAAO,CAACS,cAAR,CAAuBX,aAAvB,EAAsCH,SAAtC,EAAiDI,OAAjD,CAAjC;AAGA,MAAIW,gBAAJ;AAEA,QAAMC,WAAW,GAAG,MAAMX,OAAO,CAACY,KAAR,CAAcC,gBAAgB,CAAC,CAAAb,OAAO,SAAP,IAAAA,OAAO,WAAP,YAAAA,OAAO,CAAEc,GAAT,KAAgB,EAAjB,EAAqB,KAArB,CAA9B,CAA1B;;AACA,MAAIH,WAAW,CAACI,EAAhB,EAAoB;AAElBL,IAAAA,gBAAgB,GAAG,MAAMV,OAAO,CAACS,cAAR,CAAuBE,WAAvB,EAAoCf,SAApC,EAA+C,EACtE,GAAGG,OADmE;AAEtEiB,MAAAA,GAAG,EAAE;AAACC,QAAAA,QAAQ,EAAEZ,GAAG,IAAI;AAAlB;AAFiE,KAA/C,CAAzB;AAID;;AAMD,MAAIa,WAAW,GAAG,CAAC,MAAMV,aAAa,CAACW,IAAd,EAAP,EAA6BC,KAA/C;;AACA,MAAIF,WAAW,IAAIA,WAAW,CAACG,SAAZ,KAA0B,UAA7C,EAAyD;AACvDH,IAAAA,WAAW,GAAG,CAAC,MAAMV,aAAa,CAACW,IAAd,EAAP,EAA6BC,KAA3C;AACD;;AAED,MAAIE,SAA+B,GAAG,EAAtC;;AACA,MAAIZ,gBAAJ,EAAsB;AACpBY,IAAAA,SAAS,GAAG,CAAC,MAAMZ,gBAAgB,CAACS,IAAjB,EAAP,EAAgCC,KAA5C;;AACA,QAAIE,SAAS,IAAIA,SAAS,CAACD,SAAV,KAAwB,UAAzC,EAAqD;AACnDC,MAAAA,SAAS,GAAG,CAAC,MAAMZ,gBAAgB,CAACS,IAAjB,EAAP,EAAgCC,KAA5C;AACD;AACF;;AAED,MAAIG,QAAJ;;AACA,MAAIb,gBAAJ,EAAsB;AACpBa,IAAAA,QAAQ,GAAG7B,iBAAiB,CAACc,aAAD,EAAgBE,gBAAhB,CAA5B;AACD,GAFD,MAEO;AACLa,IAAAA,QAAQ,GAAGf,aAAX;AACD;;AAED,aAAW,MAAMgB,IAAjB,IAAyBD,QAAzB,EAAmC;AACjC,QAAIE,UAAJ;AACA,QAAIC,UAAJ;;AACA,QAAI,CAAChB,gBAAL,EAAuB;AACrBe,MAAAA,UAAU,GAAGD,IAAb;AACD,KAFD,MAEO;AACL,OAACC,UAAD,EAAaC,UAAb,IAA2BF,IAA3B;AACD;;AAED,UAAMG,iBAAiB,GAAGC,eAAe,CAACH,UAAD,CAAzC;AACA,QAAII,QAAQ,GAAGC,cAAc,CAACH,iBAAD,EAAoBD,UAApB,CAA7B;;AACA,QAAIzB,SAAJ,EAAe;AAEb4B,MAAAA,QAAQ,GAAGE,iBAAiB,CAACF,QAAD,EAAWvB,GAAX,EAAgBJ,UAAhB,CAA5B;AACD;;AACD,UAAM;AACJe,MAAAA,QAAQ,EAAEZ,GADN;AAEJC,MAAAA,GAFI;AAGJF,MAAAA,GAHI;AAIJ4B,MAAAA,MAAM,EAAEd,WAJJ;AAKJe,MAAAA,IAAI,EAAEJ;AALF,KAAN;AAOD;AACF;AAUD,OAAO,eAAeK,cAAf,CACLC,WADK,EAELpC,OAFK,EAGLC,OAHK,EAIqB;AAC1B,QAAM;AAACC,IAAAA,SAAS,GAAG,KAAb;AAAoBC,IAAAA,UAAU,GAAG;AAAjC,MAA4C,CAAAH,OAAO,SAAP,IAAAA,OAAO,WAAP,YAAAA,OAAO,CAAEI,GAAT,KAAgB,EAAlE;AACA,QAAM;AAACC,IAAAA,GAAD;AAAMC,IAAAA,GAAN;AAAWC,IAAAA;AAAX,MAAkB,MAAMC,yBAAyB,CAACR,OAAD,EAAUC,OAAV,CAAvD;AAIA,QAAM;AAACgC,IAAAA,MAAD;AAASP,IAAAA;AAAT,MAAuB,MAAMzB,OAAO,CAACoC,KAAR,CAAcD,WAAd,EAA2BxC,SAA3B,EAAsCI,OAAtC,CAAnC;AAEA,QAAM4B,iBAAiB,GAAGC,eAAe,CAACH,UAAD,CAAzC;AAGA,MAAIC,UAAU,GAAG,EAAjB;AAGA,QAAMf,WAAW,GAAG,MAAMX,OAAO,CAACY,KAAR,CAAcC,gBAAgB,CAACb,OAAO,CAACc,GAAT,EAAc,KAAd,CAA9B,CAA1B;;AACA,MAAIH,WAAW,CAACI,EAAhB,EAAoB;AAElBW,IAAAA,UAAU,GAAG,MAAM1B,OAAO,CAACoC,KAAR,CAAczB,WAAd,EAA2Bf,SAA3B,EAAsC;AAACoB,MAAAA,GAAG,EAAE;AAACC,QAAAA,QAAQ,EAAEZ,GAAG,IAAI;AAAlB;AAAN,KAAtC,CAAnB;AACD;;AAED,MAAIwB,QAAQ,GAAGC,cAAc,CAACH,iBAAD,EAAoBD,UAApB,CAA7B;;AACA,MAAIzB,SAAJ,EAAe;AACb4B,IAAAA,QAAQ,GAAGE,iBAAiB,CAACF,QAAD,EAAWvB,GAAX,EAAgBJ,UAAhB,CAA5B;AACD;;AAED,SAAO;AACLe,IAAAA,QAAQ,EAAEZ,GADL;AAELC,IAAAA,GAFK;AAGLF,IAAAA,GAHK;AAIL4B,IAAAA,MAJK;AAKLC,IAAAA,IAAI,EAAEJ;AALD,GAAP;AAOD;;AAQD,SAASD,eAAT,CAAyBH,UAAzB,EAAmD;AACjD,QAAME,iBAAwB,GAAG,EAAjC;;AACA,OAAK,MAAMU,IAAX,IAAmBZ,UAAnB,EAA+B;AAC7BE,IAAAA,iBAAiB,CAACW,IAAlB,CAAuBhD,gBAAgB,CAAC+C,IAAD,CAAvC;AACD;;AACD,SAAOV,iBAAP;AACD;;AASD,SAASG,cAAT,CAAwBL,UAAxB,EAA8CC,UAA9C,EAA+E;AAC7E,QAAMG,QAAmB,GAAG,EAA5B;;AACA,OAAK,IAAIU,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGd,UAAU,CAACe,MAA/B,EAAuCD,CAAC,EAAxC,EAA4C;AAC1C,UAAME,QAAQ,GAAGhB,UAAU,CAACc,CAAD,CAA3B;AACA,UAAMG,OAAgB,GAAG;AACvBC,MAAAA,IAAI,EAAE,SADiB;AAEvBF,MAAAA,QAFuB;AAIvBf,MAAAA,UAAU,EAAGA,UAAU,IAAIA,UAAU,CAACa,CAAD,CAAzB,IAAiC;AAJtB,KAAzB;AAMAV,IAAAA,QAAQ,CAACS,IAAT,CAAcI,OAAd;AACD;;AAED,SAAOb,QAAP;AACD;;AAUD,SAASE,iBAAT,CAA2BF,QAA3B,EAAgDe,SAAhD,EAAoEC,SAApE,EAAmG;AACjG,MAAI,CAACD,SAAD,IAAc,CAACC,SAAnB,EAA8B;AAC5B,WAAOhB,QAAP;AACD;;AAED,QAAMiB,UAAU,GAAG,IAAItD,eAAJ,CAAoB;AAACuD,IAAAA,IAAI,EAAEH,SAAS,IAAI,OAApB;AAA6BI,IAAAA,EAAE,EAAEH,SAAS,IAAI;AAA9C,GAApB,CAAnB;AACA,SAAOtD,sBAAsB,CAACsC,QAAD,EAAYoB,KAAD,IAAWH,UAAU,CAACI,OAAX,CAAmBD,KAAnB,CAAtB,CAA7B;AACD;;AASD,OAAO,eAAe1C,yBAAf,CACLR,OADK,EAELC,OAFK,EAOJ;AAGD,QAAM;AAACc,IAAAA,GAAD;AAAMF,IAAAA;AAAN,MAAeZ,OAArB;AACA,QAAMmD,UAAU,GAAGvC,KAAK,CAACC,gBAAgB,CAACC,GAAD,EAAM,KAAN,CAAjB,CAAxB;AACA,QAAMsC,UAAU,GAAGxC,KAAK,CAACC,gBAAgB,CAACC,GAAD,EAAM,KAAN,CAAjB,CAAxB;AACA,QAAMuC,UAAU,GAAGzC,KAAK,CAACC,gBAAgB,CAACC,GAAD,EAAM,KAAN,CAAjB,CAAxB;AACA,QAAMwC,OAAO,CAACC,GAAR,CAAY,CAACJ,UAAD,EAAaC,UAAb,EAAyBC,UAAzB,CAAZ,CAAN;AAEA,MAAIjD,GAAJ;AACA,MAAIC,GAAJ;AACA,MAAIC,GAAJ;AAEA,QAAMkD,WAAW,GAAG,MAAML,UAA1B;;AACA,MAAIK,WAAW,CAACzC,EAAhB,EAAoB;AAClB,UAAMoB,WAAW,GAAG,MAAMqB,WAAW,CAACrB,WAAZ,EAA1B;AACA/B,IAAAA,GAAG,GAAGX,QAAQ,CAAC0C,WAAD,CAAd;AACD;;AAED,QAAMsB,WAAW,GAAG,MAAML,UAA1B;;AACA,MAAIK,WAAW,CAAC1C,EAAhB,EAAoB;AAClBV,IAAAA,GAAG,GAAG,MAAMoD,WAAW,CAACC,IAAZ,EAAZ;AACD;;AAED,QAAMC,WAAW,GAAG,MAAMN,UAA1B;;AACA,MAAIM,WAAW,CAAC5C,EAAhB,EAAoB;AAClBT,IAAAA,GAAG,GAAG,MAAMqD,WAAW,CAACD,IAAZ,EAAZ;AACD;;AAED,SAAO;AACLtD,IAAAA,GADK;AAELC,IAAAA,GAFK;AAGLC,IAAAA;AAHK,GAAP;AAKD;AAaD,OAAO,SAASO,gBAAT,CAA0BC,GAA1B,EAAuC8C,YAAvC,EAAqE;AAC1E,QAAMC,QAAQ,GAAGC,QAAQ,CAAChD,GAAD,CAAzB;AACA,QAAMiD,SAAS,GAAGC,OAAO,CAAClD,GAAD,CAAzB;AACA,QAAMmD,WAAW,GAAGF,SAAS,KAAKA,SAAS,CAACG,WAAV,EAAlC;;AACA,MAAID,WAAJ,EAAiB;AACfL,IAAAA,YAAY,GAAGA,YAAY,CAACM,WAAb,EAAf;AACD;;AACD,mBAAUL,QAAV,cAAsBD,YAAtB;AACD;;AAOD,SAASE,QAAT,CAAkBhD,GAAlB,EAAuC;AACrC,QAAMqD,QAAQ,GAAGrD,GAAG,IAAIA,GAAG,CAACsD,WAAJ,CAAgB,GAAhB,CAAxB;;AACA,MAAI,OAAOD,QAAP,KAAoB,QAAxB,EAAkC;AAChC,WAAOA,QAAQ,IAAI,CAAZ,GAAgBrD,GAAG,CAACuD,MAAJ,CAAW,CAAX,EAAcF,QAAd,CAAhB,GAA0C,EAAjD;AACD;;AACD,SAAOA,QAAP;AACD;;AAKD,SAASH,OAAT,CAAiBlD,GAAjB,EAAsC;AACpC,QAAMqD,QAAQ,GAAGrD,GAAG,IAAIA,GAAG,CAACsD,WAAJ,CAAgB,GAAhB,CAAxB;;AACA,MAAI,OAAOD,QAAP,KAAoB,QAAxB,EAAkC;AAChC,WAAOA,QAAQ,IAAI,CAAZ,GAAgBrD,GAAG,CAACuD,MAAJ,CAAWF,QAAQ,GAAG,CAAtB,CAAhB,GAA2C,EAAlD;AACD;;AACD,SAAOA,QAAP;AACD","sourcesContent":["// import type {Feature} from '@loaders.gl/gis';\nimport type {SHXOutput} from './parse-shx';\nimport type {SHPHeader} from './parse-shp-header';\nimport type {LoaderContext} from '@loaders.gl/loader-utils';\n\nimport {binaryToGeometry, transformGeoJsonCoords} from '@loaders.gl/gis';\nimport {Proj4Projection} from '@math.gl/proj4';\nimport {parseShx} from './parse-shx';\nimport {zipBatchIterators} from '../streaming/zip-batch-iterators';\nimport {SHPLoader} from '../../shp-loader';\nimport {DBFLoader} from '../../dbf-loader';\n\ntype Feature = any;\ninterface ShapefileOutput {\n encoding?: string;\n prj?: string;\n shx?: SHXOutput;\n header: SHPHeader;\n data: object[];\n}\n/**\n * Parsing of file in batches\n *\n * @param asyncIterator\n * @param options\n * @param context\n */\n// eslint-disable-next-line max-statements, complexity\nexport async function* parseShapefileInBatches(\n asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n options?: any,\n context?: LoaderContext\n): AsyncIterable<ShapefileOutput> {\n const {reproject = false, _targetCrs = 'WGS84'} = options?.gis || {};\n const {shx, cpg, prj} = await loadShapefileSidecarFiles(options, context);\n\n // parse geometries\n // @ts-ignore context must be defined\n const shapeIterable: any = await context.parseInBatches(asyncIterator, SHPLoader, options);\n\n // parse properties\n let propertyIterable: any;\n // @ts-ignore context must be defined\n const dbfResponse = await context.fetch(replaceExtension(context?.url || '', 'dbf'));\n if (dbfResponse.ok) {\n // @ts-ignore context must be defined\n propertyIterable = await context.parseInBatches(dbfResponse, DBFLoader, {\n ...options,\n dbf: {encoding: cpg || 'latin1'}\n });\n }\n\n // When `options.metadata` is `true`, there's an extra initial `metadata`\n // object before the iterator starts. zipBatchIterators expects to receive\n // batches of Array objects, and will fail with non-iterable batches, so it's\n // important to skip over the first batch.\n let shapeHeader = (await shapeIterable.next()).value;\n if (shapeHeader && shapeHeader.batchType === 'metadata') {\n shapeHeader = (await shapeIterable.next()).value;\n }\n\n let dbfHeader: {batchType?: string} = {};\n if (propertyIterable) {\n dbfHeader = (await propertyIterable.next()).value;\n if (dbfHeader && dbfHeader.batchType === 'metadata') {\n dbfHeader = (await propertyIterable.next()).value;\n }\n }\n\n let iterator: any;\n if (propertyIterable) {\n iterator = zipBatchIterators(shapeIterable, propertyIterable);\n } else {\n iterator = shapeIterable;\n }\n\n for await (const item of iterator) {\n let geometries: any;\n let properties: any;\n if (!propertyIterable) {\n geometries = item;\n } else {\n [geometries, properties] = item;\n }\n\n const geojsonGeometries = parseGeometries(geometries);\n let features = joinProperties(geojsonGeometries, properties);\n if (reproject) {\n // @ts-ignore\n features = reprojectFeatures(features, prj, _targetCrs);\n }\n yield {\n encoding: cpg,\n prj,\n shx,\n header: shapeHeader,\n data: features\n };\n }\n}\n\n/**\n * Parse shapefile\n *\n * @param arrayBuffer\n * @param options\n * @param context\n * @returns output of shapefile\n */\nexport async function parseShapefile(\n arrayBuffer: ArrayBuffer,\n options?: {[key: string]: any},\n context?: LoaderContext\n): Promise<ShapefileOutput> {\n const {reproject = false, _targetCrs = 'WGS84'} = options?.gis || {};\n const {shx, cpg, prj} = await loadShapefileSidecarFiles(options, context);\n\n // parse geometries\n // @ts-ignore context must be defined\n const {header, geometries} = await context.parse(arrayBuffer, SHPLoader, options); // {shp: shx}\n\n const geojsonGeometries = parseGeometries(geometries);\n\n // parse properties\n let properties = [];\n\n // @ts-ignore context must be defined\n const dbfResponse = await context.fetch(replaceExtension(context.url, 'dbf'));\n if (dbfResponse.ok) {\n // @ts-ignore context must be defined\n properties = await context.parse(dbfResponse, DBFLoader, {dbf: {encoding: cpg || 'latin1'}});\n }\n\n let features = joinProperties(geojsonGeometries, properties);\n if (reproject) {\n features = reprojectFeatures(features, prj, _targetCrs);\n }\n\n return {\n encoding: cpg,\n prj,\n shx,\n header,\n data: features\n };\n}\n\n/**\n * Parse geometries\n *\n * @param geometries\n * @returns geometries as an array\n */\nfunction parseGeometries(geometries: any[]): any[] {\n const geojsonGeometries: any[] = [];\n for (const geom of geometries) {\n geojsonGeometries.push(binaryToGeometry(geom));\n }\n return geojsonGeometries;\n}\n\n/**\n * Join properties and geometries into features\n *\n * @param geometries [description]\n * @param properties [description]\n * @return [description]\n */\nfunction joinProperties(geometries: object[], properties: object[]): Feature[] {\n const features: Feature[] = [];\n for (let i = 0; i < geometries.length; i++) {\n const geometry = geometries[i];\n const feature: Feature = {\n type: 'Feature',\n geometry,\n // properties can be undefined if dbfResponse above was empty\n properties: (properties && properties[i]) || {}\n };\n features.push(feature);\n }\n\n return features;\n}\n\n/**\n * Reproject GeoJSON features to output CRS\n *\n * @param features parsed GeoJSON features\n * @param sourceCrs source coordinate reference system\n * @param targetCrs †arget coordinate reference system\n * @return Reprojected Features\n */\nfunction reprojectFeatures(features: Feature[], sourceCrs?: string, targetCrs?: string): Feature[] {\n if (!sourceCrs && !targetCrs) {\n return features;\n }\n\n const projection = new Proj4Projection({from: sourceCrs || 'WGS84', to: targetCrs || 'WGS84'});\n return transformGeoJsonCoords(features, (coord) => projection.project(coord));\n}\n\n/**\n *\n * @param options\n * @param context\n * @returns Promise\n */\n// eslint-disable-next-line max-statements\nexport async function loadShapefileSidecarFiles(\n options?: object,\n context?: LoaderContext\n): Promise<{\n shx?: SHXOutput;\n cpg?: string;\n prj?: string;\n}> {\n // Attempt a parallel load of the small sidecar files\n // @ts-ignore context must be defined\n const {url, fetch} = context;\n const shxPromise = fetch(replaceExtension(url, 'shx'));\n const cpgPromise = fetch(replaceExtension(url, 'cpg'));\n const prjPromise = fetch(replaceExtension(url, 'prj'));\n await Promise.all([shxPromise, cpgPromise, prjPromise]);\n\n let shx: SHXOutput | undefined;\n let cpg: string | undefined;\n let prj: string | undefined;\n\n const shxResponse = await shxPromise;\n if (shxResponse.ok) {\n const arrayBuffer = await shxResponse.arrayBuffer();\n shx = parseShx(arrayBuffer);\n }\n\n const cpgResponse = await cpgPromise;\n if (cpgResponse.ok) {\n cpg = await cpgResponse.text();\n }\n\n const prjResponse = await prjPromise;\n if (prjResponse.ok) {\n prj = await prjResponse.text();\n }\n\n return {\n shx,\n cpg,\n prj\n };\n}\n\n/**\n * Replace the extension at the end of a path.\n *\n * Matches the case of new extension with the case of the original file extension,\n * to increase the chance of finding files without firing off a request storm looking for various case combinations\n *\n * NOTE: Extensions can be both lower and uppercase\n * per spec, extensions should be lower case, but that doesn't mean they always are. See:\n * calvinmetcalf/shapefile-js#64, mapserver/mapserver#4712\n * https://trac.osgeo.org/mapserver/ticket/166\n */\nexport function replaceExtension(url: string, newExtension: string): string {\n const baseName = basename(url);\n const extension = extname(url);\n const isUpperCase = extension === extension.toUpperCase();\n if (isUpperCase) {\n newExtension = newExtension.toUpperCase();\n }\n return `${baseName}.${newExtension}`;\n}\n\n// NOTE - this gives the entire path minus extension (i.e. NOT same as path.basename)\n/**\n * @param url\n * @returns string\n */\nfunction basename(url: string): string {\n const extIndex = url && url.lastIndexOf('.');\n if (typeof extIndex === 'number') {\n return extIndex >= 0 ? url.substr(0, extIndex) : '';\n }\n return extIndex;\n}\n/**\n * @param url\n * @returns string\n */\nfunction extname(url: string): string {\n const extIndex = url && url.lastIndexOf('.');\n if (typeof extIndex === 'number') {\n return extIndex >= 0 ? url.substr(extIndex + 1) : '';\n }\n return extIndex;\n}\n"],"file":"parse-shapefile.js"}
|