@nshiab/simple-data-analysis-core 0.0.27 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/class/SimpleDB.d.ts +3 -8
- package/esm/class/SimpleDB.d.ts.map +1 -1
- package/esm/class/SimpleDB.js +8 -10
- package/esm/class/SimpleTable.d.ts +29 -48
- package/esm/class/SimpleTable.d.ts.map +1 -1
- package/esm/class/SimpleTable.js +87 -179
- package/esm/helpers/convertForJS.js +1 -1
- package/esm/helpers/findGeoColumn.js +2 -2
- package/esm/helpers/hasGeometryColumn.js +1 -1
- package/esm/helpers/logData.d.ts.map +1 -1
- package/esm/helpers/logData.js +7 -2
- package/esm/helpers/parseTypes.d.ts +1 -1
- package/esm/helpers/parseTypes.d.ts.map +1 -1
- package/esm/helpers/parseTypes.js +3 -1
- package/esm/helpers/setDbProps.d.ts +1 -1
- package/esm/helpers/setDbProps.d.ts.map +1 -1
- package/esm/helpers/setDbProps.js +5 -9
- package/esm/helpers/unifyColumns.d.ts +0 -2
- package/esm/helpers/unifyColumns.d.ts.map +1 -1
- package/esm/helpers/unifyColumns.js +2 -6
- package/esm/helpers/writeGeoData.d.ts.map +1 -1
- package/esm/helpers/writeGeoData.js +11 -41
- package/esm/helpers/writeIndexes.d.ts +3 -0
- package/esm/helpers/writeIndexes.d.ts.map +1 -0
- package/esm/helpers/writeIndexes.js +16 -0
- package/esm/methods/bm25.d.ts.map +1 -1
- package/esm/methods/bm25.js +1 -1
- package/esm/methods/cache.js +1 -1
- package/esm/methods/distanceQuery.d.ts.map +1 -1
- package/esm/methods/distanceQuery.js +4 -2
- package/esm/methods/fuzzyJoin.d.ts.map +1 -1
- package/esm/methods/fuzzyJoin.js +1 -6
- package/esm/methods/getGeoData.d.ts.map +1 -1
- package/esm/methods/getGeoData.js +1 -10
- package/esm/methods/join.d.ts.map +1 -1
- package/esm/methods/join.js +1 -7
- package/esm/methods/joinGeo.d.ts.map +1 -1
- package/esm/methods/joinGeo.js +1 -14
- package/esm/methods/joinGeoQuery.js +1 -1
- package/esm/methods/summarizeQuery.js +1 -1
- package/esm/methods/writeGeoDataQuery.js +2 -2
- package/package.json +2 -2
- package/script/class/SimpleDB.d.ts +3 -8
- package/script/class/SimpleDB.d.ts.map +1 -1
- package/script/class/SimpleDB.js +8 -10
- package/script/class/SimpleTable.d.ts +29 -48
- package/script/class/SimpleTable.d.ts.map +1 -1
- package/script/class/SimpleTable.js +87 -179
- package/script/helpers/convertForJS.js +1 -1
- package/script/helpers/findGeoColumn.js +2 -2
- package/script/helpers/hasGeometryColumn.js +1 -1
- package/script/helpers/logData.d.ts.map +1 -1
- package/script/helpers/logData.js +7 -2
- package/script/helpers/parseTypes.d.ts +1 -1
- package/script/helpers/parseTypes.d.ts.map +1 -1
- package/script/helpers/parseTypes.js +3 -1
- package/script/helpers/setDbProps.d.ts +1 -1
- package/script/helpers/setDbProps.d.ts.map +1 -1
- package/script/helpers/setDbProps.js +5 -9
- package/script/helpers/unifyColumns.d.ts +0 -2
- package/script/helpers/unifyColumns.d.ts.map +1 -1
- package/script/helpers/unifyColumns.js +2 -6
- package/script/helpers/writeGeoData.d.ts.map +1 -1
- package/script/helpers/writeGeoData.js +11 -41
- package/script/helpers/writeIndexes.d.ts +3 -0
- package/script/helpers/writeIndexes.d.ts.map +1 -0
- package/script/helpers/writeIndexes.js +19 -0
- package/script/methods/bm25.d.ts.map +1 -1
- package/script/methods/bm25.js +1 -1
- package/script/methods/cache.js +1 -1
- package/script/methods/distanceQuery.d.ts.map +1 -1
- package/script/methods/distanceQuery.js +4 -2
- package/script/methods/fuzzyJoin.d.ts.map +1 -1
- package/script/methods/fuzzyJoin.js +1 -6
- package/script/methods/getGeoData.d.ts.map +1 -1
- package/script/methods/getGeoData.js +1 -10
- package/script/methods/join.d.ts.map +1 -1
- package/script/methods/join.js +1 -7
- package/script/methods/joinGeo.d.ts.map +1 -1
- package/script/methods/joinGeo.js +1 -14
- package/script/methods/joinGeoQuery.js +1 -1
- package/script/methods/summarizeQuery.js +1 -1
- package/script/methods/writeGeoDataQuery.js +2 -2
- package/esm/helpers/getProjection.d.ts +0 -3
- package/esm/helpers/getProjection.d.ts.map +0 -1
- package/esm/helpers/getProjection.js +0 -25
- package/esm/helpers/getProjectionParquet.d.ts +0 -3
- package/esm/helpers/getProjectionParquet.d.ts.map +0 -1
- package/esm/helpers/getProjectionParquet.js +0 -24
- package/esm/helpers/shouldFlipBeforeExport.d.ts +0 -2
- package/esm/helpers/shouldFlipBeforeExport.d.ts.map +0 -1
- package/esm/helpers/shouldFlipBeforeExport.js +0 -3
- package/esm/helpers/writeProjectionsAndIndexes.d.ts +0 -3
- package/esm/helpers/writeProjectionsAndIndexes.d.ts.map +0 -1
- package/esm/helpers/writeProjectionsAndIndexes.js +0 -29
- package/script/helpers/getProjection.d.ts +0 -3
- package/script/helpers/getProjection.d.ts.map +0 -1
- package/script/helpers/getProjection.js +0 -31
- package/script/helpers/getProjectionParquet.d.ts +0 -3
- package/script/helpers/getProjectionParquet.d.ts.map +0 -1
- package/script/helpers/getProjectionParquet.js +0 -30
- package/script/helpers/shouldFlipBeforeExport.d.ts +0 -2
- package/script/helpers/shouldFlipBeforeExport.d.ts.map +0 -1
- package/script/helpers/shouldFlipBeforeExport.js +0 -6
- package/script/helpers/writeProjectionsAndIndexes.d.ts +0 -3
- package/script/helpers/writeProjectionsAndIndexes.d.ts.map +0 -1
- package/script/helpers/writeProjectionsAndIndexes.js +0 -32
|
@@ -27,7 +27,7 @@ function convertForJS(rows, types) {
|
|
|
27
27
|
row[key] = row[key] === null ? null : Number(row[key]);
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
-
else if (types[key]
|
|
30
|
+
else if (types[key].toLowerCase().includes("geometry")) {
|
|
31
31
|
for (const row of rows) {
|
|
32
32
|
row[key] = row[key] === null ? null : "<Geometry>";
|
|
33
33
|
}
|
|
@@ -4,7 +4,7 @@ exports.default = findGeoColumn;
|
|
|
4
4
|
async function findGeoColumn(SimpleTable) {
|
|
5
5
|
let column;
|
|
6
6
|
const types = await SimpleTable.getTypes();
|
|
7
|
-
const geometries = Object.values(types).filter((d) => d.toLowerCase()
|
|
7
|
+
const geometries = Object.values(types).filter((d) => d.toLowerCase().includes("geometry"));
|
|
8
8
|
if (geometries.length === 0) {
|
|
9
9
|
throw new Error("Table contains no geometry columns.");
|
|
10
10
|
}
|
|
@@ -12,7 +12,7 @@ async function findGeoColumn(SimpleTable) {
|
|
|
12
12
|
throw new Error("More than one column storing geometries. If the method allows to specify one, do it. Otherwise, use the selectColumns methods beforehand.");
|
|
13
13
|
}
|
|
14
14
|
else {
|
|
15
|
-
column = Object.keys(types).find((d) => types[d].toLowerCase()
|
|
15
|
+
column = Object.keys(types).find((d) => types[d].toLowerCase().includes("geometry"));
|
|
16
16
|
}
|
|
17
17
|
if (typeof column !== "string") {
|
|
18
18
|
throw new Error("No column");
|
|
@@ -20,5 +20,5 @@ exports.default = hasGeometryColumn;
|
|
|
20
20
|
*/
|
|
21
21
|
async function hasGeometryColumn(table) {
|
|
22
22
|
const types = await table.getTypes();
|
|
23
|
-
return Object.values(types).some((t) => t.toLowerCase()
|
|
23
|
+
return Object.values(types).some((t) => t.toLowerCase().includes("geometry"));
|
|
24
24
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logData.d.ts","sourceRoot":"","sources":["../../src/helpers/logData.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GAAG,IAAI,EACvC,IAAI,EACA;IACA,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC;CACxD,EAAE,GACD,IAAI,EACR,iBAAiB,CAAC,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"logData.d.ts","sourceRoot":"","sources":["../../src/helpers/logData.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GAAG,IAAI,EACvC,IAAI,EACA;IACA,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC;CACxD,EAAE,GACD,IAAI,EACR,iBAAiB,CAAC,EAAE,MAAM,QA4D3B"}
|
|
@@ -35,8 +35,13 @@ function logData(types, data, nbCharactersToLog) {
|
|
|
35
35
|
const columns = Object.keys(types);
|
|
36
36
|
if (columns.length > 0) {
|
|
37
37
|
for (const col of columns) {
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
if (data[0][col] === "<Geometry>") {
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
types[col] = types[col] + "/" +
|
|
43
|
+
(data[0][col] === null ? null : typeof data[0][col]);
|
|
44
|
+
}
|
|
40
45
|
}
|
|
41
46
|
// Add types as the first row
|
|
42
47
|
dataToBeLogged.unshift(types);
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export default function parseType(type: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" |
|
|
1
|
+
export default function parseType(type: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | `geometry('${string}')` | `GEOMETRY('${string}')`): string;
|
|
2
2
|
//# sourceMappingURL=parseTypes.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parseTypes.d.ts","sourceRoot":"","sources":["../../src/helpers/parseTypes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,IAAI,EACA,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,WAAW,GACX,0BAA0B,GAC1B,SAAS,GACT,
|
|
1
|
+
{"version":3,"file":"parseTypes.d.ts","sourceRoot":"","sources":["../../src/helpers/parseTypes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,IAAI,EACA,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,WAAW,GACX,0BAA0B,GAC1B,SAAS,GACT,aAAa,MAAM,IAAI,GACvB,aAAa,MAAM,IAAI,UAgC5B"}
|
|
@@ -28,10 +28,12 @@ function parseType(type) {
|
|
|
28
28
|
"timestamp",
|
|
29
29
|
"timestamp with time zone",
|
|
30
30
|
"boolean",
|
|
31
|
-
"geometry",
|
|
32
31
|
].includes(typeLowerCase)) {
|
|
33
32
|
return type.toUpperCase();
|
|
34
33
|
}
|
|
34
|
+
else if (typeLowerCase.includes("geometry")) {
|
|
35
|
+
return type.toUpperCase();
|
|
36
|
+
}
|
|
35
37
|
else {
|
|
36
38
|
throw new Error(`Unknown type ${type}`);
|
|
37
39
|
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type SimpleDB from "../class/SimpleDB.js";
|
|
2
|
-
export default function setDbProps(simpleDB: SimpleDB,
|
|
2
|
+
export default function setDbProps(simpleDB: SimpleDB, allIndexesFile: string): Promise<void>;
|
|
3
3
|
//# sourceMappingURL=setDbProps.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"setDbProps.d.ts","sourceRoot":"","sources":["../../src/helpers/setDbProps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAEjD,wBAA8B,UAAU,CACtC,QAAQ,EAAE,QAAQ,EAClB,
|
|
1
|
+
{"version":3,"file":"setDbProps.d.ts","sourceRoot":"","sources":["../../src/helpers/setDbProps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAEjD,wBAA8B,UAAU,CACtC,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,iBA2BvB"}
|
|
@@ -2,19 +2,15 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.default = setDbProps;
|
|
4
4
|
const node_fs_1 = require("node:fs");
|
|
5
|
-
async function setDbProps(simpleDB,
|
|
5
|
+
async function setDbProps(simpleDB, allIndexesFile) {
|
|
6
6
|
for (const table of await simpleDB.getTableNames()) {
|
|
7
7
|
simpleDB.newTable(table);
|
|
8
8
|
}
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
if (projections[table.name]) {
|
|
14
|
-
table.projections = projections[table.name];
|
|
15
|
-
}
|
|
9
|
+
for (const table of await simpleDB.getTables()) {
|
|
10
|
+
const types = await table.getTypes();
|
|
11
|
+
if (Object.values(types).some((type) => type.toLowerCase().includes("geometry"))) {
|
|
12
|
+
await simpleDB.customQuery(`INSTALL SPATIAL; LOAD SPATIAL;`);
|
|
16
13
|
}
|
|
17
|
-
await simpleDB.customQuery(`INSTALL spatial; LOAD spatial;`);
|
|
18
14
|
}
|
|
19
15
|
if ((0, node_fs_1.existsSync)(allIndexesFile)) {
|
|
20
16
|
const indexes = JSON.parse((0, node_fs_1.readFileSync)(allIndexesFile, "utf-8"));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unifyColumns.d.ts","sourceRoot":"","sources":["../../src/helpers/unifyColumns.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,YAAY,CACxC,SAAS,EAAE,WAAW,EAAE,EACxB,QAAQ,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE
|
|
1
|
+
{"version":3,"file":"unifyColumns.d.ts","sourceRoot":"","sources":["../../src/helpers/unifyColumns.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,YAAY,CACxC,SAAS,EAAE,WAAW,EAAE,EACxB,QAAQ,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE;;GAwCpC"}
|
|
@@ -1,18 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.default = unifyColumns;
|
|
4
|
-
async function unifyColumns(allTables, allTypes
|
|
4
|
+
async function unifyColumns(allTables, allTypes) {
|
|
5
5
|
const columnsAdded = {};
|
|
6
6
|
for (const column in allTypes) {
|
|
7
7
|
for (const table of allTables) {
|
|
8
8
|
if (!(await table.hasColumn(column))) {
|
|
9
9
|
await table.addColumn(column,
|
|
10
10
|
// Could be improved
|
|
11
|
-
allTypes[column].toLowerCase(), `null
|
|
12
|
-
projection: allTypes[column] === "GEOMETRY"
|
|
13
|
-
? allProjections[column]
|
|
14
|
-
: undefined,
|
|
15
|
-
});
|
|
11
|
+
allTypes[column].toLowerCase(), `null`);
|
|
16
12
|
if (!columnsAdded[table.name]) {
|
|
17
13
|
columnsAdded[table.name] = [];
|
|
18
14
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writeGeoData.d.ts","sourceRoot":"","sources":["../../src/helpers/writeGeoData.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"writeGeoData.d.ts","sourceRoot":"","sources":["../../src/helpers/writeGeoData.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,YAAY,CACxC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CAClB,GACL,OAAO,CAAC,IAAI,CAAC,CAmGf"}
|
|
@@ -7,9 +7,7 @@ exports.default = writeGeoData;
|
|
|
7
7
|
const node_fs_1 = require("node:fs");
|
|
8
8
|
const createDirectory_js_1 = __importDefault(require("./createDirectory.js"));
|
|
9
9
|
const getExtension_js_1 = __importDefault(require("./getExtension.js"));
|
|
10
|
-
const findGeoColumn_js_1 = __importDefault(require("./findGeoColumn.js"));
|
|
11
10
|
const hasGeometryColumn_js_1 = __importDefault(require("./hasGeometryColumn.js"));
|
|
12
|
-
const shouldFlipBeforeExport_js_1 = __importDefault(require("./shouldFlipBeforeExport.js"));
|
|
13
11
|
const queryDB_js_1 = __importDefault(require("./queryDB.js"));
|
|
14
12
|
const writeGeoDataQuery_js_1 = __importDefault(require("../methods/writeGeoDataQuery.js"));
|
|
15
13
|
const mergeOptions_js_1 = __importDefault(require("./mergeOptions.js"));
|
|
@@ -35,24 +33,11 @@ async function writeGeoData(table, file, options = {}) {
|
|
|
35
33
|
if (typeof options.compression === "boolean") {
|
|
36
34
|
throw new Error("The compression option is not supported for writing GeoJSON files.");
|
|
37
35
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
table: table.name,
|
|
44
|
-
method: "writeGeoData()",
|
|
45
|
-
parameters: { file, options },
|
|
46
|
-
}));
|
|
47
|
-
await table.flipCoordinates(geoColumn);
|
|
48
|
-
}
|
|
49
|
-
else {
|
|
50
|
-
await (0, queryDB_js_1.default)(table, (0, writeGeoDataQuery_js_1.default)(table.name, file, fileExtension, options), (0, mergeOptions_js_1.default)(table, {
|
|
51
|
-
table: table.name,
|
|
52
|
-
method: "writeGeoData()",
|
|
53
|
-
parameters: { file, options },
|
|
54
|
-
}));
|
|
55
|
-
}
|
|
36
|
+
await (0, queryDB_js_1.default)(table, (0, writeGeoDataQuery_js_1.default)(table.name, file, fileExtension, options), (0, mergeOptions_js_1.default)(table, {
|
|
37
|
+
table: table.name,
|
|
38
|
+
method: "writeGeoData()",
|
|
39
|
+
parameters: { file, options },
|
|
40
|
+
}));
|
|
56
41
|
if (options.metadata) {
|
|
57
42
|
const fileData = JSON.parse((0, node_fs_1.readFileSync)(file, "utf-8"));
|
|
58
43
|
fileData.metadata = options.metadata;
|
|
@@ -76,24 +61,11 @@ async function writeGeoData(table, file, options = {}) {
|
|
|
76
61
|
options.formatDates === true) {
|
|
77
62
|
throw new Error("The following options are not supported for writing SHAPEFILE files: precision, compression, rewind, metadata, and formatDates.");
|
|
78
63
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
table: table.name,
|
|
85
|
-
method: "writeGeoData()",
|
|
86
|
-
parameters: { file, options },
|
|
87
|
-
}));
|
|
88
|
-
await table.flipCoordinates(geoColumn);
|
|
89
|
-
}
|
|
90
|
-
else {
|
|
91
|
-
await (0, queryDB_js_1.default)(table, (0, writeGeoDataQuery_js_1.default)(table.name, file, fileExtension, options), (0, mergeOptions_js_1.default)(table, {
|
|
92
|
-
table: table.name,
|
|
93
|
-
method: "writeGeoData()",
|
|
94
|
-
parameters: { file, options },
|
|
95
|
-
}));
|
|
96
|
-
}
|
|
64
|
+
await (0, queryDB_js_1.default)(table, (0, writeGeoDataQuery_js_1.default)(table.name, file, fileExtension, options), (0, mergeOptions_js_1.default)(table, {
|
|
65
|
+
table: table.name,
|
|
66
|
+
method: "writeGeoData()",
|
|
67
|
+
parameters: { file, options },
|
|
68
|
+
}));
|
|
97
69
|
}
|
|
98
70
|
else if (fileExtension === "geoparquet") {
|
|
99
71
|
if (typeof options.precision === "number") {
|
|
@@ -102,9 +74,7 @@ async function writeGeoData(table, file, options = {}) {
|
|
|
102
74
|
if (typeof options.rewind === "boolean") {
|
|
103
75
|
throw new Error("The rewind option is not supported for writing PARQUET files.");
|
|
104
76
|
}
|
|
105
|
-
await (0, queryDB_js_1.default)(table, `COPY "${table.name}" TO '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT PARQUET${options.compression === true ? ", COMPRESSION 'zstd'" : ""},
|
|
106
|
-
projections: '${JSON.stringify(table.projections)}'
|
|
107
|
-
});`, (0, mergeOptions_js_1.default)(table, {
|
|
77
|
+
await (0, queryDB_js_1.default)(table, `COPY "${table.name}" TO '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT PARQUET${options.compression === true ? ", COMPRESSION 'zstd'" : ""});`, (0, mergeOptions_js_1.default)(table, {
|
|
108
78
|
table: table.name,
|
|
109
79
|
method: "writeGeoData()",
|
|
110
80
|
parameters: { file, options },
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"writeIndexes.d.ts","sourceRoot":"","sources":["../../src/helpers/writeIndexes.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,CAAC,OAAO,UAAU,YAAY,CAClC,QAAQ,EAAE,QAAQ,EAClB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,QAeb"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.default = writeIndexes;
|
|
4
|
+
const node_fs_1 = require("node:fs");
|
|
5
|
+
function writeIndexes(simpleDB, extension, file) {
|
|
6
|
+
const allIndexes = {};
|
|
7
|
+
for (const table of simpleDB.tables) {
|
|
8
|
+
if (table.indexes.length > 0) {
|
|
9
|
+
allIndexes[table.name] = table.indexes;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
const allIndexesFile = `${file.replace(`.${extension}`, "")}_indexes.json`;
|
|
13
|
+
if ((0, node_fs_1.existsSync)(allIndexesFile)) {
|
|
14
|
+
(0, node_fs_1.rmSync)(allIndexesFile);
|
|
15
|
+
}
|
|
16
|
+
if (Object.keys(allIndexes).length > 0) {
|
|
17
|
+
(0, node_fs_1.writeFileSync)(allIndexesFile, JSON.stringify(allIndexes));
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/methods/bm25.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,IAAI,CAChC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,
|
|
1
|
+
{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/methods/bm25.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,IAAI,CAChC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,wBA+DP"}
|
package/script/methods/bm25.js
CHANGED
|
@@ -46,7 +46,7 @@ async function bm25(simpleTable, text, columnId, columnText, nbResults, options
|
|
|
46
46
|
},
|
|
47
47
|
}));
|
|
48
48
|
if (typeof options.outputTable === "string") {
|
|
49
|
-
return simpleTable.sdb.newTable(options.outputTable
|
|
49
|
+
return simpleTable.sdb.newTable(options.outputTable);
|
|
50
50
|
}
|
|
51
51
|
else {
|
|
52
52
|
return simpleTable;
|
package/script/methods/cache.js
CHANGED
|
@@ -97,7 +97,7 @@ async function runAndWrite(table, run, cacheSources, cacheSourcesPath, cachePath
|
|
|
97
97
|
}
|
|
98
98
|
else {
|
|
99
99
|
const types = await table.getTypes();
|
|
100
|
-
const geometriesColumns = Object.values(types).filter((d) => d
|
|
100
|
+
const geometriesColumns = Object.values(types).filter((d) => d.toLowerCase().includes("geometry")).length;
|
|
101
101
|
if (geometriesColumns > 0) {
|
|
102
102
|
const file = `${cachePath}/${id}.geoparquet`;
|
|
103
103
|
const writeStart = Date.now();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"distanceQuery.d.ts","sourceRoot":"","sources":["../../src/methods/distanceQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,aAAa,CACnC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC;IAClB,MAAM,CAAC,EAAE,KAAK,GAAG,UAAU,GAAG,WAAW,CAAC;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;CACd,
|
|
1
|
+
{"version":3,"file":"distanceQuery.d.ts","sourceRoot":"","sources":["../../src/methods/distanceQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,aAAa,CACnC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC;IAClB,MAAM,CAAC,EAAE,KAAK,GAAG,UAAU,GAAG,WAAW,CAAC;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;CACd,UAwDP"}
|
|
@@ -32,10 +32,12 @@ function distanceQuery(table, column1, column2, newColumn, options = {}) {
|
|
|
32
32
|
}
|
|
33
33
|
else if (options.method === "spheroid") {
|
|
34
34
|
if (typeof options.decimals === "number") {
|
|
35
|
-
query +=
|
|
35
|
+
query +=
|
|
36
|
+
`ROUND(ST_Distance_Spheroid("${column1}"::GEOMETRY, "${column2}"::GEOMETRY) ${options.unit === "km" ? "/ 1000" : ""}, ${options.decimals});`;
|
|
36
37
|
}
|
|
37
38
|
else {
|
|
38
|
-
query +=
|
|
39
|
+
query +=
|
|
40
|
+
`ST_Distance_Spheroid("${column1}"::GEOMETRY, "${column2}"::GEOMETRY) ${options.unit === "km" ? "/ 1000" : ""};`;
|
|
39
41
|
}
|
|
40
42
|
}
|
|
41
43
|
else {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,
|
|
1
|
+
{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,wBAiFP"}
|
|
@@ -44,14 +44,9 @@ async function fuzzyJoin(leftTable, rightTable, leftColumn, rightColumn, thresho
|
|
|
44
44
|
options,
|
|
45
45
|
},
|
|
46
46
|
}));
|
|
47
|
-
const allProjections = {
|
|
48
|
-
...leftTable.projections,
|
|
49
|
-
...rightTable.projections,
|
|
50
|
-
};
|
|
51
47
|
const outputTable = typeof options.outputTable === "string"
|
|
52
|
-
? leftTable.sdb.newTable(options.outputTable
|
|
48
|
+
? leftTable.sdb.newTable(options.outputTable)
|
|
53
49
|
: leftTable;
|
|
54
|
-
outputTable.projections = allProjections;
|
|
55
50
|
// Remove the duplicate right-column produced when leftColumn === rightColumn
|
|
56
51
|
// (DuckDB suffixes it with _1 in SELECT *)
|
|
57
52
|
const outputCols = await outputTable.getColumns();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getGeoData.d.ts","sourceRoot":"","sources":["../../src/methods/getGeoData.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"getGeoData.d.ts","sourceRoot":"","sources":["../../src/methods/getGeoData.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,UAAU,CACtC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO;;;;;kBAuBtB,MAAM;yBACC,OAAO,EAAE;;;;;;GAgC7B"}
|
|
@@ -7,17 +7,8 @@ exports.default = getGeoData;
|
|
|
7
7
|
const rewind_js_1 = __importDefault(require("../helpers/rewind.js"));
|
|
8
8
|
const mergeOptions_js_1 = __importDefault(require("../helpers/mergeOptions.js"));
|
|
9
9
|
const queryDB_js_1 = __importDefault(require("../helpers/queryDB.js"));
|
|
10
|
-
const shouldFlipBeforeExport_js_1 = __importDefault(require("../helpers/shouldFlipBeforeExport.js"));
|
|
11
10
|
async function getGeoData(SimpleTable, column, options = {}) {
|
|
12
|
-
|
|
13
|
-
if ((0, shouldFlipBeforeExport_js_1.default)(SimpleTable.projections[column])) {
|
|
14
|
-
query =
|
|
15
|
-
`SELECT * EXCLUDE ${column}, ST_AsGeoJSON(ST_FlipCoordinates(${column})) as geoJsonFragment from "${SimpleTable.name}";`;
|
|
16
|
-
}
|
|
17
|
-
else {
|
|
18
|
-
query =
|
|
19
|
-
`SELECT * EXCLUDE ${column}, ST_AsGeoJSON(${column}) as geoJsonFragment from "${SimpleTable.name}";`;
|
|
20
|
-
}
|
|
11
|
+
const query = `SELECT * EXCLUDE ${column}, ST_AsGeoJSON(${column}) as geoJsonFragment from "${SimpleTable.name}";`;
|
|
21
12
|
const queryResult = await (0, queryDB_js_1.default)(SimpleTable, query, (0, mergeOptions_js_1.default)(SimpleTable, {
|
|
22
13
|
table: null,
|
|
23
14
|
method: "getGeoData()",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"join.d.ts","sourceRoot":"","sources":["../../src/methods/join.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,IAAI,CAChC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,
|
|
1
|
+
{"version":3,"file":"join.d.ts","sourceRoot":"","sources":["../../src/methods/join.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,IAAI,CAChC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,wBAqFP"}
|
package/script/methods/join.js
CHANGED
|
@@ -52,14 +52,8 @@ async function join(leftTable, rightTable, options = {}) {
|
|
|
52
52
|
},
|
|
53
53
|
}));
|
|
54
54
|
const outputTable = typeof options.outputTable === "string"
|
|
55
|
-
? leftTable.sdb.newTable(options.outputTable
|
|
55
|
+
? leftTable.sdb.newTable(options.outputTable)
|
|
56
56
|
: leftTable;
|
|
57
|
-
// So we reassign here
|
|
58
|
-
const allProjections = {
|
|
59
|
-
...leftTable.projections,
|
|
60
|
-
...rightTable.projections,
|
|
61
|
-
};
|
|
62
|
-
outputTable.projections = allProjections;
|
|
63
57
|
const columns = await outputTable.getColumns();
|
|
64
58
|
const extraCommonColumns = columns.filter((d) => commonColumn.map((c) => `${c}_1`).includes(d));
|
|
65
59
|
if (extraCommonColumns.length > 0) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"joinGeo.d.ts","sourceRoot":"","sources":["../../src/methods/joinGeo.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAOvD,wBAA8B,OAAO,CACnC,SAAS,EAAE,WAAW,EACtB,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,QAAQ,EACzC,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,UAAU,CAAC;IAClD,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,
|
|
1
|
+
{"version":3,"file":"joinGeo.d.ts","sourceRoot":"","sources":["../../src/methods/joinGeo.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAOvD,wBAA8B,OAAO,CACnC,SAAS,EAAE,WAAW,EACtB,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,QAAQ,EACzC,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,UAAU,CAAC;IAClD,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,wBA4FP"}
|
|
@@ -52,31 +52,18 @@ async function joinGeo(leftTable, method, rightTable, options = {}) {
|
|
|
52
52
|
},
|
|
53
53
|
}));
|
|
54
54
|
// We bring back the column names for geometries
|
|
55
|
-
let allProjections = {};
|
|
56
55
|
if (leftTableColumn === rightTableColumn) {
|
|
57
56
|
const leftObj = {};
|
|
58
57
|
leftObj[leftTableColumnForQuery] = leftTableColumn;
|
|
59
58
|
await leftTable.renameColumns(leftObj);
|
|
60
|
-
// Before renaming columns in original tables
|
|
61
|
-
allProjections = {
|
|
62
|
-
...leftTable.projections,
|
|
63
|
-
...rightTable.projections,
|
|
64
|
-
};
|
|
65
59
|
const rightObj = {};
|
|
66
60
|
rightObj[rightTableColumnForQuery] = rightTableColumn;
|
|
67
61
|
await rightTable.renameColumns(rightObj);
|
|
68
62
|
}
|
|
69
|
-
else {
|
|
70
|
-
allProjections = {
|
|
71
|
-
...leftTable.projections,
|
|
72
|
-
...rightTable.projections,
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
63
|
if (typeof options.outputTable === "string") {
|
|
76
|
-
return leftTable.sdb.newTable(options.outputTable
|
|
64
|
+
return leftTable.sdb.newTable(options.outputTable);
|
|
77
65
|
}
|
|
78
66
|
else {
|
|
79
|
-
leftTable.projections = allProjections;
|
|
80
67
|
return leftTable;
|
|
81
68
|
}
|
|
82
69
|
}
|
|
@@ -41,7 +41,7 @@ function joinGeoQuery(leftTable, leftTableColumn, method, rightTable, rightTable
|
|
|
41
41
|
else if (distanceMethod === "spheroid") {
|
|
42
42
|
// Should be using ST_DWithin_Spheroid but doesn't work?
|
|
43
43
|
query +=
|
|
44
|
-
` ON ST_Distance_Spheroid("${leftTable}"."${leftTableColumn}", "${rightTable}"."${rightTableColumn}") < ${distance}`;
|
|
44
|
+
` ON ST_Distance_Spheroid("${leftTable}"."${leftTableColumn}"::GEOMETRY, "${rightTable}"."${rightTableColumn}"::GEOMETRY) < ${distance}`;
|
|
45
45
|
}
|
|
46
46
|
else {
|
|
47
47
|
throw new Error(`Unknown ${distanceMethod}`);
|
|
@@ -49,7 +49,7 @@ function summarizeQuery(table, types, outputTable, values, categories, summaries
|
|
|
49
49
|
aggregates[summary] !== "count") {
|
|
50
50
|
return `\nNULL as '${columns ? columns[i] : summary}'`;
|
|
51
51
|
}
|
|
52
|
-
else if (types[value]
|
|
52
|
+
else if (types[value].toLowerCase().includes("geometry")) {
|
|
53
53
|
return `\nNULL AS '${columns ? columns[i] : summary}'`;
|
|
54
54
|
}
|
|
55
55
|
else if (types[value] === "VARCHAR" &&
|
|
@@ -12,12 +12,12 @@ function writeGeoDataQuery(table, file, fileExtension, options = {}) {
|
|
|
12
12
|
layerOptions.push(`COORDINATE_PRECISION=${options.precision}`);
|
|
13
13
|
}
|
|
14
14
|
layerOptions.push(`RFC7946=YES`);
|
|
15
|
-
return `INSTALL spatial; LOAD spatial; COPY "${table}" to '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${layerOptions.length > 0
|
|
15
|
+
return `INSTALL spatial; LOAD spatial; SET geometry_always_xy = true; COPY "${table}" to '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${layerOptions.length > 0
|
|
16
16
|
? `, LAYER_CREATION_OPTIONS ('WRITE_NAME=NO', ${layerOptions.map((d) => `'${d}'`).join(", ")})`
|
|
17
17
|
: ""})`;
|
|
18
18
|
}
|
|
19
19
|
else if (fileExtension === "shp") {
|
|
20
|
-
return `INSTALL spatial; LOAD spatial; COPY "${table}" TO '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT GDAL, DRIVER 'ESRI Shapefile', LAYER_CREATION_OPTIONS 'ENCODING=UTF-8')`;
|
|
20
|
+
return `INSTALL spatial; LOAD spatial; SET geometry_always_xy = true; COPY "${table}" TO '${(0, cleanPath_js_1.default)(file)}' WITH (FORMAT GDAL, DRIVER 'ESRI Shapefile', LAYER_CREATION_OPTIONS 'ENCODING=UTF-8')`;
|
|
21
21
|
}
|
|
22
22
|
else {
|
|
23
23
|
throw new Error(`Unknown extension ${fileExtension}`);
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"getProjection.d.ts","sourceRoot":"","sources":["../../src/helpers/getProjection.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAGjD,wBAA8B,aAAa,CACzC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,mBAiCb"}
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
import mergeOptions from "./mergeOptions.js";
|
|
2
|
-
import queryDB from "./queryDB.js";
|
|
3
|
-
import cleanPath from "./cleanPath.js";
|
|
4
|
-
export default async function getProjection(simpleDB, file) {
|
|
5
|
-
// Load spatial may not be necessary if we change how this works
|
|
6
|
-
const queryResult = await queryDB(simpleDB, `INSTALL spatial;
|
|
7
|
-
LOAD spatial;
|
|
8
|
-
SELECT layers[1].geometry_fields[1].crs.proj4 as proj4 FROM st_read_meta('${cleanPath(file)}')`, mergeOptions(simpleDB, {
|
|
9
|
-
table: null,
|
|
10
|
-
method: "getProjection()",
|
|
11
|
-
parameters: { file },
|
|
12
|
-
returnDataFrom: "query",
|
|
13
|
-
}));
|
|
14
|
-
if (!queryResult) {
|
|
15
|
-
throw new Error("No queryResults");
|
|
16
|
-
}
|
|
17
|
-
const proj4 = queryResult[0].proj4;
|
|
18
|
-
if (proj4 === null) {
|
|
19
|
-
return "UNKNOWN";
|
|
20
|
-
}
|
|
21
|
-
if (typeof proj4 !== "string") {
|
|
22
|
-
throw new Error(`Expected proj4 to be a string, got ${typeof proj4}`);
|
|
23
|
-
}
|
|
24
|
-
return proj4;
|
|
25
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"getProjectionParquet.d.ts","sourceRoot":"","sources":["../../src/helpers/getProjectionParquet.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAKvD,wBAA8B,oBAAoB,CAChD,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,gBA+Bb"}
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import cleanPath from "./cleanPath.js";
|
|
2
|
-
import mergeOptions from "./mergeOptions.js";
|
|
3
|
-
import queryDB from "./queryDB.js";
|
|
4
|
-
export default async function getProjectionParquet(SimpleTable, file) {
|
|
5
|
-
const queryResult = await queryDB(SimpleTable, `SELECT * FROM parquet_kv_metadata('${cleanPath(file)}');`, mergeOptions(SimpleTable, {
|
|
6
|
-
table: SimpleTable.name,
|
|
7
|
-
method: "getProjectionParquet()",
|
|
8
|
-
parameters: { file },
|
|
9
|
-
returnDataFrom: "query",
|
|
10
|
-
}));
|
|
11
|
-
if (!queryResult) {
|
|
12
|
-
throw new Error(`Could not get metadata from parquet file: ${file}`);
|
|
13
|
-
}
|
|
14
|
-
const projection = queryResult.find((d) => {
|
|
15
|
-
return d.key?.toString() === "projections";
|
|
16
|
-
});
|
|
17
|
-
if (!projection || projection.value === null) {
|
|
18
|
-
console.warn(`\nCould not get projection from parquet file: ${file}\n`);
|
|
19
|
-
return {};
|
|
20
|
-
}
|
|
21
|
-
else {
|
|
22
|
-
return JSON.parse(projection.value.replaceAll("\\x22", '"'));
|
|
23
|
-
}
|
|
24
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"shouldFlipBeforeExport.d.ts","sourceRoot":"","sources":["../../src/helpers/shouldFlipBeforeExport.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,sBAAsB,CAAC,UAAU,EAAE,MAAM,WAEhE"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"writeProjectionsAndIndexes.d.ts","sourceRoot":"","sources":["../../src/helpers/writeProjectionsAndIndexes.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,CAAC,OAAO,UAAU,0BAA0B,CAChD,QAAQ,EAAE,QAAQ,EAClB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,QA+Bb"}
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import { existsSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
-
export default function writeProjectionsAndIndexes(simpleDB, extension, file) {
|
|
3
|
-
const allProjections = {};
|
|
4
|
-
for (const table of simpleDB.tables) {
|
|
5
|
-
if (Object.keys(table.projections).length > 0) {
|
|
6
|
-
allProjections[table.name] = table.projections;
|
|
7
|
-
}
|
|
8
|
-
}
|
|
9
|
-
const allProjectionsFile = `${file.replace(`.${extension}`, "")}_projections.json`;
|
|
10
|
-
if (existsSync(allProjectionsFile)) {
|
|
11
|
-
rmSync(allProjectionsFile);
|
|
12
|
-
}
|
|
13
|
-
if (Object.keys(allProjections).length > 0) {
|
|
14
|
-
writeFileSync(allProjectionsFile, JSON.stringify(allProjections));
|
|
15
|
-
}
|
|
16
|
-
const allIndexes = {};
|
|
17
|
-
for (const table of simpleDB.tables) {
|
|
18
|
-
if (table.indexes.length > 0) {
|
|
19
|
-
allIndexes[table.name] = table.indexes;
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
const allIndexesFile = `${file.replace(`.${extension}`, "")}_indexes.json`;
|
|
23
|
-
if (existsSync(allIndexesFile)) {
|
|
24
|
-
rmSync(allIndexesFile);
|
|
25
|
-
}
|
|
26
|
-
if (Object.keys(allIndexes).length > 0) {
|
|
27
|
-
writeFileSync(allIndexesFile, JSON.stringify(allIndexes));
|
|
28
|
-
}
|
|
29
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"getProjection.d.ts","sourceRoot":"","sources":["../../src/helpers/getProjection.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAGjD,wBAA8B,aAAa,CACzC,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,MAAM,mBAiCb"}
|