@nshiab/simple-data-analysis-core 0.0.26 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/class/SimpleDB.d.ts +3 -8
- package/esm/class/SimpleDB.d.ts.map +1 -1
- package/esm/class/SimpleDB.js +8 -10
- package/esm/class/SimpleTable.d.ts +61 -51
- package/esm/class/SimpleTable.d.ts.map +1 -1
- package/esm/class/SimpleTable.js +132 -188
- package/esm/helpers/convertForJS.js +1 -1
- package/esm/helpers/findGeoColumn.js +2 -2
- package/esm/helpers/hasGeometryColumn.js +1 -1
- package/esm/helpers/logData.d.ts.map +1 -1
- package/esm/helpers/logData.js +7 -2
- package/esm/helpers/parseTypes.d.ts +1 -1
- package/esm/helpers/parseTypes.d.ts.map +1 -1
- package/esm/helpers/parseTypes.js +3 -1
- package/esm/helpers/setDbProps.d.ts +1 -1
- package/esm/helpers/setDbProps.d.ts.map +1 -1
- package/esm/helpers/setDbProps.js +5 -9
- package/esm/helpers/unifyColumns.d.ts +0 -2
- package/esm/helpers/unifyColumns.d.ts.map +1 -1
- package/esm/helpers/unifyColumns.js +2 -6
- package/esm/helpers/writeGeoData.d.ts.map +1 -1
- package/esm/helpers/writeGeoData.js +11 -41
- package/esm/helpers/writeIndexes.d.ts +3 -0
- package/esm/helpers/writeIndexes.d.ts.map +1 -0
- package/esm/helpers/writeIndexes.js +16 -0
- package/esm/methods/bm25.d.ts.map +1 -1
- package/esm/methods/bm25.js +1 -1
- package/esm/methods/cache.js +1 -1
- package/esm/methods/distanceQuery.d.ts.map +1 -1
- package/esm/methods/distanceQuery.js +4 -2
- package/esm/methods/fuzzyJoin.d.ts.map +1 -1
- package/esm/methods/fuzzyJoin.js +1 -6
- package/esm/methods/getGeoData.d.ts.map +1 -1
- package/esm/methods/getGeoData.js +1 -10
- package/esm/methods/getNbRows.d.ts +3 -1
- package/esm/methods/getNbRows.d.ts.map +1 -1
- package/esm/methods/getNbRows.js +5 -5
- package/esm/methods/join.d.ts.map +1 -1
- package/esm/methods/join.js +1 -7
- package/esm/methods/joinGeo.d.ts.map +1 -1
- package/esm/methods/joinGeo.js +1 -14
- package/esm/methods/joinGeoQuery.js +1 -1
- package/esm/methods/randomPointQuery.d.ts.map +1 -1
- package/esm/methods/randomPointQuery.js +60 -22
- package/esm/methods/summarizeQuery.js +1 -1
- package/esm/methods/writeGeoDataQuery.js +2 -2
- package/package.json +2 -2
- package/script/class/SimpleDB.d.ts +3 -8
- package/script/class/SimpleDB.d.ts.map +1 -1
- package/script/class/SimpleDB.js +8 -10
- package/script/class/SimpleTable.d.ts +61 -51
- package/script/class/SimpleTable.d.ts.map +1 -1
- package/script/class/SimpleTable.js +132 -188
- package/script/helpers/convertForJS.js +1 -1
- package/script/helpers/findGeoColumn.js +2 -2
- package/script/helpers/hasGeometryColumn.js +1 -1
- package/script/helpers/logData.d.ts.map +1 -1
- package/script/helpers/logData.js +7 -2
- package/script/helpers/parseTypes.d.ts +1 -1
- package/script/helpers/parseTypes.d.ts.map +1 -1
- package/script/helpers/parseTypes.js +3 -1
- package/script/helpers/setDbProps.d.ts +1 -1
- package/script/helpers/setDbProps.d.ts.map +1 -1
- package/script/helpers/setDbProps.js +5 -9
- package/script/helpers/unifyColumns.d.ts +0 -2
- package/script/helpers/unifyColumns.d.ts.map +1 -1
- package/script/helpers/unifyColumns.js +2 -6
- package/script/helpers/writeGeoData.d.ts.map +1 -1
- package/script/helpers/writeGeoData.js +11 -41
- package/script/helpers/writeIndexes.d.ts +3 -0
- package/script/helpers/writeIndexes.d.ts.map +1 -0
- package/script/helpers/writeIndexes.js +19 -0
- package/script/methods/bm25.d.ts.map +1 -1
- package/script/methods/bm25.js +1 -1
- package/script/methods/cache.js +1 -1
- package/script/methods/distanceQuery.d.ts.map +1 -1
- package/script/methods/distanceQuery.js +4 -2
- package/script/methods/fuzzyJoin.d.ts.map +1 -1
- package/script/methods/fuzzyJoin.js +1 -6
- package/script/methods/getGeoData.d.ts.map +1 -1
- package/script/methods/getGeoData.js +1 -10
- package/script/methods/getNbRows.d.ts +3 -1
- package/script/methods/getNbRows.d.ts.map +1 -1
- package/script/methods/getNbRows.js +5 -5
- package/script/methods/join.d.ts.map +1 -1
- package/script/methods/join.js +1 -7
- package/script/methods/joinGeo.d.ts.map +1 -1
- package/script/methods/joinGeo.js +1 -14
- package/script/methods/joinGeoQuery.js +1 -1
- package/script/methods/randomPointQuery.d.ts.map +1 -1
- package/script/methods/randomPointQuery.js +60 -22
- package/script/methods/summarizeQuery.js +1 -1
- package/script/methods/writeGeoDataQuery.js +2 -2
- package/esm/helpers/getProjection.d.ts +0 -3
- package/esm/helpers/getProjection.d.ts.map +0 -1
- package/esm/helpers/getProjection.js +0 -25
- package/esm/helpers/getProjectionParquet.d.ts +0 -3
- package/esm/helpers/getProjectionParquet.d.ts.map +0 -1
- package/esm/helpers/getProjectionParquet.js +0 -24
- package/esm/helpers/shouldFlipBeforeExport.d.ts +0 -2
- package/esm/helpers/shouldFlipBeforeExport.d.ts.map +0 -1
- package/esm/helpers/shouldFlipBeforeExport.js +0 -3
- package/esm/helpers/writeProjectionsAndIndexes.d.ts +0 -3
- package/esm/helpers/writeProjectionsAndIndexes.d.ts.map +0 -1
- package/esm/helpers/writeProjectionsAndIndexes.js +0 -29
- package/script/helpers/getProjection.d.ts +0 -3
- package/script/helpers/getProjection.d.ts.map +0 -1
- package/script/helpers/getProjection.js +0 -31
- package/script/helpers/getProjectionParquet.d.ts +0 -3
- package/script/helpers/getProjectionParquet.d.ts.map +0 -1
- package/script/helpers/getProjectionParquet.js +0 -30
- package/script/helpers/shouldFlipBeforeExport.d.ts +0 -2
- package/script/helpers/shouldFlipBeforeExport.d.ts.map +0 -1
- package/script/helpers/shouldFlipBeforeExport.js +0 -6
- package/script/helpers/writeProjectionsAndIndexes.d.ts +0 -3
- package/script/helpers/writeProjectionsAndIndexes.d.ts.map +0 -1
- package/script/helpers/writeProjectionsAndIndexes.js +0 -32
|
@@ -24,7 +24,7 @@ export default function convertForJS(rows, types) {
|
|
|
24
24
|
row[key] = row[key] === null ? null : Number(row[key]);
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
|
-
else if (types[key]
|
|
27
|
+
else if (types[key].toLowerCase().includes("geometry")) {
|
|
28
28
|
for (const row of rows) {
|
|
29
29
|
row[key] = row[key] === null ? null : "<Geometry>";
|
|
30
30
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export default async function findGeoColumn(SimpleTable) {
|
|
2
2
|
let column;
|
|
3
3
|
const types = await SimpleTable.getTypes();
|
|
4
|
-
const geometries = Object.values(types).filter((d) => d.toLowerCase()
|
|
4
|
+
const geometries = Object.values(types).filter((d) => d.toLowerCase().includes("geometry"));
|
|
5
5
|
if (geometries.length === 0) {
|
|
6
6
|
throw new Error("Table contains no geometry columns.");
|
|
7
7
|
}
|
|
@@ -9,7 +9,7 @@ export default async function findGeoColumn(SimpleTable) {
|
|
|
9
9
|
throw new Error("More than one column storing geometries. If the method allows to specify one, do it. Otherwise, use the selectColumns methods beforehand.");
|
|
10
10
|
}
|
|
11
11
|
else {
|
|
12
|
-
column = Object.keys(types).find((d) => types[d].toLowerCase()
|
|
12
|
+
column = Object.keys(types).find((d) => types[d].toLowerCase().includes("geometry"));
|
|
13
13
|
}
|
|
14
14
|
if (typeof column !== "string") {
|
|
15
15
|
throw new Error("No column");
|
|
@@ -17,5 +17,5 @@
|
|
|
17
17
|
*/
|
|
18
18
|
export default async function hasGeometryColumn(table) {
|
|
19
19
|
const types = await table.getTypes();
|
|
20
|
-
return Object.values(types).some((t) => t.toLowerCase()
|
|
20
|
+
return Object.values(types).some((t) => t.toLowerCase().includes("geometry"));
|
|
21
21
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logData.d.ts","sourceRoot":"","sources":["../../src/helpers/logData.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GAAG,IAAI,EACvC,IAAI,EACA;IACA,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC;CACxD,EAAE,GACD,IAAI,EACR,iBAAiB,CAAC,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"logData.d.ts","sourceRoot":"","sources":["../../src/helpers/logData.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,UAAU,OAAO,CAC7B,KAAK,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,GAAG,IAAI,EACvC,IAAI,EACA;IACA,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC;CACxD,EAAE,GACD,IAAI,EACR,iBAAiB,CAAC,EAAE,MAAM,QA4D3B"}
|
package/esm/helpers/logData.js
CHANGED
|
@@ -29,8 +29,13 @@ export default function logData(types, data, nbCharactersToLog) {
|
|
|
29
29
|
const columns = Object.keys(types);
|
|
30
30
|
if (columns.length > 0) {
|
|
31
31
|
for (const col of columns) {
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
if (data[0][col] === "<Geometry>") {
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
types[col] = types[col] + "/" +
|
|
37
|
+
(data[0][col] === null ? null : typeof data[0][col]);
|
|
38
|
+
}
|
|
34
39
|
}
|
|
35
40
|
// Add types as the first row
|
|
36
41
|
dataToBeLogged.unshift(types);
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export default function parseType(type: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" |
|
|
1
|
+
export default function parseType(type: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | `geometry('${string}')` | `GEOMETRY('${string}')`): string;
|
|
2
2
|
//# sourceMappingURL=parseTypes.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parseTypes.d.ts","sourceRoot":"","sources":["../../src/helpers/parseTypes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,IAAI,EACA,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,WAAW,GACX,0BAA0B,GAC1B,SAAS,GACT,
|
|
1
|
+
{"version":3,"file":"parseTypes.d.ts","sourceRoot":"","sources":["../../src/helpers/parseTypes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,IAAI,EACA,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,GACV,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,WAAW,GACX,0BAA0B,GAC1B,SAAS,GACT,aAAa,MAAM,IAAI,GACvB,aAAa,MAAM,IAAI,UAgC5B"}
|
|
@@ -25,10 +25,12 @@ export default function parseType(type) {
|
|
|
25
25
|
"timestamp",
|
|
26
26
|
"timestamp with time zone",
|
|
27
27
|
"boolean",
|
|
28
|
-
"geometry",
|
|
29
28
|
].includes(typeLowerCase)) {
|
|
30
29
|
return type.toUpperCase();
|
|
31
30
|
}
|
|
31
|
+
else if (typeLowerCase.includes("geometry")) {
|
|
32
|
+
return type.toUpperCase();
|
|
33
|
+
}
|
|
32
34
|
else {
|
|
33
35
|
throw new Error(`Unknown type ${type}`);
|
|
34
36
|
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type SimpleDB from "../class/SimpleDB.js";
|
|
2
|
-
export default function setDbProps(simpleDB: SimpleDB,
|
|
2
|
+
export default function setDbProps(simpleDB: SimpleDB, allIndexesFile: string): Promise<void>;
|
|
3
3
|
//# sourceMappingURL=setDbProps.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"setDbProps.d.ts","sourceRoot":"","sources":["../../src/helpers/setDbProps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAEjD,wBAA8B,UAAU,CACtC,QAAQ,EAAE,QAAQ,EAClB,
|
|
1
|
+
{"version":3,"file":"setDbProps.d.ts","sourceRoot":"","sources":["../../src/helpers/setDbProps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,QAAQ,MAAM,sBAAsB,CAAC;AAEjD,wBAA8B,UAAU,CACtC,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,MAAM,iBA2BvB"}
|
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
-
export default async function setDbProps(simpleDB,
|
|
2
|
+
export default async function setDbProps(simpleDB, allIndexesFile) {
|
|
3
3
|
for (const table of await simpleDB.getTableNames()) {
|
|
4
4
|
simpleDB.newTable(table);
|
|
5
5
|
}
|
|
6
|
-
const
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
if (projections[table.name]) {
|
|
11
|
-
table.projections = projections[table.name];
|
|
12
|
-
}
|
|
6
|
+
for (const table of await simpleDB.getTables()) {
|
|
7
|
+
const types = await table.getTypes();
|
|
8
|
+
if (Object.values(types).some((type) => type.toLowerCase().includes("geometry"))) {
|
|
9
|
+
await simpleDB.customQuery(`INSTALL SPATIAL; LOAD SPATIAL;`);
|
|
13
10
|
}
|
|
14
|
-
await simpleDB.customQuery(`INSTALL spatial; LOAD spatial;`);
|
|
15
11
|
}
|
|
16
12
|
if (existsSync(allIndexesFile)) {
|
|
17
13
|
const indexes = JSON.parse(readFileSync(allIndexesFile, "utf-8"));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unifyColumns.d.ts","sourceRoot":"","sources":["../../src/helpers/unifyColumns.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,YAAY,CACxC,SAAS,EAAE,WAAW,EAAE,EACxB,QAAQ,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE
|
|
1
|
+
{"version":3,"file":"unifyColumns.d.ts","sourceRoot":"","sources":["../../src/helpers/unifyColumns.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,YAAY,CACxC,SAAS,EAAE,WAAW,EAAE,EACxB,QAAQ,EAAE;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE;;GAwCpC"}
|
|
@@ -1,15 +1,11 @@
|
|
|
1
|
-
export default async function unifyColumns(allTables, allTypes
|
|
1
|
+
export default async function unifyColumns(allTables, allTypes) {
|
|
2
2
|
const columnsAdded = {};
|
|
3
3
|
for (const column in allTypes) {
|
|
4
4
|
for (const table of allTables) {
|
|
5
5
|
if (!(await table.hasColumn(column))) {
|
|
6
6
|
await table.addColumn(column,
|
|
7
7
|
// Could be improved
|
|
8
|
-
allTypes[column].toLowerCase(), `null
|
|
9
|
-
projection: allTypes[column] === "GEOMETRY"
|
|
10
|
-
? allProjections[column]
|
|
11
|
-
: undefined,
|
|
12
|
-
});
|
|
8
|
+
allTypes[column].toLowerCase(), `null`);
|
|
13
9
|
if (!columnsAdded[table.name]) {
|
|
14
10
|
columnsAdded[table.name] = [];
|
|
15
11
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writeGeoData.d.ts","sourceRoot":"","sources":["../../src/helpers/writeGeoData.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"writeGeoData.d.ts","sourceRoot":"","sources":["../../src/helpers/writeGeoData.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,YAAY,CACxC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CAClB,GACL,OAAO,CAAC,IAAI,CAAC,CAmGf"}
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
import { readFileSync, writeFileSync } from "node:fs";
|
|
2
2
|
import createDirectory from "./createDirectory.js";
|
|
3
3
|
import getExtension from "./getExtension.js";
|
|
4
|
-
import findGeoColumn from "./findGeoColumn.js";
|
|
5
4
|
import hasGeometryColumn from "./hasGeometryColumn.js";
|
|
6
|
-
import shouldFlipBeforeExport from "./shouldFlipBeforeExport.js";
|
|
7
5
|
import queryDB from "./queryDB.js";
|
|
8
6
|
import writeGeoDataQuery from "../methods/writeGeoDataQuery.js";
|
|
9
7
|
import mergeOptions from "./mergeOptions.js";
|
|
@@ -29,24 +27,11 @@ export default async function writeGeoData(table, file, options = {}) {
|
|
|
29
27
|
if (typeof options.compression === "boolean") {
|
|
30
28
|
throw new Error("The compression option is not supported for writing GeoJSON files.");
|
|
31
29
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
table: table.name,
|
|
38
|
-
method: "writeGeoData()",
|
|
39
|
-
parameters: { file, options },
|
|
40
|
-
}));
|
|
41
|
-
await table.flipCoordinates(geoColumn);
|
|
42
|
-
}
|
|
43
|
-
else {
|
|
44
|
-
await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
|
|
45
|
-
table: table.name,
|
|
46
|
-
method: "writeGeoData()",
|
|
47
|
-
parameters: { file, options },
|
|
48
|
-
}));
|
|
49
|
-
}
|
|
30
|
+
await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
|
|
31
|
+
table: table.name,
|
|
32
|
+
method: "writeGeoData()",
|
|
33
|
+
parameters: { file, options },
|
|
34
|
+
}));
|
|
50
35
|
if (options.metadata) {
|
|
51
36
|
const fileData = JSON.parse(readFileSync(file, "utf-8"));
|
|
52
37
|
fileData.metadata = options.metadata;
|
|
@@ -70,24 +55,11 @@ export default async function writeGeoData(table, file, options = {}) {
|
|
|
70
55
|
options.formatDates === true) {
|
|
71
56
|
throw new Error("The following options are not supported for writing SHAPEFILE files: precision, compression, rewind, metadata, and formatDates.");
|
|
72
57
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
table: table.name,
|
|
79
|
-
method: "writeGeoData()",
|
|
80
|
-
parameters: { file, options },
|
|
81
|
-
}));
|
|
82
|
-
await table.flipCoordinates(geoColumn);
|
|
83
|
-
}
|
|
84
|
-
else {
|
|
85
|
-
await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
|
|
86
|
-
table: table.name,
|
|
87
|
-
method: "writeGeoData()",
|
|
88
|
-
parameters: { file, options },
|
|
89
|
-
}));
|
|
90
|
-
}
|
|
58
|
+
await queryDB(table, writeGeoDataQuery(table.name, file, fileExtension, options), mergeOptions(table, {
|
|
59
|
+
table: table.name,
|
|
60
|
+
method: "writeGeoData()",
|
|
61
|
+
parameters: { file, options },
|
|
62
|
+
}));
|
|
91
63
|
}
|
|
92
64
|
else if (fileExtension === "geoparquet") {
|
|
93
65
|
if (typeof options.precision === "number") {
|
|
@@ -96,9 +68,7 @@ export default async function writeGeoData(table, file, options = {}) {
|
|
|
96
68
|
if (typeof options.rewind === "boolean") {
|
|
97
69
|
throw new Error("The rewind option is not supported for writing PARQUET files.");
|
|
98
70
|
}
|
|
99
|
-
await queryDB(table, `COPY "${table.name}" TO '${cleanPath(file)}' WITH (FORMAT PARQUET${options.compression === true ? ", COMPRESSION 'zstd'" : ""},
|
|
100
|
-
projections: '${JSON.stringify(table.projections)}'
|
|
101
|
-
});`, mergeOptions(table, {
|
|
71
|
+
await queryDB(table, `COPY "${table.name}" TO '${cleanPath(file)}' WITH (FORMAT PARQUET${options.compression === true ? ", COMPRESSION 'zstd'" : ""});`, mergeOptions(table, {
|
|
102
72
|
table: table.name,
|
|
103
73
|
method: "writeGeoData()",
|
|
104
74
|
parameters: { file, options },
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"writeIndexes.d.ts","sourceRoot":"","sources":["../../src/helpers/writeIndexes.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,CAAC,OAAO,UAAU,YAAY,CAClC,QAAQ,EAAE,QAAQ,EAClB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,QAeb"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { existsSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
export default function writeIndexes(simpleDB, extension, file) {
|
|
3
|
+
const allIndexes = {};
|
|
4
|
+
for (const table of simpleDB.tables) {
|
|
5
|
+
if (table.indexes.length > 0) {
|
|
6
|
+
allIndexes[table.name] = table.indexes;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
const allIndexesFile = `${file.replace(`.${extension}`, "")}_indexes.json`;
|
|
10
|
+
if (existsSync(allIndexesFile)) {
|
|
11
|
+
rmSync(allIndexesFile);
|
|
12
|
+
}
|
|
13
|
+
if (Object.keys(allIndexes).length > 0) {
|
|
14
|
+
writeFileSync(allIndexesFile, JSON.stringify(allIndexes));
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/methods/bm25.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,IAAI,CAChC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,
|
|
1
|
+
{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/methods/bm25.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAIvD,wBAA8B,IAAI,CAChC,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,OAAO,CAAC,EACJ,QAAQ,GACR,QAAQ,GACR,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,OAAO,GACP,WAAW,GACX,YAAY,GACZ,OAAO,GACP,SAAS,GACT,YAAY,GACZ,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,YAAY,GACZ,UAAU,GACV,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,SAAS,GACT,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACd,wBA+DP"}
|
package/esm/methods/bm25.js
CHANGED
|
@@ -40,7 +40,7 @@ export default async function bm25(simpleTable, text, columnId, columnText, nbRe
|
|
|
40
40
|
},
|
|
41
41
|
}));
|
|
42
42
|
if (typeof options.outputTable === "string") {
|
|
43
|
-
return simpleTable.sdb.newTable(options.outputTable
|
|
43
|
+
return simpleTable.sdb.newTable(options.outputTable);
|
|
44
44
|
}
|
|
45
45
|
else {
|
|
46
46
|
return simpleTable;
|
package/esm/methods/cache.js
CHANGED
|
@@ -91,7 +91,7 @@ async function runAndWrite(table, run, cacheSources, cacheSourcesPath, cachePath
|
|
|
91
91
|
}
|
|
92
92
|
else {
|
|
93
93
|
const types = await table.getTypes();
|
|
94
|
-
const geometriesColumns = Object.values(types).filter((d) => d
|
|
94
|
+
const geometriesColumns = Object.values(types).filter((d) => d.toLowerCase().includes("geometry")).length;
|
|
95
95
|
if (geometriesColumns > 0) {
|
|
96
96
|
const file = `${cachePath}/${id}.geoparquet`;
|
|
97
97
|
const writeStart = Date.now();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"distanceQuery.d.ts","sourceRoot":"","sources":["../../src/methods/distanceQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,aAAa,CACnC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC;IAClB,MAAM,CAAC,EAAE,KAAK,GAAG,UAAU,GAAG,WAAW,CAAC;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;CACd,
|
|
1
|
+
{"version":3,"file":"distanceQuery.d.ts","sourceRoot":"","sources":["../../src/methods/distanceQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,aAAa,CACnC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC;IAClB,MAAM,CAAC,EAAE,KAAK,GAAG,UAAU,GAAG,WAAW,CAAC;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;CACd,UAwDP"}
|
|
@@ -29,10 +29,12 @@ export default function distanceQuery(table, column1, column2, newColumn, option
|
|
|
29
29
|
}
|
|
30
30
|
else if (options.method === "spheroid") {
|
|
31
31
|
if (typeof options.decimals === "number") {
|
|
32
|
-
query +=
|
|
32
|
+
query +=
|
|
33
|
+
`ROUND(ST_Distance_Spheroid("${column1}"::GEOMETRY, "${column2}"::GEOMETRY) ${options.unit === "km" ? "/ 1000" : ""}, ${options.decimals});`;
|
|
33
34
|
}
|
|
34
35
|
else {
|
|
35
|
-
query +=
|
|
36
|
+
query +=
|
|
37
|
+
`ST_Distance_Spheroid("${column1}"::GEOMETRY, "${column2}"::GEOMETRY) ${options.unit === "km" ? "/ 1000" : ""};`;
|
|
36
38
|
}
|
|
37
39
|
}
|
|
38
40
|
else {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,
|
|
1
|
+
{"version":3,"file":"fuzzyJoin.d.ts","sourceRoot":"","sources":["../../src/methods/fuzzyJoin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,SAAS,CACrC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,MAAM,CAAC,EACH,OAAO,GACP,eAAe,GACf,kBAAkB,GAClB,iBAAiB,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/B,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACxB,wBAiFP"}
|
package/esm/methods/fuzzyJoin.js
CHANGED
|
@@ -38,14 +38,9 @@ export default async function fuzzyJoin(leftTable, rightTable, leftColumn, right
|
|
|
38
38
|
options,
|
|
39
39
|
},
|
|
40
40
|
}));
|
|
41
|
-
const allProjections = {
|
|
42
|
-
...leftTable.projections,
|
|
43
|
-
...rightTable.projections,
|
|
44
|
-
};
|
|
45
41
|
const outputTable = typeof options.outputTable === "string"
|
|
46
|
-
? leftTable.sdb.newTable(options.outputTable
|
|
42
|
+
? leftTable.sdb.newTable(options.outputTable)
|
|
47
43
|
: leftTable;
|
|
48
|
-
outputTable.projections = allProjections;
|
|
49
44
|
// Remove the duplicate right-column produced when leftColumn === rightColumn
|
|
50
45
|
// (DuckDB suffixes it with _1 in SELECT *)
|
|
51
46
|
const outputCols = await outputTable.getColumns();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getGeoData.d.ts","sourceRoot":"","sources":["../../src/methods/getGeoData.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"getGeoData.d.ts","sourceRoot":"","sources":["../../src/methods/getGeoData.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,UAAU,CACtC,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO;;;;;kBAuBtB,MAAM;yBACC,OAAO,EAAE;;;;;;GAgC7B"}
|
|
@@ -1,17 +1,8 @@
|
|
|
1
1
|
import rewind from "../helpers/rewind.js";
|
|
2
2
|
import mergeOptions from "../helpers/mergeOptions.js";
|
|
3
3
|
import queryDB from "../helpers/queryDB.js";
|
|
4
|
-
import shouldFlipBeforeExport from "../helpers/shouldFlipBeforeExport.js";
|
|
5
4
|
export default async function getGeoData(SimpleTable, column, options = {}) {
|
|
6
|
-
|
|
7
|
-
if (shouldFlipBeforeExport(SimpleTable.projections[column])) {
|
|
8
|
-
query =
|
|
9
|
-
`SELECT * EXCLUDE ${column}, ST_AsGeoJSON(ST_FlipCoordinates(${column})) as geoJsonFragment from "${SimpleTable.name}";`;
|
|
10
|
-
}
|
|
11
|
-
else {
|
|
12
|
-
query =
|
|
13
|
-
`SELECT * EXCLUDE ${column}, ST_AsGeoJSON(${column}) as geoJsonFragment from "${SimpleTable.name}";`;
|
|
14
|
-
}
|
|
5
|
+
const query = `SELECT * EXCLUDE ${column}, ST_AsGeoJSON(${column}) as geoJsonFragment from "${SimpleTable.name}";`;
|
|
15
6
|
const queryResult = await queryDB(SimpleTable, query, mergeOptions(SimpleTable, {
|
|
16
7
|
table: null,
|
|
17
8
|
method: "getGeoData()",
|
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
import type SimpleTable from "../class/SimpleTable.js";
|
|
2
|
-
export default function getNbRows(SimpleTable: SimpleTable
|
|
2
|
+
export default function getNbRows(SimpleTable: SimpleTable, options?: {
|
|
3
|
+
conditions?: string;
|
|
4
|
+
}): Promise<number>;
|
|
3
5
|
//# sourceMappingURL=getNbRows.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getNbRows.d.ts","sourceRoot":"","sources":["../../src/methods/getNbRows.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,SAAS,
|
|
1
|
+
{"version":3,"file":"getNbRows.d.ts","sourceRoot":"","sources":["../../src/methods/getNbRows.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAEvD,wBAA8B,SAAS,CACrC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,mBAqBtC"}
|
package/esm/methods/getNbRows.js
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import mergeOptions from "../helpers/mergeOptions.js";
|
|
2
2
|
import queryDB from "../helpers/queryDB.js";
|
|
3
|
-
export default async function getNbRows(SimpleTable) {
|
|
4
|
-
const queryResult = await queryDB(SimpleTable, `SELECT CAST(COUNT(*) AS INTEGER) FROM "${SimpleTable.name}"`, mergeOptions(SimpleTable, {
|
|
3
|
+
export default async function getNbRows(SimpleTable, options = {}) {
|
|
4
|
+
const queryResult = await queryDB(SimpleTable, `SELECT CAST(COUNT(*) AS INTEGER) AS nbRows FROM "${SimpleTable.name}"${options.conditions ? ` WHERE ${options.conditions}` : ""}`, mergeOptions(SimpleTable, {
|
|
5
5
|
table: SimpleTable.name,
|
|
6
6
|
returnDataFrom: "query",
|
|
7
|
-
method: "
|
|
8
|
-
parameters: {},
|
|
7
|
+
method: "getNbRows()",
|
|
8
|
+
parameters: { options },
|
|
9
9
|
}));
|
|
10
10
|
if (!queryResult) {
|
|
11
11
|
throw new Error("No result");
|
|
12
12
|
}
|
|
13
|
-
const length = queryResult[0]
|
|
13
|
+
const length = queryResult[0].nbRows;
|
|
14
14
|
return length;
|
|
15
15
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"join.d.ts","sourceRoot":"","sources":["../../src/methods/join.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,IAAI,CAChC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,
|
|
1
|
+
{"version":3,"file":"join.d.ts","sourceRoot":"","sources":["../../src/methods/join.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAMvD,wBAA8B,IAAI,CAChC,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,wBAqFP"}
|
package/esm/methods/join.js
CHANGED
|
@@ -46,14 +46,8 @@ export default async function join(leftTable, rightTable, options = {}) {
|
|
|
46
46
|
},
|
|
47
47
|
}));
|
|
48
48
|
const outputTable = typeof options.outputTable === "string"
|
|
49
|
-
? leftTable.sdb.newTable(options.outputTable
|
|
49
|
+
? leftTable.sdb.newTable(options.outputTable)
|
|
50
50
|
: leftTable;
|
|
51
|
-
// So we reassign here
|
|
52
|
-
const allProjections = {
|
|
53
|
-
...leftTable.projections,
|
|
54
|
-
...rightTable.projections,
|
|
55
|
-
};
|
|
56
|
-
outputTable.projections = allProjections;
|
|
57
51
|
const columns = await outputTable.getColumns();
|
|
58
52
|
const extraCommonColumns = columns.filter((d) => commonColumn.map((c) => `${c}_1`).includes(d));
|
|
59
53
|
if (extraCommonColumns.length > 0) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"joinGeo.d.ts","sourceRoot":"","sources":["../../src/methods/joinGeo.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAOvD,wBAA8B,OAAO,CACnC,SAAS,EAAE,WAAW,EACtB,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,QAAQ,EACzC,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,UAAU,CAAC;IAClD,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,
|
|
1
|
+
{"version":3,"file":"joinGeo.d.ts","sourceRoot":"","sources":["../../src/methods/joinGeo.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAOvD,wBAA8B,OAAO,CACnC,SAAS,EAAE,WAAW,EACtB,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,QAAQ,EACzC,UAAU,EAAE,WAAW,EACvB,OAAO,GAAE;IACP,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,UAAU,CAAC;IAClD,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,wBA4FP"}
|
package/esm/methods/joinGeo.js
CHANGED
|
@@ -46,31 +46,18 @@ export default async function joinGeo(leftTable, method, rightTable, options = {
|
|
|
46
46
|
},
|
|
47
47
|
}));
|
|
48
48
|
// We bring back the column names for geometries
|
|
49
|
-
let allProjections = {};
|
|
50
49
|
if (leftTableColumn === rightTableColumn) {
|
|
51
50
|
const leftObj = {};
|
|
52
51
|
leftObj[leftTableColumnForQuery] = leftTableColumn;
|
|
53
52
|
await leftTable.renameColumns(leftObj);
|
|
54
|
-
// Before renaming columns in original tables
|
|
55
|
-
allProjections = {
|
|
56
|
-
...leftTable.projections,
|
|
57
|
-
...rightTable.projections,
|
|
58
|
-
};
|
|
59
53
|
const rightObj = {};
|
|
60
54
|
rightObj[rightTableColumnForQuery] = rightTableColumn;
|
|
61
55
|
await rightTable.renameColumns(rightObj);
|
|
62
56
|
}
|
|
63
|
-
else {
|
|
64
|
-
allProjections = {
|
|
65
|
-
...leftTable.projections,
|
|
66
|
-
...rightTable.projections,
|
|
67
|
-
};
|
|
68
|
-
}
|
|
69
57
|
if (typeof options.outputTable === "string") {
|
|
70
|
-
return leftTable.sdb.newTable(options.outputTable
|
|
58
|
+
return leftTable.sdb.newTable(options.outputTable);
|
|
71
59
|
}
|
|
72
60
|
else {
|
|
73
|
-
leftTable.projections = allProjections;
|
|
74
61
|
return leftTable;
|
|
75
62
|
}
|
|
76
63
|
}
|
|
@@ -38,7 +38,7 @@ export default function joinGeoQuery(leftTable, leftTableColumn, method, rightTa
|
|
|
38
38
|
else if (distanceMethod === "spheroid") {
|
|
39
39
|
// Should be using ST_DWithin_Spheroid but doesn't work?
|
|
40
40
|
query +=
|
|
41
|
-
` ON ST_Distance_Spheroid("${leftTable}"."${leftTableColumn}", "${rightTable}"."${rightTableColumn}") < ${distance}`;
|
|
41
|
+
` ON ST_Distance_Spheroid("${leftTable}"."${leftTableColumn}"::GEOMETRY, "${rightTable}"."${rightTableColumn}"::GEOMETRY) < ${distance}`;
|
|
42
42
|
}
|
|
43
43
|
else {
|
|
44
44
|
throw new Error(`Unknown ${distanceMethod}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"randomPointQuery.d.ts","sourceRoot":"","sources":["../../src/methods/randomPointQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,gBAAgB,CACtC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"randomPointQuery.d.ts","sourceRoot":"","sources":["../../src/methods/randomPointQuery.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,gBAAgB,CACtC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,UAmEtB"}
|
|
@@ -1,28 +1,66 @@
|
|
|
1
1
|
export default function randomPointQuery(table, column, newColumn, nbPointsToTry) {
|
|
2
|
+
const addColumn = newColumn === column
|
|
3
|
+
? ""
|
|
4
|
+
: `ALTER TABLE "${table}" ADD COLUMN "${newColumn}" GEOMETRY;`;
|
|
5
|
+
// Recursive CTE approach: each iteration only carries forward the rows that
|
|
6
|
+
// have NOT yet found a valid interior point, so DuckDB stops per-row as soon
|
|
7
|
+
// as ST_Within is satisfied — giving O(1) behaviour for easy polygons instead
|
|
8
|
+
// of the O(N) materialisation that occurs with LATERAL + range() + WHERE.
|
|
9
|
+
//
|
|
10
|
+
// Zero-tries contract: `WHERE 1 <= ${nbPointsToTry}` in the base case
|
|
11
|
+
// produces no rows when nbPointsToTry is 0, so the LEFT JOIN returns NULL for
|
|
12
|
+
// every row and the UPDATE sets the column to NULL — no special-case branch
|
|
13
|
+
// needed and the normal UPDATE code path is always exercised.
|
|
14
|
+
//
|
|
15
|
+
// Volatility guard: `CASE WHEN (rid + n) IS NOT NULL THEN random() END`
|
|
16
|
+
// prevents the planner from constant-folding random() across iterations while
|
|
17
|
+
// still evaluating to random() for every non-null row.
|
|
2
18
|
return `INSTALL spatial;
|
|
3
19
|
LOAD spatial;
|
|
4
|
-
${
|
|
5
|
-
|
|
6
|
-
|
|
20
|
+
${addColumn}
|
|
21
|
+
WITH RECURSIVE
|
|
22
|
+
base AS (
|
|
23
|
+
SELECT
|
|
24
|
+
rowid AS rid,
|
|
25
|
+
"${column}" AS geom,
|
|
26
|
+
ST_XMin("${column}") AS xmin,
|
|
27
|
+
ST_XMax("${column}") - ST_XMin("${column}") AS xdiff,
|
|
28
|
+
ST_YMin("${column}") AS ymin,
|
|
29
|
+
ST_YMax("${column}") - ST_YMin("${column}") AS ydiff
|
|
30
|
+
FROM "${table}"
|
|
31
|
+
),
|
|
32
|
+
attempts(rid, geom, xmin, xdiff, ymin, ydiff, pt, n) AS (
|
|
33
|
+
SELECT
|
|
34
|
+
rid, geom, xmin, xdiff, ymin, ydiff,
|
|
35
|
+
ST_Point(
|
|
36
|
+
xmin + CASE WHEN rid IS NOT NULL THEN random() END * xdiff,
|
|
37
|
+
ymin + CASE WHEN rid IS NOT NULL THEN random() END * ydiff
|
|
38
|
+
),
|
|
39
|
+
1
|
|
40
|
+
FROM base
|
|
41
|
+
WHERE 1 <= ${nbPointsToTry}
|
|
42
|
+
UNION ALL
|
|
43
|
+
SELECT
|
|
44
|
+
rid, geom, xmin, xdiff, ymin, ydiff,
|
|
45
|
+
ST_Point(
|
|
46
|
+
xmin + CASE WHEN (rid + n) IS NOT NULL THEN random() END * xdiff,
|
|
47
|
+
ymin + CASE WHEN (rid + n) IS NOT NULL THEN random() END * ydiff
|
|
48
|
+
),
|
|
49
|
+
n + 1
|
|
50
|
+
FROM attempts
|
|
51
|
+
WHERE NOT ST_Within(pt, geom) AND n < ${nbPointsToTry}
|
|
52
|
+
)
|
|
7
53
|
UPDATE "${table}" AS t
|
|
8
|
-
SET "${newColumn}" =
|
|
54
|
+
SET "${newColumn}" = v.pt
|
|
9
55
|
FROM (
|
|
10
|
-
SELECT
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
) AS pt
|
|
21
|
-
FROM range(${nbPointsToTry}) AS r(id)
|
|
22
|
-
)
|
|
23
|
-
WHERE ST_Within(pt, t_inner."${column}")
|
|
24
|
-
LIMIT 1
|
|
25
|
-
) AS p
|
|
26
|
-
) AS sub
|
|
27
|
-
WHERE t.rowid = sub.rid;`;
|
|
56
|
+
SELECT b.rid, vp.pt
|
|
57
|
+
FROM (SELECT rowid AS rid FROM "${table}") b
|
|
58
|
+
LEFT JOIN (
|
|
59
|
+
SELECT rid, pt
|
|
60
|
+
FROM attempts
|
|
61
|
+
WHERE ST_Within(pt, geom)
|
|
62
|
+
QUALIFY ROW_NUMBER() OVER (PARTITION BY rid ORDER BY n) = 1
|
|
63
|
+
) vp ON b.rid = vp.rid
|
|
64
|
+
) v
|
|
65
|
+
WHERE t.rowid = v.rid;`;
|
|
28
66
|
}
|
|
@@ -46,7 +46,7 @@ export default function summarizeQuery(table, types, outputTable, values, catego
|
|
|
46
46
|
aggregates[summary] !== "count") {
|
|
47
47
|
return `\nNULL as '${columns ? columns[i] : summary}'`;
|
|
48
48
|
}
|
|
49
|
-
else if (types[value]
|
|
49
|
+
else if (types[value].toLowerCase().includes("geometry")) {
|
|
50
50
|
return `\nNULL AS '${columns ? columns[i] : summary}'`;
|
|
51
51
|
}
|
|
52
52
|
else if (types[value] === "VARCHAR" &&
|
|
@@ -6,12 +6,12 @@ export default function writeGeoDataQuery(table, file, fileExtension, options =
|
|
|
6
6
|
layerOptions.push(`COORDINATE_PRECISION=${options.precision}`);
|
|
7
7
|
}
|
|
8
8
|
layerOptions.push(`RFC7946=YES`);
|
|
9
|
-
return `INSTALL spatial; LOAD spatial; COPY "${table}" to '${cleanPath(file)}' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${layerOptions.length > 0
|
|
9
|
+
return `INSTALL spatial; LOAD spatial; SET geometry_always_xy = true; COPY "${table}" to '${cleanPath(file)}' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${layerOptions.length > 0
|
|
10
10
|
? `, LAYER_CREATION_OPTIONS ('WRITE_NAME=NO', ${layerOptions.map((d) => `'${d}'`).join(", ")})`
|
|
11
11
|
: ""})`;
|
|
12
12
|
}
|
|
13
13
|
else if (fileExtension === "shp") {
|
|
14
|
-
return `INSTALL spatial; LOAD spatial; COPY "${table}" TO '${cleanPath(file)}' WITH (FORMAT GDAL, DRIVER 'ESRI Shapefile', LAYER_CREATION_OPTIONS 'ENCODING=UTF-8')`;
|
|
14
|
+
return `INSTALL spatial; LOAD spatial; SET geometry_always_xy = true; COPY "${table}" TO '${cleanPath(file)}' WITH (FORMAT GDAL, DRIVER 'ESRI Shapefile', LAYER_CREATION_OPTIONS 'ENCODING=UTF-8')`;
|
|
15
15
|
}
|
|
16
16
|
else {
|
|
17
17
|
throw new Error(`Unknown extension ${fileExtension}`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nshiab/simple-data-analysis-core",
|
|
3
|
-
"version": "0.0
|
|
3
|
+
"version": "0.1.0",
|
|
4
4
|
"description": "The core functionalities of the simple-data-analysis library, with only one dependency: DuckDB.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
"access": "public"
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
|
-
"@duckdb/node-api": "1.
|
|
27
|
+
"@duckdb/node-api": "1.5.3-r.2"
|
|
28
28
|
},
|
|
29
29
|
"_generatedBy": "dnt@dev"
|
|
30
30
|
}
|