@nshiab/simple-data-analysis-core 2.0.4 → 2.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -12
- package/esm/class/Simple.d.ts +8 -2
- package/esm/class/Simple.d.ts.map +1 -1
- package/esm/class/Simple.js +9 -3
- package/esm/class/SimpleDB.d.ts +72 -1
- package/esm/class/SimpleDB.d.ts.map +1 -1
- package/esm/class/SimpleDB.js +81 -1
- package/esm/class/SimpleTable.d.ts +434 -263
- package/esm/class/SimpleTable.d.ts.map +1 -1
- package/esm/class/SimpleTable.js +441 -316
- package/esm/helpers/LogValue.d.ts +8 -0
- package/esm/helpers/LogValue.d.ts.map +1 -0
- package/esm/helpers/LogValue.js +23 -0
- package/esm/helpers/cacheTableDependencies.d.ts +9 -0
- package/esm/helpers/cacheTableDependencies.d.ts.map +1 -1
- package/esm/helpers/cacheTableDependencies.js +131 -21
- package/esm/helpers/cleanSQL.d.ts +1 -1
- package/esm/helpers/cleanSQL.d.ts.map +1 -1
- package/esm/helpers/cleanSQL.js +268 -23
- package/esm/helpers/flushAllTables.d.ts.map +1 -1
- package/esm/helpers/flushAllTables.js +23 -28
- package/esm/helpers/foldIdentifier.d.ts +3 -0
- package/esm/helpers/foldIdentifier.d.ts.map +1 -0
- package/esm/helpers/foldIdentifier.js +4 -0
- package/esm/helpers/geometryFromJSON.d.ts +3 -0
- package/esm/helpers/geometryFromJSON.d.ts.map +1 -0
- package/esm/helpers/geometryFromJSON.js +4 -0
- package/esm/helpers/index.d.ts +1 -0
- package/esm/helpers/index.d.ts.map +1 -1
- package/esm/helpers/index.js +1 -0
- package/esm/helpers/logData.d.ts.map +1 -1
- package/esm/helpers/logData.js +45 -44
- package/esm/helpers/makeLogConverter.d.ts +5 -0
- package/esm/helpers/makeLogConverter.d.ts.map +1 -0
- package/esm/helpers/makeLogConverter.js +64 -0
- package/esm/helpers/osmFiles.d.ts.map +1 -1
- package/esm/helpers/parseTypes.d.ts +1 -1
- package/esm/helpers/parseTypes.d.ts.map +1 -1
- package/esm/helpers/parseTypes.js +4 -0
- package/esm/helpers/pendingOps.d.ts +4 -2
- package/esm/helpers/pendingOps.d.ts.map +1 -1
- package/esm/helpers/prepareGeometry.d.ts +3 -0
- package/esm/helpers/prepareGeometry.d.ts.map +1 -0
- package/esm/helpers/prepareGeometry.js +99 -0
- package/esm/helpers/prepareJSON.d.ts +3 -0
- package/esm/helpers/prepareJSON.d.ts.map +1 -0
- package/esm/helpers/prepareJSON.js +50 -0
- package/esm/helpers/printTable.d.ts +5 -4
- package/esm/helpers/printTable.d.ts.map +1 -1
- package/esm/helpers/printTable.js +32 -36
- package/esm/helpers/queryDB.js +1 -1
- package/esm/helpers/queueAsyncBarrier.d.ts +4 -0
- package/esm/helpers/queueAsyncBarrier.d.ts.map +1 -1
- package/esm/helpers/queueAsyncBarrier.js +4 -0
- package/esm/helpers/queueOp.d.ts.map +1 -1
- package/esm/helpers/queueOp.js +5 -3
- package/esm/helpers/readMutationRows.d.ts +9 -0
- package/esm/helpers/readMutationRows.d.ts.map +1 -0
- package/esm/helpers/readMutationRows.js +94 -0
- package/esm/helpers/referencedTables.d.ts +9 -0
- package/esm/helpers/referencedTables.d.ts.map +1 -0
- package/esm/helpers/referencedTables.js +22 -0
- package/esm/helpers/runQuery.d.ts.map +1 -1
- package/esm/helpers/runQuery.js +11 -13
- package/esm/helpers/scanSQL.d.ts +7 -0
- package/esm/helpers/scanSQL.d.ts.map +1 -0
- package/esm/helpers/scanSQL.js +71 -0
- package/esm/helpers/tableRegistry.d.ts +2 -0
- package/esm/helpers/tableRegistry.d.ts.map +1 -1
- package/esm/helpers/tableRegistry.js +12 -4
- package/esm/helpers/updateColumnsWithJS.d.ts +52 -0
- package/esm/helpers/updateColumnsWithJS.d.ts.map +1 -0
- package/esm/helpers/updateColumnsWithJS.js +139 -0
- package/esm/methods/addCharacterCount.d.ts +3 -0
- package/esm/methods/addCharacterCount.d.ts.map +1 -0
- package/esm/methods/addCharacterCount.js +15 -0
- package/esm/methods/addColumn.d.ts +1 -1
- package/esm/methods/addColumn.d.ts.map +1 -1
- package/esm/methods/addGeoNoise.d.ts +7 -0
- package/esm/methods/addGeoNoise.d.ts.map +1 -0
- package/esm/methods/addGeoNoise.js +135 -0
- package/esm/methods/addNoise.d.ts +5 -0
- package/esm/methods/addNoise.d.ts.map +1 -0
- package/esm/methods/addNoise.js +53 -0
- package/esm/methods/addWordCount.d.ts +3 -0
- package/esm/methods/addWordCount.d.ts.map +1 -0
- package/esm/methods/addWordCount.js +15 -0
- package/esm/methods/bm25.d.ts.map +1 -1
- package/esm/methods/bm25.js +1 -2
- package/esm/methods/cache.d.ts.map +1 -1
- package/esm/methods/cache.js +4 -6
- package/esm/methods/columnProportions.d.ts +1 -0
- package/esm/methods/columnProportions.d.ts.map +1 -1
- package/esm/methods/columnProportions.js +7 -2
- package/esm/methods/convert.d.ts +2 -2
- package/esm/methods/convert.d.ts.map +1 -1
- package/esm/methods/convert.js +5 -0
- package/esm/methods/getBottom.d.ts +1 -1
- package/esm/methods/getBottom.d.ts.map +1 -1
- package/esm/methods/getBottom.js +2 -2
- package/esm/methods/getData.d.ts +1 -1
- package/esm/methods/getData.d.ts.map +1 -1
- package/esm/methods/getData.js +2 -2
- package/esm/methods/join.d.ts.map +1 -1
- package/esm/methods/join.js +8 -16
- package/esm/methods/joinGeo.d.ts.map +1 -1
- package/esm/methods/joinGeo.js +10 -19
- package/esm/methods/loadArray.d.ts +4 -2
- package/esm/methods/loadArray.d.ts.map +1 -1
- package/esm/methods/loadArray.js +182 -18
- package/esm/methods/loadData.js +3 -0
- package/esm/methods/loadYahooFinanceData.d.ts +10 -0
- package/esm/methods/loadYahooFinanceData.d.ts.map +1 -0
- package/esm/methods/loadYahooFinanceData.js +133 -0
- package/esm/methods/log.d.ts +7 -0
- package/esm/methods/log.d.ts.map +1 -0
- package/esm/methods/log.js +58 -0
- package/esm/methods/logBottom.d.ts +5 -0
- package/esm/methods/logBottom.d.ts.map +1 -0
- package/esm/methods/logBottom.js +9 -0
- package/esm/methods/removeTable.d.ts.map +1 -1
- package/esm/methods/removeTable.js +2 -0
- package/esm/methods/removeTables.d.ts.map +1 -1
- package/esm/methods/removeTables.js +7 -0
- package/esm/methods/renameTable.d.ts.map +1 -1
- package/esm/methods/renameTable.js +4 -0
- package/esm/methods/rowProportions.d.ts +1 -0
- package/esm/methods/rowProportions.d.ts.map +1 -1
- package/esm/methods/rowProportions.js +5 -1
- package/esm/methods/rowToText.js +2 -2
- package/esm/methods/selectTables.d.ts.map +1 -1
- package/esm/methods/selectTables.js +6 -0
- package/esm/methods/setTypes.d.ts +1 -1
- package/esm/methods/setTypes.d.ts.map +1 -1
- package/esm/methods/stream.d.ts.map +1 -1
- package/esm/methods/stream.js +2 -1
- package/esm/methods/updateWithJS.d.ts +1 -3
- package/esm/methods/updateWithJS.d.ts.map +1 -1
- package/esm/methods/updateWithJS.js +123 -37
- package/llm.md +600 -430
- package/llms.txt +1 -1
- package/package.json +1 -1
- package/script/class/Simple.d.ts +8 -2
- package/script/class/Simple.d.ts.map +1 -1
- package/script/class/Simple.js +9 -3
- package/script/class/SimpleDB.d.ts +72 -1
- package/script/class/SimpleDB.d.ts.map +1 -1
- package/script/class/SimpleDB.js +81 -1
- package/script/class/SimpleTable.d.ts +434 -263
- package/script/class/SimpleTable.d.ts.map +1 -1
- package/script/class/SimpleTable.js +441 -316
- package/script/helpers/LogValue.d.ts +8 -0
- package/script/helpers/LogValue.d.ts.map +1 -0
- package/script/helpers/LogValue.js +26 -0
- package/script/helpers/cacheTableDependencies.d.ts +9 -0
- package/script/helpers/cacheTableDependencies.d.ts.map +1 -1
- package/script/helpers/cacheTableDependencies.js +170 -20
- package/script/helpers/cleanSQL.d.ts +1 -1
- package/script/helpers/cleanSQL.d.ts.map +1 -1
- package/script/helpers/cleanSQL.js +271 -23
- package/script/helpers/flushAllTables.d.ts.map +1 -1
- package/script/helpers/flushAllTables.js +24 -29
- package/script/helpers/foldIdentifier.d.ts +3 -0
- package/script/helpers/foldIdentifier.d.ts.map +1 -0
- package/script/helpers/foldIdentifier.js +7 -0
- package/script/helpers/geometryFromJSON.d.ts +3 -0
- package/script/helpers/geometryFromJSON.d.ts.map +1 -0
- package/script/helpers/geometryFromJSON.js +7 -0
- package/script/helpers/index.d.ts +1 -0
- package/script/helpers/index.d.ts.map +1 -1
- package/script/helpers/index.js +3 -1
- package/script/helpers/logData.d.ts.map +1 -1
- package/script/helpers/logData.js +45 -44
- package/script/helpers/makeLogConverter.d.ts +5 -0
- package/script/helpers/makeLogConverter.d.ts.map +1 -0
- package/script/helpers/makeLogConverter.js +70 -0
- package/script/helpers/osmFiles.d.ts.map +1 -1
- package/script/helpers/parseTypes.d.ts +1 -1
- package/script/helpers/parseTypes.d.ts.map +1 -1
- package/script/helpers/parseTypes.js +4 -0
- package/script/helpers/pendingOps.d.ts +4 -2
- package/script/helpers/pendingOps.d.ts.map +1 -1
- package/script/helpers/prepareGeometry.d.ts +3 -0
- package/script/helpers/prepareGeometry.d.ts.map +1 -0
- package/script/helpers/prepareGeometry.js +102 -0
- package/script/helpers/prepareJSON.d.ts +3 -0
- package/script/helpers/prepareJSON.d.ts.map +1 -0
- package/script/helpers/prepareJSON.js +53 -0
- package/script/helpers/printTable.d.ts +5 -4
- package/script/helpers/printTable.d.ts.map +1 -1
- package/script/helpers/printTable.js +32 -36
- package/script/helpers/queryDB.js +1 -1
- package/script/helpers/queueAsyncBarrier.d.ts +4 -0
- package/script/helpers/queueAsyncBarrier.d.ts.map +1 -1
- package/script/helpers/queueAsyncBarrier.js +4 -0
- package/script/helpers/queueOp.d.ts.map +1 -1
- package/script/helpers/queueOp.js +4 -2
- package/script/helpers/readMutationRows.d.ts +9 -0
- package/script/helpers/readMutationRows.d.ts.map +1 -0
- package/script/helpers/readMutationRows.js +97 -0
- package/script/helpers/referencedTables.d.ts +9 -0
- package/script/helpers/referencedTables.d.ts.map +1 -0
- package/script/helpers/referencedTables.js +28 -0
- package/script/helpers/runQuery.d.ts.map +1 -1
- package/script/helpers/runQuery.js +11 -13
- package/script/helpers/scanSQL.d.ts +7 -0
- package/script/helpers/scanSQL.d.ts.map +1 -0
- package/script/helpers/scanSQL.js +74 -0
- package/script/helpers/tableRegistry.d.ts +2 -0
- package/script/helpers/tableRegistry.d.ts.map +1 -1
- package/script/helpers/tableRegistry.js +16 -4
- package/script/helpers/updateColumnsWithJS.d.ts +52 -0
- package/script/helpers/updateColumnsWithJS.d.ts.map +1 -0
- package/script/helpers/updateColumnsWithJS.js +145 -0
- package/script/methods/addCharacterCount.d.ts +3 -0
- package/script/methods/addCharacterCount.d.ts.map +1 -0
- package/script/methods/addCharacterCount.js +21 -0
- package/script/methods/addColumn.d.ts +1 -1
- package/script/methods/addColumn.d.ts.map +1 -1
- package/script/methods/addGeoNoise.d.ts +7 -0
- package/script/methods/addGeoNoise.d.ts.map +1 -0
- package/script/methods/addGeoNoise.js +141 -0
- package/script/methods/addNoise.d.ts +5 -0
- package/script/methods/addNoise.d.ts.map +1 -0
- package/script/methods/addNoise.js +59 -0
- package/script/methods/addWordCount.d.ts +3 -0
- package/script/methods/addWordCount.d.ts.map +1 -0
- package/script/methods/addWordCount.js +21 -0
- package/script/methods/bm25.d.ts.map +1 -1
- package/script/methods/bm25.js +1 -2
- package/script/methods/cache.d.ts.map +1 -1
- package/script/methods/cache.js +3 -5
- package/script/methods/columnProportions.d.ts +1 -0
- package/script/methods/columnProportions.d.ts.map +1 -1
- package/script/methods/columnProportions.js +7 -2
- package/script/methods/convert.d.ts +2 -2
- package/script/methods/convert.d.ts.map +1 -1
- package/script/methods/convert.js +5 -0
- package/script/methods/getBottom.d.ts +1 -1
- package/script/methods/getBottom.d.ts.map +1 -1
- package/script/methods/getBottom.js +2 -2
- package/script/methods/getData.d.ts +1 -1
- package/script/methods/getData.d.ts.map +1 -1
- package/script/methods/getData.js +2 -2
- package/script/methods/join.d.ts.map +1 -1
- package/script/methods/join.js +8 -16
- package/script/methods/joinGeo.d.ts.map +1 -1
- package/script/methods/joinGeo.js +10 -19
- package/script/methods/loadArray.d.ts +4 -2
- package/script/methods/loadArray.d.ts.map +1 -1
- package/script/methods/loadArray.js +181 -17
- package/script/methods/loadData.js +3 -0
- package/script/methods/loadYahooFinanceData.d.ts +10 -0
- package/script/methods/loadYahooFinanceData.d.ts.map +1 -0
- package/script/methods/loadYahooFinanceData.js +139 -0
- package/script/methods/log.d.ts +7 -0
- package/script/methods/log.d.ts.map +1 -0
- package/script/methods/log.js +64 -0
- package/script/methods/logBottom.d.ts +5 -0
- package/script/methods/logBottom.d.ts.map +1 -0
- package/script/methods/logBottom.js +15 -0
- package/script/methods/removeTable.d.ts.map +1 -1
- package/script/methods/removeTable.js +2 -0
- package/script/methods/removeTables.d.ts.map +1 -1
- package/script/methods/removeTables.js +7 -0
- package/script/methods/renameTable.d.ts.map +1 -1
- package/script/methods/renameTable.js +4 -0
- package/script/methods/rowProportions.d.ts +1 -0
- package/script/methods/rowProportions.d.ts.map +1 -1
- package/script/methods/rowProportions.js +5 -1
- package/script/methods/rowToText.js +2 -2
- package/script/methods/selectTables.d.ts.map +1 -1
- package/script/methods/selectTables.js +6 -0
- package/script/methods/setTypes.d.ts +1 -1
- package/script/methods/setTypes.d.ts.map +1 -1
- package/script/methods/stream.d.ts.map +1 -1
- package/script/methods/stream.js +2 -1
- package/script/methods/updateWithJS.d.ts +1 -3
- package/script/methods/updateWithJS.d.ts.map +1 -1
- package/script/methods/updateWithJS.js +122 -36
package/esm/class/SimpleTable.js
CHANGED
|
@@ -10,6 +10,8 @@ var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (
|
|
|
10
10
|
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
11
11
|
};
|
|
12
12
|
var _SimpleTable_name;
|
|
13
|
+
import logBottom from "../methods/logBottom.js";
|
|
14
|
+
import log from "../methods/log.js";
|
|
13
15
|
import quoteIdentifier from "../helpers/quoteIdentifier.js";
|
|
14
16
|
import csvFormat from "../helpers/csvFormat.js";
|
|
15
17
|
import getDescription from "../methods/getDescription.js";
|
|
@@ -17,6 +19,8 @@ import removeMissing from "../methods/removeMissing.js";
|
|
|
17
19
|
import getColumns from "../methods/getColumns.js";
|
|
18
20
|
import getRowCount from "../methods/getRowCount.js";
|
|
19
21
|
import getCharacterCount from "../methods/getCharacterCount.js";
|
|
22
|
+
import addCharacterCount from "../methods/addCharacterCount.js";
|
|
23
|
+
import addWordCount from "../methods/addWordCount.js";
|
|
20
24
|
import getTypes from "../methods/getTypes.js";
|
|
21
25
|
import getHash from "../methods/getHash.js";
|
|
22
26
|
import getValues from "../methods/getValues.js";
|
|
@@ -118,6 +122,7 @@ import length from "../methods/length.js";
|
|
|
118
122
|
import area from "../methods/area.js";
|
|
119
123
|
import reproject from "../methods/reproject.js";
|
|
120
124
|
import reducePrecision from "../methods/reducePrecision.js";
|
|
125
|
+
import addGeoNoise from "../methods/addGeoNoise.js";
|
|
121
126
|
import flipCoordinates from "../methods/flipCoordinates.js";
|
|
122
127
|
import addGeoType from "../methods/addGeoType.js";
|
|
123
128
|
import addGeoClosedStatus from "../methods/addGeoClosedStatus.js";
|
|
@@ -132,6 +137,7 @@ import getSchema from "../methods/getSchema.js";
|
|
|
132
137
|
import outliersIQR from "../methods/outliersIQR.js";
|
|
133
138
|
import bins from "../methods/bins.js";
|
|
134
139
|
import round from "../methods/round.js";
|
|
140
|
+
import addNoise from "../methods/addNoise.js";
|
|
135
141
|
import rowToText from "../methods/rowToText.js";
|
|
136
142
|
import replaceNulls from "../methods/replaceNulls.js";
|
|
137
143
|
import pad from "../methods/pad.js";
|
|
@@ -153,6 +159,7 @@ import insertRows from "../methods/insertRows.js";
|
|
|
153
159
|
import loadGeoData from "../methods/loadGeoData.js";
|
|
154
160
|
import loadOpenStreetMap from "../methods/loadOpenStreetMap.js";
|
|
155
161
|
import loadStatCanData from "../methods/loadStatCanData.js";
|
|
162
|
+
import loadYahooFinanceData from "../methods/loadYahooFinanceData.js";
|
|
156
163
|
import setTypes from "../methods/setTypes.js";
|
|
157
164
|
import flushAllTables from "../helpers/flushAllTables.js";
|
|
158
165
|
import queueOp from "../helpers/queueOp.js";
|
|
@@ -218,7 +225,7 @@ class SimpleTable extends Simple {
|
|
|
218
225
|
* @param simpleDB - The SimpleDB instance that this table belongs to.
|
|
219
226
|
* @param options - An optional object with configuration options:
|
|
220
227
|
* @param options.rowsToLog - The number of rows to log when displaying table data.
|
|
221
|
-
* @param options.charsToLog - The maximum
|
|
228
|
+
* @param options.charsToLog - The maximum characters to display for text and stringified nested cells, including truncation markers (default: 75). Independent of column width. Use Infinity for unlimited text.
|
|
222
229
|
* @param options.typesToLog - A boolean indicating whether to include data types when logging a table.
|
|
223
230
|
* @category Constructor
|
|
224
231
|
*/
|
|
@@ -294,6 +301,7 @@ class SimpleTable extends Simple {
|
|
|
294
301
|
}
|
|
295
302
|
/**
|
|
296
303
|
* Renames the current table.
|
|
304
|
+
* The new name must not belong to another registered table, ignoring ASCII letter case.
|
|
297
305
|
*
|
|
298
306
|
* @param name - The new name for the table.
|
|
299
307
|
* @returns A promise that resolves to the renamed table.
|
|
@@ -315,9 +323,7 @@ class SimpleTable extends Simple {
|
|
|
315
323
|
* Sets the data types for columns in a new table. If the table already exists, it will be replaced.
|
|
316
324
|
* To convert the types of an existing table, use the `.convert()` method instead.
|
|
317
325
|
*
|
|
318
|
-
*
|
|
319
|
-
*
|
|
320
|
-
* @param types - An object specifying the column names and their target data types (JavaScript or SQL types).
|
|
326
|
+
* @param types - An object specifying the column names and their target data types (JavaScript or SQL types), including JSON, FLOAT[n] vectors, and GEOMETRY with a CRS.
|
|
321
327
|
* @returns The table, so methods can be chained.
|
|
322
328
|
* @category Table Management
|
|
323
329
|
*
|
|
@@ -330,26 +336,43 @@ class SimpleTable extends Simple {
|
|
|
330
336
|
* raise: "float",
|
|
331
337
|
* }).log();
|
|
332
338
|
* ```
|
|
339
|
+
*
|
|
340
|
+
* @example
|
|
341
|
+
* ```ts
|
|
342
|
+
* // Create an empty table with JSON, embedding, and geometry columns.
|
|
343
|
+
* await table.setTypes({
|
|
344
|
+
* details: "JSON",
|
|
345
|
+
* embedding: "FLOAT[3]",
|
|
346
|
+
* geom: "GEOMETRY('EPSG:4326')",
|
|
347
|
+
* }).log();
|
|
348
|
+
* ```
|
|
333
349
|
*/
|
|
334
350
|
setTypes(types) {
|
|
335
351
|
setTypes(this, types);
|
|
336
352
|
return this;
|
|
337
353
|
}
|
|
338
354
|
/**
|
|
339
|
-
* Loads an array of JavaScript objects into the table.
|
|
355
|
+
* Loads an array of JavaScript objects into the table. Types are inferred for
|
|
356
|
+
* numbers, bigints, strings, booleans, and Date values. Array and object cells
|
|
357
|
+
* require an explicit supported type in columnTypes. Types can also be
|
|
358
|
+
* specified for scalar columns instead of inferred from their values.
|
|
340
359
|
*
|
|
341
360
|
* JavaScript `Date` values are inferred as DuckDB `TIMESTAMP` values. Their
|
|
342
361
|
* instant is preserved, but JavaScript `Date` does not retain the timezone or
|
|
343
|
-
* offset originally used to construct it.
|
|
344
|
-
*
|
|
362
|
+
* offset originally used to construct it.
|
|
363
|
+
*
|
|
364
|
+
* Declare `JSON` for general nested data, `FLOAT[n]` for a fixed-size float
|
|
365
|
+
* vector, or `GEOMETRY('EPSG:4326')` for GeoJSON geometries in WGS84.
|
|
345
366
|
*
|
|
346
367
|
* @param rows - An array of objects, where each object represents a row and its properties represent columns.
|
|
368
|
+
* @param options - Options for loading the array, captured when called.
|
|
369
|
+
* @param options.columnTypes - Types for specific columns. Required for array and object cells; omitted scalar columns are inferred, and all-null columns default to VARCHAR. Values must be compatible with the selected type without losing information.
|
|
347
370
|
* @returns The table, so methods can be chained.
|
|
348
371
|
* @category Importing Data
|
|
349
372
|
*
|
|
350
373
|
* @example
|
|
351
374
|
* ```ts
|
|
352
|
-
* // Load
|
|
375
|
+
* // Load scalar columns without columnTypes; strings and numbers are inferred.
|
|
353
376
|
* const data = [
|
|
354
377
|
* { letter: "a", number: 1 },
|
|
355
378
|
* { letter: "b", number: 2 }
|
|
@@ -359,31 +382,68 @@ class SimpleTable extends Simple {
|
|
|
359
382
|
*
|
|
360
383
|
* @example
|
|
361
384
|
* ```ts
|
|
362
|
-
* //
|
|
363
|
-
*
|
|
385
|
+
* // Choose a numeric type when all-null values provide nothing to infer.
|
|
386
|
+
* await table.loadArray(
|
|
387
|
+
* [{ name: "A", value: null }, { name: "B", value: null }],
|
|
388
|
+
* { columnTypes: { value: "DOUBLE" } },
|
|
389
|
+
* ).log();
|
|
390
|
+
* ```
|
|
391
|
+
*
|
|
392
|
+
* @example
|
|
393
|
+
* ```ts
|
|
394
|
+
* // Store the Date as a UTC timestamp with JavaScript's millisecond precision.
|
|
395
|
+
* // The original -04:00 offset is not retained.
|
|
364
396
|
* await table.loadArray([{
|
|
365
397
|
* observedAt: new Date("2024-04-07T13:00:00-04:00"),
|
|
366
398
|
* }]).log();
|
|
367
399
|
* ```
|
|
400
|
+
*
|
|
401
|
+
* @example
|
|
402
|
+
* ```ts
|
|
403
|
+
* // Declare JSON to load nested objects and variable-length arrays directly.
|
|
404
|
+
* await table.loadArray([{
|
|
405
|
+
* name: "Station",
|
|
406
|
+
* metadata: { active: true, tags: ["weather", "urban"] },
|
|
407
|
+
* measurements: [1, 2.5, null],
|
|
408
|
+
* }], { columnTypes: { metadata: "JSON", measurements: "JSON" } }).log();
|
|
409
|
+
* ```
|
|
410
|
+
*
|
|
411
|
+
* @example
|
|
412
|
+
* ```ts
|
|
413
|
+
* // Declare FLOAT[3] to store each three-element array as a fixed-size vector.
|
|
414
|
+
* await table.loadArray([{ vector: [1, 2, 3] }], {
|
|
415
|
+
* columnTypes: { vector: "FLOAT[3]" },
|
|
416
|
+
* }).log();
|
|
417
|
+
* ```
|
|
418
|
+
*
|
|
419
|
+
* @example
|
|
420
|
+
* ```ts
|
|
421
|
+
* // Declare a geometry type to store WGS84 GeoJSON as a spatial column.
|
|
422
|
+
* await table.loadArray([{
|
|
423
|
+
* name: "Montreal",
|
|
424
|
+
* geom: { type: "Point", coordinates: [-73.57, 45.50] },
|
|
425
|
+
* }], { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } }).log();
|
|
426
|
+
* ```
|
|
368
427
|
*/
|
|
369
|
-
loadArray(rows) {
|
|
370
|
-
loadArray(this, rows);
|
|
428
|
+
loadArray(rows, options = {}) {
|
|
429
|
+
loadArray(this, rows, options);
|
|
371
430
|
return this;
|
|
372
431
|
}
|
|
373
432
|
/**
|
|
374
433
|
* Loads data from one or more local or remote files into the table.
|
|
375
434
|
* Supported file formats include CSV, JSON, Parquet, and Excel.
|
|
376
|
-
*
|
|
435
|
+
*
|
|
436
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
377
437
|
*
|
|
378
438
|
* @param files - The path(s) or URL(s) of the file(s) containing the data to be loaded.
|
|
379
439
|
* @param options - An optional object with configuration options:
|
|
380
440
|
* @param options.fileType - The type of file to load ("csv", "dsv", "json", "parquet", "excel"). Defaults to being inferred from the file extension.
|
|
381
441
|
* @param options.autoDetect - A boolean indicating whether to automatically detect the data format. Defaults to `true`.
|
|
382
|
-
* @param options.conditions - A SQL `WHERE` clause expression, without the `WHERE` keyword, to filter source rows before applying `limit`.
|
|
442
|
+
* @param options.conditions - A SQL `WHERE` clause expression, without the `WHERE` keyword, to filter source rows before applying `limit`. Can reference source columns excluded from `columns`.
|
|
383
443
|
* @param options.limit - A number indicating the maximum number of matching rows to load, after applying `conditions` if provided. Defaults to all matching rows.
|
|
384
444
|
* @param options.includeFilename - A boolean indicating whether to include the filename as a new column in the loaded data. Defaults to `false`.
|
|
385
445
|
* @param options.unifyColumns - A boolean indicating whether to unify columns across multiple files when their structures differ. Missing columns will be filled with `NULL` values. Defaults to `false`.
|
|
386
|
-
* @param options.columnTypes - An object mapping column names to their expected data types. By default, types are inferred.
|
|
446
|
+
* @param options.columnTypes - An object mapping column names to their expected data types. By default, types are inferred. Geometry types are not supported; use loadGeoData() instead.
|
|
387
447
|
* @param options.columns - An array of column names to load. When provided, only the specified columns are loaded, reducing memory usage and improving load times. Not supported for Excel files — combining `columns` with Excel files throws an error. If an invalid column name is provided, DuckDB will throw its native error. An empty array behaves the same as omitting the option (loads all columns). Defaults to loading all columns.
|
|
388
448
|
* @param options.header - A boolean indicating whether the file has a header row. Applicable to CSV files. Defaults to `true`.
|
|
389
449
|
* @param options.allText - A boolean indicating whether all columns should be treated as text. Applicable to CSV files. Defaults to `false`.
|
|
@@ -460,8 +520,6 @@ class SimpleTable extends Simple {
|
|
|
460
520
|
}
|
|
461
521
|
/**
|
|
462
522
|
* Downloads a complete Statistics Canada table and loads it into this table.
|
|
463
|
-
* The method queues the download and load; they run when an async observer
|
|
464
|
-
* method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
465
523
|
*
|
|
466
524
|
* Results are cached as Parquet files in `.sda-cache/statcan` by default.
|
|
467
525
|
* Cached data does not expire unless a TTL is provided.
|
|
@@ -499,15 +557,75 @@ class SimpleTable extends Simple {
|
|
|
499
557
|
return this;
|
|
500
558
|
}
|
|
501
559
|
/**
|
|
502
|
-
*
|
|
560
|
+
* Downloads historical market data from Yahoo Finance and loads it into this
|
|
561
|
+
* table as `datetime`, `open`, `high`, `low`, `close`, `adjustedClose`, and
|
|
562
|
+
* `volume` columns. `datetime` contains Yahoo's timestamps as JavaScript
|
|
563
|
+
* `Date` values. Unavailable values are preserved as `null`.
|
|
564
|
+
*
|
|
565
|
+
* All range boundaries are evaluated in UTC. Use ISO date-time strings ending
|
|
566
|
+
* in `Z`. Yahoo trading sessions may occur on a different UTC date from their
|
|
567
|
+
* exchange-local date, so choose the range appropriate for the market and
|
|
568
|
+
* interval. Avoid numeric constructors such as `new Date(2025, 2, 15)`, which
|
|
569
|
+
* use the runtime's local timezone.
|
|
570
|
+
*
|
|
571
|
+
* This method uses an undocumented Yahoo Finance endpoint and is not
|
|
572
|
+
* affiliated with or endorsed by Yahoo. It is provided for educational,
|
|
573
|
+
* research, and journalistic purposes. Before using it, review Yahoo's terms
|
|
574
|
+
* and any applicable data-provider restrictions.
|
|
575
|
+
*
|
|
576
|
+
* @param symbol - The stock or index symbol, such as `"AAPL"` or `"^GSPTSE"`.
|
|
577
|
+
* @param startDate - The inclusive UTC start of the requested range.
|
|
578
|
+
* @param endDate - The inclusive UTC end of the requested range. The UTC day,
|
|
579
|
+
* hour, or minute containing this instant is included, according to
|
|
580
|
+
* `interval`.
|
|
581
|
+
* @param interval - The interval between observations: daily, hourly, or every
|
|
582
|
+
* minute.
|
|
583
|
+
* @returns The table, so methods can be chained.
|
|
584
|
+
* @throws {RangeError} If either date is invalid or `endDate` is before `startDate`.
|
|
585
|
+
* @throws {Error} If Yahoo rejects the request or returns no data.
|
|
586
|
+
* @see https://legal.yahoo.com/us/en/yahoo/terms/otos/index.html
|
|
587
|
+
* @see https://help.yahoo.com/kb/finance/SLN2310.html
|
|
588
|
+
* @category Importing Data
|
|
503
589
|
*
|
|
504
|
-
*
|
|
590
|
+
* @example
|
|
591
|
+
* ```ts
|
|
592
|
+
* // Request daily observations using explicit UTC boundaries.
|
|
593
|
+
* await table
|
|
594
|
+
* .loadYahooFinanceData(
|
|
595
|
+
* "^GSPTSE",
|
|
596
|
+
* new Date("2025-03-01T00:00:00Z"),
|
|
597
|
+
* new Date("2025-03-15T00:00:00Z"),
|
|
598
|
+
* "1d",
|
|
599
|
+
* )
|
|
600
|
+
* .log();
|
|
601
|
+
* ```
|
|
602
|
+
*
|
|
603
|
+
* @example
|
|
604
|
+
* ```ts
|
|
605
|
+
* // Include the UTC hours from 13:00 through 16:00.
|
|
606
|
+
* await table
|
|
607
|
+
* .loadYahooFinanceData(
|
|
608
|
+
* "AAPL",
|
|
609
|
+
* new Date("2025-03-14T13:00:00Z"),
|
|
610
|
+
* new Date("2025-03-14T16:00:00Z"),
|
|
611
|
+
* "1h",
|
|
612
|
+
* )
|
|
613
|
+
* .log();
|
|
614
|
+
* ```
|
|
615
|
+
*/
|
|
616
|
+
loadYahooFinanceData(symbol, startDate, endDate, interval) {
|
|
617
|
+
loadYahooFinanceData(this, symbol, startDate, endDate, interval);
|
|
618
|
+
return this;
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* Loads geospatial data from an external file or URL into the table.
|
|
622
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
505
623
|
*
|
|
506
624
|
* @param file - The path or URL of the external file containing the geospatial data.
|
|
507
625
|
* @param options - An optional object with configuration options:
|
|
508
626
|
* @param options.toEPSG4326 - If `true`, the method will attempt to reproject the data to EPSG:4326 (WGS84).
|
|
509
627
|
* @param options.columns - The columns to load. Include the geometry column that should remain in the resulting table, usually `"geom"`. By default, all columns are loaded.
|
|
510
|
-
* @param options.conditions - A SQL `WHERE` clause expression, without the `WHERE` keyword, to filter source rows before materialization and reprojection.
|
|
628
|
+
* @param options.conditions - A SQL `WHERE` clause expression, without the `WHERE` keyword, to filter source rows before materialization and reprojection. Can reference source columns excluded from `columns`. Geometry conditions use the source coordinate system.
|
|
511
629
|
* @returns The table, so methods can be chained.
|
|
512
630
|
* @category Geospatial
|
|
513
631
|
*
|
|
@@ -561,7 +679,7 @@ class SimpleTable extends Simple {
|
|
|
561
679
|
return this;
|
|
562
680
|
}
|
|
563
681
|
/**
|
|
564
|
-
* Loads OpenStreetMap data into the table from a local `.osm` or `.osm.pbf` file, a remote file URL, or an Overpass bounding-box query. Pass a path or URL string to load an existing file, or pass a bounding box with `filters` to download matching features.
|
|
682
|
+
* Loads OpenStreetMap data into the table from a local `.osm` or `.osm.pbf` file, a remote file URL, or an Overpass bounding-box query. Pass a path or URL string to load an existing file, or pass a bounding box with `filters` to download matching features.
|
|
565
683
|
*
|
|
566
684
|
* DuckDB's [Osmium community extension](https://duckdb.org/community_extensions/extensions/osmium) materializes the complete result and reconstructs `geom` as EPSG:4326. By default, that canonical result is cached as GeoParquet in `.sda-cache/osm`. Distinct paths, local file fingerprints, normalized URLs, endpoints, queries, and bounding boxes retain independent entries. The cache has no expiration unless `ttl` is set and can be cleared by removing `.sda-cache/osm`. Immutable URLs can safely use the default indefinite cache; mutable URLs should set a TTL.
|
|
567
685
|
*
|
|
@@ -648,8 +766,6 @@ class SimpleTable extends Simple {
|
|
|
648
766
|
* DuckDB FTS indexes do not update automatically when the table changes; use
|
|
649
767
|
* `overwrite: true` to rebuild the index after modifying the table.
|
|
650
768
|
*
|
|
651
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
652
|
-
*
|
|
653
769
|
* @param idColumn - The column containing the document identifiers.
|
|
654
770
|
* @param textColumn - The column containing the text to search.
|
|
655
771
|
* @param options - An optional object with configuration options:
|
|
@@ -708,8 +824,6 @@ class SimpleTable extends Simple {
|
|
|
708
824
|
* If a VSS index already exists on the table, this method will skip creation and log a message (when verbose is enabled), unless the `overwrite` option is set to `true`.
|
|
709
825
|
* The index definition is recorded in {@link indexes}.
|
|
710
826
|
*
|
|
711
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
712
|
-
*
|
|
713
827
|
* @param column - The name of the column containing vector embeddings (must be FLOAT array type).
|
|
714
828
|
* @param options - An optional object with configuration options:
|
|
715
829
|
* @param options.overwrite - If `true`, drops and recreates the index even if it already exists. Defaults to `false`.
|
|
@@ -770,8 +884,6 @@ class SimpleTable extends Simple {
|
|
|
770
884
|
* changes; use `overwriteIndex: true` to rebuild the index after modifying
|
|
771
885
|
* the table.
|
|
772
886
|
*
|
|
773
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
774
|
-
*
|
|
775
887
|
* @param text - The search query text to match against the text column.
|
|
776
888
|
* @param idColumn - The name of the column containing unique identifiers for each row.
|
|
777
889
|
* @param textColumn - The name of the column containing the text to search.
|
|
@@ -873,8 +985,6 @@ class SimpleTable extends Simple {
|
|
|
873
985
|
/**
|
|
874
986
|
* Inserts rows, provided as an array of JavaScript objects, into the table.
|
|
875
987
|
*
|
|
876
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
877
|
-
*
|
|
878
988
|
* @param rows - An array of objects, where each object represents a row to be inserted and its properties correspond to column names.
|
|
879
989
|
* @returns The table, so methods can be chained.
|
|
880
990
|
* @category Importing Data
|
|
@@ -894,7 +1004,7 @@ class SimpleTable extends Simple {
|
|
|
894
1004
|
return this;
|
|
895
1005
|
}
|
|
896
1006
|
/**
|
|
897
|
-
* Inserts all rows from one or more other tables into this table. If tables do not have the same columns, an error will be thrown unless the `unifyColumns` option is set to `true`.
|
|
1007
|
+
* Inserts all rows from one or more other tables into this table. If tables do not have the same columns, an error will be thrown unless the `unifyColumns` option is set to `true`.
|
|
898
1008
|
*
|
|
899
1009
|
* @param tables - The name(s) of the table(s) or SimpleTable instance(s) from which rows will be inserted.
|
|
900
1010
|
* @param options - An optional object with configuration options:
|
|
@@ -927,8 +1037,6 @@ class SimpleTable extends Simple {
|
|
|
927
1037
|
/**
|
|
928
1038
|
* Fetches sample data from the simple-data-analysis-core GitHub repository.
|
|
929
1039
|
*
|
|
930
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
931
|
-
*
|
|
932
1040
|
* @param sample - The name of the sample to load.
|
|
933
1041
|
*
|
|
934
1042
|
* Tabular data:
|
|
@@ -954,13 +1062,12 @@ class SimpleTable extends Simple {
|
|
|
954
1062
|
}
|
|
955
1063
|
/**
|
|
956
1064
|
* Returns a new table with the same structure and data as this table. The data can be optionally filtered, limited to a specific number of rows, and offset.
|
|
1065
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
957
1066
|
*
|
|
958
1067
|
* If `conditions`, `limit`, and `offset` are all used, they are applied in this order: `conditions` (WHERE clause) first, then `offset`, and finally `limit` (LIMIT).
|
|
959
1068
|
*
|
|
960
1069
|
* Note that cloning large tables can be a slow operation.
|
|
961
1070
|
*
|
|
962
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
963
|
-
*
|
|
964
1071
|
* @param nameOrOptions - Either a string specifying the name of the new table, or an optional object with configuration options. If not provided, a default name (e.g., "table1", "table2") will be generated.
|
|
965
1072
|
* @param nameOrOptions.name - The name of the new table to be created in the database. If not provided, a default name (e.g., "table1", "table2") will be generated.
|
|
966
1073
|
* @param nameOrOptions.conditions - A SQL `WHERE` clause condition to filter the data during cloning. Defaults to no condition (clones all rows).
|
|
@@ -1029,8 +1136,6 @@ class SimpleTable extends Simple {
|
|
|
1029
1136
|
/**
|
|
1030
1137
|
* Clones an existing column in this table, creating a new column with identical values.
|
|
1031
1138
|
*
|
|
1032
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1033
|
-
*
|
|
1034
1139
|
* @param column - The name of the original column to clone.
|
|
1035
1140
|
* @param newColumn - The name of the new column to be created.
|
|
1036
1141
|
* @returns The table, so methods can be chained.
|
|
@@ -1052,8 +1157,6 @@ class SimpleTable extends Simple {
|
|
|
1052
1157
|
*
|
|
1053
1158
|
* **Important:** The offset is applied based on the current row order in the table. For meaningful results, ensure your data is sorted appropriately (e.g., by date/time for time-series analysis) before calling this method.
|
|
1054
1159
|
*
|
|
1055
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1056
|
-
*
|
|
1057
1160
|
* @param column - The name of the original column.
|
|
1058
1161
|
* @param newColumn - The name of the new column to be created with offset values.
|
|
1059
1162
|
* @param options - An optional object with configuration options:
|
|
@@ -1099,8 +1202,6 @@ class SimpleTable extends Simple {
|
|
|
1099
1202
|
/**
|
|
1100
1203
|
* Fills `NULL` values in specified columns. By default, each `NULL` is replaced with the last non-`NULL` value from the preceding row. When `interpolate` is `true`, `NULL` values are replaced using linear interpolation (or extrapolation at the ends). Pass `interpolateBy` with a real numeric or date column name to use it as the X-axis, so that interpolated values are proportional to the actual distances between X-axis values rather than treating every row as equidistant. When `interpolateBy` is set, `interpolate` is automatically assumed `true`.
|
|
1101
1204
|
*
|
|
1102
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1103
|
-
*
|
|
1104
1205
|
* @param columns - The column(s) for which to fill `NULL` values.
|
|
1105
1206
|
* @param options - An optional object with configuration options:
|
|
1106
1207
|
* @param options.by - A column name or an array of column names to partition by. The fill is applied independently within each group.
|
|
@@ -1159,7 +1260,7 @@ class SimpleTable extends Simple {
|
|
|
1159
1260
|
* Sorts the rows of the table based on specified column(s) and order(s).
|
|
1160
1261
|
* If no columns are specified, all columns are sorted from left to right in ascending order.
|
|
1161
1262
|
*
|
|
1162
|
-
*
|
|
1263
|
+
* Order-preserving transformations after a sort retain that order. Operations such as joins, grouping, aggregation, and sampling do not guarantee input order; chain `sort()` after them when deterministic output order matters.
|
|
1163
1264
|
*
|
|
1164
1265
|
* @param order - An object mapping column names to their sorting order: `"asc"` for ascending or `"desc"` for descending. If `null`, all columns are sorted ascendingly.
|
|
1165
1266
|
* @param options - An optional object with configuration options:
|
|
@@ -1196,7 +1297,7 @@ class SimpleTable extends Simple {
|
|
|
1196
1297
|
return this;
|
|
1197
1298
|
}
|
|
1198
1299
|
/**
|
|
1199
|
-
* Selects specific columns in the table, removing all others.
|
|
1300
|
+
* Selects specific columns in the table, removing all others.
|
|
1200
1301
|
*
|
|
1201
1302
|
* @param columns - The name or an array of names of the columns to be selected.
|
|
1202
1303
|
* @returns The table, so methods can be chained.
|
|
@@ -1221,8 +1322,6 @@ class SimpleTable extends Simple {
|
|
|
1221
1322
|
/**
|
|
1222
1323
|
* Skips the first `n` rows of the table, effectively removing them.
|
|
1223
1324
|
*
|
|
1224
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1225
|
-
*
|
|
1226
1325
|
* @param count - The number of rows to skip from the beginning of the table.
|
|
1227
1326
|
* @returns The table, so methods can be chained.
|
|
1228
1327
|
* @category Selecting or Filtering Data
|
|
@@ -1258,8 +1357,6 @@ class SimpleTable extends Simple {
|
|
|
1258
1357
|
/**
|
|
1259
1358
|
* Selects random rows from the table, removing all others. You can optionally specify a seed to ensure repeatable sampling.
|
|
1260
1359
|
*
|
|
1261
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1262
|
-
*
|
|
1263
1360
|
* @param count - The number of rows to select (e.g., `100`) or a percentage string (e.g., `"10%"`) specifying the sampling size.
|
|
1264
1361
|
* @param options - An optional object with configuration options:
|
|
1265
1362
|
* @param options.seed - A number specifying the seed for repeatable sampling. Using the same seed will always yield the same random rows. Defaults to a random seed.
|
|
@@ -1291,8 +1388,6 @@ class SimpleTable extends Simple {
|
|
|
1291
1388
|
/**
|
|
1292
1389
|
* Selects a specified number of rows from this table. An offset can be applied to skip initial rows, and the results can be output to a new table.
|
|
1293
1390
|
*
|
|
1294
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1295
|
-
*
|
|
1296
1391
|
* @param count - The number of rows to select.
|
|
1297
1392
|
* @param options - An optional object with configuration options:
|
|
1298
1393
|
* @param options.offset - The number of rows to skip from the beginning of the table before selecting. Defaults to `0`.
|
|
@@ -1331,8 +1426,6 @@ class SimpleTable extends Simple {
|
|
|
1331
1426
|
* Removes duplicate rows from this table, keeping only unique rows.
|
|
1332
1427
|
* Note that the resulting data order might differ from the original.
|
|
1333
1428
|
*
|
|
1334
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1335
|
-
*
|
|
1336
1429
|
* @param options - An optional object with configuration options:
|
|
1337
1430
|
* @param options.on - A column name or an array of column names to consider when identifying duplicates. If specified, duplicates are determined based only on the values in these columns. If omitted, all columns are considered.
|
|
1338
1431
|
* @returns The table, so methods can be chained.
|
|
@@ -1363,7 +1456,6 @@ class SimpleTable extends Simple {
|
|
|
1363
1456
|
/**
|
|
1364
1457
|
* Removes rows with missing values from this table.
|
|
1365
1458
|
* By default, missing values include SQL `NULL`, as well as string representations like `"NULL"`, `"null"`, `"NaN"`, `"undefined"`, and empty strings `""`.
|
|
1366
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1367
1459
|
*
|
|
1368
1460
|
* @param options - An optional object with configuration options:
|
|
1369
1461
|
* @param options.columns - A string or an array of strings specifying the columns to consider for missing values. If omitted, all columns are considered.
|
|
@@ -1401,7 +1493,7 @@ class SimpleTable extends Simple {
|
|
|
1401
1493
|
return this;
|
|
1402
1494
|
}
|
|
1403
1495
|
/**
|
|
1404
|
-
* Trims specified characters from the beginning, end, or both sides of string values in the given columns.
|
|
1496
|
+
* Trims specified characters from the beginning, end, or both sides of string values in the given columns.
|
|
1405
1497
|
*
|
|
1406
1498
|
* @param columns - The column name or an array of column names to trim.
|
|
1407
1499
|
* @param options - An optional object with configuration options:
|
|
@@ -1434,8 +1526,7 @@ class SimpleTable extends Simple {
|
|
|
1434
1526
|
}
|
|
1435
1527
|
/**
|
|
1436
1528
|
* Filters rows from this table based on SQL conditions. Note that it's often faster to use the `removeRows` method for simple removals.
|
|
1437
|
-
*
|
|
1438
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1529
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
1439
1530
|
*
|
|
1440
1531
|
* @param conditions - The filtering conditions specified as a SQL `WHERE` clause (e.g., `"column1 > 10 AND column2 = 'value'"`).
|
|
1441
1532
|
* @returns The table, so methods can be chained.
|
|
@@ -1472,8 +1563,6 @@ class SimpleTable extends Simple {
|
|
|
1472
1563
|
/**
|
|
1473
1564
|
* Keeps rows in this table that have specific values in specified columns, removing all other rows.
|
|
1474
1565
|
*
|
|
1475
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1476
|
-
*
|
|
1477
1566
|
* @param columnsAndValues - An object where keys are column names and values are the specific values (or an array of values) to keep in those columns. Use `null` to keep rows where a column is `NULL`.
|
|
1478
1567
|
* @returns The table, so methods can be chained.
|
|
1479
1568
|
* @category Selecting or Filtering Data
|
|
@@ -1503,8 +1592,6 @@ class SimpleTable extends Simple {
|
|
|
1503
1592
|
/**
|
|
1504
1593
|
* Removes rows from this table that have specific values in specified columns.
|
|
1505
1594
|
*
|
|
1506
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1507
|
-
*
|
|
1508
1595
|
* @param columnsAndValues - An object where keys are column names and values are the specific values (or an array of values) to remove from those columns. Use `null` to remove rows where a column is `NULL`; otherwise, `NULL` rows are retained.
|
|
1509
1596
|
* @returns The table, so methods can be chained.
|
|
1510
1597
|
* @category Selecting or Filtering Data
|
|
@@ -1533,9 +1620,7 @@ class SimpleTable extends Simple {
|
|
|
1533
1620
|
}
|
|
1534
1621
|
/**
|
|
1535
1622
|
* Removes rows from this table based on SQL conditions. This method is similar to `filter()`, but removes rows instead of keeping them.
|
|
1536
|
-
*
|
|
1537
|
-
*
|
|
1538
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1623
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
1539
1624
|
*
|
|
1540
1625
|
* @param conditions - The filtering conditions specified as a SQL `WHERE` clause (e.g., `"fruit = 'apple'"`).
|
|
1541
1626
|
* @returns The table, so methods can be chained.
|
|
@@ -1573,8 +1658,6 @@ class SimpleTable extends Simple {
|
|
|
1573
1658
|
* Renames one or more columns in the table. Throws if a source column does
|
|
1574
1659
|
* not exist, so a typo fails loudly instead of being silently ignored.
|
|
1575
1660
|
*
|
|
1576
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1577
|
-
*
|
|
1578
1661
|
* @param names - An object mapping old column names to their new column names (e.g., `{ "oldName": "newName", "anotherOld": "anotherNew" }`).
|
|
1579
1662
|
* @param options - Configuration options.
|
|
1580
1663
|
* @param options.strict - Whether to verify the source columns exist before renaming. Defaults to `true`. Set to `false` to skip the check and its schema lookup when you know the columns exist and are renaming across many tables where the extra round-trip adds up.
|
|
@@ -1606,8 +1689,6 @@ class SimpleTable extends Simple {
|
|
|
1606
1689
|
/**
|
|
1607
1690
|
* Cleans column names by removing non-alphanumeric characters and formatting them to camel case.
|
|
1608
1691
|
*
|
|
1609
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1610
|
-
*
|
|
1611
1692
|
* @returns The table, so methods can be chained.
|
|
1612
1693
|
* @category Column Operations
|
|
1613
1694
|
*
|
|
@@ -1660,8 +1741,6 @@ class SimpleTable extends Simple {
|
|
|
1660
1741
|
* | Sales | 2022 | 75 |
|
|
1661
1742
|
* | Sales | 2023 | 98 |
|
|
1662
1743
|
*
|
|
1663
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1664
|
-
*
|
|
1665
1744
|
* @param columns - An array of strings representing the names of the columns to be stacked (unpivoted).
|
|
1666
1745
|
* @param namesTo - The name of the new column that will contain the original column names (e.g., "Year").
|
|
1667
1746
|
* @param valuesTo - The name of the new column that will contain the values from the stacked columns (e.g., "Employees").
|
|
@@ -1703,8 +1782,6 @@ class SimpleTable extends Simple {
|
|
|
1703
1782
|
*
|
|
1704
1783
|
* When multiple rows share the same `namesFrom`/grouping combination, their `valuesFrom` values are combined with the `options.stat` function (`"sum"` by default).
|
|
1705
1784
|
*
|
|
1706
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1707
|
-
*
|
|
1708
1785
|
* @param namesFrom - The name of the column containing the values that will be transformed into new column headers (e.g., "Year").
|
|
1709
1786
|
* @param valuesFrom - The name of the column containing the values to be spread across the new columns (e.g., "Employees").
|
|
1710
1787
|
* @param options - An optional object with configuration options:
|
|
@@ -1729,9 +1806,7 @@ class SimpleTable extends Simple {
|
|
|
1729
1806
|
*
|
|
1730
1807
|
* When converting strings to numbers, commas (often used as thousand separators) will be automatically removed before conversion.
|
|
1731
1808
|
*
|
|
1732
|
-
*
|
|
1733
|
-
*
|
|
1734
|
-
* @param types - An object mapping column names to their target data types for conversion.
|
|
1809
|
+
* @param types - An object mapping column names to their target data types for conversion, including JSON, FLOAT[n] vectors, and GEOMETRY with a CRS.
|
|
1735
1810
|
* @param options - An optional object with configuration options:
|
|
1736
1811
|
* @param options.strict - If `false`, values that cannot be converted will be replaced by `NULL` instead of throwing an error. Defaults to `true`.
|
|
1737
1812
|
* @param options.datetimeFormat - A string specifying the format for date and time conversions. Uses `strftime` and `strptime` functions from DuckDB. For format specifiers, see [DuckDB's documentation](https://duckdb.org/docs/sql/functions/dateformat).
|
|
@@ -1778,6 +1853,18 @@ class SimpleTable extends Simple {
|
|
|
1778
1853
|
* // Convert 'amount' to float, replacing unconvertible values with NULL
|
|
1779
1854
|
* await table.convert({ amount: "float" }, { strict: false }).log();
|
|
1780
1855
|
* ```
|
|
1856
|
+
*
|
|
1857
|
+
* @example
|
|
1858
|
+
* ```ts
|
|
1859
|
+
* // Parse JSON, a fixed-size float vector, and WKT geometry from text.
|
|
1860
|
+
* await table.loadArray([{
|
|
1861
|
+
* details: '{"count":3}',
|
|
1862
|
+
* embedding: "[0.25,0.5,0.75]",
|
|
1863
|
+
* geom: "POINT (-73 45)",
|
|
1864
|
+
* }])
|
|
1865
|
+
* .convert({ details: "JSON", embedding: "FLOAT[3]", geom: "GEOMETRY('EPSG:4326')" })
|
|
1866
|
+
* .log();
|
|
1867
|
+
* ```
|
|
1781
1868
|
*/
|
|
1782
1869
|
convert(types, options = {}) {
|
|
1783
1870
|
convert(this, types, options);
|
|
@@ -1800,7 +1887,7 @@ class SimpleTable extends Simple {
|
|
|
1800
1887
|
return this;
|
|
1801
1888
|
}
|
|
1802
1889
|
/**
|
|
1803
|
-
* Removes one or more columns from this table.
|
|
1890
|
+
* Removes one or more columns from this table.
|
|
1804
1891
|
*
|
|
1805
1892
|
* @param columns - The name or an array of names of the columns to be removed.
|
|
1806
1893
|
* @returns The table, so methods can be chained.
|
|
@@ -1823,10 +1910,11 @@ class SimpleTable extends Simple {
|
|
|
1823
1910
|
return this;
|
|
1824
1911
|
}
|
|
1825
1912
|
/**
|
|
1826
|
-
* Adds a new column to the table based on a specified data type (JavaScript or SQL types) and a SQL definition.
|
|
1913
|
+
* Adds a new column to the table based on a specified data type (JavaScript or SQL types) and a SQL definition.
|
|
1914
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, expressions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
1827
1915
|
*
|
|
1828
1916
|
* @param newColumn - The name of the new column to be added.
|
|
1829
|
-
* @param type - The data type for the new column. Can be a JavaScript type (e.g., `"number"`, `"string"`) or a SQL type (e.g., `"integer"`, `"varchar"`).
|
|
1917
|
+
* @param type - The data type for the new column. Can be a JavaScript type (e.g., `"number"`, `"string"`) or a SQL type (e.g., `"integer"`, `"varchar"`, `"JSON"`, `"FLOAT[3]"`, `"GEOMETRY('EPSG:4326')"`).
|
|
1830
1918
|
* @param definition - A SQL expression defining how the values for the new column should be computed (e.g., `"column1 + column2"`, `"ST_Centroid(geom_column)"`).
|
|
1831
1919
|
* @returns The table, so methods can be chained.
|
|
1832
1920
|
* @category Column Operations
|
|
@@ -1842,6 +1930,25 @@ class SimpleTable extends Simple {
|
|
|
1842
1930
|
* // Add a new geometry column 'centroid' using the centroid of an existing 'country' geometry column
|
|
1843
1931
|
* await table.addColumn("centroid", "geometry('EPSG:4326')", `ST_Centroid("country")`).log();
|
|
1844
1932
|
* ```
|
|
1933
|
+
*
|
|
1934
|
+
* @example
|
|
1935
|
+
* ```ts
|
|
1936
|
+
* // Read a number from a JSON object into a new column using dot notation.
|
|
1937
|
+
* await table.loadArray([{ details: { count: 3 } }], {
|
|
1938
|
+
* columnTypes: { details: "JSON" },
|
|
1939
|
+
* })
|
|
1940
|
+
* .addColumn("count", "integer", "details.count")
|
|
1941
|
+
* .log();
|
|
1942
|
+
* ```
|
|
1943
|
+
*
|
|
1944
|
+
* @example
|
|
1945
|
+
* ```ts
|
|
1946
|
+
* // Create JSON and fixed-size vector columns with DuckDB expressions.
|
|
1947
|
+
* await table.loadArray([{ count: 3 }])
|
|
1948
|
+
* .addColumn("details", "JSON", "json_object('count', count)")
|
|
1949
|
+
* .addColumn("embedding", "FLOAT[3]", "[0.25, 0.5, 0.75]")
|
|
1950
|
+
* .log();
|
|
1951
|
+
* ```
|
|
1845
1952
|
*/
|
|
1846
1953
|
addColumn(newColumn, type, definition) {
|
|
1847
1954
|
addColumn(this, newColumn, type, definition);
|
|
@@ -1861,9 +1968,6 @@ class SimpleTable extends Simple {
|
|
|
1861
1968
|
* values produce `NULL` extracted values. Parts extracted from
|
|
1862
1969
|
* `TIMESTAMP WITH TIME ZONE` values use UTC.
|
|
1863
1970
|
*
|
|
1864
|
-
* This method queues the operation; it runs when an async observer method
|
|
1865
|
-
* (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1866
|
-
*
|
|
1867
1971
|
* @param column - The temporal column from which to extract components.
|
|
1868
1972
|
* @param parts - A part to extract using its name as the new column, or an object mapping each custom new-column name to the part it should contain.
|
|
1869
1973
|
* @returns The table, so methods can be chained.
|
|
@@ -1891,8 +1995,6 @@ class SimpleTable extends Simple {
|
|
|
1891
1995
|
/**
|
|
1892
1996
|
* Adds a new column to the table containing the row number, starting at 0 (like an index).
|
|
1893
1997
|
*
|
|
1894
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1895
|
-
*
|
|
1896
1998
|
* @param newColumn - The name of the new column that will store the row number.
|
|
1897
1999
|
* @param options - An optional object with configuration options:
|
|
1898
2000
|
* @param options.by - A column name or an array of column names to partition by. The row number restarts at 0 within each group.
|
|
@@ -1919,8 +2021,6 @@ class SimpleTable extends Simple {
|
|
|
1919
2021
|
* Performs a cross join operation with another table. A cross join returns the Cartesian product of the rows from both tables, meaning all possible pairs of rows will be in the resulting table.
|
|
1920
2022
|
* This means that if the left table has `n` rows and the right table has `m` rows, the result will have `n * m` rows.
|
|
1921
2023
|
*
|
|
1922
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1923
|
-
*
|
|
1924
2024
|
* @param rightTable - The SimpleTable instance to cross join with.
|
|
1925
2025
|
* @param options - An optional object with configuration options:
|
|
1926
2026
|
* @param options.outputTable - If `true`, the results will be stored in a new table with a generated name. If a string, it will be used as the name for the new table. If `false` or omitted, the current table will be overwritten. Defaults to `false`.
|
|
@@ -1952,7 +2052,6 @@ class SimpleTable extends Simple {
|
|
|
1952
2052
|
* Merges the data of this table (considered the left table) with another table (the right table) based on a common column or multiple columns.
|
|
1953
2053
|
* Note that the order of rows in the returned data is not guaranteed to be the same as in the original tables.
|
|
1954
2054
|
* This operation might create temporary files in a `.tmp` folder; consider adding `.tmp` to your `.gitignore`.
|
|
1955
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called. The join uses the other table's state as of this call: operations queued on it afterwards run after the join.
|
|
1956
2055
|
*
|
|
1957
2056
|
* @param rightTable - The SimpleTable instance to be joined with this table.
|
|
1958
2057
|
* @param options - An optional object with configuration options:
|
|
@@ -2000,8 +2099,6 @@ class SimpleTable extends Simple {
|
|
|
2000
2099
|
*
|
|
2001
2100
|
* This operation might create temporary files in a `.tmp` folder; consider adding `.tmp` to your `.gitignore`.
|
|
2002
2101
|
*
|
|
2003
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called. The join uses the other table's state as of this call: operations queued on it afterwards run after the join.
|
|
2004
|
-
*
|
|
2005
2102
|
* @param rightTable - The SimpleTable instance to be joined with this table.
|
|
2006
2103
|
* @param leftColumn - The name of the column in this (left) table containing the text to compare.
|
|
2007
2104
|
* @param rightColumn - The name of the column in the right table containing the text to compare.
|
|
@@ -2068,8 +2165,6 @@ class SimpleTable extends Simple {
|
|
|
2068
2165
|
* Similarity is computed using the [rapidfuzz](https://query.farm/duckdb_extension_rapidfuzz) DuckDB community extension,
|
|
2069
2166
|
* which is installed and loaded automatically.
|
|
2070
2167
|
*
|
|
2071
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2072
|
-
*
|
|
2073
2168
|
* @param column - The name of the column containing the strings to normalize.
|
|
2074
2169
|
* @param newColumn - The name of the column to write the normalized values to. Use the same name as `column` to normalize in-place.
|
|
2075
2170
|
* @param threshold - The minimum similarity score (0–100) for two strings to be considered duplicates. For `method: "ratio"`, a length-based pre-filter is automatically applied based on the threshold to improve performance without losing accuracy.
|
|
@@ -2123,8 +2218,6 @@ class SimpleTable extends Simple {
|
|
|
2123
2218
|
/**
|
|
2124
2219
|
* Replaces specified strings in the selected columns.
|
|
2125
2220
|
*
|
|
2126
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2127
|
-
*
|
|
2128
2221
|
* @param columns - The column name, an array of column names, or `"all"` to apply the replacement to every column in the table.
|
|
2129
2222
|
* @param replacements - An object mapping old strings to new strings (e.g., `{ "oldValue": "newValue" }`).
|
|
2130
2223
|
* @param options - An optional object with configuration options:
|
|
@@ -2168,7 +2261,7 @@ class SimpleTable extends Simple {
|
|
|
2168
2261
|
return this;
|
|
2169
2262
|
}
|
|
2170
2263
|
/**
|
|
2171
|
-
* Converts string values in the specified columns to lowercase.
|
|
2264
|
+
* Converts string values in the specified columns to lowercase.
|
|
2172
2265
|
*
|
|
2173
2266
|
* @param columns - The column name or an array of column names to be converted to lowercase.
|
|
2174
2267
|
* @returns The table, so methods can be chained.
|
|
@@ -2191,7 +2284,7 @@ class SimpleTable extends Simple {
|
|
|
2191
2284
|
return this;
|
|
2192
2285
|
}
|
|
2193
2286
|
/**
|
|
2194
|
-
* Converts string values in the specified columns to uppercase.
|
|
2287
|
+
* Converts string values in the specified columns to uppercase.
|
|
2195
2288
|
*
|
|
2196
2289
|
* @param columns - The column name or an array of column names to be converted to uppercase.
|
|
2197
2290
|
* @returns The table, so methods can be chained.
|
|
@@ -2214,7 +2307,7 @@ class SimpleTable extends Simple {
|
|
|
2214
2307
|
return this;
|
|
2215
2308
|
}
|
|
2216
2309
|
/**
|
|
2217
|
-
* Capitalizes the first letter of each string in the specified columns and converts the rest of the string to lowercase.
|
|
2310
|
+
* Capitalizes the first letter of each string in the specified columns and converts the rest of the string to lowercase.
|
|
2218
2311
|
*
|
|
2219
2312
|
* @param columns - The column name or an array of column names to be capitalized.
|
|
2220
2313
|
* @returns The table, so methods can be chained.
|
|
@@ -2239,8 +2332,6 @@ class SimpleTable extends Simple {
|
|
|
2239
2332
|
/**
|
|
2240
2333
|
* Truncates string values in a specified column to a maximum number of characters.
|
|
2241
2334
|
*
|
|
2242
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2243
|
-
*
|
|
2244
2335
|
* @param column - The column name containing strings to be truncated.
|
|
2245
2336
|
* @param length - The maximum number of characters to keep.
|
|
2246
2337
|
* @returns The table, so methods can be chained.
|
|
@@ -2269,8 +2360,6 @@ class SimpleTable extends Simple {
|
|
|
2269
2360
|
* column is of a different type. `null` values remain `null`. If any string
|
|
2270
2361
|
* already exceeds the target length, an error is thrown (no silent truncation).
|
|
2271
2362
|
*
|
|
2272
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2273
|
-
*
|
|
2274
2363
|
* @param columns - The column name(s) containing strings to be padded.
|
|
2275
2364
|
* @param length - The target length of the padded strings.
|
|
2276
2365
|
* @param options - An optional object with configuration options:
|
|
@@ -2310,8 +2399,6 @@ class SimpleTable extends Simple {
|
|
|
2310
2399
|
* Splits strings in a specified column by a separator and extracts a substring at a given index, storing the result in a new or existing column.
|
|
2311
2400
|
* If the index is out of bounds, an empty string will be returned for that row.
|
|
2312
2401
|
*
|
|
2313
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2314
|
-
*
|
|
2315
2402
|
* @param column - The name of the column containing the strings to be split.
|
|
2316
2403
|
* @param separator - The substring to use as a delimiter for splitting the strings.
|
|
2317
2404
|
* @param index - The zero-based index of the substring to extract after splitting. For example, `0` for the first part, `1` for the second, etc.
|
|
@@ -2344,8 +2431,6 @@ class SimpleTable extends Simple {
|
|
|
2344
2431
|
* If a row has fewer parts than the number of new columns, a warning will be logged and the extra columns will contain empty strings (unless `strict` is set to `false`).
|
|
2345
2432
|
* If a row has more parts than the number of new columns, an error will be thrown unless `strict` is set to `false`.
|
|
2346
2433
|
*
|
|
2347
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2348
|
-
*
|
|
2349
2434
|
* @param column - The name of the column containing the strings to be split.
|
|
2350
2435
|
* @param separator - The substring to use as a delimiter for splitting the strings.
|
|
2351
2436
|
* @param newColumns - An array of column names for the extracted parts.
|
|
@@ -2381,8 +2466,6 @@ class SimpleTable extends Simple {
|
|
|
2381
2466
|
/**
|
|
2382
2467
|
* Extracts a specific number of characters from the beginning (left side) of string values in the specified column.
|
|
2383
2468
|
*
|
|
2384
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2385
|
-
*
|
|
2386
2469
|
* @param column - The name of the column containing the strings to be modified.
|
|
2387
2470
|
* @param count - The number of characters to extract from the left side of each string.
|
|
2388
2471
|
* @returns The table, so methods can be chained.
|
|
@@ -2402,8 +2485,6 @@ class SimpleTable extends Simple {
|
|
|
2402
2485
|
/**
|
|
2403
2486
|
* Extracts a specific number of characters from the end (right side) of string values in the specified column.
|
|
2404
2487
|
*
|
|
2405
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2406
|
-
*
|
|
2407
2488
|
* @param column - The name of the column containing the strings to be modified.
|
|
2408
2489
|
* @param count - The number of characters to extract from the right side of each string.
|
|
2409
2490
|
* @returns The table, so methods can be chained.
|
|
@@ -2423,8 +2504,6 @@ class SimpleTable extends Simple {
|
|
|
2423
2504
|
/**
|
|
2424
2505
|
* Replaces `NULL` values in the specified columns with a given value.
|
|
2425
2506
|
*
|
|
2426
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2427
|
-
*
|
|
2428
2507
|
* @param columns - The column name, an array of column names, or `"all"` to apply the replacement to every column in the table.
|
|
2429
2508
|
* @param value - The value to replace `NULL` occurrences with.
|
|
2430
2509
|
* @returns The table, so methods can be chained.
|
|
@@ -2461,8 +2540,6 @@ class SimpleTable extends Simple {
|
|
|
2461
2540
|
/**
|
|
2462
2541
|
* Concatenates values from specified columns into a new column.
|
|
2463
2542
|
*
|
|
2464
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2465
|
-
*
|
|
2466
2543
|
* @param columns - An array of column names whose values will be concatenated.
|
|
2467
2544
|
* @param newColumn - The name of the new column to store the concatenated values.
|
|
2468
2545
|
* @param options - An optional object with configuration options:
|
|
@@ -2497,8 +2574,6 @@ class SimpleTable extends Simple {
|
|
|
2497
2574
|
*
|
|
2498
2575
|
* If a column value is `NULL`, it will be replaced by `'Unknown'` in the concatenated result.
|
|
2499
2576
|
*
|
|
2500
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2501
|
-
*
|
|
2502
2577
|
* @param columns - An array of column names whose values will be concatenated with labels.
|
|
2503
2578
|
* @param newColumn - The name of the new column to create with the concatenated values.
|
|
2504
2579
|
* @returns The table, so methods can be chained.
|
|
@@ -2546,8 +2621,6 @@ class SimpleTable extends Simple {
|
|
|
2546
2621
|
*
|
|
2547
2622
|
* Each value in the specified column is split using the provided separator, and a new row is created for each resulting substring. All other column values are duplicated across the newly created rows.
|
|
2548
2623
|
*
|
|
2549
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2550
|
-
*
|
|
2551
2624
|
* @param column - The name of the column containing string values to be split and unnested.
|
|
2552
2625
|
* @param separator - The delimiter string used to split the column values.
|
|
2553
2626
|
* @returns The table, so methods can be chained.
|
|
@@ -2580,8 +2653,6 @@ class SimpleTable extends Simple {
|
|
|
2580
2653
|
*
|
|
2581
2654
|
* If a row has a value of 3 in the specified column, it will be repeated 3 times. If the value is 0 or negative, the row will be removed.
|
|
2582
2655
|
*
|
|
2583
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2584
|
-
*
|
|
2585
2656
|
* @param column - The name of the column containing the number of times each row should be repeated.
|
|
2586
2657
|
* @param options - An optional object with configuration options:
|
|
2587
2658
|
* @param options.index - The name of a new column to store the index of the repeated row (starting at 0).
|
|
@@ -2613,8 +2684,6 @@ class SimpleTable extends Simple {
|
|
|
2613
2684
|
*
|
|
2614
2685
|
* This is the inverse operation of `unnest()`. Multiple rows are combined into fewer rows by grouping on specified category columns and concatenating the target column values with a separator.
|
|
2615
2686
|
*
|
|
2616
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2617
|
-
*
|
|
2618
2687
|
* @param column - The name of the column whose values will be aggregated and concatenated.
|
|
2619
2688
|
* @param separator - The delimiter string used to join the column values.
|
|
2620
2689
|
* @param by - The column name or an array of column names to group by.
|
|
@@ -2647,8 +2716,6 @@ class SimpleTable extends Simple {
|
|
|
2647
2716
|
/**
|
|
2648
2717
|
* Rounds numeric values in specified columns.
|
|
2649
2718
|
*
|
|
2650
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2651
|
-
*
|
|
2652
2719
|
* @param columns - The column name or an array of column names containing numeric values to be rounded.
|
|
2653
2720
|
* @param options - An optional integer specifying the number of decimal places, or an object with configuration options:
|
|
2654
2721
|
* @param options.decimals - The number of decimal places to round to. Defaults to `0` (rounds to the nearest integer).
|
|
@@ -2691,9 +2758,37 @@ class SimpleTable extends Simple {
|
|
|
2691
2758
|
return this;
|
|
2692
2759
|
}
|
|
2693
2760
|
/**
|
|
2694
|
-
*
|
|
2761
|
+
* Adds independently generated uniform random noise to numeric values in one or more columns.
|
|
2762
|
+
* Each changed value receives an offset between `-max` and `max`. Selected integer and decimal columns become `DOUBLE` columns so fractional noise is retained. This method adds random jitter; it does not provide anonymization or differential privacy guarantees.
|
|
2763
|
+
*
|
|
2764
|
+
* @param columns - The numeric column name or array of numeric column names to which noise will be added. When multiple columns are provided, each value receives an independent random offset.
|
|
2765
|
+
* @param max - The maximum absolute offset, expressed in each column's units. Must be a finite number greater than or equal to `0`.
|
|
2766
|
+
* @param options - An optional object with configuration options:
|
|
2767
|
+
* @param options.onlyDuplicates - If `true`, adds noise only to values that occur more than once in their column. Each selected column is evaluated independently, and every occurrence of a duplicated value is changed. Defaults to `false`.
|
|
2768
|
+
* @returns The table, so methods can be chained.
|
|
2769
|
+
* @category Updating Data
|
|
2770
|
+
*
|
|
2771
|
+
* @example
|
|
2772
|
+
* ```ts
|
|
2773
|
+
* // Add an offset between -0.01 and 0.01 to every value
|
|
2774
|
+
* await table.addNoise("measurement", 0.01).log();
|
|
2775
|
+
* ```
|
|
2695
2776
|
*
|
|
2696
|
-
*
|
|
2777
|
+
* @example
|
|
2778
|
+
* ```ts
|
|
2779
|
+
* // Add noise independently to duplicated x values and duplicated y values
|
|
2780
|
+
* await table.addNoise(["x", "y"], 0.1, {
|
|
2781
|
+
* onlyDuplicates: true,
|
|
2782
|
+
* }).log();
|
|
2783
|
+
* ```
|
|
2784
|
+
*/
|
|
2785
|
+
addNoise(columns, max, options = {}) {
|
|
2786
|
+
addNoise(this, columns, max, options);
|
|
2787
|
+
return this;
|
|
2788
|
+
}
|
|
2789
|
+
/**
|
|
2790
|
+
* Updates values in a specified column using a SQL expression.
|
|
2791
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, expressions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
2697
2792
|
*
|
|
2698
2793
|
* @param column - The name of the column to be updated.
|
|
2699
2794
|
* @param definition - The SQL expression used to set the new values in the column (e.g., `"column1 * 2"`, `"UPPER(column_name)"`).
|
|
@@ -2717,6 +2812,16 @@ class SimpleTable extends Simple {
|
|
|
2717
2812
|
* // Set 'status' to 'active' where 'isActive' is true
|
|
2718
2813
|
* await table.updateColumn("status", `CASE WHEN isActive THEN 'active' ELSE 'inactive' END`).log();
|
|
2719
2814
|
* ```
|
|
2815
|
+
*
|
|
2816
|
+
* @example
|
|
2817
|
+
* ```ts
|
|
2818
|
+
* // Update an existing column with a value from a JSON object using dot notation.
|
|
2819
|
+
* await table.loadArray([{ count: 0, details: { count: 3 } }], {
|
|
2820
|
+
* columnTypes: { details: "JSON" },
|
|
2821
|
+
* })
|
|
2822
|
+
* .updateColumn("count", "details.count")
|
|
2823
|
+
* .log();
|
|
2824
|
+
* ```
|
|
2720
2825
|
*/
|
|
2721
2826
|
updateColumn(column, definition) {
|
|
2722
2827
|
updateColumn(this, column, definition);
|
|
@@ -2725,8 +2830,6 @@ class SimpleTable extends Simple {
|
|
|
2725
2830
|
/**
|
|
2726
2831
|
* Assigns ranks to rows in a new column based on the values of a specified column.
|
|
2727
2832
|
*
|
|
2728
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2729
|
-
*
|
|
2730
2833
|
* @param column - The column containing the values to be used for ranking.
|
|
2731
2834
|
* @param newColumn - The name of the new column where the ranks will be stored.
|
|
2732
2835
|
* @param options - An optional object with configuration options:
|
|
@@ -2767,8 +2870,6 @@ class SimpleTable extends Simple {
|
|
|
2767
2870
|
/**
|
|
2768
2871
|
* Assigns quantiles to rows in a new column based on specified column values.
|
|
2769
2872
|
*
|
|
2770
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2771
|
-
*
|
|
2772
2873
|
* @param column - The column containing values from which quantiles will be assigned.
|
|
2773
2874
|
* @param count - The number of quantiles to divide the data into (e.g., `4` for quartiles, `10` for deciles).
|
|
2774
2875
|
* @param newColumn - The name of the new column where the assigned quantiles will be stored.
|
|
@@ -2802,8 +2903,6 @@ class SimpleTable extends Simple {
|
|
|
2802
2903
|
/**
|
|
2803
2904
|
* Assigns bins for specified column values based on an interval size.
|
|
2804
2905
|
*
|
|
2805
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2806
|
-
*
|
|
2807
2906
|
* @param column - The column containing values from which bins will be computed.
|
|
2808
2907
|
* @param interval - The interval size for binning the values.
|
|
2809
2908
|
* @param newColumn - The name of the new column where the bins will be stored.
|
|
@@ -2873,11 +2972,16 @@ class SimpleTable extends Simple {
|
|
|
2873
2972
|
* | 2022 | 354 | 278 | 56 | 0.51 | 0.4 | 0.08 |
|
|
2874
2973
|
* | 2023 | 856 | 321 | 221 | 0.61 | 0.23 | 0.16 |
|
|
2875
2974
|
*
|
|
2876
|
-
*
|
|
2975
|
+
* @example
|
|
2976
|
+
* ```ts
|
|
2977
|
+
* // Compute percentages that sum to 100 on each row before rounding
|
|
2978
|
+
* await table.rowProportions(["Men", "Women", "NonBinary"], { base: 100, decimals: 1 }).log();
|
|
2979
|
+
* ```
|
|
2877
2980
|
*
|
|
2878
2981
|
* @param columns - An array of column names for which proportions will be computed on each row.
|
|
2879
2982
|
* @param options - An optional object with configuration options:
|
|
2880
2983
|
* @param options.suffix - A string suffix to append to the names of the new columns storing the computed proportions. Defaults to `"Perc"`.
|
|
2984
|
+
* @param options.base - A finite positive value that the proportions in each row sum to before rounding. Defaults to `1`.
|
|
2881
2985
|
* @param options.decimals - The number of decimal places to round the computed proportions. Defaults to `undefined` (no rounding).
|
|
2882
2986
|
* @returns The table, so methods can be chained.
|
|
2883
2987
|
* @category Analyzing Data
|
|
@@ -2897,9 +3001,6 @@ class SimpleTable extends Simple {
|
|
|
2897
3001
|
* option can therefore increase the table's row count. If null values leave
|
|
2898
3002
|
* a row without the requested rank, the new columns contain null.
|
|
2899
3003
|
*
|
|
2900
|
-
* This method queues the operation; it runs when an async observer method
|
|
2901
|
-
* (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2902
|
-
*
|
|
2903
3004
|
* @param columns - The numeric columns to rank within each row.
|
|
2904
3005
|
* @param options - The output columns and ranking configuration. At least one of `nameColumn` or `valueColumn` is required.
|
|
2905
3006
|
* @param options.nameColumn - The name of a new column containing the selected source column's name.
|
|
@@ -2936,12 +3037,11 @@ class SimpleTable extends Simple {
|
|
|
2936
3037
|
/**
|
|
2937
3038
|
* Computes proportions vertically over a column's values, relative to the sum of all values in that column or group.
|
|
2938
3039
|
*
|
|
2939
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2940
|
-
*
|
|
2941
3040
|
* @param column - The column containing values for which proportions will be computed. The proportions are calculated based on the sum of values in the specified column.
|
|
2942
3041
|
* @param newColumn - The name of the new column where the proportions will be stored.
|
|
2943
3042
|
* @param options - An optional object with configuration options:
|
|
2944
3043
|
* @param options.by - The column name or an array of column names to partition by. Proportions are calculated independently within each group.
|
|
3044
|
+
* @param options.base - A finite positive value that the proportions in the column or each group sum to before rounding. Defaults to `1`.
|
|
2945
3045
|
* @param options.decimals - The number of decimal places to round the computed proportions. Defaults to `undefined` (no rounding).
|
|
2946
3046
|
* @returns The table, so methods can be chained.
|
|
2947
3047
|
* @category Analyzing Data
|
|
@@ -2960,6 +3060,12 @@ class SimpleTable extends Simple {
|
|
|
2960
3060
|
*
|
|
2961
3061
|
* @example
|
|
2962
3062
|
* ```ts
|
|
3063
|
+
* // Compute percentages that sum to 100 before rounding
|
|
3064
|
+
* await table.columnProportions("sales", "sales_percentage", { base: 100, decimals: 1 }).log();
|
|
3065
|
+
* ```
|
|
3066
|
+
*
|
|
3067
|
+
* @example
|
|
3068
|
+
* ```ts
|
|
2963
3069
|
* // Compute proportions for 'sales' by 'region' and 'product_type'
|
|
2964
3070
|
* await table.columnProportions("sales", "sales_proportion", { by: ["region", "product_type"] }).log();
|
|
2965
3071
|
* ```
|
|
@@ -2972,8 +3078,6 @@ class SimpleTable extends Simple {
|
|
|
2972
3078
|
* Creates a summary table from selected columns, optionally grouped by other columns.
|
|
2973
3079
|
* This method allows you to aggregate data, calculate statistics (e.g., count, mean, sum), and group results by categorical columns.
|
|
2974
3080
|
*
|
|
2975
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2976
|
-
*
|
|
2977
3081
|
* @param options - An object with configuration options for summarization:
|
|
2978
3082
|
* @param options.columns - The column name or an array of column names to summarize. If omitted, only the row count is returned.
|
|
2979
3083
|
* @param options.by - The column name or an array of column names to group by.
|
|
@@ -3072,9 +3176,6 @@ class SimpleTable extends Simple {
|
|
|
3072
3176
|
* string is also used as its row label; pass an object to customize that
|
|
3073
3177
|
* label. If `options.stats` is omitted, every supported stat is added.
|
|
3074
3178
|
*
|
|
3075
|
-
* This method queues the operation; it runs when an async observer method
|
|
3076
|
-
* (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3077
|
-
*
|
|
3078
3179
|
* @param columns - The numeric column name, an array of numeric column names, or `"all"` to summarize every numeric column.
|
|
3079
3180
|
* @param labelColumn - The existing string column in which stat row labels will be written.
|
|
3080
3181
|
* @param options - An optional object with configuration options:
|
|
@@ -3116,8 +3217,6 @@ class SimpleTable extends Simple {
|
|
|
3116
3217
|
/**
|
|
3117
3218
|
* Computes the cumulative sum of values in a column. For this method to work properly, ensure your data is sorted first.
|
|
3118
3219
|
*
|
|
3119
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3120
|
-
*
|
|
3121
3220
|
* @param column - The name of the column storing the values to be accumulated.
|
|
3122
3221
|
* @param newColumn - The name of the new column in which the computed cumulative values will be stored.
|
|
3123
3222
|
* @param options - An optional object with configuration options:
|
|
@@ -3154,8 +3253,6 @@ class SimpleTable extends Simple {
|
|
|
3154
3253
|
* For rows without enough preceding or following rows to form a complete window, `NULL` will be returned.
|
|
3155
3254
|
* For this method to work properly, ensure your data is sorted by the relevant column(s) first.
|
|
3156
3255
|
*
|
|
3157
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3158
|
-
*
|
|
3159
3256
|
* @param column - The name of the column storing the values to be aggregated.
|
|
3160
3257
|
* @param newColumn - The name of the new column in which the computed rolling values will be stored.
|
|
3161
3258
|
* @param stat - The aggregation function to apply: `"min"`, `"max"`, `"mean"`, `"median"`, or `"sum"`.
|
|
@@ -3194,8 +3291,6 @@ class SimpleTable extends Simple {
|
|
|
3194
3291
|
* Calculates correlations between columns. If no `x` and `y` columns are specified, the method computes the correlations for all numeric column combinations.
|
|
3195
3292
|
* Note that correlation is symmetrical: the correlation of `x` with `y` is the same as `y` with `x`.
|
|
3196
3293
|
*
|
|
3197
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3198
|
-
*
|
|
3199
3294
|
* @param options - An optional object with configuration options:
|
|
3200
3295
|
* @param options.x - The name of the column for the x-values. If omitted, correlations will be computed for all numeric columns.
|
|
3201
3296
|
* @param options.y - The name of the column for the y-values. It can be provided only when `options.x` is also set. If both are omitted, correlations will be computed for all numeric column pairs.
|
|
@@ -3243,8 +3338,6 @@ class SimpleTable extends Simple {
|
|
|
3243
3338
|
* If no `x` and `y` columns are specified, the method computes linear regression analysis for all numeric column permutations.
|
|
3244
3339
|
* Note that linear regression analysis is asymmetrical: the linear regression of `x` over `y` is not the same as `y` over `x`.
|
|
3245
3340
|
*
|
|
3246
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3247
|
-
*
|
|
3248
3341
|
* @param options - An optional object with configuration options:
|
|
3249
3342
|
* @param options.x - The name of the column for the independent variable (x-values). If omitted, linear regressions will be computed for all numeric columns as x.
|
|
3250
3343
|
* @param options.y - The name of the column for the dependent variable (y-values). It can be provided only when `options.x` is also set. If both are omitted, linear regressions will be computed for all numeric column permutations.
|
|
@@ -3290,8 +3383,6 @@ class SimpleTable extends Simple {
|
|
|
3290
3383
|
/**
|
|
3291
3384
|
* Identifies outliers in a specified column using the Interquartile Range (IQR) method.
|
|
3292
3385
|
*
|
|
3293
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3294
|
-
*
|
|
3295
3386
|
* @param column - The name of the column in which outliers will be identified.
|
|
3296
3387
|
* @param newColumn - The name of the new column where the boolean results (`TRUE` for outlier, `FALSE` otherwise) will be stored.
|
|
3297
3388
|
* @param options - An optional object with configuration options:
|
|
@@ -3318,8 +3409,6 @@ class SimpleTable extends Simple {
|
|
|
3318
3409
|
/**
|
|
3319
3410
|
* Computes the Z-score for values in a specified column.
|
|
3320
3411
|
*
|
|
3321
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3322
|
-
*
|
|
3323
3412
|
* @param column - The name of the column for which Z-scores will be calculated.
|
|
3324
3413
|
* @param newColumn - The name of the new column where the computed Z-scores will be stored.
|
|
3325
3414
|
* @param options - An optional object with configuration options:
|
|
@@ -3353,8 +3442,6 @@ class SimpleTable extends Simple {
|
|
|
3353
3442
|
/**
|
|
3354
3443
|
* Normalizes the values in a column using min-max normalization.
|
|
3355
3444
|
*
|
|
3356
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3357
|
-
*
|
|
3358
3445
|
* @param column - The name of the column in which values will be normalized.
|
|
3359
3446
|
* @param newColumn - The name of the new column where normalized values will be stored.
|
|
3360
3447
|
* @param options - An optional object with configuration options:
|
|
@@ -3399,8 +3486,6 @@ class SimpleTable extends Simple {
|
|
|
3399
3486
|
*
|
|
3400
3487
|
* Exact temporal references are compared at their full DuckDB precision. JavaScript `Date` objects only have millisecond precision and always represent an instant. Construct them with an explicit timezone, such as `new Date("2001-01-01T00:00:00Z")`; date-time strings without `Z` or an offset use the user's local timezone.
|
|
3401
3488
|
*
|
|
3402
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3403
|
-
*
|
|
3404
3489
|
* @param column - The numeric column containing the values to index.
|
|
3405
3490
|
* @param newColumn - The name of the new column where indexed values will be stored.
|
|
3406
3491
|
* @param reference - A statistic calculated from `column`, a column and exact non-null `equals` value selecting a row, or a column and `at` set to `min` or `max` selecting its unique extreme row. The selected row's `column` value becomes the reference.
|
|
@@ -3458,50 +3543,84 @@ class SimpleTable extends Simple {
|
|
|
3458
3543
|
/**
|
|
3459
3544
|
* Updates data in the table using a JavaScript function. The function receives the existing rows as an array of objects and must return the modified rows as an array of objects.
|
|
3460
3545
|
* This method offers high flexibility for data manipulation but can be slow for large tables as it involves transferring data between DuckDB and JavaScript.
|
|
3461
|
-
*
|
|
3546
|
+
* Before writing a JavaScript callback, check for an existing SDA method that
|
|
3547
|
+
* performs the same operation; it will usually be faster and more efficient.
|
|
3462
3548
|
*
|
|
3463
|
-
*
|
|
3549
|
+
* If the table has geometry columns, the callback can read and modify their
|
|
3550
|
+
* GeoJSON geometry objects directly. Extra properties added to these objects
|
|
3551
|
+
* are ignored; create new columns on the returned rows to store additional
|
|
3552
|
+
* attributes. All geometry columns must use EPSG:4326; use reproject() first
|
|
3553
|
+
* if needed.
|
|
3464
3554
|
*
|
|
3465
3555
|
* @param dataModifier - A synchronous or asynchronous function that takes the existing rows (as an array of objects) and returns the modified rows (as an array of objects).
|
|
3466
3556
|
* @param options - An optional object with configuration options:
|
|
3467
|
-
* @param options.batchSize - If provided, rows are processed in batches of this size instead of all at once, so large tables don't have to be materialized entirely in memory. The modifier function is called once per batch.
|
|
3557
|
+
* @param options.batchSize - If provided, rows are processed in batches of this size instead of all at once, so large tables don't have to be materialized entirely in memory. This limits input rows per batch, but does not eliminate conversion work or bound geometry size or callback output expansion. The modifier function is called once per batch. Tables with a column named `rowid` or `__sda_rowid` (case-insensitive) are rejected before the modifier runs; rename the column or omit batchSize.
|
|
3558
|
+
* @param options.columnTypes - Explicit types for newly added columns, following loadArray's columnTypes contract. Declare geometry as GEOMETRY('EPSG:4326'); geometry is never inferred from object shape. Existing columns cannot be redeclared.
|
|
3468
3559
|
* @returns The table, so methods can be chained.
|
|
3469
3560
|
* @category Updating Data
|
|
3470
3561
|
*
|
|
3471
3562
|
* @example
|
|
3472
3563
|
* ```ts
|
|
3473
|
-
* //
|
|
3474
|
-
*
|
|
3475
|
-
*
|
|
3476
|
-
*
|
|
3477
|
-
*
|
|
3478
|
-
* .updateWithJS((rows) => {
|
|
3479
|
-
* return rows.map((row) => ({
|
|
3480
|
-
* ...row,
|
|
3481
|
-
* wordCount: typeof row.text === "string"
|
|
3482
|
-
* ? [...segmenter.segment(row.text)].filter((part) => part.isWordLike)
|
|
3483
|
-
* .length
|
|
3484
|
-
* : null,
|
|
3485
|
-
* }));
|
|
3486
|
-
* })
|
|
3564
|
+
* // Double the value in each row with a JavaScript callback.
|
|
3565
|
+
* await table.loadArray([{ value: 1 }, { value: 2 }])
|
|
3566
|
+
* .updateWithJS((rows) => rows.map((row) => ({
|
|
3567
|
+
* value: Number(row.value) * 2,
|
|
3568
|
+
* })))
|
|
3487
3569
|
* .log();
|
|
3488
3570
|
* ```
|
|
3489
3571
|
*
|
|
3490
3572
|
* @example
|
|
3491
3573
|
* ```ts
|
|
3492
|
-
* //
|
|
3493
|
-
*
|
|
3494
|
-
* .updateWithJS(
|
|
3495
|
-
*
|
|
3496
|
-
*
|
|
3497
|
-
* headers: { "content-type": "application/json" },
|
|
3498
|
-
* body: JSON.stringify(rows.map((row) => row.review)),
|
|
3499
|
-
* });
|
|
3500
|
-
* const scores = await response.json() as number[];
|
|
3501
|
-
* return rows.map((row, index) => ({ ...row, score: scores[index] }));
|
|
3502
|
-
* }, { batchSize: 100 })
|
|
3574
|
+
* // Process two rows per callback to limit how many rows are handled at once.
|
|
3575
|
+
* await table.loadArray([{ value: 1 }, { value: 2 }, { value: 3 }])
|
|
3576
|
+
* .updateWithJS((rows) => rows.map((row) => ({
|
|
3577
|
+
* value: Number(row.value) * 2,
|
|
3578
|
+
* })), { batchSize: 2 })
|
|
3503
3579
|
* .log();
|
|
3504
3580
|
* ```
|
|
3581
|
+
*
|
|
3582
|
+
* @example
|
|
3583
|
+
* ```ts
|
|
3584
|
+
* // Add a JSON object column to keep related attributes together.
|
|
3585
|
+
* await table.loadArray([{ name: "Station" }])
|
|
3586
|
+
* .updateWithJS((rows) => rows.map((row) => ({
|
|
3587
|
+
* ...row, details: { active: true, category: "weather" },
|
|
3588
|
+
* })), { columnTypes: { details: "JSON" } })
|
|
3589
|
+
* .log();
|
|
3590
|
+
* ```
|
|
3591
|
+
*
|
|
3592
|
+
* @example
|
|
3593
|
+
* ```ts
|
|
3594
|
+
* // Add a three-element float vector, such as an embedding.
|
|
3595
|
+
* await table.loadArray([{ name: "Station" }])
|
|
3596
|
+
* .updateWithJS((rows) => rows.map((row) => ({
|
|
3597
|
+
* ...row, embedding: [0.25, 0.5, 0.75],
|
|
3598
|
+
* })), { columnTypes: { embedding: "FLOAT[3]" } })
|
|
3599
|
+
* .log();
|
|
3600
|
+
* ```
|
|
3601
|
+
*
|
|
3602
|
+
* @example
|
|
3603
|
+
* ```ts
|
|
3604
|
+
* // Move each point slightly east by editing its longitude.
|
|
3605
|
+
* await sdb.newTable().loadArray([{
|
|
3606
|
+
* geom: { type: "Point", coordinates: [-73, 45] },
|
|
3607
|
+
* }], { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } })
|
|
3608
|
+
* .updateWithJS((rows) => rows.map((row) => {
|
|
3609
|
+
* // This example loads only Points. Narrow the type for coordinate access.
|
|
3610
|
+
* const geom = row.geom as { type: "Point"; coordinates: number[] } | null;
|
|
3611
|
+
* if (geom !== null) geom.coordinates[0] += 0.01;
|
|
3612
|
+
* return row;
|
|
3613
|
+
* })).log();
|
|
3614
|
+
* ```
|
|
3615
|
+
*
|
|
3616
|
+
* @example
|
|
3617
|
+
* ```ts
|
|
3618
|
+
* // Create a geometry column from longitude and latitude values.
|
|
3619
|
+
* await sdb.newTable().loadArray([{ longitude: -73, latitude: 45 }])
|
|
3620
|
+
* .updateWithJS((rows) => rows.map((row) => ({
|
|
3621
|
+
* ...row, geom: { type: "Point", coordinates: [row.longitude, row.latitude] },
|
|
3622
|
+
* })), { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } }).log();
|
|
3623
|
+
* ```
|
|
3505
3624
|
*/
|
|
3506
3625
|
updateWithJS(dataModifier, options = {}) {
|
|
3507
3626
|
updateWithJS(this, dataModifier, options);
|
|
@@ -3582,8 +3701,6 @@ class SimpleTable extends Simple {
|
|
|
3582
3701
|
* Produces identical output to `journalism-format`'s `normalizeString()` function
|
|
3583
3702
|
* for all common cases including accented Latin characters.
|
|
3584
3703
|
*
|
|
3585
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3586
|
-
*
|
|
3587
3704
|
* @param column The column containing the text to normalize
|
|
3588
3705
|
* @param newColumn The column to store the normalized results
|
|
3589
3706
|
* @param options Configuration options
|
|
@@ -3629,8 +3746,48 @@ class SimpleTable extends Simple {
|
|
|
3629
3746
|
const result = (await getColumns(this)).length;
|
|
3630
3747
|
return result;
|
|
3631
3748
|
}
|
|
3749
|
+
/**
|
|
3750
|
+
* Adds a new column containing the number of characters in each string in the specified column.
|
|
3751
|
+
* Counts are based on Unicode code points, not grapheme clusters. A user-perceived character composed of multiple code points, such as some emoji or decomposed accented letters, counts as multiple characters.
|
|
3752
|
+
* `NULL` input values produce `NULL` counts.
|
|
3753
|
+
*
|
|
3754
|
+
* @param column - The name of the column containing the strings to count.
|
|
3755
|
+
* @param newColumn - The name of the new column where the character counts will be stored.
|
|
3756
|
+
* @returns The table, so methods can be chained.
|
|
3757
|
+
* @category Text Processing
|
|
3758
|
+
*
|
|
3759
|
+
* @example
|
|
3760
|
+
* ```ts
|
|
3761
|
+
* // Add a character count for each value in the 'name' column
|
|
3762
|
+
* await table.addCharacterCount("name", "nameCharacterCount").log();
|
|
3763
|
+
* ```
|
|
3764
|
+
*/
|
|
3765
|
+
addCharacterCount(column, newColumn) {
|
|
3766
|
+
addCharacterCount(this, column, newColumn);
|
|
3767
|
+
return this;
|
|
3768
|
+
}
|
|
3769
|
+
/**
|
|
3770
|
+
* Adds a new column containing the word count for each string in the specified column.
|
|
3771
|
+
* A word is any contiguous sequence of non-whitespace characters. Spaces, tabs, and line breaks separate words. Punctuation is not removed, so a standalone punctuation sequence counts as a word. Text without whitespace counts as one word, regardless of language. Empty or whitespace-only strings produce `0`, and `NULL` input values produce `NULL` counts.
|
|
3772
|
+
*
|
|
3773
|
+
* @param column - The name of the column containing the strings to count.
|
|
3774
|
+
* @param newColumn - The name of the new column where the word counts will be stored.
|
|
3775
|
+
* @returns The table, so methods can be chained.
|
|
3776
|
+
* @category Text Processing
|
|
3777
|
+
*
|
|
3778
|
+
* @example
|
|
3779
|
+
* ```ts
|
|
3780
|
+
* // Add a word count for each value in the 'article' column
|
|
3781
|
+
* await table.addWordCount("article", "wordCount").log();
|
|
3782
|
+
* ```
|
|
3783
|
+
*/
|
|
3784
|
+
addWordCount(column, newColumn) {
|
|
3785
|
+
addWordCount(this, column, newColumn);
|
|
3786
|
+
return this;
|
|
3787
|
+
}
|
|
3632
3788
|
/**
|
|
3633
3789
|
* Returns the total number of characters in a column storing strings.
|
|
3790
|
+
* Counts are based on Unicode code points, not grapheme clusters. A user-perceived character composed of multiple code points, such as some emoji or decomposed accented letters, counts as multiple characters.
|
|
3634
3791
|
*
|
|
3635
3792
|
* @param column - The name of the string column to count characters from.
|
|
3636
3793
|
* @returns A promise that resolves to the total number of characters across all rows in the specified column.
|
|
@@ -3648,6 +3805,7 @@ class SimpleTable extends Simple {
|
|
|
3648
3805
|
}
|
|
3649
3806
|
/**
|
|
3650
3807
|
* Returns the number of rows in the table.
|
|
3808
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
3651
3809
|
*
|
|
3652
3810
|
* @param options - An optional object with configuration options:
|
|
3653
3811
|
* @param options.conditions - The filtering conditions specified as a SQL `WHERE` clause (e.g., `"category = 'Book'"`).
|
|
@@ -3988,7 +4146,7 @@ class SimpleTable extends Simple {
|
|
|
3988
4146
|
}
|
|
3989
4147
|
/**
|
|
3990
4148
|
* Returns the first row of the table, optionally filtered by SQL conditions.
|
|
3991
|
-
*
|
|
4149
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
3992
4150
|
* Temporal values use the same JavaScript representations as `getData()`.
|
|
3993
4151
|
*
|
|
3994
4152
|
* @param options - An optional object with configuration options:
|
|
@@ -4015,7 +4173,7 @@ class SimpleTable extends Simple {
|
|
|
4015
4173
|
}
|
|
4016
4174
|
/**
|
|
4017
4175
|
* Returns the last row of the table, optionally filtered by SQL conditions.
|
|
4018
|
-
*
|
|
4176
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
4019
4177
|
* Temporal values use the same JavaScript representations as `getData()`.
|
|
4020
4178
|
*
|
|
4021
4179
|
* @param options - An optional object with configuration options:
|
|
@@ -4042,7 +4200,7 @@ class SimpleTable extends Simple {
|
|
|
4042
4200
|
}
|
|
4043
4201
|
/**
|
|
4044
4202
|
* Returns the top `n` rows of the table, optionally filtered by SQL conditions.
|
|
4045
|
-
*
|
|
4203
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
4046
4204
|
* Temporal values use the same JavaScript representations as `getData()`.
|
|
4047
4205
|
*
|
|
4048
4206
|
* @param count - The number of rows to return from the top of the table.
|
|
@@ -4071,7 +4229,7 @@ class SimpleTable extends Simple {
|
|
|
4071
4229
|
/**
|
|
4072
4230
|
* Returns the bottom `n` rows of the table, optionally filtered by SQL conditions.
|
|
4073
4231
|
* By default, the last row will be returned first. To preserve the original order, use the `originalOrder` option.
|
|
4074
|
-
*
|
|
4232
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
4075
4233
|
* Temporal values use the same JavaScript representations as `getData()`.
|
|
4076
4234
|
*
|
|
4077
4235
|
* @param count - The number of rows to return from the bottom of the table.
|
|
@@ -4107,7 +4265,7 @@ class SimpleTable extends Simple {
|
|
|
4107
4265
|
}
|
|
4108
4266
|
/**
|
|
4109
4267
|
* Returns a single row that matches the specified conditions. If no row matches or if more than one row matches, an error is thrown by default.
|
|
4110
|
-
*
|
|
4268
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
4111
4269
|
* Temporal values use the same JavaScript representations as `getData()`.
|
|
4112
4270
|
*
|
|
4113
4271
|
* @param conditions - The conditions to match, specified as a SQL `WHERE` clause.
|
|
@@ -4152,13 +4310,26 @@ class SimpleTable extends Simple {
|
|
|
4152
4310
|
}
|
|
4153
4311
|
/**
|
|
4154
4312
|
* Returns the data from the table as an array of objects, optionally filtered by SQL conditions.
|
|
4155
|
-
*
|
|
4313
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
4156
4314
|
*
|
|
4157
4315
|
* Top-level DuckDB `DATE` and `TIMESTAMP` columns are returned as JavaScript
|
|
4158
4316
|
* `Date` objects interpreted in UTC. `TIMESTAMP WITH TIME ZONE` values are
|
|
4159
4317
|
* returned as UTC strings, preserving DuckDB's microsecond precision;
|
|
4160
4318
|
* JavaScript `Date` supports only milliseconds.
|
|
4161
4319
|
*
|
|
4320
|
+
* Top-level `BIGINT`, `UBIGINT`, `HUGEINT`, and `UHUGEINT` columns are
|
|
4321
|
+
* returned as JavaScript numbers. Values outside `Number.MIN_SAFE_INTEGER`
|
|
4322
|
+
* through `Number.MAX_SAFE_INTEGER` throw instead of losing precision.
|
|
4323
|
+
* Use `convert({ columnName: "string" })` before `getData()` to retrieve
|
|
4324
|
+
* their exact digits as strings.
|
|
4325
|
+
*
|
|
4326
|
+
* SQL arrays and lists, including float vectors, return their actual elements
|
|
4327
|
+
* as JavaScript arrays. Null cells and null elements remain null.
|
|
4328
|
+
*
|
|
4329
|
+
* Nested SQL lists and structs become JavaScript arrays and objects. Within
|
|
4330
|
+
* them, `BIGINT`, `UBIGINT`, `HUGEINT`, `UHUGEINT`, and `DECIMAL` values remain
|
|
4331
|
+
* strings to preserve precision, even for small integers.
|
|
4332
|
+
*
|
|
4162
4333
|
* @param options - An optional object with configuration options:
|
|
4163
4334
|
* @param options.columns - An array of column names to include in the result. If omitted, all columns will be included.
|
|
4164
4335
|
* @param options.conditions - The filtering conditions specified as a SQL `WHERE` clause (e.g., `"category = 'Book'"`).
|
|
@@ -4192,6 +4363,18 @@ class SimpleTable extends Simple {
|
|
|
4192
4363
|
* // Return at most two rows.
|
|
4193
4364
|
* const preview = await table.getData({ limit: 2 });
|
|
4194
4365
|
* ```
|
|
4366
|
+
*
|
|
4367
|
+
* @example
|
|
4368
|
+
* ```ts
|
|
4369
|
+
* // Read nested integers without losing digits. SQL creates the nested types.
|
|
4370
|
+
* await sdb.customQuery(`CREATE TABLE nested_example AS SELECT
|
|
4371
|
+
* [42::BIGINT, 9007199254740993::BIGINT] AS ids,
|
|
4372
|
+
* {'count': 2::INTEGER, 'amount': 123.450::DECIMAL(6,3)} AS details`);
|
|
4373
|
+
* const nested = sdb.newTable("nested_example");
|
|
4374
|
+
* const rows = await nested.getData();
|
|
4375
|
+
* // [{ ids: ["42", "9007199254740993"], details: { count: 2, amount: "123.450" } }]
|
|
4376
|
+
* await nested.log();
|
|
4377
|
+
* ```
|
|
4195
4378
|
*/
|
|
4196
4379
|
async getData(options = {}) {
|
|
4197
4380
|
return await getData(this, options);
|
|
@@ -4200,6 +4383,7 @@ class SimpleTable extends Simple {
|
|
|
4200
4383
|
* Streams the table rows one by one as an async iterator, without
|
|
4201
4384
|
* materializing the whole table in memory. Values are converted to
|
|
4202
4385
|
* JavaScript types the same way as `getData()`.
|
|
4386
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
4203
4387
|
*
|
|
4204
4388
|
* The underlying DuckDB result is streamed chunk by chunk, so tables
|
|
4205
4389
|
* larger than the available memory can be iterated. Avoid running other
|
|
@@ -4232,7 +4416,7 @@ class SimpleTable extends Simple {
|
|
|
4232
4416
|
}
|
|
4233
4417
|
/**
|
|
4234
4418
|
* Returns the data from the table as a CSV string, optionally filtered by SQL conditions.
|
|
4235
|
-
*
|
|
4419
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
4236
4420
|
* Temporal values are first converted as they are in `getData()`, then
|
|
4237
4421
|
* serialized using UTC date and timestamp text.
|
|
4238
4422
|
*
|
|
@@ -4271,8 +4455,6 @@ class SimpleTable extends Simple {
|
|
|
4271
4455
|
/**
|
|
4272
4456
|
* Creates point geometries from latitude (y) and longitude (x) columns.
|
|
4273
4457
|
*
|
|
4274
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4275
|
-
*
|
|
4276
4458
|
* @param latColumn - The name of the column storing the latitude (y-coordinate) values.
|
|
4277
4459
|
* @param lonColumn - The name of the column storing the longitude (x-coordinate) values.
|
|
4278
4460
|
* @param newColumn - The name of the new column where the point geometries will be stored.
|
|
@@ -4302,8 +4484,6 @@ class SimpleTable extends Simple {
|
|
|
4302
4484
|
/**
|
|
4303
4485
|
* Adds a column with boolean values indicating the validity of geometries.
|
|
4304
4486
|
*
|
|
4305
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4306
|
-
*
|
|
4307
4487
|
* @param newColumn - The name of the new column where the boolean results (`TRUE` for valid, `FALSE` for invalid) will be stored.
|
|
4308
4488
|
* @param options - An optional object with configuration options:
|
|
4309
4489
|
* @param options.column - The name of the column storing the geometries to be checked. If omitted, the method will automatically attempt to find a geometry column.
|
|
@@ -4330,8 +4510,6 @@ class SimpleTable extends Simple {
|
|
|
4330
4510
|
/**
|
|
4331
4511
|
* Adds a column with the number of vertices (points) in each geometry.
|
|
4332
4512
|
*
|
|
4333
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4334
|
-
*
|
|
4335
4513
|
* @param newColumn - The name of the new column where the vertex counts will be stored.
|
|
4336
4514
|
* @param options - An optional object with configuration options:
|
|
4337
4515
|
* @param options.column - The name of the column storing the geometries. If omitted, the method will automatically attempt to find a geometry column.
|
|
@@ -4358,8 +4536,6 @@ class SimpleTable extends Simple {
|
|
|
4358
4536
|
/**
|
|
4359
4537
|
* Attempts to make invalid geometries valid without removing any vertices.
|
|
4360
4538
|
*
|
|
4361
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4362
|
-
*
|
|
4363
4539
|
* @param column - The name of the column storing the geometries to be fixed. If omitted, the method will automatically attempt to find a geometry column.
|
|
4364
4540
|
* @returns The table, so methods can be chained.
|
|
4365
4541
|
* @category Geospatial
|
|
@@ -4383,8 +4559,6 @@ class SimpleTable extends Simple {
|
|
|
4383
4559
|
/**
|
|
4384
4560
|
* Adds a column with boolean values indicating whether geometries are closed (e.g., polygons) or open (e.g., linestrings).
|
|
4385
4561
|
*
|
|
4386
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4387
|
-
*
|
|
4388
4562
|
* @param newColumn - The name of the new column where the boolean results (`TRUE` for closed, `FALSE` for open) will be stored.
|
|
4389
4563
|
* @param options - An optional object with configuration options:
|
|
4390
4564
|
* @param options.column - The name of the column storing the geometries. If omitted, the method will automatically attempt to find a geometry column.
|
|
@@ -4410,8 +4584,6 @@ class SimpleTable extends Simple {
|
|
|
4410
4584
|
/**
|
|
4411
4585
|
* Adds a column with the geometry type (e.g., `"POINT"`, `"LINESTRING"`, `"POLYGON"`) for each geometry.
|
|
4412
4586
|
*
|
|
4413
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4414
|
-
*
|
|
4415
4587
|
* @param newColumn - The name of the new column where the geometry types will be stored.
|
|
4416
4588
|
* @param options - An optional object with configuration options:
|
|
4417
4589
|
* @param options.column - The name of the column storing the geometries. If omitted, the method will automatically attempt to find a geometry column.
|
|
@@ -4438,8 +4610,6 @@ class SimpleTable extends Simple {
|
|
|
4438
4610
|
* Flips the coordinate order of geometries in a specified column (e.g., from `[longitude (x), latitude (y)]` to `[latitude (y), longitude (x)]` or vice-versa).
|
|
4439
4611
|
* **Warning:** This method should be used with caution as it directly manipulates coordinate order and can affect the accuracy of geospatial operations if not used correctly.
|
|
4440
4612
|
*
|
|
4441
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4442
|
-
*
|
|
4443
4613
|
* @param column - The name of the column storing the geometries. If omitted, the method will automatically attempt to find a geometry column.
|
|
4444
4614
|
* @returns The table, so methods can be chained.
|
|
4445
4615
|
* @category Geospatial
|
|
@@ -4461,9 +4631,42 @@ class SimpleTable extends Simple {
|
|
|
4461
4631
|
return this;
|
|
4462
4632
|
}
|
|
4463
4633
|
/**
|
|
4464
|
-
*
|
|
4634
|
+
* Moves EPSG:4326 point geometries to random locations within a maximum great-circle distance of their original positions.
|
|
4635
|
+
*
|
|
4636
|
+
* Points are sampled uniformly within the requested distance using the spherical direct geodesic (destination-point) formula and the same spherical Earth model as DuckDB's `ST_Distance_Sphere()`. This accounts for longitude scale changing with latitude. This method adds random jitter; it does not provide anonymization or differential privacy guarantees.
|
|
4637
|
+
*
|
|
4638
|
+
* This method supports only `POINT` geometries in `EPSG:4326`. Null and empty geometries are preserved.
|
|
4465
4639
|
*
|
|
4466
|
-
*
|
|
4640
|
+
* @param maxDistance - The maximum great-circle displacement in the selected unit. Must be a finite number greater than or equal to `0` and no greater than half Earth's circumference.
|
|
4641
|
+
* @param options - An optional object with configuration options:
|
|
4642
|
+
* @param options.column - The name of the EPSG:4326 point geometry column. If omitted, the method will automatically attempt to find a geometry column.
|
|
4643
|
+
* @param options.unit - The unit of `maxDistance`: `"m"` for metres or `"km"` for kilometres. Defaults to `"m"`.
|
|
4644
|
+
* @param options.onlyDuplicates - If `true`, moves only points whose complete original geometry occurs more than once in the selected geometry column. Every point in a duplicated group is moved. Defaults to `false`.
|
|
4645
|
+
* @returns The table, so methods can be chained.
|
|
4646
|
+
* @category Geospatial
|
|
4647
|
+
*
|
|
4648
|
+
* @example
|
|
4649
|
+
* ```ts
|
|
4650
|
+
* // Move every point by no more than 500 metres
|
|
4651
|
+
* await table.addGeoNoise(500).log();
|
|
4652
|
+
* ```
|
|
4653
|
+
*
|
|
4654
|
+
* @example
|
|
4655
|
+
* ```ts
|
|
4656
|
+
* // Separate duplicated points by up to 0.1 kilometres
|
|
4657
|
+
* await table.addGeoNoise(0.1, {
|
|
4658
|
+
* column: "geom",
|
|
4659
|
+
* unit: "km",
|
|
4660
|
+
* onlyDuplicates: true,
|
|
4661
|
+
* }).log();
|
|
4662
|
+
* ```
|
|
4663
|
+
*/
|
|
4664
|
+
addGeoNoise(maxDistance, options = {}) {
|
|
4665
|
+
addGeoNoise(this, maxDistance, options);
|
|
4666
|
+
return this;
|
|
4667
|
+
}
|
|
4668
|
+
/**
|
|
4669
|
+
* Reduces the precision of geometries in a specified column to a given number of decimal places.
|
|
4467
4670
|
*
|
|
4468
4671
|
* @param decimals - The number of decimal places to keep in the coordinates of the geometries.
|
|
4469
4672
|
* @param options - An optional object with configuration options:
|
|
@@ -4490,8 +4693,6 @@ class SimpleTable extends Simple {
|
|
|
4490
4693
|
/**
|
|
4491
4694
|
* Reprojects the geometries in a specified column to another Spatial Reference System (SRS).
|
|
4492
4695
|
*
|
|
4493
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4494
|
-
*
|
|
4495
4696
|
* @param crs - The target SRS (e.g., `"EPSG:3347"`, or `"EPSG:4326"` for EPSG:4326 (WGS84)).
|
|
4496
4697
|
* @param options - An optional object with configuration options:
|
|
4497
4698
|
* @param options.column - The name of the column storing the geometries. If omitted, the method will automatically attempt to find a geometry column.
|
|
@@ -4518,8 +4719,6 @@ class SimpleTable extends Simple {
|
|
|
4518
4719
|
* Computes the area of geometries in square meters (`"m2"`) or optionally square kilometers (`"km2"`).
|
|
4519
4720
|
* The input geometry is assumed to be in EPSG:4326 (WGS84).
|
|
4520
4721
|
*
|
|
4521
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4522
|
-
*
|
|
4523
4722
|
* @param newColumn - The name of the new column where the computed areas will be stored.
|
|
4524
4723
|
* @param options - An optional object with configuration options:
|
|
4525
4724
|
* @param options.unit - The unit for the computed area: `"m2"` (square meters) or `"km2"` (square kilometers). Defaults to `"m2"`.
|
|
@@ -4560,8 +4759,6 @@ class SimpleTable extends Simple {
|
|
|
4560
4759
|
* Computes the length of line geometries in meters (`"m"`) or optionally kilometers (`"km"`).
|
|
4561
4760
|
* The input geometry is assumed to be in EPSG:4326 (WGS84).
|
|
4562
4761
|
*
|
|
4563
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4564
|
-
*
|
|
4565
4762
|
* @param newColumn - The name of the new column where the computed lengths will be stored.
|
|
4566
4763
|
* @param options - An optional object with configuration options:
|
|
4567
4764
|
* @param options.unit - The unit for the computed length: `"m"` (meters) or `"km"` (kilometers). Defaults to `"m"`.
|
|
@@ -4602,8 +4799,6 @@ class SimpleTable extends Simple {
|
|
|
4602
4799
|
* Computes the perimeter of polygon geometries in meters (`"m"`) or optionally kilometers (`"km"`).
|
|
4603
4800
|
* The input geometry is assumed to be in EPSG:4326 (WGS84).
|
|
4604
4801
|
*
|
|
4605
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4606
|
-
*
|
|
4607
4802
|
* @param newColumn - The name of the new column where the computed perimeters will be stored.
|
|
4608
4803
|
* @param options - An optional object with configuration options:
|
|
4609
4804
|
* @param options.unit - The unit for the computed perimeter: `"m"` (meters) or `"km"` (kilometers). Defaults to `"m"`.
|
|
@@ -4644,8 +4839,6 @@ class SimpleTable extends Simple {
|
|
|
4644
4839
|
* Computes a buffer (a polygon representing a specified distance around a geometry) for geometries in a specified column.
|
|
4645
4840
|
* The distance is in the Spatial Reference System (SRS) unit of the input geometries.
|
|
4646
4841
|
*
|
|
4647
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4648
|
-
*
|
|
4649
4842
|
* @param newColumn - The name of the new column where the buffered geometries will be stored.
|
|
4650
4843
|
* @param distance - The distance for the buffer. This value is in the units of the geometry's SRS.
|
|
4651
4844
|
* @param options - An optional object with configuration options:
|
|
@@ -4673,7 +4866,6 @@ class SimpleTable extends Simple {
|
|
|
4673
4866
|
* Merges the data of this table (considered the left table) with another table (the right table) based on a spatial relationship.
|
|
4674
4867
|
* Note that the order of rows in the returned data is not guaranteed to be the same as in the original tables.
|
|
4675
4868
|
* This operation might create temporary files in a `.tmp` folder; consider adding `.tmp` to your `.gitignore`.
|
|
4676
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called. The join uses the other table's state as of this call: operations queued on it afterwards run after the join.
|
|
4677
4869
|
*
|
|
4678
4870
|
* @param rightTable - The SimpleTable instance to be joined with this table.
|
|
4679
4871
|
* @param method - The spatial join method to use: `"intersect"` (geometries overlap), `"inside"` (geometries of the left table are entirely within geometries of the right table), or `"withinDistance"` (geometries of the left table are within a specified distance of geometries in the right table).
|
|
@@ -4745,8 +4937,6 @@ class SimpleTable extends Simple {
|
|
|
4745
4937
|
/**
|
|
4746
4938
|
* Computes the intersection of two sets of geometries, creating new geometries where they overlap.
|
|
4747
4939
|
*
|
|
4748
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4749
|
-
*
|
|
4750
4940
|
* @param column1 - The name of the first column storing geometries.
|
|
4751
4941
|
* @param column2 - The name of the second column storing geometries. Both columns must have the same projection.
|
|
4752
4942
|
* @param newColumn - The name of the new column where the computed intersection geometries will be stored.
|
|
@@ -4766,8 +4956,6 @@ class SimpleTable extends Simple {
|
|
|
4766
4956
|
/**
|
|
4767
4957
|
* Computes the geometric difference between two geometries, returning the portion of the first geometry that does not intersect the second.
|
|
4768
4958
|
*
|
|
4769
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4770
|
-
*
|
|
4771
4959
|
* @param column1 - The name of the column storing the geometries from which the second geometries will be subtracted.
|
|
4772
4960
|
* @param column2 - The name of the column storing the geometries to subtract. Both columns must have the same projection.
|
|
4773
4961
|
* @param newColumn - The name of the new column where the geometric differences will be stored.
|
|
@@ -4787,8 +4975,6 @@ class SimpleTable extends Simple {
|
|
|
4787
4975
|
/**
|
|
4788
4976
|
* Fills holes in polygon geometries.
|
|
4789
4977
|
*
|
|
4790
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4791
|
-
*
|
|
4792
4978
|
* @param column - The name of the column storing the geometries. If omitted, the method will automatically attempt to find a geometry column.
|
|
4793
4979
|
* @returns The table, so methods can be chained.
|
|
4794
4980
|
* @category Geospatial
|
|
@@ -4812,8 +4998,6 @@ class SimpleTable extends Simple {
|
|
|
4812
4998
|
/**
|
|
4813
4999
|
* Returns `TRUE` if two geometries intersect (overlap in any way), and `FALSE` otherwise.
|
|
4814
5000
|
*
|
|
4815
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4816
|
-
*
|
|
4817
5001
|
* @param column1 - The name of the first column storing geometries.
|
|
4818
5002
|
* @param column2 - The name of the second column storing geometries. Both columns must have the same projection.
|
|
4819
5003
|
* @param newColumn - The name of the new column where the boolean results (`TRUE` for intersection, `FALSE` otherwise) will be stored.
|
|
@@ -4833,8 +5017,6 @@ class SimpleTable extends Simple {
|
|
|
4833
5017
|
/**
|
|
4834
5018
|
* Returns `TRUE` if every point of a geometry in `column` is covered by a geometry in `containerColumn`, including their boundaries, and `FALSE` otherwise.
|
|
4835
5019
|
*
|
|
4836
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4837
|
-
*
|
|
4838
5020
|
* @param column - The name of the column storing the geometries to be tested for containment.
|
|
4839
5021
|
* @param containerColumn - The name of the column storing the geometries to be tested as containers. Both columns must have the same projection.
|
|
4840
5022
|
* @param newColumn - The name of the new column where the boolean results (`TRUE` when covered, `FALSE` otherwise) will be stored.
|
|
@@ -4854,8 +5036,6 @@ class SimpleTable extends Simple {
|
|
|
4854
5036
|
/**
|
|
4855
5037
|
* Computes the union of two geometries, creating a new geometry that represents the merged area of both.
|
|
4856
5038
|
*
|
|
4857
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4858
|
-
*
|
|
4859
5039
|
* @param column1 - The name of the first column storing geometries.
|
|
4860
5040
|
* @param column2 - The name of the second column storing geometries. Both columns must have the same projection.
|
|
4861
5041
|
* @param newColumn - The name of the new column where the computed union geometries will be stored.
|
|
@@ -4876,8 +5056,6 @@ class SimpleTable extends Simple {
|
|
|
4876
5056
|
* Extracts the latitude (y) and longitude (x) coordinates from point geometries.
|
|
4877
5057
|
* The input geometry is assumed to be in EPSG:4326 (WGS84).
|
|
4878
5058
|
*
|
|
4879
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4880
|
-
*
|
|
4881
5059
|
* @param column - The name of the column storing the point geometries.
|
|
4882
5060
|
* @param latColumn - The name of the new column where the extracted latitude (y-coordinate) values will be stored.
|
|
4883
5061
|
* @param lonColumn - The name of the new column where the extracted longitude (x-coordinate) values will be stored.
|
|
@@ -4897,8 +5075,6 @@ class SimpleTable extends Simple {
|
|
|
4897
5075
|
/**
|
|
4898
5076
|
* Simplifies geometries while preserving their overall coverage. A higher tolerance results in more significant simplification.
|
|
4899
5077
|
*
|
|
4900
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4901
|
-
*
|
|
4902
5078
|
* @param tolerance - A numeric value representing the simplification tolerance. A higher value leads to greater simplification.
|
|
4903
5079
|
* @param options - An optional object with configuration options:
|
|
4904
5080
|
* @param options.column - The name of the column storing the geometries. If omitted, the method will automatically attempt to find a geometry column.
|
|
@@ -4926,8 +5102,6 @@ class SimpleTable extends Simple {
|
|
|
4926
5102
|
* Computes the centroid of geometries.
|
|
4927
5103
|
* The values are returned in the SRS unit of the input geometries.
|
|
4928
5104
|
*
|
|
4929
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4930
|
-
*
|
|
4931
5105
|
* @param newColumn - The name of the new column where the computed centroid geometries will be stored.
|
|
4932
5106
|
* @param options - An optional object with configuration options:
|
|
4933
5107
|
* @param options.column - The name of the column storing the geometries. If omitted, the method will automatically attempt to find a geometry column.
|
|
@@ -4953,8 +5127,6 @@ class SimpleTable extends Simple {
|
|
|
4953
5127
|
/**
|
|
4954
5128
|
* Generates a random point within the geometries of a specified column.
|
|
4955
5129
|
*
|
|
4956
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4957
|
-
*
|
|
4958
5130
|
* @param newColumn - The name of the new column where the random points will be stored.
|
|
4959
5131
|
* @param tries - The number of points to generate within the bounding box of each geometry to find one that is within the geometry itself.
|
|
4960
5132
|
* @param options - An optional object with configuration options:
|
|
@@ -4989,8 +5161,6 @@ class SimpleTable extends Simple {
|
|
|
4989
5161
|
* You can optionally specify `"spheroid"` or `"haversine"` methods to get results in meters or kilometers.
|
|
4990
5162
|
* If using `"spheroid"` or `"haversine"`, the input geometries must be in EPSG:4326 (WGS84).
|
|
4991
5163
|
*
|
|
4992
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4993
|
-
*
|
|
4994
5164
|
* @param column1 - The name of the first column storing geometries.
|
|
4995
5165
|
* @param column2 - The name of the second column storing geometries.
|
|
4996
5166
|
* @param newColumn - The name of the new column where the computed distances will be stored.
|
|
@@ -5035,8 +5205,6 @@ class SimpleTable extends Simple {
|
|
|
5035
5205
|
/**
|
|
5036
5206
|
* Unnests geometries recursively, transforming multi-part geometries (e.g., MultiPolygon) into individual single-part geometries (e.g., Polygon).
|
|
5037
5207
|
*
|
|
5038
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
5039
|
-
*
|
|
5040
5208
|
* @param column - The name of the column storing the geometries to be unnested. If omitted, the method will automatically attempt to find a geometry column.
|
|
5041
5209
|
* @returns The table, so methods can be chained.
|
|
5042
5210
|
* @category Geospatial
|
|
@@ -5060,8 +5228,6 @@ class SimpleTable extends Simple {
|
|
|
5060
5228
|
/**
|
|
5061
5229
|
* Adds the bounding box coordinates of geometries in a specified column as four new columns: `minLon`, `minLat`, `maxLon`, and `maxLat`.
|
|
5062
5230
|
*
|
|
5063
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
5064
|
-
*
|
|
5065
5231
|
* @param options - An optional object with configuration options:
|
|
5066
5232
|
* @param options.column - The name of the column storing the geometries for which the bounding box will be computed. If omitted, the method will automatically attempt to find a geometry column.
|
|
5067
5233
|
* @param options.decimals - The number of decimal places to round the bounding box coordinates. Defaults to `undefined` (no rounding).
|
|
@@ -5089,8 +5255,6 @@ class SimpleTable extends Simple {
|
|
|
5089
5255
|
/**
|
|
5090
5256
|
* Aggregates geometries in a specified column based on a chosen aggregation method.
|
|
5091
5257
|
*
|
|
5092
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
5093
|
-
*
|
|
5094
5258
|
* @param method - The aggregation method to apply: `"union"` (combines all geometries into a single multi-geometry) or `"intersection"` (computes the intersection of all geometries).
|
|
5095
5259
|
* @param options - An optional object with configuration options:
|
|
5096
5260
|
* @param options.column - The name of the column storing the geometries to be aggregated. If omitted, the method will automatically attempt to find a geometry column.
|
|
@@ -5123,8 +5287,6 @@ class SimpleTable extends Simple {
|
|
|
5123
5287
|
/**
|
|
5124
5288
|
* Transforms closed linestring geometries into polygon geometries.
|
|
5125
5289
|
*
|
|
5126
|
-
* This method queues the operation; it runs when an async observer method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
5127
|
-
*
|
|
5128
5290
|
* @param column - The name of the column storing the linestring geometries. If omitted, the method will automatically attempt to find a geometry column.
|
|
5129
5291
|
* @returns The table, so methods can be chained.
|
|
5130
5292
|
* @category Geospatial
|
|
@@ -5171,7 +5333,9 @@ class SimpleTable extends Simple {
|
|
|
5171
5333
|
return await getBoundingBox(this, column);
|
|
5172
5334
|
}
|
|
5173
5335
|
/**
|
|
5174
|
-
* Returns the table's geospatial data as a GeoJSON
|
|
5336
|
+
* Returns the table's geospatial data as a GeoJSON FeatureCollection.
|
|
5337
|
+
* Each row becomes a Feature: the selected geometry column becomes its geometry,
|
|
5338
|
+
* and the remaining columns become its properties.
|
|
5175
5339
|
* If the table has multiple geometry columns, you must specify which one to use.
|
|
5176
5340
|
*
|
|
5177
5341
|
* @param column - The name of the column storing the geometries. If omitted, the method will automatically attempt to find a geometry column.
|
|
@@ -5209,12 +5373,13 @@ class SimpleTable extends Simple {
|
|
|
5209
5373
|
}
|
|
5210
5374
|
/**
|
|
5211
5375
|
* Writes the table's data to a file in various formats (CSV, JSON, Parquet, DuckDB, SQLite).
|
|
5376
|
+
* Tables with geometry columns must use `writeGeoData()` instead.
|
|
5212
5377
|
* If the specified path does not exist, it will be created.
|
|
5213
5378
|
*
|
|
5214
5379
|
* @param file - The absolute path to the output file (e.g., `"./output.csv"`, `"./output.json"`).
|
|
5215
5380
|
* @param options - An optional object with configuration options:
|
|
5216
5381
|
* @param options.compression - A boolean indicating whether to compress the output file. If `true`, CSV and JSON files will be compressed with GZIP, while Parquet files will use ZSTD. Defaults to `false`.
|
|
5217
|
-
* @param options.dataAsArrays - For JSON files only. If `true`, JSON files are written as a single object with arrays for each column (e.g., `{ "col1": [v1, v2], "col2": [v3, v4] }`) instead of an array of objects. This can reduce file size for web projects. You can use the `arraysToData` function from the [journalism library](https://jsr.io/@nshiab/journalism/doc/~/arraysToData) to convert it back.
|
|
5382
|
+
* @param options.dataAsArrays - For JSON files only. If `true`, JSON files are written as a single object with arrays for each column (e.g., `{ "col1": [v1, v2], "col2": [v3, v4] }`) instead of an array of objects. This can reduce file size for web projects. You can use the `arraysToData` function from the [journalism-format library](https://jsr.io/@nshiab/journalism-format/doc/~/arraysToData) to convert it back.
|
|
5218
5383
|
* @param options.formatDates - For CSV and JSON files only. If `true`, date and timestamp columns will be formatted as ISO 8601 strings (e.g., `"2025-01-01T01:00:00.000Z"`). Defaults to `false`.
|
|
5219
5384
|
* @returns A promise that resolves to the table, so methods can be chained.
|
|
5220
5385
|
* @category File Operations
|
|
@@ -5315,28 +5480,28 @@ class SimpleTable extends Simple {
|
|
|
5315
5480
|
* Caches the results of computations in `./.sda-cache`.
|
|
5316
5481
|
* You should add `./.sda-cache` to your `.gitignore` file.
|
|
5317
5482
|
*
|
|
5318
|
-
* Cache entries are stored as DuckDB database files. Full-text search (FTS)
|
|
5319
|
-
* indexes are persisted in the cache file and restored directly on a cache
|
|
5320
|
-
* hit. Vector similarity search (VSS/HNSW) indexes are not persisted in the
|
|
5321
|
-
* cache file; their definitions are stored as metadata and used to rebuild
|
|
5322
|
-
* the indexes on every cache hit. If loading the entry or restoring its
|
|
5323
|
-
* indexes fails, the computation runs again and replaces the cache entry.
|
|
5324
|
-
*
|
|
5325
5483
|
* `cache()` automatically tracks whether earlier SDA operations changed the
|
|
5326
5484
|
* table. It also records every other already registered `SimpleTable` read
|
|
5327
5485
|
* through `SimpleTable` methods while `compute` runs and invalidates the
|
|
5328
5486
|
* cached step when any of their generations change. Tables created inside
|
|
5329
5487
|
* `compute` are part of the computation itself and are not dependencies.
|
|
5330
5488
|
*
|
|
5489
|
+
* Inside `compute`, you may modify the table being cached and create or
|
|
5490
|
+
* modify temporary tables, provided you remove them before the callback
|
|
5491
|
+
* finishes. Other tables that already existed may be read but not modified,
|
|
5492
|
+
* renamed, or removed. `customQuery()` bypasses enforcement of these rules.
|
|
5493
|
+
*
|
|
5331
5494
|
* `SimpleDB.customQuery()` bypasses this tracking. Reading or changing a
|
|
5332
5495
|
* table with `customQuery()` can therefore return stale cached data. Include
|
|
5333
5496
|
* a value that identifies the custom query's dependencies in `options.inputs`
|
|
5334
5497
|
* (such as a table content hash), or use tracked `SimpleTable` methods.
|
|
5335
5498
|
*
|
|
5336
|
-
*
|
|
5337
|
-
*
|
|
5338
|
-
*
|
|
5339
|
-
*
|
|
5499
|
+
* Cache entries are stored as DuckDB database files. Full-text search (FTS)
|
|
5500
|
+
* indexes are persisted in the cache file and restored directly on a cache
|
|
5501
|
+
* hit. Vector similarity search (VSS/HNSW) indexes are not persisted in the
|
|
5502
|
+
* cache file; their definitions are stored as metadata and used to rebuild
|
|
5503
|
+
* the indexes on every cache hit. If loading the entry or restoring its
|
|
5504
|
+
* indexes fails, the computation runs again and replaces the cache entry.
|
|
5340
5505
|
*
|
|
5341
5506
|
* @param compute - A function wrapping the computations to be cached. It receives the table on which `cache()` was called. This function will be executed on the first run or if the cached data is invalid/expired.
|
|
5342
5507
|
* @param options - An optional object with configuration options:
|
|
@@ -5421,7 +5586,16 @@ class SimpleTable extends Simple {
|
|
|
5421
5586
|
/**
|
|
5422
5587
|
* Logs a specified number of rows from the table to the console. By default, the first 10 rows are logged.
|
|
5423
5588
|
* You can optionally log the column types and filter the data based on conditions.
|
|
5424
|
-
*
|
|
5589
|
+
* SQL dates and timestamps retain their native precision for display, including
|
|
5590
|
+
* temporal infinities. Lists and objects are stringified using the same nested
|
|
5591
|
+
* representations as `getData()`, then truncated according to `charsToLog`.
|
|
5592
|
+
* Unsafe top-level large integers throw, just as with `getData()`.
|
|
5593
|
+
* Type annotations describe the SQL type and the JavaScript extraction type
|
|
5594
|
+
* before display formatting; nulls do not determine the column's annotation.
|
|
5595
|
+
* Colors reflect the SQL value's meaning, so exact decimal strings use numeric
|
|
5596
|
+
* coloring. Column width is independent of the content truncation budget.
|
|
5597
|
+
*
|
|
5598
|
+
* With the default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
5425
5599
|
*
|
|
5426
5600
|
* @param options - Either the number of rows to log (a specific number or `"all"`) or an object with configuration options:
|
|
5427
5601
|
* @param options.count - The number of rows to log. Defaults to 10 or the value set in the SimpleDB instance. Use `"all"` to log all rows.
|
|
@@ -5461,57 +5635,7 @@ class SimpleTable extends Simple {
|
|
|
5461
5635
|
* ```
|
|
5462
5636
|
*/
|
|
5463
5637
|
async log(options = {}) {
|
|
5464
|
-
|
|
5465
|
-
await this.sdb.start();
|
|
5466
|
-
this.db = this.sdb.db;
|
|
5467
|
-
this.connection = this.sdb.connection;
|
|
5468
|
-
}
|
|
5469
|
-
if (this.connection === undefined) {
|
|
5470
|
-
throw new Error("this.connection is undefined");
|
|
5471
|
-
}
|
|
5472
|
-
let count;
|
|
5473
|
-
if (typeof options === "number") {
|
|
5474
|
-
count = options;
|
|
5475
|
-
}
|
|
5476
|
-
else if (options === "all") {
|
|
5477
|
-
count = await this.getRowCount();
|
|
5478
|
-
}
|
|
5479
|
-
else if (typeof options === "object") {
|
|
5480
|
-
if (options.count === "all") {
|
|
5481
|
-
count = await this.getRowCount();
|
|
5482
|
-
}
|
|
5483
|
-
else if (typeof options.count === "number") {
|
|
5484
|
-
count = options.count;
|
|
5485
|
-
}
|
|
5486
|
-
else {
|
|
5487
|
-
count = this.rowsToLog;
|
|
5488
|
-
}
|
|
5489
|
-
}
|
|
5490
|
-
else {
|
|
5491
|
-
count = this.rowsToLog;
|
|
5492
|
-
}
|
|
5493
|
-
const types = typeof options === "object"
|
|
5494
|
-
? options.types ?? this.typesToLog
|
|
5495
|
-
: this.typesToLog;
|
|
5496
|
-
const conditions = typeof options === "object"
|
|
5497
|
-
? options.conditions ?? undefined
|
|
5498
|
-
: undefined;
|
|
5499
|
-
if (this.connection === undefined ||
|
|
5500
|
-
!(await this.sdb.hasTable(this.name))) {
|
|
5501
|
-
console.log(`\nTable ${this.name}: no data`);
|
|
5502
|
-
}
|
|
5503
|
-
else {
|
|
5504
|
-
console.log(`\nTable ${this.name}:`);
|
|
5505
|
-
conditions && console.log(`Conditions: ${conditions}`);
|
|
5506
|
-
const data = await this.getTop(count, { conditions });
|
|
5507
|
-
logData(types ? await this.getTypes() : null, data, this.charsToLog);
|
|
5508
|
-
const rowCount = conditions
|
|
5509
|
-
? parseInt((await this.sdb.customQuery(`select count(*) as count from ${quoteIdentifier(this.name)} where ${conditions}`, { returnData: true }))[0].count)
|
|
5510
|
-
: await this.getRowCount();
|
|
5511
|
-
console.log(`${formatNumber(rowCount)} rows in total ${`(count: ${count}${typeof this.charsToLog === "number"
|
|
5512
|
-
? `, charsToLog: ${this.charsToLog}`
|
|
5513
|
-
: ""})`}`);
|
|
5514
|
-
}
|
|
5638
|
+
await log(this, options);
|
|
5515
5639
|
return this;
|
|
5516
5640
|
}
|
|
5517
5641
|
/**
|
|
@@ -5713,6 +5837,10 @@ class SimpleTable extends Simple {
|
|
|
5713
5837
|
/**
|
|
5714
5838
|
* Logs the bottom `n` rows of the table to the console. By default, the last row will be returned first. To preserve the original order, use the `originalOrder` option.
|
|
5715
5839
|
*
|
|
5840
|
+
* Uses the same precision-preserving temporal display, nested stringification,
|
|
5841
|
+
* numeric coloring, and `charsToLog` truncation as `log()`. Unsafe top-level
|
|
5842
|
+
* large integers throw, just as with `getData()`.
|
|
5843
|
+
*
|
|
5716
5844
|
* @param count - The number of rows to log from the bottom of the table. Defaults to the table's `rowsToLog` option if not specified.
|
|
5717
5845
|
* @param options - An optional object with logging preferences.
|
|
5718
5846
|
* @param options.originalOrder - If true, the rows are displayed in their original order (top to bottom). Defaults to false.
|
|
@@ -5738,10 +5866,7 @@ class SimpleTable extends Simple {
|
|
|
5738
5866
|
* ```
|
|
5739
5867
|
*/
|
|
5740
5868
|
async logBottom(count, options = {}) {
|
|
5741
|
-
|
|
5742
|
-
console.log(`\nTable ${this.name} (${rows} bottom rows):`);
|
|
5743
|
-
const data = await this.getBottom(rows, options);
|
|
5744
|
-
logData(null, data, this.charsToLog);
|
|
5869
|
+
await logBottom(this, count, options);
|
|
5745
5870
|
return this;
|
|
5746
5871
|
}
|
|
5747
5872
|
/**
|