@nshiab/simple-data-analysis-core 2.0.4 → 2.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -12
- package/esm/class/Simple.d.ts +8 -2
- package/esm/class/Simple.d.ts.map +1 -1
- package/esm/class/Simple.js +9 -3
- package/esm/class/SimpleDB.d.ts +72 -1
- package/esm/class/SimpleDB.d.ts.map +1 -1
- package/esm/class/SimpleDB.js +81 -1
- package/esm/class/SimpleTable.d.ts +434 -263
- package/esm/class/SimpleTable.d.ts.map +1 -1
- package/esm/class/SimpleTable.js +441 -316
- package/esm/helpers/LogValue.d.ts +8 -0
- package/esm/helpers/LogValue.d.ts.map +1 -0
- package/esm/helpers/LogValue.js +23 -0
- package/esm/helpers/cacheTableDependencies.d.ts +9 -0
- package/esm/helpers/cacheTableDependencies.d.ts.map +1 -1
- package/esm/helpers/cacheTableDependencies.js +131 -21
- package/esm/helpers/cleanSQL.d.ts +1 -1
- package/esm/helpers/cleanSQL.d.ts.map +1 -1
- package/esm/helpers/cleanSQL.js +268 -23
- package/esm/helpers/flushAllTables.d.ts.map +1 -1
- package/esm/helpers/flushAllTables.js +23 -28
- package/esm/helpers/foldIdentifier.d.ts +3 -0
- package/esm/helpers/foldIdentifier.d.ts.map +1 -0
- package/esm/helpers/foldIdentifier.js +4 -0
- package/esm/helpers/geometryFromJSON.d.ts +3 -0
- package/esm/helpers/geometryFromJSON.d.ts.map +1 -0
- package/esm/helpers/geometryFromJSON.js +4 -0
- package/esm/helpers/index.d.ts +1 -0
- package/esm/helpers/index.d.ts.map +1 -1
- package/esm/helpers/index.js +1 -0
- package/esm/helpers/logData.d.ts.map +1 -1
- package/esm/helpers/logData.js +45 -44
- package/esm/helpers/makeLogConverter.d.ts +5 -0
- package/esm/helpers/makeLogConverter.d.ts.map +1 -0
- package/esm/helpers/makeLogConverter.js +64 -0
- package/esm/helpers/osmFiles.d.ts.map +1 -1
- package/esm/helpers/parseTypes.d.ts +1 -1
- package/esm/helpers/parseTypes.d.ts.map +1 -1
- package/esm/helpers/parseTypes.js +4 -0
- package/esm/helpers/pendingOps.d.ts +4 -2
- package/esm/helpers/pendingOps.d.ts.map +1 -1
- package/esm/helpers/prepareGeometry.d.ts +3 -0
- package/esm/helpers/prepareGeometry.d.ts.map +1 -0
- package/esm/helpers/prepareGeometry.js +99 -0
- package/esm/helpers/prepareJSON.d.ts +3 -0
- package/esm/helpers/prepareJSON.d.ts.map +1 -0
- package/esm/helpers/prepareJSON.js +50 -0
- package/esm/helpers/printTable.d.ts +5 -4
- package/esm/helpers/printTable.d.ts.map +1 -1
- package/esm/helpers/printTable.js +32 -36
- package/esm/helpers/queryDB.js +1 -1
- package/esm/helpers/queueAsyncBarrier.d.ts +4 -0
- package/esm/helpers/queueAsyncBarrier.d.ts.map +1 -1
- package/esm/helpers/queueAsyncBarrier.js +4 -0
- package/esm/helpers/queueOp.d.ts.map +1 -1
- package/esm/helpers/queueOp.js +5 -3
- package/esm/helpers/readMutationRows.d.ts +9 -0
- package/esm/helpers/readMutationRows.d.ts.map +1 -0
- package/esm/helpers/readMutationRows.js +94 -0
- package/esm/helpers/referencedTables.d.ts +9 -0
- package/esm/helpers/referencedTables.d.ts.map +1 -0
- package/esm/helpers/referencedTables.js +22 -0
- package/esm/helpers/runQuery.d.ts.map +1 -1
- package/esm/helpers/runQuery.js +11 -13
- package/esm/helpers/scanSQL.d.ts +7 -0
- package/esm/helpers/scanSQL.d.ts.map +1 -0
- package/esm/helpers/scanSQL.js +71 -0
- package/esm/helpers/tableRegistry.d.ts +2 -0
- package/esm/helpers/tableRegistry.d.ts.map +1 -1
- package/esm/helpers/tableRegistry.js +12 -4
- package/esm/helpers/updateColumnsWithJS.d.ts +52 -0
- package/esm/helpers/updateColumnsWithJS.d.ts.map +1 -0
- package/esm/helpers/updateColumnsWithJS.js +139 -0
- package/esm/methods/addCharacterCount.d.ts +3 -0
- package/esm/methods/addCharacterCount.d.ts.map +1 -0
- package/esm/methods/addCharacterCount.js +15 -0
- package/esm/methods/addColumn.d.ts +1 -1
- package/esm/methods/addColumn.d.ts.map +1 -1
- package/esm/methods/addGeoNoise.d.ts +7 -0
- package/esm/methods/addGeoNoise.d.ts.map +1 -0
- package/esm/methods/addGeoNoise.js +135 -0
- package/esm/methods/addNoise.d.ts +5 -0
- package/esm/methods/addNoise.d.ts.map +1 -0
- package/esm/methods/addNoise.js +53 -0
- package/esm/methods/addWordCount.d.ts +3 -0
- package/esm/methods/addWordCount.d.ts.map +1 -0
- package/esm/methods/addWordCount.js +15 -0
- package/esm/methods/bm25.d.ts.map +1 -1
- package/esm/methods/bm25.js +1 -2
- package/esm/methods/cache.d.ts.map +1 -1
- package/esm/methods/cache.js +4 -6
- package/esm/methods/columnProportions.d.ts +1 -0
- package/esm/methods/columnProportions.d.ts.map +1 -1
- package/esm/methods/columnProportions.js +7 -2
- package/esm/methods/convert.d.ts +2 -2
- package/esm/methods/convert.d.ts.map +1 -1
- package/esm/methods/convert.js +5 -0
- package/esm/methods/getBottom.d.ts +1 -1
- package/esm/methods/getBottom.d.ts.map +1 -1
- package/esm/methods/getBottom.js +2 -2
- package/esm/methods/getData.d.ts +1 -1
- package/esm/methods/getData.d.ts.map +1 -1
- package/esm/methods/getData.js +2 -2
- package/esm/methods/join.d.ts.map +1 -1
- package/esm/methods/join.js +8 -16
- package/esm/methods/joinGeo.d.ts.map +1 -1
- package/esm/methods/joinGeo.js +10 -19
- package/esm/methods/loadArray.d.ts +4 -2
- package/esm/methods/loadArray.d.ts.map +1 -1
- package/esm/methods/loadArray.js +182 -18
- package/esm/methods/loadData.js +3 -0
- package/esm/methods/loadYahooFinanceData.d.ts +10 -0
- package/esm/methods/loadYahooFinanceData.d.ts.map +1 -0
- package/esm/methods/loadYahooFinanceData.js +133 -0
- package/esm/methods/log.d.ts +7 -0
- package/esm/methods/log.d.ts.map +1 -0
- package/esm/methods/log.js +58 -0
- package/esm/methods/logBottom.d.ts +5 -0
- package/esm/methods/logBottom.d.ts.map +1 -0
- package/esm/methods/logBottom.js +9 -0
- package/esm/methods/removeTable.d.ts.map +1 -1
- package/esm/methods/removeTable.js +2 -0
- package/esm/methods/removeTables.d.ts.map +1 -1
- package/esm/methods/removeTables.js +7 -0
- package/esm/methods/renameTable.d.ts.map +1 -1
- package/esm/methods/renameTable.js +4 -0
- package/esm/methods/rowProportions.d.ts +1 -0
- package/esm/methods/rowProportions.d.ts.map +1 -1
- package/esm/methods/rowProportions.js +5 -1
- package/esm/methods/rowToText.js +2 -2
- package/esm/methods/selectTables.d.ts.map +1 -1
- package/esm/methods/selectTables.js +6 -0
- package/esm/methods/setTypes.d.ts +1 -1
- package/esm/methods/setTypes.d.ts.map +1 -1
- package/esm/methods/stream.d.ts.map +1 -1
- package/esm/methods/stream.js +2 -1
- package/esm/methods/updateWithJS.d.ts +1 -3
- package/esm/methods/updateWithJS.d.ts.map +1 -1
- package/esm/methods/updateWithJS.js +123 -37
- package/llm.md +600 -430
- package/llms.txt +1 -1
- package/package.json +1 -1
- package/script/class/Simple.d.ts +8 -2
- package/script/class/Simple.d.ts.map +1 -1
- package/script/class/Simple.js +9 -3
- package/script/class/SimpleDB.d.ts +72 -1
- package/script/class/SimpleDB.d.ts.map +1 -1
- package/script/class/SimpleDB.js +81 -1
- package/script/class/SimpleTable.d.ts +434 -263
- package/script/class/SimpleTable.d.ts.map +1 -1
- package/script/class/SimpleTable.js +441 -316
- package/script/helpers/LogValue.d.ts +8 -0
- package/script/helpers/LogValue.d.ts.map +1 -0
- package/script/helpers/LogValue.js +26 -0
- package/script/helpers/cacheTableDependencies.d.ts +9 -0
- package/script/helpers/cacheTableDependencies.d.ts.map +1 -1
- package/script/helpers/cacheTableDependencies.js +170 -20
- package/script/helpers/cleanSQL.d.ts +1 -1
- package/script/helpers/cleanSQL.d.ts.map +1 -1
- package/script/helpers/cleanSQL.js +271 -23
- package/script/helpers/flushAllTables.d.ts.map +1 -1
- package/script/helpers/flushAllTables.js +24 -29
- package/script/helpers/foldIdentifier.d.ts +3 -0
- package/script/helpers/foldIdentifier.d.ts.map +1 -0
- package/script/helpers/foldIdentifier.js +7 -0
- package/script/helpers/geometryFromJSON.d.ts +3 -0
- package/script/helpers/geometryFromJSON.d.ts.map +1 -0
- package/script/helpers/geometryFromJSON.js +7 -0
- package/script/helpers/index.d.ts +1 -0
- package/script/helpers/index.d.ts.map +1 -1
- package/script/helpers/index.js +3 -1
- package/script/helpers/logData.d.ts.map +1 -1
- package/script/helpers/logData.js +45 -44
- package/script/helpers/makeLogConverter.d.ts +5 -0
- package/script/helpers/makeLogConverter.d.ts.map +1 -0
- package/script/helpers/makeLogConverter.js +70 -0
- package/script/helpers/osmFiles.d.ts.map +1 -1
- package/script/helpers/parseTypes.d.ts +1 -1
- package/script/helpers/parseTypes.d.ts.map +1 -1
- package/script/helpers/parseTypes.js +4 -0
- package/script/helpers/pendingOps.d.ts +4 -2
- package/script/helpers/pendingOps.d.ts.map +1 -1
- package/script/helpers/prepareGeometry.d.ts +3 -0
- package/script/helpers/prepareGeometry.d.ts.map +1 -0
- package/script/helpers/prepareGeometry.js +102 -0
- package/script/helpers/prepareJSON.d.ts +3 -0
- package/script/helpers/prepareJSON.d.ts.map +1 -0
- package/script/helpers/prepareJSON.js +53 -0
- package/script/helpers/printTable.d.ts +5 -4
- package/script/helpers/printTable.d.ts.map +1 -1
- package/script/helpers/printTable.js +32 -36
- package/script/helpers/queryDB.js +1 -1
- package/script/helpers/queueAsyncBarrier.d.ts +4 -0
- package/script/helpers/queueAsyncBarrier.d.ts.map +1 -1
- package/script/helpers/queueAsyncBarrier.js +4 -0
- package/script/helpers/queueOp.d.ts.map +1 -1
- package/script/helpers/queueOp.js +4 -2
- package/script/helpers/readMutationRows.d.ts +9 -0
- package/script/helpers/readMutationRows.d.ts.map +1 -0
- package/script/helpers/readMutationRows.js +97 -0
- package/script/helpers/referencedTables.d.ts +9 -0
- package/script/helpers/referencedTables.d.ts.map +1 -0
- package/script/helpers/referencedTables.js +28 -0
- package/script/helpers/runQuery.d.ts.map +1 -1
- package/script/helpers/runQuery.js +11 -13
- package/script/helpers/scanSQL.d.ts +7 -0
- package/script/helpers/scanSQL.d.ts.map +1 -0
- package/script/helpers/scanSQL.js +74 -0
- package/script/helpers/tableRegistry.d.ts +2 -0
- package/script/helpers/tableRegistry.d.ts.map +1 -1
- package/script/helpers/tableRegistry.js +16 -4
- package/script/helpers/updateColumnsWithJS.d.ts +52 -0
- package/script/helpers/updateColumnsWithJS.d.ts.map +1 -0
- package/script/helpers/updateColumnsWithJS.js +145 -0
- package/script/methods/addCharacterCount.d.ts +3 -0
- package/script/methods/addCharacterCount.d.ts.map +1 -0
- package/script/methods/addCharacterCount.js +21 -0
- package/script/methods/addColumn.d.ts +1 -1
- package/script/methods/addColumn.d.ts.map +1 -1
- package/script/methods/addGeoNoise.d.ts +7 -0
- package/script/methods/addGeoNoise.d.ts.map +1 -0
- package/script/methods/addGeoNoise.js +141 -0
- package/script/methods/addNoise.d.ts +5 -0
- package/script/methods/addNoise.d.ts.map +1 -0
- package/script/methods/addNoise.js +59 -0
- package/script/methods/addWordCount.d.ts +3 -0
- package/script/methods/addWordCount.d.ts.map +1 -0
- package/script/methods/addWordCount.js +21 -0
- package/script/methods/bm25.d.ts.map +1 -1
- package/script/methods/bm25.js +1 -2
- package/script/methods/cache.d.ts.map +1 -1
- package/script/methods/cache.js +3 -5
- package/script/methods/columnProportions.d.ts +1 -0
- package/script/methods/columnProportions.d.ts.map +1 -1
- package/script/methods/columnProportions.js +7 -2
- package/script/methods/convert.d.ts +2 -2
- package/script/methods/convert.d.ts.map +1 -1
- package/script/methods/convert.js +5 -0
- package/script/methods/getBottom.d.ts +1 -1
- package/script/methods/getBottom.d.ts.map +1 -1
- package/script/methods/getBottom.js +2 -2
- package/script/methods/getData.d.ts +1 -1
- package/script/methods/getData.d.ts.map +1 -1
- package/script/methods/getData.js +2 -2
- package/script/methods/join.d.ts.map +1 -1
- package/script/methods/join.js +8 -16
- package/script/methods/joinGeo.d.ts.map +1 -1
- package/script/methods/joinGeo.js +10 -19
- package/script/methods/loadArray.d.ts +4 -2
- package/script/methods/loadArray.d.ts.map +1 -1
- package/script/methods/loadArray.js +181 -17
- package/script/methods/loadData.js +3 -0
- package/script/methods/loadYahooFinanceData.d.ts +10 -0
- package/script/methods/loadYahooFinanceData.d.ts.map +1 -0
- package/script/methods/loadYahooFinanceData.js +139 -0
- package/script/methods/log.d.ts +7 -0
- package/script/methods/log.d.ts.map +1 -0
- package/script/methods/log.js +64 -0
- package/script/methods/logBottom.d.ts +5 -0
- package/script/methods/logBottom.d.ts.map +1 -0
- package/script/methods/logBottom.js +15 -0
- package/script/methods/removeTable.d.ts.map +1 -1
- package/script/methods/removeTable.js +2 -0
- package/script/methods/removeTables.d.ts.map +1 -1
- package/script/methods/removeTables.js +7 -0
- package/script/methods/renameTable.d.ts.map +1 -1
- package/script/methods/renameTable.js +4 -0
- package/script/methods/rowProportions.d.ts +1 -0
- package/script/methods/rowProportions.d.ts.map +1 -1
- package/script/methods/rowProportions.js +5 -1
- package/script/methods/rowToText.js +2 -2
- package/script/methods/selectTables.d.ts.map +1 -1
- package/script/methods/selectTables.js +6 -0
- package/script/methods/setTypes.d.ts +1 -1
- package/script/methods/setTypes.d.ts.map +1 -1
- package/script/methods/stream.d.ts.map +1 -1
- package/script/methods/stream.js +2 -1
- package/script/methods/updateWithJS.d.ts +1 -3
- package/script/methods/updateWithJS.d.ts.map +1 -1
- package/script/methods/updateWithJS.js +122 -36
package/llm.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# The Simple Data Analysis (Core) Library
|
|
2
2
|
|
|
3
3
|
- Package: `@nshiab/simple-data-analysis-core`
|
|
4
|
-
- Version: `2.0.
|
|
4
|
+
- Version: `2.0.6`
|
|
5
5
|
|
|
6
6
|
To install the library with Deno, use:
|
|
7
7
|
|
|
@@ -91,12 +91,16 @@ schema is reserved for versioned SDA metadata. SQLite files can be imported with
|
|
|
91
91
|
- **`options.logDuration`**: A flag indicating whether to log the total
|
|
92
92
|
execution duration.
|
|
93
93
|
- **`options.rowsToLog`**: The number of rows to display when logging a table.
|
|
94
|
-
- **`options.charsToLog`**: The maximum
|
|
95
|
-
|
|
94
|
+
- **`options.charsToLog`**: The maximum characters to display for text and
|
|
95
|
+
stringified nested cells, including truncation markers (default: 75).
|
|
96
|
+
Independent of column width. Use Infinity for unlimited text.
|
|
96
97
|
- **`options.typesToLog`**: A flag indicating whether to include data types when
|
|
97
98
|
logging a table.
|
|
98
99
|
- **`options.cacheVerbose`**: Whether to log cache hits and misses, code and
|
|
99
100
|
input changes, TTL status, and cache read/write timing.
|
|
101
|
+
- **`options.expressionSyntax`**: Operator syntax for expressions and custom
|
|
102
|
+
queries: `"js"` (default) translates JavaScript-style operators in SQL
|
|
103
|
+
expressions; `"sql"` preserves SQL unchanged.
|
|
100
104
|
- **`options.logSQL`**: A flag indicating whether to log SQL immediately before
|
|
101
105
|
execution.
|
|
102
106
|
- **`options.explainSQL`**: A flag indicating whether to log DuckDB query plans
|
|
@@ -116,7 +120,8 @@ schema is reserved for versioned SDA metadata. SQLite files can be imported with
|
|
|
116
120
|
|
|
117
121
|
#### `newTable`
|
|
118
122
|
|
|
119
|
-
Creates a new SimpleTable instance within the database.
|
|
123
|
+
Creates a new SimpleTable instance within the database. The name must be unique
|
|
124
|
+
among registered tables, ignoring ASCII letter case.
|
|
120
125
|
|
|
121
126
|
##### Signature
|
|
122
127
|
|
|
@@ -385,9 +390,11 @@ console.log(extensions); // Output: [{ extension_name: "spatial", loaded: true,
|
|
|
385
390
|
|
|
386
391
|
#### `customQuery`
|
|
387
392
|
|
|
388
|
-
Executes a custom SQL query directly against the DuckDB instance.
|
|
389
|
-
|
|
390
|
-
|
|
393
|
+
Executes a custom SQL query directly against the DuckDB instance. With the
|
|
394
|
+
default `SimpleDB.expressionSyntax: "js"`, queries support JavaScript-style
|
|
395
|
+
operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for
|
|
396
|
+
unchanged SQL. Queries run in UTC. When data is returned, temporal values use
|
|
397
|
+
the same JavaScript representations as `SimpleTable.getData()`.
|
|
391
398
|
|
|
392
399
|
`customQuery()` bypasses the dependency and table-generation tracking used by
|
|
393
400
|
`SimpleTable.cache()`. Reading or changing a table with `customQuery()` can
|
|
@@ -643,8 +650,9 @@ Creates an instance of SimpleTable.
|
|
|
643
650
|
- **`simpleDB`**: The SimpleDB instance that this table belongs to.
|
|
644
651
|
- **`options`**: An optional object with configuration options:
|
|
645
652
|
- **`options.rowsToLog`**: The number of rows to log when displaying table data.
|
|
646
|
-
- **`options.charsToLog`**: The maximum
|
|
647
|
-
|
|
653
|
+
- **`options.charsToLog`**: The maximum characters to display for text and
|
|
654
|
+
stringified nested cells, including truncation markers (default: 75).
|
|
655
|
+
Independent of column width. Use Infinity for unlimited text.
|
|
648
656
|
- **`options.typesToLog`**: A boolean indicating whether to include data types
|
|
649
657
|
when logging a table.
|
|
650
658
|
|
|
@@ -701,7 +709,8 @@ await table.log();
|
|
|
701
709
|
|
|
702
710
|
#### `renameTable`
|
|
703
711
|
|
|
704
|
-
Renames the current table.
|
|
712
|
+
Renames the current table. The new name must not belong to another registered
|
|
713
|
+
table, ignoring ASCII letter case.
|
|
705
714
|
|
|
706
715
|
##### Signature
|
|
707
716
|
|
|
@@ -731,19 +740,17 @@ Sets the data types for columns in a new table. If the table already exists, it
|
|
|
731
740
|
will be replaced. To convert the types of an existing table, use the
|
|
732
741
|
`.convert()` method instead.
|
|
733
742
|
|
|
734
|
-
This method queues the operation; it runs when an async observer method (like
|
|
735
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
736
|
-
|
|
737
743
|
##### Signature
|
|
738
744
|
|
|
739
745
|
```typescript
|
|
740
|
-
setTypes(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | geometry('${string}') | GEOMETRY('${string}')>): this;
|
|
746
|
+
setTypes(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | float[${number}] | FLOAT[${number}] | geometry('${string}') | GEOMETRY('${string}')>): this;
|
|
741
747
|
```
|
|
742
748
|
|
|
743
749
|
##### Parameters
|
|
744
750
|
|
|
745
751
|
- **`types`**: An object specifying the column names and their target data types
|
|
746
|
-
(JavaScript or SQL types)
|
|
752
|
+
(JavaScript or SQL types), including JSON, FLOAT[n] vectors, and GEOMETRY with
|
|
753
|
+
a CRS.
|
|
747
754
|
|
|
748
755
|
##### Returns
|
|
749
756
|
|
|
@@ -760,27 +767,44 @@ await table.setTypes({
|
|
|
760
767
|
}).log();
|
|
761
768
|
```
|
|
762
769
|
|
|
770
|
+
```ts
|
|
771
|
+
// Create an empty table with JSON, embedding, and geometry columns.
|
|
772
|
+
await table.setTypes({
|
|
773
|
+
details: "JSON",
|
|
774
|
+
embedding: "FLOAT[3]",
|
|
775
|
+
geom: "GEOMETRY('EPSG:4326')",
|
|
776
|
+
}).log();
|
|
777
|
+
```
|
|
778
|
+
|
|
763
779
|
#### `loadArray`
|
|
764
780
|
|
|
765
|
-
Loads an array of JavaScript objects into the table.
|
|
766
|
-
|
|
767
|
-
|
|
781
|
+
Loads an array of JavaScript objects into the table. Types are inferred for
|
|
782
|
+
numbers, bigints, strings, booleans, and Date values. Array and object cells
|
|
783
|
+
require an explicit supported type in columnTypes. Types can also be specified
|
|
784
|
+
for scalar columns instead of inferred from their values.
|
|
768
785
|
|
|
769
786
|
JavaScript `Date` values are inferred as DuckDB `TIMESTAMP` values. Their
|
|
770
787
|
instant is preserved, but JavaScript `Date` does not retain the timezone or
|
|
771
|
-
offset originally used to construct it.
|
|
772
|
-
|
|
788
|
+
offset originally used to construct it.
|
|
789
|
+
|
|
790
|
+
Declare `JSON` for general nested data, `FLOAT[n]` for a fixed-size float
|
|
791
|
+
vector, or `GEOMETRY('EPSG:4326')` for GeoJSON geometries in WGS84.
|
|
773
792
|
|
|
774
793
|
##### Signature
|
|
775
794
|
|
|
776
795
|
```typescript
|
|
777
|
-
loadArray(rows: Record<string, unknown>[]): this;
|
|
796
|
+
loadArray(rows: Record<string, unknown>[], options?: { columnTypes?: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "INTEGER" | "BIGINT" | "DOUBLE" | "VARCHAR" | "BOOLEAN" | "DATE" | "TIME" | "TIMESTAMP" | "TIMESTAMP WITH TIME ZONE" | "GEOMETRY('EPSG:4326')" | "geometry('EPSG:4326')" | "JSON" | "json" | FLOAT[${number}] | float[${number}]> }): this;
|
|
778
797
|
```
|
|
779
798
|
|
|
780
799
|
##### Parameters
|
|
781
800
|
|
|
782
801
|
- **`rows`**: An array of objects, where each object represents a row and its
|
|
783
802
|
properties represent columns.
|
|
803
|
+
- **`options`**: Options for loading the array, captured when called.
|
|
804
|
+
- **`options.columnTypes`**: Types for specific columns. Required for array and
|
|
805
|
+
object cells; omitted scalar columns are inferred, and all-null columns
|
|
806
|
+
default to VARCHAR. Values must be compatible with the selected type without
|
|
807
|
+
losing information.
|
|
784
808
|
|
|
785
809
|
##### Returns
|
|
786
810
|
|
|
@@ -789,7 +813,7 @@ The table, so methods can be chained.
|
|
|
789
813
|
##### Examples
|
|
790
814
|
|
|
791
815
|
```ts
|
|
792
|
-
// Load
|
|
816
|
+
// Load scalar columns without columnTypes; strings and numbers are inferred.
|
|
793
817
|
const data = [
|
|
794
818
|
{ letter: "a", number: 1 },
|
|
795
819
|
{ letter: "b", number: 2 },
|
|
@@ -798,19 +822,53 @@ await table.loadArray(data).log();
|
|
|
798
822
|
```
|
|
799
823
|
|
|
800
824
|
```ts
|
|
801
|
-
//
|
|
802
|
-
|
|
825
|
+
// Choose a numeric type when all-null values provide nothing to infer.
|
|
826
|
+
await table.loadArray(
|
|
827
|
+
[{ name: "A", value: null }, { name: "B", value: null }],
|
|
828
|
+
{ columnTypes: { value: "DOUBLE" } },
|
|
829
|
+
).log();
|
|
830
|
+
```
|
|
831
|
+
|
|
832
|
+
```ts
|
|
833
|
+
// Store the Date as a UTC timestamp with JavaScript's millisecond precision.
|
|
834
|
+
// The original -04:00 offset is not retained.
|
|
803
835
|
await table.loadArray([{
|
|
804
836
|
observedAt: new Date("2024-04-07T13:00:00-04:00"),
|
|
805
837
|
}]).log();
|
|
806
838
|
```
|
|
807
839
|
|
|
840
|
+
```ts
|
|
841
|
+
// Declare JSON to load nested objects and variable-length arrays directly.
|
|
842
|
+
await table.loadArray([{
|
|
843
|
+
name: "Station",
|
|
844
|
+
metadata: { active: true, tags: ["weather", "urban"] },
|
|
845
|
+
measurements: [1, 2.5, null],
|
|
846
|
+
}], { columnTypes: { metadata: "JSON", measurements: "JSON" } }).log();
|
|
847
|
+
```
|
|
848
|
+
|
|
849
|
+
```ts
|
|
850
|
+
// Declare FLOAT[3] to store each three-element array as a fixed-size vector.
|
|
851
|
+
await table.loadArray([{ vector: [1, 2, 3] }], {
|
|
852
|
+
columnTypes: { vector: "FLOAT[3]" },
|
|
853
|
+
}).log();
|
|
854
|
+
```
|
|
855
|
+
|
|
856
|
+
```ts
|
|
857
|
+
// Declare a geometry type to store WGS84 GeoJSON as a spatial column.
|
|
858
|
+
await table.loadArray([{
|
|
859
|
+
name: "Montreal",
|
|
860
|
+
geom: { type: "Point", coordinates: [-73.57, 45.50] },
|
|
861
|
+
}], { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } }).log();
|
|
862
|
+
```
|
|
863
|
+
|
|
808
864
|
#### `loadData`
|
|
809
865
|
|
|
810
866
|
Loads data from one or more local or remote files into the table. Supported file
|
|
811
|
-
formats include CSV, JSON, Parquet, and Excel.
|
|
812
|
-
|
|
813
|
-
|
|
867
|
+
formats include CSV, JSON, Parquet, and Excel.
|
|
868
|
+
|
|
869
|
+
With the default `SimpleDB.expressionSyntax: "js"`, conditions support
|
|
870
|
+
JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
|
|
871
|
+
`expressionSyntax: "sql"` for unchanged SQL.
|
|
814
872
|
|
|
815
873
|
##### Signature
|
|
816
874
|
|
|
@@ -828,10 +886,8 @@ loadData(files: string | string[], options?: { fileType?: "csv" | "dsv" | "json"
|
|
|
828
886
|
- **`options.autoDetect`**: A boolean indicating whether to automatically detect
|
|
829
887
|
the data format. Defaults to `true`.
|
|
830
888
|
- **`options.conditions`**: A SQL `WHERE` clause expression, without the `WHERE`
|
|
831
|
-
keyword, to filter source rows before applying `limit`.
|
|
832
|
-
|
|
833
|
-
excluded from `columns`. Defaults to no filtering; an empty string behaves the
|
|
834
|
-
same as omitting this option.
|
|
889
|
+
keyword, to filter source rows before applying `limit`. Can reference source
|
|
890
|
+
columns excluded from `columns`.
|
|
835
891
|
- **`options.limit`**: A number indicating the maximum number of matching rows
|
|
836
892
|
to load, after applying `conditions` if provided. Defaults to all matching
|
|
837
893
|
rows.
|
|
@@ -841,7 +897,8 @@ loadData(files: string | string[], options?: { fileType?: "csv" | "dsv" | "json"
|
|
|
841
897
|
across multiple files when their structures differ. Missing columns will be
|
|
842
898
|
filled with `NULL` values. Defaults to `false`.
|
|
843
899
|
- **`options.columnTypes`**: An object mapping column names to their expected
|
|
844
|
-
data types. By default, types are inferred.
|
|
900
|
+
data types. By default, types are inferred. Geometry types are not supported;
|
|
901
|
+
use loadGeoData() instead.
|
|
845
902
|
- **`options.columns`**: An array of column names to load. When provided, only
|
|
846
903
|
the specified columns are loaded, reducing memory usage and improving load
|
|
847
904
|
times. Not supported for Excel files — combining `columns` with Excel files
|
|
@@ -931,9 +988,7 @@ await table
|
|
|
931
988
|
|
|
932
989
|
#### `loadStatCanData`
|
|
933
990
|
|
|
934
|
-
Downloads a complete Statistics Canada table and loads it into this table.
|
|
935
|
-
method queues the download and load; they run when an async observer method
|
|
936
|
-
(like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
991
|
+
Downloads a complete Statistics Canada table and loads it into this table.
|
|
937
992
|
|
|
938
993
|
Results are cached as Parquet files in `.sda-cache/statcan` by default. Cached
|
|
939
994
|
data does not expire unless a TTL is provided.
|
|
@@ -979,12 +1034,81 @@ await table
|
|
|
979
1034
|
.log();
|
|
980
1035
|
```
|
|
981
1036
|
|
|
982
|
-
#### `
|
|
1037
|
+
#### `loadYahooFinanceData`
|
|
983
1038
|
|
|
984
|
-
|
|
1039
|
+
Downloads historical market data from Yahoo Finance and loads it into this table
|
|
1040
|
+
as `datetime`, `open`, `high`, `low`, `close`, `adjustedClose`, and `volume`
|
|
1041
|
+
columns. `datetime` contains Yahoo's timestamps as JavaScript `Date` values.
|
|
1042
|
+
Unavailable values are preserved as `null`.
|
|
985
1043
|
|
|
986
|
-
|
|
987
|
-
`
|
|
1044
|
+
All range boundaries are evaluated in UTC. Use ISO date-time strings ending in
|
|
1045
|
+
`Z`. Yahoo trading sessions may occur on a different UTC date from their
|
|
1046
|
+
exchange-local date, so choose the range appropriate for the market and
|
|
1047
|
+
interval. Avoid numeric constructors such as `new Date(2025, 2, 15)`, which use
|
|
1048
|
+
the runtime's local timezone.
|
|
1049
|
+
|
|
1050
|
+
This method uses an undocumented Yahoo Finance endpoint and is not affiliated
|
|
1051
|
+
with or endorsed by Yahoo. It is provided for educational, research, and
|
|
1052
|
+
journalistic purposes. Before using it, review Yahoo's terms and any applicable
|
|
1053
|
+
data-provider restrictions.
|
|
1054
|
+
|
|
1055
|
+
##### Signature
|
|
1056
|
+
|
|
1057
|
+
```typescript
|
|
1058
|
+
loadYahooFinanceData(symbol: string, startDate: Date, endDate: Date, interval: "1d" | "1h" | "1m"): this;
|
|
1059
|
+
```
|
|
1060
|
+
|
|
1061
|
+
##### Parameters
|
|
1062
|
+
|
|
1063
|
+
- **`symbol`**: The stock or index symbol, such as `"AAPL"` or `"^GSPTSE"`.
|
|
1064
|
+
- **`startDate`**: The inclusive UTC start of the requested range.
|
|
1065
|
+
- **`endDate`**: The inclusive UTC end of the requested range. The UTC day,
|
|
1066
|
+
hour, or minute containing this instant is included, according to `interval`.
|
|
1067
|
+
- **`interval`**: The interval between observations: daily, hourly, or every
|
|
1068
|
+
minute.
|
|
1069
|
+
|
|
1070
|
+
##### Returns
|
|
1071
|
+
|
|
1072
|
+
The table, so methods can be chained.
|
|
1073
|
+
|
|
1074
|
+
##### Throws
|
|
1075
|
+
|
|
1076
|
+
- **`RangeError`**: If either date is invalid or `endDate` is before
|
|
1077
|
+
`startDate`.
|
|
1078
|
+
- **`Error`**: If Yahoo rejects the request or returns no data.
|
|
1079
|
+
|
|
1080
|
+
##### Examples
|
|
1081
|
+
|
|
1082
|
+
```ts
|
|
1083
|
+
// Request daily observations using explicit UTC boundaries.
|
|
1084
|
+
await table
|
|
1085
|
+
.loadYahooFinanceData(
|
|
1086
|
+
"^GSPTSE",
|
|
1087
|
+
new Date("2025-03-01T00:00:00Z"),
|
|
1088
|
+
new Date("2025-03-15T00:00:00Z"),
|
|
1089
|
+
"1d",
|
|
1090
|
+
)
|
|
1091
|
+
.log();
|
|
1092
|
+
```
|
|
1093
|
+
|
|
1094
|
+
```ts
|
|
1095
|
+
// Include the UTC hours from 13:00 through 16:00.
|
|
1096
|
+
await table
|
|
1097
|
+
.loadYahooFinanceData(
|
|
1098
|
+
"AAPL",
|
|
1099
|
+
new Date("2025-03-14T13:00:00Z"),
|
|
1100
|
+
new Date("2025-03-14T16:00:00Z"),
|
|
1101
|
+
"1h",
|
|
1102
|
+
)
|
|
1103
|
+
.log();
|
|
1104
|
+
```
|
|
1105
|
+
|
|
1106
|
+
#### `loadGeoData`
|
|
1107
|
+
|
|
1108
|
+
Loads geospatial data from an external file or URL into the table. With the
|
|
1109
|
+
default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style
|
|
1110
|
+
operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for
|
|
1111
|
+
unchanged SQL.
|
|
988
1112
|
|
|
989
1113
|
##### Signature
|
|
990
1114
|
|
|
@@ -1003,11 +1127,9 @@ loadGeoData(file: string, options?: { toEPSG4326?: boolean; columns?: string[];
|
|
|
1003
1127
|
should remain in the resulting table, usually `"geom"`. By default, all
|
|
1004
1128
|
columns are loaded.
|
|
1005
1129
|
- **`options.conditions`**: A SQL `WHERE` clause expression, without the `WHERE`
|
|
1006
|
-
keyword, to filter source rows before materialization and reprojection.
|
|
1007
|
-
|
|
1008
|
-
source
|
|
1009
|
-
coordinate system. Defaults to no filtering; an empty string behaves the same
|
|
1010
|
-
as omitting this option.
|
|
1130
|
+
keyword, to filter source rows before materialization and reprojection. Can
|
|
1131
|
+
reference source columns excluded from `columns`. Geometry conditions use the
|
|
1132
|
+
source coordinate system.
|
|
1011
1133
|
|
|
1012
1134
|
##### Returns
|
|
1013
1135
|
|
|
@@ -1060,8 +1182,7 @@ await table
|
|
|
1060
1182
|
Loads OpenStreetMap data into the table from a local `.osm` or `.osm.pbf` file,
|
|
1061
1183
|
a remote file URL, or an Overpass bounding-box query. Pass a path or URL string
|
|
1062
1184
|
to load an existing file, or pass a bounding box with `filters` to download
|
|
1063
|
-
matching features.
|
|
1064
|
-
method (like `getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1185
|
+
matching features.
|
|
1065
1186
|
|
|
1066
1187
|
DuckDB's
|
|
1067
1188
|
[Osmium community extension](https://duckdb.org/community_extensions/extensions/osmium)
|
|
@@ -1194,9 +1315,6 @@ method requires an FTS index and creates one automatically when needed. DuckDB
|
|
|
1194
1315
|
FTS indexes do not update automatically when the table changes; use
|
|
1195
1316
|
`overwrite: true` to rebuild the index after modifying the table.
|
|
1196
1317
|
|
|
1197
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1198
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1199
|
-
|
|
1200
1318
|
##### Signature
|
|
1201
1319
|
|
|
1202
1320
|
```typescript
|
|
@@ -1270,9 +1388,6 @@ If a VSS index already exists on the table, this method will skip creation and
|
|
|
1270
1388
|
log a message (when verbose is enabled), unless the `overwrite` option is set to
|
|
1271
1389
|
`true`. The index definition is recorded in {@link indexes}.
|
|
1272
1390
|
|
|
1273
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1274
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1275
|
-
|
|
1276
1391
|
##### Signature
|
|
1277
1392
|
|
|
1278
1393
|
```typescript
|
|
@@ -1347,9 +1462,6 @@ It reuses the table's existing FTS index unless `overwriteIndex` is `true`.
|
|
|
1347
1462
|
DuckDB FTS indexes do not update automatically when the source table changes;
|
|
1348
1463
|
use `overwriteIndex: true` to rebuild the index after modifying the table.
|
|
1349
1464
|
|
|
1350
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1351
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1352
|
-
|
|
1353
1465
|
##### Signature
|
|
1354
1466
|
|
|
1355
1467
|
```typescript
|
|
@@ -1472,9 +1584,6 @@ await table.bm25("italian sauce", "Dish", "Recipe", 5, {
|
|
|
1472
1584
|
|
|
1473
1585
|
Inserts rows, provided as an array of JavaScript objects, into the table.
|
|
1474
1586
|
|
|
1475
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1476
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1477
|
-
|
|
1478
1587
|
##### Signature
|
|
1479
1588
|
|
|
1480
1589
|
```typescript
|
|
@@ -1505,9 +1614,7 @@ await table.insertRows(newRows).log();
|
|
|
1505
1614
|
|
|
1506
1615
|
Inserts all rows from one or more other tables into this table. If tables do not
|
|
1507
1616
|
have the same columns, an error will be thrown unless the `unifyColumns` option
|
|
1508
|
-
is set to `true`.
|
|
1509
|
-
observer method (like `getData()` or `log()`) is awaited, or when `run()` is
|
|
1510
|
-
called.
|
|
1617
|
+
is set to `true`.
|
|
1511
1618
|
|
|
1512
1619
|
##### Signature
|
|
1513
1620
|
|
|
@@ -1549,9 +1656,6 @@ await tableA.insertTables(["tableB", "tableC"], { unifyColumns: true }).log();
|
|
|
1549
1656
|
|
|
1550
1657
|
Fetches sample data from the simple-data-analysis-core GitHub repository.
|
|
1551
1658
|
|
|
1552
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1553
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1554
|
-
|
|
1555
1659
|
##### Signature
|
|
1556
1660
|
|
|
1557
1661
|
```typescript
|
|
@@ -1583,7 +1687,10 @@ await table.loadSample("fires").log();
|
|
|
1583
1687
|
#### `clone`
|
|
1584
1688
|
|
|
1585
1689
|
Returns a new table with the same structure and data as this table. The data can
|
|
1586
|
-
be optionally filtered, limited to a specific number of rows, and offset.
|
|
1690
|
+
be optionally filtered, limited to a specific number of rows, and offset. With
|
|
1691
|
+
the default `SimpleDB.expressionSyntax: "js"`, conditions support
|
|
1692
|
+
JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
|
|
1693
|
+
`expressionSyntax: "sql"` for unchanged SQL.
|
|
1587
1694
|
|
|
1588
1695
|
If `conditions`, `limit`, and `offset` are all used, they are applied in this
|
|
1589
1696
|
order: `conditions` (WHERE clause) first, then `offset`, and finally `limit`
|
|
@@ -1591,9 +1698,6 @@ order: `conditions` (WHERE clause) first, then `offset`, and finally `limit`
|
|
|
1591
1698
|
|
|
1592
1699
|
Note that cloning large tables can be a slow operation.
|
|
1593
1700
|
|
|
1594
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1595
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1596
|
-
|
|
1597
1701
|
##### Signature
|
|
1598
1702
|
|
|
1599
1703
|
```typescript
|
|
@@ -1674,9 +1778,6 @@ const tableB = await tableA.clone({
|
|
|
1674
1778
|
Clones an existing column in this table, creating a new column with identical
|
|
1675
1779
|
values.
|
|
1676
1780
|
|
|
1677
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1678
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1679
|
-
|
|
1680
1781
|
##### Signature
|
|
1681
1782
|
|
|
1682
1783
|
```typescript
|
|
@@ -1709,9 +1810,6 @@ different time points.
|
|
|
1709
1810
|
table. For meaningful results, ensure your data is sorted appropriately (e.g.,
|
|
1710
1811
|
by date/time for time-series analysis) before calling this method.
|
|
1711
1812
|
|
|
1712
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1713
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1714
|
-
|
|
1715
1813
|
##### Signature
|
|
1716
1814
|
|
|
1717
1815
|
```typescript
|
|
@@ -1776,9 +1874,6 @@ use it as the X-axis, so that interpolated values are proportional to the actual
|
|
|
1776
1874
|
distances between X-axis values rather than treating every row as equidistant.
|
|
1777
1875
|
When `interpolateBy` is set, `interpolate` is automatically assumed `true`.
|
|
1778
1876
|
|
|
1779
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1780
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1781
|
-
|
|
1782
1877
|
##### Signature
|
|
1783
1878
|
|
|
1784
1879
|
```typescript
|
|
@@ -1852,11 +1947,9 @@ Sorts the rows of the table based on specified column(s) and order(s). If no
|
|
|
1852
1947
|
columns are specified, all columns are sorted from left to right in ascending
|
|
1853
1948
|
order.
|
|
1854
1949
|
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
grouping, aggregation, and sampling do not guarantee input order; chain `sort()`
|
|
1859
|
-
after them when deterministic output order matters.
|
|
1950
|
+
Order-preserving transformations after a sort retain that order. Operations such
|
|
1951
|
+
as joins, grouping, aggregation, and sampling do not guarantee input order;
|
|
1952
|
+
chain `sort()` after them when deterministic output order matters.
|
|
1860
1953
|
|
|
1861
1954
|
##### Signature
|
|
1862
1955
|
|
|
@@ -1902,9 +1995,7 @@ await table.sort({ column1: "asc" }, { lang: { column1: "fr" } }).log();
|
|
|
1902
1995
|
|
|
1903
1996
|
#### `selectColumns`
|
|
1904
1997
|
|
|
1905
|
-
Selects specific columns in the table, removing all others.
|
|
1906
|
-
the operation; it runs when an async observer method (like `getData()` or
|
|
1907
|
-
`log()`) is awaited, or when `run()` is called.
|
|
1998
|
+
Selects specific columns in the table, removing all others.
|
|
1908
1999
|
|
|
1909
2000
|
##### Signature
|
|
1910
2001
|
|
|
@@ -1936,9 +2027,6 @@ await table.selectColumns("productName").log();
|
|
|
1936
2027
|
|
|
1937
2028
|
Skips the first `n` rows of the table, effectively removing them.
|
|
1938
2029
|
|
|
1939
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1940
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1941
|
-
|
|
1942
2030
|
##### Signature
|
|
1943
2031
|
|
|
1944
2032
|
```typescript
|
|
@@ -1991,9 +2079,6 @@ console.log(hasAgeColumn); // Output: true or false
|
|
|
1991
2079
|
Selects random rows from the table, removing all others. You can optionally
|
|
1992
2080
|
specify a seed to ensure repeatable sampling.
|
|
1993
2081
|
|
|
1994
|
-
This method queues the operation; it runs when an async observer method (like
|
|
1995
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
1996
|
-
|
|
1997
2082
|
##### Signature
|
|
1998
2083
|
|
|
1999
2084
|
```typescript
|
|
@@ -2035,9 +2120,6 @@ await table.sample("10%", { seed: 123 }).log();
|
|
|
2035
2120
|
Selects a specified number of rows from this table. An offset can be applied to
|
|
2036
2121
|
skip initial rows, and the results can be output to a new table.
|
|
2037
2122
|
|
|
2038
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2039
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2040
|
-
|
|
2041
2123
|
##### Signature
|
|
2042
2124
|
|
|
2043
2125
|
```typescript
|
|
@@ -2089,9 +2171,6 @@ const topCustomersTable = await table.selectRows(75, {
|
|
|
2089
2171
|
Removes duplicate rows from this table, keeping only unique rows. Note that the
|
|
2090
2172
|
resulting data order might differ from the original.
|
|
2091
2173
|
|
|
2092
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2093
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2094
|
-
|
|
2095
2174
|
##### Signature
|
|
2096
2175
|
|
|
2097
2176
|
```typescript
|
|
@@ -2130,9 +2209,7 @@ await table.removeDuplicates({ on: ["firstName", "lastName"] }).log();
|
|
|
2130
2209
|
|
|
2131
2210
|
Removes rows with missing values from this table. By default, missing values
|
|
2132
2211
|
include SQL `NULL`, as well as string representations like `"NULL"`, `"null"`,
|
|
2133
|
-
`"NaN"`, `"undefined"`, and empty strings `""`.
|
|
2134
|
-
operation; it runs when an async observer method (like `getData()` or `log()`)
|
|
2135
|
-
is awaited, or when `run()` is called.
|
|
2212
|
+
`"NaN"`, `"undefined"`, and empty strings `""`.
|
|
2136
2213
|
|
|
2137
2214
|
##### Signature
|
|
2138
2215
|
|
|
@@ -2180,9 +2257,7 @@ await table.removeMissing({ columns: "age", missingValues: [-1] }).log();
|
|
|
2180
2257
|
#### `trim`
|
|
2181
2258
|
|
|
2182
2259
|
Trims specified characters from the beginning, end, or both sides of string
|
|
2183
|
-
values in the given columns.
|
|
2184
|
-
async observer method (like `getData()` or `log()`) is awaited, or when `run()`
|
|
2185
|
-
is called.
|
|
2260
|
+
values in the given columns.
|
|
2186
2261
|
|
|
2187
2262
|
##### Signature
|
|
2188
2263
|
|
|
@@ -2224,10 +2299,9 @@ await table.trim(["description", "notes"], { side: "right" }).log();
|
|
|
2224
2299
|
#### `filter`
|
|
2225
2300
|
|
|
2226
2301
|
Filters rows from this table based on SQL conditions. Note that it's often
|
|
2227
|
-
faster to use the `removeRows` method for simple removals.
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
`log()`) is awaited, or when `run()` is called.
|
|
2302
|
+
faster to use the `removeRows` method for simple removals. With the default
|
|
2303
|
+
`SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
|
|
2304
|
+
(`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
2231
2305
|
|
|
2232
2306
|
##### Signature
|
|
2233
2307
|
|
|
@@ -2272,9 +2346,6 @@ await table.filter(`lastPurchaseDate >= '2023-01-01'`).log();
|
|
|
2272
2346
|
Keeps rows in this table that have specific values in specified columns,
|
|
2273
2347
|
removing all other rows.
|
|
2274
2348
|
|
|
2275
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2276
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2277
|
-
|
|
2278
2349
|
##### Signature
|
|
2279
2350
|
|
|
2280
2351
|
```typescript
|
|
@@ -2313,9 +2384,6 @@ await table.keepValues({ status: null }).log();
|
|
|
2313
2384
|
|
|
2314
2385
|
Removes rows from this table that have specific values in specified columns.
|
|
2315
2386
|
|
|
2316
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2317
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2318
|
-
|
|
2319
2387
|
##### Signature
|
|
2320
2388
|
|
|
2321
2389
|
```typescript
|
|
@@ -2354,11 +2422,9 @@ await table.removeValues({ status: null }).log();
|
|
|
2354
2422
|
#### `removeRows`
|
|
2355
2423
|
|
|
2356
2424
|
Removes rows from this table based on SQL conditions. This method is similar to
|
|
2357
|
-
`filter()`, but removes rows instead of keeping them.
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2361
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2425
|
+
`filter()`, but removes rows instead of keeping them. With the default
|
|
2426
|
+
`SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
|
|
2427
|
+
(`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
2362
2428
|
|
|
2363
2429
|
##### Signature
|
|
2364
2430
|
|
|
@@ -2404,9 +2470,6 @@ await table.removeRows(
|
|
|
2404
2470
|
Renames one or more columns in the table. Throws if a source column does not
|
|
2405
2471
|
exist, so a typo fails loudly instead of being silently ignored.
|
|
2406
2472
|
|
|
2407
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2408
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2409
|
-
|
|
2410
2473
|
##### Signature
|
|
2411
2474
|
|
|
2412
2475
|
```typescript
|
|
@@ -2450,9 +2513,6 @@ await table.renameColumns({ "product_id": "productId" }, { strict: false })
|
|
|
2450
2513
|
Cleans column names by removing non-alphanumeric characters and formatting them
|
|
2451
2514
|
to camel case.
|
|
2452
2515
|
|
|
2453
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2454
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2455
|
-
|
|
2456
2516
|
##### Signature
|
|
2457
2517
|
|
|
2458
2518
|
```typescript
|
|
@@ -2523,9 +2583,6 @@ The table will then look like this:
|
|
|
2523
2583
|
| Sales | 2022 | 75 |
|
|
2524
2584
|
| Sales | 2023 | 98 |
|
|
2525
2585
|
|
|
2526
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2527
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2528
|
-
|
|
2529
2586
|
#### `wider`
|
|
2530
2587
|
|
|
2531
2588
|
Restructures this table by unstacking (pivoting) values, transforming data from
|
|
@@ -2584,9 +2641,6 @@ When multiple rows share the same `namesFrom`/grouping combination, their
|
|
|
2584
2641
|
`valuesFrom` values are combined with the `options.stat` function (`"sum"` by
|
|
2585
2642
|
default).
|
|
2586
2643
|
|
|
2587
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2588
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2589
|
-
|
|
2590
2644
|
#### `convert`
|
|
2591
2645
|
|
|
2592
2646
|
Converts data types of specified columns to target types (JavaScript or SQL
|
|
@@ -2606,20 +2660,16 @@ UTC).
|
|
|
2606
2660
|
When converting strings to numbers, commas (often used as thousand separators)
|
|
2607
2661
|
will be automatically removed before conversion.
|
|
2608
2662
|
|
|
2609
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2610
|
-
`getData()` or `log()`) is awaited, or when `run()` is called. If a column
|
|
2611
|
-
doesn't exist, the error is thrown at that point too.
|
|
2612
|
-
|
|
2613
2663
|
##### Signature
|
|
2614
2664
|
|
|
2615
2665
|
```typescript
|
|
2616
|
-
convert(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean">, options?: { strict?: boolean; datetimeFormat?: string }): this;
|
|
2666
|
+
convert(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | float[${number}] | FLOAT[${number}] | geometry('${string}') | GEOMETRY('${string}')>, options?: { strict?: boolean; datetimeFormat?: string }): this;
|
|
2617
2667
|
```
|
|
2618
2668
|
|
|
2619
2669
|
##### Parameters
|
|
2620
2670
|
|
|
2621
2671
|
- **`types`**: An object mapping column names to their target data types for
|
|
2622
|
-
conversion.
|
|
2672
|
+
conversion, including JSON, FLOAT[n] vectors, and GEOMETRY with a CRS.
|
|
2623
2673
|
- **`options`**: An optional object with configuration options:
|
|
2624
2674
|
- **`options.strict`**: If `false`, values that cannot be converted will be
|
|
2625
2675
|
replaced by `NULL` instead of throwing an error. Defaults to `true`.
|
|
@@ -2672,6 +2722,21 @@ await table.convert({ column3: "string" }, {
|
|
|
2672
2722
|
await table.convert({ amount: "float" }, { strict: false }).log();
|
|
2673
2723
|
```
|
|
2674
2724
|
|
|
2725
|
+
```ts
|
|
2726
|
+
// Parse JSON, a fixed-size float vector, and WKT geometry from text.
|
|
2727
|
+
await table.loadArray([{
|
|
2728
|
+
details: '{"count":3}',
|
|
2729
|
+
embedding: "[0.25,0.5,0.75]",
|
|
2730
|
+
geom: "POINT (-73 45)",
|
|
2731
|
+
}])
|
|
2732
|
+
.convert({
|
|
2733
|
+
details: "JSON",
|
|
2734
|
+
embedding: "FLOAT[3]",
|
|
2735
|
+
geom: "GEOMETRY('EPSG:4326')",
|
|
2736
|
+
})
|
|
2737
|
+
.log();
|
|
2738
|
+
```
|
|
2739
|
+
|
|
2675
2740
|
#### `removeTable`
|
|
2676
2741
|
|
|
2677
2742
|
Removes the table from the database. After this operation, invoking methods on
|
|
@@ -2696,9 +2761,7 @@ await table.removeTable();
|
|
|
2696
2761
|
|
|
2697
2762
|
#### `removeColumns`
|
|
2698
2763
|
|
|
2699
|
-
Removes one or more columns from this table.
|
|
2700
|
-
it runs when an async observer method (like `getData()` or `log()`) is awaited,
|
|
2701
|
-
or when `run()` is called.
|
|
2764
|
+
Removes one or more columns from this table.
|
|
2702
2765
|
|
|
2703
2766
|
##### Signature
|
|
2704
2767
|
|
|
@@ -2729,21 +2792,22 @@ await table.removeColumns("tempColumn").log();
|
|
|
2729
2792
|
#### `addColumn`
|
|
2730
2793
|
|
|
2731
2794
|
Adds a new column to the table based on a specified data type (JavaScript or SQL
|
|
2732
|
-
types) and a SQL definition.
|
|
2733
|
-
|
|
2734
|
-
|
|
2795
|
+
types) and a SQL definition. With the default `SimpleDB.expressionSyntax: "js"`,
|
|
2796
|
+
expressions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
|
|
2797
|
+
`expressionSyntax: "sql"` for unchanged SQL.
|
|
2735
2798
|
|
|
2736
2799
|
##### Signature
|
|
2737
2800
|
|
|
2738
2801
|
```typescript
|
|
2739
|
-
addColumn(newColumn: string, type: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | geometry('${string}') | GEOMETRY('${string}'), definition: string): this;
|
|
2802
|
+
addColumn(newColumn: string, type: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | float[${number}] | FLOAT[${number}] | geometry('${string}') | GEOMETRY('${string}'), definition: string): this;
|
|
2740
2803
|
```
|
|
2741
2804
|
|
|
2742
2805
|
##### Parameters
|
|
2743
2806
|
|
|
2744
2807
|
- **`newColumn`**: The name of the new column to be added.
|
|
2745
2808
|
- **`type`**: The data type for the new column. Can be a JavaScript type (e.g.,
|
|
2746
|
-
`"number"`, `"string"`) or a SQL type (e.g., `"integer"`, `"varchar"
|
|
2809
|
+
`"number"`, `"string"`) or a SQL type (e.g., `"integer"`, `"varchar"`,
|
|
2810
|
+
`"JSON"`, `"FLOAT[3]"`, `"GEOMETRY('EPSG:4326')"`).
|
|
2747
2811
|
- **`definition`**: A SQL expression defining how the values for the new column
|
|
2748
2812
|
should be computed (e.g., `"column1 + column2"`,
|
|
2749
2813
|
`"ST_Centroid(geom_column)"`).
|
|
@@ -2768,6 +2832,23 @@ await table.addColumn(
|
|
|
2768
2832
|
).log();
|
|
2769
2833
|
```
|
|
2770
2834
|
|
|
2835
|
+
```ts
|
|
2836
|
+
// Read a number from a JSON object into a new column using dot notation.
|
|
2837
|
+
await table.loadArray([{ details: { count: 3 } }], {
|
|
2838
|
+
columnTypes: { details: "JSON" },
|
|
2839
|
+
})
|
|
2840
|
+
.addColumn("count", "integer", "details.count")
|
|
2841
|
+
.log();
|
|
2842
|
+
```
|
|
2843
|
+
|
|
2844
|
+
```ts
|
|
2845
|
+
// Create JSON and fixed-size vector columns with DuckDB expressions.
|
|
2846
|
+
await table.loadArray([{ count: 3 }])
|
|
2847
|
+
.addColumn("details", "JSON", "json_object('count', count)")
|
|
2848
|
+
.addColumn("embedding", "FLOAT[3]", "[0.25, 0.5, 0.75]")
|
|
2849
|
+
.log();
|
|
2850
|
+
```
|
|
2851
|
+
|
|
2771
2852
|
#### `extractDatePart`
|
|
2772
2853
|
|
|
2773
2854
|
Extracts one or more components from a temporal column into new columns. Pass a
|
|
@@ -2782,9 +2863,6 @@ time parts apply to times and timestamps. `NULL` input values produce `NULL`
|
|
|
2782
2863
|
extracted values. Parts extracted from `TIMESTAMP WITH TIME ZONE` values use
|
|
2783
2864
|
UTC.
|
|
2784
2865
|
|
|
2785
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2786
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2787
|
-
|
|
2788
2866
|
##### Signature
|
|
2789
2867
|
|
|
2790
2868
|
```typescript
|
|
@@ -2821,9 +2899,6 @@ await table.extractDatePart("publishedAt", {
|
|
|
2821
2899
|
Adds a new column to the table containing the row number, starting at 0 (like an
|
|
2822
2900
|
index).
|
|
2823
2901
|
|
|
2824
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2825
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2826
|
-
|
|
2827
2902
|
##### Signature
|
|
2828
2903
|
|
|
2829
2904
|
```typescript
|
|
@@ -2860,9 +2935,6 @@ Cartesian product of the rows from both tables, meaning all possible pairs of
|
|
|
2860
2935
|
rows will be in the resulting table. This means that if the left table has `n`
|
|
2861
2936
|
rows and the right table has `m` rows, the result will have `n * m` rows.
|
|
2862
2937
|
|
|
2863
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2864
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
2865
|
-
|
|
2866
2938
|
##### Signature
|
|
2867
2939
|
|
|
2868
2940
|
```typescript
|
|
@@ -2906,10 +2978,7 @@ Merges the data of this table (considered the left table) with another table
|
|
|
2906
2978
|
(the right table) based on a common column or multiple columns. Note that the
|
|
2907
2979
|
order of rows in the returned data is not guaranteed to be the same as in the
|
|
2908
2980
|
original tables. This operation might create temporary files in a `.tmp` folder;
|
|
2909
|
-
consider adding `.tmp` to your `.gitignore`.
|
|
2910
|
-
it runs when an async observer method (like `getData()` or `log()`) is awaited,
|
|
2911
|
-
or when `run()` is called. The join uses the other table's state as of this
|
|
2912
|
-
call: operations queued on it afterwards run after the join.
|
|
2981
|
+
consider adding `.tmp` to your `.gitignore`.
|
|
2913
2982
|
|
|
2914
2983
|
##### Signature
|
|
2915
2984
|
|
|
@@ -2972,11 +3041,6 @@ order alphabetically by the left column and then by the right column.
|
|
|
2972
3041
|
This operation might create temporary files in a `.tmp` folder; consider adding
|
|
2973
3042
|
`.tmp` to your `.gitignore`.
|
|
2974
3043
|
|
|
2975
|
-
This method queues the operation; it runs when an async observer method (like
|
|
2976
|
-
`getData()` or `log()`) is awaited, or when `run()` is called. The join uses the
|
|
2977
|
-
other table's state as of this call: operations queued on it afterwards run
|
|
2978
|
-
after the join.
|
|
2979
|
-
|
|
2980
3044
|
##### Signature
|
|
2981
3045
|
|
|
2982
3046
|
```typescript
|
|
@@ -3061,9 +3125,6 @@ Similarity is computed using the
|
|
|
3061
3125
|
[rapidfuzz](https://query.farm/duckdb_extension_rapidfuzz) DuckDB community
|
|
3062
3126
|
extension, which is installed and loaded automatically.
|
|
3063
3127
|
|
|
3064
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3065
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3066
|
-
|
|
3067
3128
|
##### Signature
|
|
3068
3129
|
|
|
3069
3130
|
```typescript
|
|
@@ -3135,9 +3196,6 @@ await table.fuzzyClean("category", "category", 80, {
|
|
|
3135
3196
|
|
|
3136
3197
|
Replaces specified strings in the selected columns.
|
|
3137
3198
|
|
|
3138
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3139
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3140
|
-
|
|
3141
3199
|
##### Signature
|
|
3142
3200
|
|
|
3143
3201
|
```typescript
|
|
@@ -3195,9 +3253,7 @@ await table.replace("all", { "%": "" }).log();
|
|
|
3195
3253
|
|
|
3196
3254
|
#### `lower`
|
|
3197
3255
|
|
|
3198
|
-
Converts string values in the specified columns to lowercase.
|
|
3199
|
-
the operation; it runs when an async observer method (like `getData()` or
|
|
3200
|
-
`log()`) is awaited, or when `run()` is called.
|
|
3256
|
+
Converts string values in the specified columns to lowercase.
|
|
3201
3257
|
|
|
3202
3258
|
##### Signature
|
|
3203
3259
|
|
|
@@ -3228,9 +3284,7 @@ await table.lower(["column1", "column2"]).log();
|
|
|
3228
3284
|
|
|
3229
3285
|
#### `upper`
|
|
3230
3286
|
|
|
3231
|
-
Converts string values in the specified columns to uppercase.
|
|
3232
|
-
the operation; it runs when an async observer method (like `getData()` or
|
|
3233
|
-
`log()`) is awaited, or when `run()` is called.
|
|
3287
|
+
Converts string values in the specified columns to uppercase.
|
|
3234
3288
|
|
|
3235
3289
|
##### Signature
|
|
3236
3290
|
|
|
@@ -3262,9 +3316,7 @@ await table.upper(["column1", "column2"]).log();
|
|
|
3262
3316
|
#### `capitalize`
|
|
3263
3317
|
|
|
3264
3318
|
Capitalizes the first letter of each string in the specified columns and
|
|
3265
|
-
converts the rest of the string to lowercase.
|
|
3266
|
-
it runs when an async observer method (like `getData()` or `log()`) is awaited,
|
|
3267
|
-
or when `run()` is called.
|
|
3319
|
+
converts the rest of the string to lowercase.
|
|
3268
3320
|
|
|
3269
3321
|
##### Signature
|
|
3270
3322
|
|
|
@@ -3296,9 +3348,6 @@ await table.capitalize(["column1", "column2"]).log();
|
|
|
3296
3348
|
|
|
3297
3349
|
Truncates string values in a specified column to a maximum number of characters.
|
|
3298
3350
|
|
|
3299
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3300
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3301
|
-
|
|
3302
3351
|
##### Signature
|
|
3303
3352
|
|
|
3304
3353
|
```typescript
|
|
@@ -3334,9 +3383,6 @@ The columns must contain string (VARCHAR) values. An error is thrown if any
|
|
|
3334
3383
|
column is of a different type. `null` values remain `null`. If any string
|
|
3335
3384
|
already exceeds the target length, an error is thrown (no silent truncation).
|
|
3336
3385
|
|
|
3337
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3338
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3339
|
-
|
|
3340
3386
|
##### Signature
|
|
3341
3387
|
|
|
3342
3388
|
```typescript
|
|
@@ -3386,9 +3432,6 @@ Splits strings in a specified column by a separator and extracts a substring at
|
|
|
3386
3432
|
a given index, storing the result in a new or existing column. If the index is
|
|
3387
3433
|
out of bounds, an empty string will be returned for that row.
|
|
3388
3434
|
|
|
3389
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3390
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3391
|
-
|
|
3392
3435
|
##### Signature
|
|
3393
3436
|
|
|
3394
3437
|
```typescript
|
|
@@ -3435,9 +3478,6 @@ extra columns will contain empty strings (unless `strict` is set to `false`). If
|
|
|
3435
3478
|
a row has more parts than the number of new columns, an error will be thrown
|
|
3436
3479
|
unless `strict` is set to `false`.
|
|
3437
3480
|
|
|
3438
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3439
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3440
|
-
|
|
3441
3481
|
##### Signature
|
|
3442
3482
|
|
|
3443
3483
|
```typescript
|
|
@@ -3482,9 +3522,6 @@ await table.splitSpread("data", "|", ["col1", "col2"], { strict: false }).log();
|
|
|
3482
3522
|
Extracts a specific number of characters from the beginning (left side) of
|
|
3483
3523
|
string values in the specified column.
|
|
3484
3524
|
|
|
3485
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3486
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3487
|
-
|
|
3488
3525
|
##### Signature
|
|
3489
3526
|
|
|
3490
3527
|
```typescript
|
|
@@ -3514,9 +3551,6 @@ await table.firstChars("productCode", 2).log();
|
|
|
3514
3551
|
Extracts a specific number of characters from the end (right side) of string
|
|
3515
3552
|
values in the specified column.
|
|
3516
3553
|
|
|
3517
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3518
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3519
|
-
|
|
3520
3554
|
##### Signature
|
|
3521
3555
|
|
|
3522
3556
|
```typescript
|
|
@@ -3545,9 +3579,6 @@ await table.lastChars("productCode", 2).log();
|
|
|
3545
3579
|
|
|
3546
3580
|
Replaces `NULL` values in the specified columns with a given value.
|
|
3547
3581
|
|
|
3548
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3549
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3550
|
-
|
|
3551
3582
|
##### Signature
|
|
3552
3583
|
|
|
3553
3584
|
```typescript
|
|
@@ -3590,9 +3621,6 @@ await table.replaceNulls("all", 0).log();
|
|
|
3590
3621
|
|
|
3591
3622
|
Concatenates values from specified columns into a new column.
|
|
3592
3623
|
|
|
3593
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3594
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3595
|
-
|
|
3596
3624
|
##### Signature
|
|
3597
3625
|
|
|
3598
3626
|
```typescript
|
|
@@ -3639,9 +3667,6 @@ All values must be string, otherwise an error will be thrown. Use the
|
|
|
3639
3667
|
If a column value is `NULL`, it will be replaced by `'Unknown'` in the
|
|
3640
3668
|
concatenated result.
|
|
3641
3669
|
|
|
3642
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3643
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3644
|
-
|
|
3645
3670
|
##### Signature
|
|
3646
3671
|
|
|
3647
3672
|
```typescript
|
|
@@ -3701,9 +3726,6 @@ Each value in the specified column is split using the provided separator, and a
|
|
|
3701
3726
|
new row is created for each resulting substring. All other column values are
|
|
3702
3727
|
duplicated across the newly created rows.
|
|
3703
3728
|
|
|
3704
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3705
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3706
|
-
|
|
3707
3729
|
##### Signature
|
|
3708
3730
|
|
|
3709
3731
|
```typescript
|
|
@@ -3745,9 +3767,6 @@ Repeats rows based on the values in a column.
|
|
|
3745
3767
|
If a row has a value of 3 in the specified column, it will be repeated 3 times.
|
|
3746
3768
|
If the value is 0 or negative, the row will be removed.
|
|
3747
3769
|
|
|
3748
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3749
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3750
|
-
|
|
3751
3770
|
##### Signature
|
|
3752
3771
|
|
|
3753
3772
|
```typescript
|
|
@@ -3791,9 +3810,6 @@ This is the inverse operation of `unnest()`. Multiple rows are combined into
|
|
|
3791
3810
|
fewer rows by grouping on specified category columns and concatenating the
|
|
3792
3811
|
target column values with a separator.
|
|
3793
3812
|
|
|
3794
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3795
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3796
|
-
|
|
3797
3813
|
##### Signature
|
|
3798
3814
|
|
|
3799
3815
|
```typescript
|
|
@@ -3834,9 +3850,6 @@ await table.nest("tags", ",", ["country", "city"]).log();
|
|
|
3834
3850
|
|
|
3835
3851
|
Rounds numeric values in specified columns.
|
|
3836
3852
|
|
|
3837
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3838
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3839
|
-
|
|
3840
3853
|
##### Signature
|
|
3841
3854
|
|
|
3842
3855
|
```typescript
|
|
@@ -3888,12 +3901,57 @@ await table.round(["columnA", "columnB"], { decimals: 1, method: "ceiling" })
|
|
|
3888
3901
|
await table.round("column1", 2).log();
|
|
3889
3902
|
```
|
|
3890
3903
|
|
|
3891
|
-
#### `
|
|
3904
|
+
#### `addNoise`
|
|
3905
|
+
|
|
3906
|
+
Adds independently generated uniform random noise to numeric values in one or
|
|
3907
|
+
more columns. Each changed value receives an offset between `-max` and `max`.
|
|
3908
|
+
Selected integer and decimal columns become `DOUBLE` columns so fractional noise
|
|
3909
|
+
is retained. This method adds random jitter; it does not provide anonymization
|
|
3910
|
+
or differential privacy guarantees.
|
|
3892
3911
|
|
|
3893
|
-
|
|
3912
|
+
##### Signature
|
|
3894
3913
|
|
|
3895
|
-
|
|
3896
|
-
|
|
3914
|
+
```typescript
|
|
3915
|
+
addNoise(columns: string | string[], max: number, options?: { onlyDuplicates?: boolean }): this;
|
|
3916
|
+
```
|
|
3917
|
+
|
|
3918
|
+
##### Parameters
|
|
3919
|
+
|
|
3920
|
+
- **`columns`**: The numeric column name or array of numeric column names to
|
|
3921
|
+
which noise will be added. When multiple columns are provided, each value
|
|
3922
|
+
receives an independent random offset.
|
|
3923
|
+
- **`max`**: The maximum absolute offset, expressed in each column's units. Must
|
|
3924
|
+
be a finite number greater than or equal to `0`.
|
|
3925
|
+
- **`options`**: An optional object with configuration options:
|
|
3926
|
+
- **`options.onlyDuplicates`**: If `true`, adds noise only to values that occur
|
|
3927
|
+
more than once in their column. Each selected column is evaluated
|
|
3928
|
+
independently, and every occurrence of a duplicated value is changed. Defaults
|
|
3929
|
+
to `false`.
|
|
3930
|
+
|
|
3931
|
+
##### Returns
|
|
3932
|
+
|
|
3933
|
+
The table, so methods can be chained.
|
|
3934
|
+
|
|
3935
|
+
##### Examples
|
|
3936
|
+
|
|
3937
|
+
```ts
|
|
3938
|
+
// Add an offset between -0.01 and 0.01 to every value
|
|
3939
|
+
await table.addNoise("measurement", 0.01).log();
|
|
3940
|
+
```
|
|
3941
|
+
|
|
3942
|
+
```ts
|
|
3943
|
+
// Add noise independently to duplicated x values and duplicated y values
|
|
3944
|
+
await table.addNoise(["x", "y"], 0.1, {
|
|
3945
|
+
onlyDuplicates: true,
|
|
3946
|
+
}).log();
|
|
3947
|
+
```
|
|
3948
|
+
|
|
3949
|
+
#### `updateColumn`
|
|
3950
|
+
|
|
3951
|
+
Updates values in a specified column using a SQL expression. With the default
|
|
3952
|
+
`SimpleDB.expressionSyntax: "js"`, expressions support JavaScript-style
|
|
3953
|
+
operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for
|
|
3954
|
+
unchanged SQL.
|
|
3897
3955
|
|
|
3898
3956
|
##### Signature
|
|
3899
3957
|
|
|
@@ -3931,13 +3989,19 @@ await table.updateColumn(
|
|
|
3931
3989
|
).log();
|
|
3932
3990
|
```
|
|
3933
3991
|
|
|
3992
|
+
```ts
|
|
3993
|
+
// Update an existing column with a value from a JSON object using dot notation.
|
|
3994
|
+
await table.loadArray([{ count: 0, details: { count: 3 } }], {
|
|
3995
|
+
columnTypes: { details: "JSON" },
|
|
3996
|
+
})
|
|
3997
|
+
.updateColumn("count", "details.count")
|
|
3998
|
+
.log();
|
|
3999
|
+
```
|
|
4000
|
+
|
|
3934
4001
|
#### `ranks`
|
|
3935
4002
|
|
|
3936
4003
|
Assigns ranks to rows in a new column based on the values of a specified column.
|
|
3937
4004
|
|
|
3938
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3939
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3940
|
-
|
|
3941
4005
|
##### Signature
|
|
3942
4006
|
|
|
3943
4007
|
```typescript
|
|
@@ -3988,9 +4052,6 @@ await table.ranks("sales", "salesRank", { by: ["department", "city"] }).log();
|
|
|
3988
4052
|
|
|
3989
4053
|
Assigns quantiles to rows in a new column based on specified column values.
|
|
3990
4054
|
|
|
3991
|
-
This method queues the operation; it runs when an async observer method (like
|
|
3992
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
3993
|
-
|
|
3994
4055
|
##### Signature
|
|
3995
4056
|
|
|
3996
4057
|
```typescript
|
|
@@ -4034,9 +4095,6 @@ await table.quantiles("sales", 4, "salesQuartile").log();
|
|
|
4034
4095
|
|
|
4035
4096
|
Assigns bins for specified column values based on an interval size.
|
|
4036
4097
|
|
|
4037
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4038
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4039
|
-
|
|
4040
4098
|
##### Signature
|
|
4041
4099
|
|
|
4042
4100
|
```typescript
|
|
@@ -4089,7 +4147,7 @@ each row, adding new columns for these proportions.
|
|
|
4089
4147
|
##### Signature
|
|
4090
4148
|
|
|
4091
4149
|
```typescript
|
|
4092
|
-
rowProportions(columns: string[], options?: { suffix?: string; decimals?: number }): this;
|
|
4150
|
+
rowProportions(columns: string[], options?: { suffix?: string; base?: number; decimals?: number }): this;
|
|
4093
4151
|
```
|
|
4094
4152
|
|
|
4095
4153
|
##### Parameters
|
|
@@ -4099,6 +4157,8 @@ rowProportions(columns: string[], options?: { suffix?: string; decimals?: number
|
|
|
4099
4157
|
- **`options`**: An optional object with configuration options:
|
|
4100
4158
|
- **`options.suffix`**: A string suffix to append to the names of the new
|
|
4101
4159
|
columns storing the computed proportions. Defaults to `"Perc"`.
|
|
4160
|
+
- **`options.base`**: A finite positive value that the proportions in each row
|
|
4161
|
+
sum to before rounding. Defaults to `1`.
|
|
4102
4162
|
- **`options.decimals`**: The number of decimal places to round the computed
|
|
4103
4163
|
proportions. Defaults to `undefined` (no rounding).
|
|
4104
4164
|
|
|
@@ -4141,8 +4201,13 @@ The table will then look like this:
|
|
|
4141
4201
|
| 2022 | 354 | 278 | 56 | 0.51 | 0.4 | 0.08 |
|
|
4142
4202
|
| 2023 | 856 | 321 | 221 | 0.61 | 0.23 | 0.16 |
|
|
4143
4203
|
|
|
4144
|
-
|
|
4145
|
-
|
|
4204
|
+
```ts
|
|
4205
|
+
// Compute percentages that sum to 100 on each row before rounding
|
|
4206
|
+
await table.rowProportions(["Men", "Women", "NonBinary"], {
|
|
4207
|
+
base: 100,
|
|
4208
|
+
decimals: 1,
|
|
4209
|
+
}).log();
|
|
4210
|
+
```
|
|
4146
4211
|
|
|
4147
4212
|
#### `rowRanks`
|
|
4148
4213
|
|
|
@@ -4156,9 +4221,6 @@ to produce one row for each tied column. The `"all"` option can therefore
|
|
|
4156
4221
|
increase the table's row count. If null values leave a row without the requested
|
|
4157
4222
|
rank, the new columns contain null.
|
|
4158
4223
|
|
|
4159
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4160
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4161
|
-
|
|
4162
4224
|
##### Signature
|
|
4163
4225
|
|
|
4164
4226
|
```typescript
|
|
@@ -4210,13 +4272,10 @@ await table.rowRanks(["CAQ", "PLQ", "PQ"], {
|
|
|
4210
4272
|
Computes proportions vertically over a column's values, relative to the sum of
|
|
4211
4273
|
all values in that column or group.
|
|
4212
4274
|
|
|
4213
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4214
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4215
|
-
|
|
4216
4275
|
##### Signature
|
|
4217
4276
|
|
|
4218
4277
|
```typescript
|
|
4219
|
-
columnProportions(column: string, newColumn: string, options?: { by?: string | string[]; decimals?: number }): this;
|
|
4278
|
+
columnProportions(column: string, newColumn: string, options?: { by?: string | string[]; base?: number; decimals?: number }): this;
|
|
4220
4279
|
```
|
|
4221
4280
|
|
|
4222
4281
|
##### Parameters
|
|
@@ -4229,6 +4288,8 @@ columnProportions(column: string, newColumn: string, options?: { by?: string | s
|
|
|
4229
4288
|
- **`options`**: An optional object with configuration options:
|
|
4230
4289
|
- **`options.by`**: The column name or an array of column names to partition by.
|
|
4231
4290
|
Proportions are calculated independently within each group.
|
|
4291
|
+
- **`options.base`**: A finite positive value that the proportions in the column
|
|
4292
|
+
or each group sum to before rounding. Defaults to `1`.
|
|
4232
4293
|
- **`options.decimals`**: The number of decimal places to round the computed
|
|
4233
4294
|
proportions. Defaults to `undefined` (no rounding).
|
|
4234
4295
|
|
|
@@ -4249,6 +4310,14 @@ await table.columnProportions("column1", "perc", { by: "column2", decimals: 2 })
|
|
|
4249
4310
|
.log();
|
|
4250
4311
|
```
|
|
4251
4312
|
|
|
4313
|
+
```ts
|
|
4314
|
+
// Compute percentages that sum to 100 before rounding
|
|
4315
|
+
await table.columnProportions("sales", "sales_percentage", {
|
|
4316
|
+
base: 100,
|
|
4317
|
+
decimals: 1,
|
|
4318
|
+
}).log();
|
|
4319
|
+
```
|
|
4320
|
+
|
|
4252
4321
|
```ts
|
|
4253
4322
|
// Compute proportions for 'sales' by 'region' and 'product_type'
|
|
4254
4323
|
await table.columnProportions("sales", "sales_proportion", {
|
|
@@ -4262,9 +4331,6 @@ Creates a summary table from selected columns, optionally grouped by other
|
|
|
4262
4331
|
columns. This method allows you to aggregate data, calculate statistics (e.g.,
|
|
4263
4332
|
count, mean, sum), and group results by categorical columns.
|
|
4264
4333
|
|
|
4265
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4266
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4267
|
-
|
|
4268
4334
|
##### Signature
|
|
4269
4335
|
|
|
4270
4336
|
```typescript
|
|
@@ -4392,9 +4458,6 @@ summarized nor used for labels contain `NULL` in the added rows. A stat string
|
|
|
4392
4458
|
is also used as its row label; pass an object to customize that label. If
|
|
4393
4459
|
`options.stats` is omitted, every supported stat is added.
|
|
4394
4460
|
|
|
4395
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4396
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4397
|
-
|
|
4398
4461
|
##### Signature
|
|
4399
4462
|
|
|
4400
4463
|
```typescript
|
|
@@ -4449,9 +4512,6 @@ await table.addSummaryRows("all", "region", {
|
|
|
4449
4512
|
Computes the cumulative sum of values in a column. For this method to work
|
|
4450
4513
|
properly, ensure your data is sorted first.
|
|
4451
4514
|
|
|
4452
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4453
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4454
|
-
|
|
4455
4515
|
##### Signature
|
|
4456
4516
|
|
|
4457
4517
|
```typescript
|
|
@@ -4500,9 +4560,6 @@ column. For rows without enough preceding or following rows to form a complete
|
|
|
4500
4560
|
window, `NULL` will be returned. For this method to work properly, ensure your
|
|
4501
4561
|
data is sorted by the relevant column(s) first.
|
|
4502
4562
|
|
|
4503
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4504
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4505
|
-
|
|
4506
4563
|
##### Signature
|
|
4507
4564
|
|
|
4508
4565
|
```typescript
|
|
@@ -4559,9 +4616,6 @@ specified, the method computes the correlations for all numeric column
|
|
|
4559
4616
|
combinations. Note that correlation is symmetrical: the correlation of `x` with
|
|
4560
4617
|
`y` is the same as `y` with `x`.
|
|
4561
4618
|
|
|
4562
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4563
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4564
|
-
|
|
4565
4619
|
##### Signature
|
|
4566
4620
|
|
|
4567
4621
|
```typescript
|
|
@@ -4628,9 +4682,6 @@ the method computes linear regression analysis for all numeric column
|
|
|
4628
4682
|
permutations. Note that linear regression analysis is asymmetrical: the linear
|
|
4629
4683
|
regression of `x` over `y` is not the same as `y` over `x`.
|
|
4630
4684
|
|
|
4631
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4632
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4633
|
-
|
|
4634
4685
|
##### Signature
|
|
4635
4686
|
|
|
4636
4687
|
```typescript
|
|
@@ -4695,9 +4746,6 @@ await table.linearRegressions({ decimals: 3 }).log();
|
|
|
4695
4746
|
Identifies outliers in a specified column using the Interquartile Range (IQR)
|
|
4696
4747
|
method.
|
|
4697
4748
|
|
|
4698
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4699
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4700
|
-
|
|
4701
4749
|
##### Signature
|
|
4702
4750
|
|
|
4703
4751
|
```typescript
|
|
@@ -4733,9 +4781,6 @@ await table.outliersIQR("salary", "salaryOutlier", { by: "gender" }).log();
|
|
|
4733
4781
|
|
|
4734
4782
|
Computes the Z-score for values in a specified column.
|
|
4735
4783
|
|
|
4736
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4737
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4738
|
-
|
|
4739
4784
|
##### Signature
|
|
4740
4785
|
|
|
4741
4786
|
```typescript
|
|
@@ -4778,9 +4823,6 @@ await table.zScore("score", "scoreZScore", { decimals: 2 }).log();
|
|
|
4778
4823
|
|
|
4779
4824
|
Normalizes the values in a column using min-max normalization.
|
|
4780
4825
|
|
|
4781
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4782
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4783
|
-
|
|
4784
4826
|
##### Signature
|
|
4785
4827
|
|
|
4786
4828
|
```typescript
|
|
@@ -4846,9 +4888,6 @@ an instant. Construct them with an explicit timezone, such as
|
|
|
4846
4888
|
`new Date("2001-01-01T00:00:00Z")`; date-time strings without `Z` or an offset
|
|
4847
4889
|
use the user's local timezone.
|
|
4848
4890
|
|
|
4849
|
-
This method queues the operation; it runs when an async observer method (like
|
|
4850
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
4851
|
-
|
|
4852
4891
|
##### Signature
|
|
4853
4892
|
|
|
4854
4893
|
```typescript
|
|
@@ -4927,16 +4966,19 @@ Updates data in the table using a JavaScript function. The function receives the
|
|
|
4927
4966
|
existing rows as an array of objects and must return the modified rows as an
|
|
4928
4967
|
array of objects. This method offers high flexibility for data manipulation but
|
|
4929
4968
|
can be slow for large tables as it involves transferring data between DuckDB and
|
|
4930
|
-
JavaScript.
|
|
4969
|
+
JavaScript. Before writing a JavaScript callback, check for an existing SDA
|
|
4970
|
+
method that performs the same operation; it will usually be faster and more
|
|
4971
|
+
efficient.
|
|
4931
4972
|
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4973
|
+
If the table has geometry columns, the callback can read and modify their
|
|
4974
|
+
GeoJSON geometry objects directly. Extra properties added to these objects are
|
|
4975
|
+
ignored; create new columns on the returned rows to store additional attributes.
|
|
4976
|
+
All geometry columns must use EPSG:4326; use reproject() first if needed.
|
|
4935
4977
|
|
|
4936
4978
|
##### Signature
|
|
4937
4979
|
|
|
4938
4980
|
```typescript
|
|
4939
|
-
updateWithJS(dataModifier: ((rows: Record<string, unknown>[]) => Promise<Record<string, unknown>[]>) | ((rows: Record<string, unknown>[]) => Record<string, unknown>[]), options?: { batchSize?: number }): this;
|
|
4981
|
+
updateWithJS(dataModifier: ((rows: Record<string, unknown>[]) => Promise<Record<string, unknown>[]>) | ((rows: Record<string, unknown>[]) => Record<string, unknown>[]), options?: { batchSize?: number; columnTypes?: Record<string, string> }): this;
|
|
4940
4982
|
```
|
|
4941
4983
|
|
|
4942
4984
|
##### Parameters
|
|
@@ -4947,7 +4989,15 @@ updateWithJS(dataModifier: ((rows: Record<string, unknown>[]) => Promise<Record<
|
|
|
4947
4989
|
- **`options`**: An optional object with configuration options:
|
|
4948
4990
|
- **`options.batchSize`**: If provided, rows are processed in batches of this
|
|
4949
4991
|
size instead of all at once, so large tables don't have to be materialized
|
|
4950
|
-
entirely in memory.
|
|
4992
|
+
entirely in memory. This limits input rows per batch, but does not eliminate
|
|
4993
|
+
conversion work or bound geometry size or callback output expansion. The
|
|
4994
|
+
modifier function is called once per batch. Tables with a column named `rowid`
|
|
4995
|
+
or `__sda_rowid` (case-insensitive) are rejected before the modifier runs;
|
|
4996
|
+
rename the column or omit batchSize.
|
|
4997
|
+
- **`options.columnTypes`**: Explicit types for newly added columns, following
|
|
4998
|
+
loadArray's columnTypes contract. Declare geometry as GEOMETRY('EPSG:4326');
|
|
4999
|
+
geometry is never inferred from object shape. Existing columns cannot be
|
|
5000
|
+
redeclared.
|
|
4951
5001
|
|
|
4952
5002
|
##### Returns
|
|
4953
5003
|
|
|
@@ -4956,38 +5006,73 @@ The table, so methods can be chained.
|
|
|
4956
5006
|
##### Examples
|
|
4957
5007
|
|
|
4958
5008
|
```ts
|
|
4959
|
-
//
|
|
4960
|
-
|
|
4961
|
-
|
|
4962
|
-
|
|
4963
|
-
|
|
4964
|
-
|
|
4965
|
-
|
|
5009
|
+
// Double the value in each row with a JavaScript callback.
|
|
5010
|
+
await table.loadArray([{ value: 1 }, { value: 2 }])
|
|
5011
|
+
.updateWithJS((rows) =>
|
|
5012
|
+
rows.map((row) => ({
|
|
5013
|
+
value: Number(row.value) * 2,
|
|
5014
|
+
}))
|
|
5015
|
+
)
|
|
5016
|
+
.log();
|
|
5017
|
+
```
|
|
5018
|
+
|
|
5019
|
+
```ts
|
|
5020
|
+
// Process two rows per callback to limit how many rows are handled at once.
|
|
5021
|
+
await table.loadArray([{ value: 1 }, { value: 2 }, { value: 3 }])
|
|
5022
|
+
.updateWithJS((rows) =>
|
|
5023
|
+
rows.map((row) => ({
|
|
5024
|
+
value: Number(row.value) * 2,
|
|
5025
|
+
})), { batchSize: 2 })
|
|
5026
|
+
.log();
|
|
5027
|
+
```
|
|
5028
|
+
|
|
5029
|
+
```ts
|
|
5030
|
+
// Add a JSON object column to keep related attributes together.
|
|
5031
|
+
await table.loadArray([{ name: "Station" }])
|
|
5032
|
+
.updateWithJS((rows) =>
|
|
5033
|
+
rows.map((row) => ({
|
|
4966
5034
|
...row,
|
|
4967
|
-
|
|
4968
|
-
|
|
4969
|
-
.length
|
|
4970
|
-
: null,
|
|
4971
|
-
}));
|
|
4972
|
-
})
|
|
5035
|
+
details: { active: true, category: "weather" },
|
|
5036
|
+
})), { columnTypes: { details: "JSON" } })
|
|
4973
5037
|
.log();
|
|
4974
5038
|
```
|
|
4975
5039
|
|
|
4976
5040
|
```ts
|
|
4977
|
-
//
|
|
4978
|
-
|
|
4979
|
-
.updateWithJS(
|
|
4980
|
-
|
|
4981
|
-
|
|
4982
|
-
|
|
4983
|
-
|
|
4984
|
-
});
|
|
4985
|
-
const scores = await response.json() as number[];
|
|
4986
|
-
return rows.map((row, index) => ({ ...row, score: scores[index] }));
|
|
4987
|
-
}, { batchSize: 100 })
|
|
5041
|
+
// Add a three-element float vector, such as an embedding.
|
|
5042
|
+
await table.loadArray([{ name: "Station" }])
|
|
5043
|
+
.updateWithJS((rows) =>
|
|
5044
|
+
rows.map((row) => ({
|
|
5045
|
+
...row,
|
|
5046
|
+
embedding: [0.25, 0.5, 0.75],
|
|
5047
|
+
})), { columnTypes: { embedding: "FLOAT[3]" } })
|
|
4988
5048
|
.log();
|
|
4989
5049
|
```
|
|
4990
5050
|
|
|
5051
|
+
```ts
|
|
5052
|
+
// Move each point slightly east by editing its longitude.
|
|
5053
|
+
await sdb.newTable().loadArray([{
|
|
5054
|
+
geom: { type: "Point", coordinates: [-73, 45] },
|
|
5055
|
+
}], { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } })
|
|
5056
|
+
.updateWithJS((rows) =>
|
|
5057
|
+
rows.map((row) => {
|
|
5058
|
+
// This example loads only Points. Narrow the type for coordinate access.
|
|
5059
|
+
const geom = row.geom as { type: "Point"; coordinates: number[] } | null;
|
|
5060
|
+
if (geom !== null) geom.coordinates[0] += 0.01;
|
|
5061
|
+
return row;
|
|
5062
|
+
})
|
|
5063
|
+
).log();
|
|
5064
|
+
```
|
|
5065
|
+
|
|
5066
|
+
```ts
|
|
5067
|
+
// Create a geometry column from longitude and latitude values.
|
|
5068
|
+
await sdb.newTable().loadArray([{ longitude: -73, latitude: 45 }])
|
|
5069
|
+
.updateWithJS((rows) =>
|
|
5070
|
+
rows.map((row) => ({
|
|
5071
|
+
...row,
|
|
5072
|
+
geom: { type: "Point", coordinates: [row.longitude, row.latitude] },
|
|
5073
|
+
})), { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } }).log();
|
|
5074
|
+
```
|
|
5075
|
+
|
|
4991
5076
|
#### `getSchema`
|
|
4992
5077
|
|
|
4993
5078
|
Returns the schema of the table, including column names and their data types.
|
|
@@ -5093,9 +5178,6 @@ Normalizes string values in a column by:
|
|
|
5093
5178
|
Produces identical output to `journalism-format`'s `normalizeString()` function
|
|
5094
5179
|
for all common cases including accented Latin characters.
|
|
5095
5180
|
|
|
5096
|
-
This method queues the operation; it runs when an async observer method (like
|
|
5097
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
5098
|
-
|
|
5099
5181
|
##### Signature
|
|
5100
5182
|
|
|
5101
5183
|
```typescript
|
|
@@ -5155,9 +5237,75 @@ const columnCount = await table.getColumnCount();
|
|
|
5155
5237
|
console.log(columnCount); // e.g., 3
|
|
5156
5238
|
```
|
|
5157
5239
|
|
|
5240
|
+
#### `addCharacterCount`
|
|
5241
|
+
|
|
5242
|
+
Adds a new column containing the number of characters in each string in the
|
|
5243
|
+
specified column. Counts are based on Unicode code points, not grapheme
|
|
5244
|
+
clusters. A user-perceived character composed of multiple code points, such as
|
|
5245
|
+
some emoji or decomposed accented letters, counts as multiple characters. `NULL`
|
|
5246
|
+
input values produce `NULL` counts.
|
|
5247
|
+
|
|
5248
|
+
##### Signature
|
|
5249
|
+
|
|
5250
|
+
```typescript
|
|
5251
|
+
addCharacterCount(column: string, newColumn: string): this;
|
|
5252
|
+
```
|
|
5253
|
+
|
|
5254
|
+
##### Parameters
|
|
5255
|
+
|
|
5256
|
+
- **`column`**: The name of the column containing the strings to count.
|
|
5257
|
+
- **`newColumn`**: The name of the new column where the character counts will be
|
|
5258
|
+
stored.
|
|
5259
|
+
|
|
5260
|
+
##### Returns
|
|
5261
|
+
|
|
5262
|
+
The table, so methods can be chained.
|
|
5263
|
+
|
|
5264
|
+
##### Examples
|
|
5265
|
+
|
|
5266
|
+
```ts
|
|
5267
|
+
// Add a character count for each value in the 'name' column
|
|
5268
|
+
await table.addCharacterCount("name", "nameCharacterCount").log();
|
|
5269
|
+
```
|
|
5270
|
+
|
|
5271
|
+
#### `addWordCount`
|
|
5272
|
+
|
|
5273
|
+
Adds a new column containing the word count for each string in the specified
|
|
5274
|
+
column. A word is any contiguous sequence of non-whitespace characters. Spaces,
|
|
5275
|
+
tabs, and line breaks separate words. Punctuation is not removed, so a
|
|
5276
|
+
standalone punctuation sequence counts as a word. Text without whitespace counts
|
|
5277
|
+
as one word, regardless of language. Empty or whitespace-only strings produce
|
|
5278
|
+
`0`, and `NULL` input values produce `NULL` counts.
|
|
5279
|
+
|
|
5280
|
+
##### Signature
|
|
5281
|
+
|
|
5282
|
+
```typescript
|
|
5283
|
+
addWordCount(column: string, newColumn: string): this;
|
|
5284
|
+
```
|
|
5285
|
+
|
|
5286
|
+
##### Parameters
|
|
5287
|
+
|
|
5288
|
+
- **`column`**: The name of the column containing the strings to count.
|
|
5289
|
+
- **`newColumn`**: The name of the new column where the word counts will be
|
|
5290
|
+
stored.
|
|
5291
|
+
|
|
5292
|
+
##### Returns
|
|
5293
|
+
|
|
5294
|
+
The table, so methods can be chained.
|
|
5295
|
+
|
|
5296
|
+
##### Examples
|
|
5297
|
+
|
|
5298
|
+
```ts
|
|
5299
|
+
// Add a word count for each value in the 'article' column
|
|
5300
|
+
await table.addWordCount("article", "wordCount").log();
|
|
5301
|
+
```
|
|
5302
|
+
|
|
5158
5303
|
#### `getCharacterCount`
|
|
5159
5304
|
|
|
5160
|
-
Returns the total number of characters in a column storing strings.
|
|
5305
|
+
Returns the total number of characters in a column storing strings. Counts are
|
|
5306
|
+
based on Unicode code points, not grapheme clusters. A user-perceived character
|
|
5307
|
+
composed of multiple code points, such as some emoji or decomposed accented
|
|
5308
|
+
letters, counts as multiple characters.
|
|
5161
5309
|
|
|
5162
5310
|
##### Signature
|
|
5163
5311
|
|
|
@@ -5184,7 +5332,9 @@ console.log(totalChars); // e.g., 523
|
|
|
5184
5332
|
|
|
5185
5333
|
#### `getRowCount`
|
|
5186
5334
|
|
|
5187
|
-
Returns the number of rows in the table.
|
|
5335
|
+
Returns the number of rows in the table. With the default
|
|
5336
|
+
`SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
|
|
5337
|
+
(`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
5188
5338
|
|
|
5189
5339
|
##### Signature
|
|
5190
5340
|
|
|
@@ -5674,9 +5824,11 @@ console.log(uniqueCategories); // e.g., ["Books", "Clothing", "Electronics"]
|
|
|
5674
5824
|
|
|
5675
5825
|
#### `getFirstRow`
|
|
5676
5826
|
|
|
5677
|
-
Returns the first row of the table, optionally filtered by SQL conditions.
|
|
5678
|
-
|
|
5679
|
-
|
|
5827
|
+
Returns the first row of the table, optionally filtered by SQL conditions. With
|
|
5828
|
+
the default `SimpleDB.expressionSyntax: "js"`, conditions support
|
|
5829
|
+
JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
|
|
5830
|
+
`expressionSyntax: "sql"` for unchanged SQL. Temporal values use the same
|
|
5831
|
+
JavaScript representations as `getData()`.
|
|
5680
5832
|
|
|
5681
5833
|
##### Signature
|
|
5682
5834
|
|
|
@@ -5713,9 +5865,11 @@ console.log(firstRowBooks);
|
|
|
5713
5865
|
|
|
5714
5866
|
#### `getLastRow`
|
|
5715
5867
|
|
|
5716
|
-
Returns the last row of the table, optionally filtered by SQL conditions.
|
|
5717
|
-
|
|
5718
|
-
|
|
5868
|
+
Returns the last row of the table, optionally filtered by SQL conditions. With
|
|
5869
|
+
the default `SimpleDB.expressionSyntax: "js"`, conditions support
|
|
5870
|
+
JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
|
|
5871
|
+
`expressionSyntax: "sql"` for unchanged SQL. Temporal values use the same
|
|
5872
|
+
JavaScript representations as `getData()`.
|
|
5719
5873
|
|
|
5720
5874
|
##### Signature
|
|
5721
5875
|
|
|
@@ -5753,8 +5907,10 @@ console.log(lastRowBooks);
|
|
|
5753
5907
|
#### `getTop`
|
|
5754
5908
|
|
|
5755
5909
|
Returns the top `n` rows of the table, optionally filtered by SQL conditions.
|
|
5756
|
-
|
|
5757
|
-
|
|
5910
|
+
With the default `SimpleDB.expressionSyntax: "js"`, conditions support
|
|
5911
|
+
JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
|
|
5912
|
+
`expressionSyntax: "sql"` for unchanged SQL. Temporal values use the same
|
|
5913
|
+
JavaScript representations as `getData()`.
|
|
5758
5914
|
|
|
5759
5915
|
##### Signature
|
|
5760
5916
|
|
|
@@ -5791,9 +5947,10 @@ console.log(top5Books);
|
|
|
5791
5947
|
|
|
5792
5948
|
Returns the bottom `n` rows of the table, optionally filtered by SQL conditions.
|
|
5793
5949
|
By default, the last row will be returned first. To preserve the original order,
|
|
5794
|
-
use the `originalOrder` option.
|
|
5795
|
-
|
|
5796
|
-
|
|
5950
|
+
use the `originalOrder` option. With the default
|
|
5951
|
+
`SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
|
|
5952
|
+
(`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
5953
|
+
Temporal values use the same JavaScript representations as `getData()`.
|
|
5797
5954
|
|
|
5798
5955
|
##### Signature
|
|
5799
5956
|
|
|
@@ -5842,9 +5999,10 @@ console.log(bottom5Books);
|
|
|
5842
5999
|
#### `getRow`
|
|
5843
6000
|
|
|
5844
6001
|
Returns a single row that matches the specified conditions. If no row matches or
|
|
5845
|
-
if more than one row matches, an error is thrown by default.
|
|
5846
|
-
|
|
5847
|
-
|
|
6002
|
+
if more than one row matches, an error is thrown by default. With the default
|
|
6003
|
+
`SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
|
|
6004
|
+
(`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
|
|
6005
|
+
Temporal values use the same JavaScript representations as `getData()`.
|
|
5848
6006
|
|
|
5849
6007
|
##### Signature
|
|
5850
6008
|
|
|
@@ -5893,14 +6051,28 @@ console.log(flexibleRow);
|
|
|
5893
6051
|
#### `getData`
|
|
5894
6052
|
|
|
5895
6053
|
Returns the data from the table as an array of objects, optionally filtered by
|
|
5896
|
-
SQL conditions.
|
|
5897
|
-
`||`, `===`, `!==`).
|
|
6054
|
+
SQL conditions. With the default `SimpleDB.expressionSyntax: "js"`, conditions
|
|
6055
|
+
support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
|
|
6056
|
+
`expressionSyntax: "sql"` for unchanged SQL.
|
|
5898
6057
|
|
|
5899
6058
|
Top-level DuckDB `DATE` and `TIMESTAMP` columns are returned as JavaScript
|
|
5900
6059
|
`Date` objects interpreted in UTC. `TIMESTAMP WITH TIME ZONE` values are
|
|
5901
6060
|
returned as UTC strings, preserving DuckDB's microsecond precision; JavaScript
|
|
5902
6061
|
`Date` supports only milliseconds.
|
|
5903
6062
|
|
|
6063
|
+
Top-level `BIGINT`, `UBIGINT`, `HUGEINT`, and `UHUGEINT` columns are returned as
|
|
6064
|
+
JavaScript numbers. Values outside `Number.MIN_SAFE_INTEGER` through
|
|
6065
|
+
`Number.MAX_SAFE_INTEGER` throw instead of losing precision. Use
|
|
6066
|
+
`convert({ columnName: "string" })` before `getData()` to retrieve their exact
|
|
6067
|
+
digits as strings.
|
|
6068
|
+
|
|
6069
|
+
SQL arrays and lists, including float vectors, return their actual elements as
|
|
6070
|
+
JavaScript arrays. Null cells and null elements remain null.
|
|
6071
|
+
|
|
6072
|
+
Nested SQL lists and structs become JavaScript arrays and objects. Within them,
|
|
6073
|
+
`BIGINT`, `UBIGINT`, `HUGEINT`, `UHUGEINT`, and `DECIMAL` values remain strings
|
|
6074
|
+
to preserve precision, even for small integers.
|
|
6075
|
+
|
|
5904
6076
|
##### Signature
|
|
5905
6077
|
|
|
5906
6078
|
```typescript
|
|
@@ -5950,11 +6122,24 @@ console.log(booksData);
|
|
|
5950
6122
|
const preview = await table.getData({ limit: 2 });
|
|
5951
6123
|
```
|
|
5952
6124
|
|
|
6125
|
+
```ts
|
|
6126
|
+
// Read nested integers without losing digits. SQL creates the nested types.
|
|
6127
|
+
await sdb.customQuery(`CREATE TABLE nested_example AS SELECT
|
|
6128
|
+
[42::BIGINT, 9007199254740993::BIGINT] AS ids,
|
|
6129
|
+
{'count': 2::INTEGER, 'amount': 123.450::DECIMAL(6,3)} AS details`);
|
|
6130
|
+
const nested = sdb.newTable("nested_example");
|
|
6131
|
+
const rows = await nested.getData();
|
|
6132
|
+
// [{ ids: ["42", "9007199254740993"], details: { count: 2, amount: "123.450" } }]
|
|
6133
|
+
await nested.log();
|
|
6134
|
+
```
|
|
6135
|
+
|
|
5953
6136
|
#### `stream`
|
|
5954
6137
|
|
|
5955
6138
|
Streams the table rows one by one as an async iterator, without materializing
|
|
5956
6139
|
the whole table in memory. Values are converted to JavaScript types the same way
|
|
5957
|
-
as `getData()`.
|
|
6140
|
+
as `getData()`. With the default `SimpleDB.expressionSyntax: "js"`, conditions
|
|
6141
|
+
support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
|
|
6142
|
+
`expressionSyntax: "sql"` for unchanged SQL.
|
|
5958
6143
|
|
|
5959
6144
|
The underlying DuckDB result is streamed chunk by chunk, so tables larger than
|
|
5960
6145
|
the available memory can be iterated. Avoid running other queries on the same
|
|
@@ -6001,9 +6186,10 @@ for await (
|
|
|
6001
6186
|
#### `getDataAsCSV`
|
|
6002
6187
|
|
|
6003
6188
|
Returns the data from the table as a CSV string, optionally filtered by SQL
|
|
6004
|
-
conditions.
|
|
6005
|
-
|
|
6006
|
-
|
|
6189
|
+
conditions. With the default `SimpleDB.expressionSyntax: "js"`, conditions
|
|
6190
|
+
support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
|
|
6191
|
+
`expressionSyntax: "sql"` for unchanged SQL. Temporal values are first converted
|
|
6192
|
+
as they are in `getData()`, then serialized using UTC date and timestamp text.
|
|
6007
6193
|
|
|
6008
6194
|
##### Signature
|
|
6009
6195
|
|
|
@@ -6053,9 +6239,6 @@ console.log(booksDataCSV);
|
|
|
6053
6239
|
|
|
6054
6240
|
Creates point geometries from latitude (y) and longitude (x) columns.
|
|
6055
6241
|
|
|
6056
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6057
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6058
|
-
|
|
6059
6242
|
##### Signature
|
|
6060
6243
|
|
|
6061
6244
|
```typescript
|
|
@@ -6096,9 +6279,6 @@ await table.createPoints("y", "x", "geom", { projection: "EPSG:3347" }).log();
|
|
|
6096
6279
|
|
|
6097
6280
|
Adds a column with boolean values indicating the validity of geometries.
|
|
6098
6281
|
|
|
6099
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6100
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6101
|
-
|
|
6102
6282
|
##### Signature
|
|
6103
6283
|
|
|
6104
6284
|
```typescript
|
|
@@ -6135,9 +6315,6 @@ await table.addGeoValidity("isValidMyGeom", { column: "myGeom" }).log();
|
|
|
6135
6315
|
|
|
6136
6316
|
Adds a column with the number of vertices (points) in each geometry.
|
|
6137
6317
|
|
|
6138
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6139
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6140
|
-
|
|
6141
6318
|
##### Signature
|
|
6142
6319
|
|
|
6143
6320
|
```typescript
|
|
@@ -6173,9 +6350,6 @@ await table.addVertexCount("myGeomVertices", { column: "myGeom" }).log();
|
|
|
6173
6350
|
|
|
6174
6351
|
Attempts to make invalid geometries valid without removing any vertices.
|
|
6175
6352
|
|
|
6176
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6177
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6178
|
-
|
|
6179
6353
|
##### Signature
|
|
6180
6354
|
|
|
6181
6355
|
```typescript
|
|
@@ -6208,9 +6382,6 @@ await table.fixGeo("myGeom").log();
|
|
|
6208
6382
|
Adds a column with boolean values indicating whether geometries are closed
|
|
6209
6383
|
(e.g., polygons) or open (e.g., linestrings).
|
|
6210
6384
|
|
|
6211
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6212
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6213
|
-
|
|
6214
6385
|
##### Signature
|
|
6215
6386
|
|
|
6216
6387
|
```typescript
|
|
@@ -6247,9 +6418,6 @@ await table.addGeoClosedStatus("boundaryClosed", { column: "boundaryGeom" })
|
|
|
6247
6418
|
Adds a column with the geometry type (e.g., `"POINT"`, `"LINESTRING"`,
|
|
6248
6419
|
`"POLYGON"`) for each geometry.
|
|
6249
6420
|
|
|
6250
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6251
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6252
|
-
|
|
6253
6421
|
##### Signature
|
|
6254
6422
|
|
|
6255
6423
|
```typescript
|
|
@@ -6288,9 +6456,6 @@ vice-versa). **Warning:** This method should be used with caution as it directly
|
|
|
6288
6456
|
manipulates coordinate order and can affect the accuracy of geospatial
|
|
6289
6457
|
operations if not used correctly.
|
|
6290
6458
|
|
|
6291
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6292
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6293
|
-
|
|
6294
6459
|
##### Signature
|
|
6295
6460
|
|
|
6296
6461
|
```typescript
|
|
@@ -6318,14 +6483,65 @@ await table.flipCoordinates().log();
|
|
|
6318
6483
|
await table.flipCoordinates("myGeom").log();
|
|
6319
6484
|
```
|
|
6320
6485
|
|
|
6486
|
+
#### `addGeoNoise`
|
|
6487
|
+
|
|
6488
|
+
Moves EPSG:4326 point geometries to random locations within a maximum
|
|
6489
|
+
great-circle distance of their original positions.
|
|
6490
|
+
|
|
6491
|
+
Points are sampled uniformly within the requested distance using the spherical
|
|
6492
|
+
direct geodesic (destination-point) formula and the same spherical Earth model
|
|
6493
|
+
as DuckDB's `ST_Distance_Sphere()`. This accounts for longitude scale changing
|
|
6494
|
+
with latitude. This method adds random jitter; it does not provide anonymization
|
|
6495
|
+
or differential privacy guarantees.
|
|
6496
|
+
|
|
6497
|
+
This method supports only `POINT` geometries in `EPSG:4326`. Null and empty
|
|
6498
|
+
geometries are preserved.
|
|
6499
|
+
|
|
6500
|
+
##### Signature
|
|
6501
|
+
|
|
6502
|
+
```typescript
|
|
6503
|
+
addGeoNoise(maxDistance: number, options?: { column?: string; unit?: "m" | "km"; onlyDuplicates?: boolean }): this;
|
|
6504
|
+
```
|
|
6505
|
+
|
|
6506
|
+
##### Parameters
|
|
6507
|
+
|
|
6508
|
+
- **`maxDistance`**: The maximum great-circle displacement in the selected unit.
|
|
6509
|
+
Must be a finite number greater than or equal to `0` and no greater than half
|
|
6510
|
+
Earth's circumference.
|
|
6511
|
+
- **`options`**: An optional object with configuration options:
|
|
6512
|
+
- **`options.column`**: The name of the EPSG:4326 point geometry column. If
|
|
6513
|
+
omitted, the method will automatically attempt to find a geometry column.
|
|
6514
|
+
- **`options.unit`**: The unit of `maxDistance`: `"m"` for metres or `"km"` for
|
|
6515
|
+
kilometres. Defaults to `"m"`.
|
|
6516
|
+
- **`options.onlyDuplicates`**: If `true`, moves only points whose complete
|
|
6517
|
+
original geometry occurs more than once in the selected geometry column. Every
|
|
6518
|
+
point in a duplicated group is moved. Defaults to `false`.
|
|
6519
|
+
|
|
6520
|
+
##### Returns
|
|
6521
|
+
|
|
6522
|
+
The table, so methods can be chained.
|
|
6523
|
+
|
|
6524
|
+
##### Examples
|
|
6525
|
+
|
|
6526
|
+
```ts
|
|
6527
|
+
// Move every point by no more than 500 metres
|
|
6528
|
+
await table.addGeoNoise(500).log();
|
|
6529
|
+
```
|
|
6530
|
+
|
|
6531
|
+
```ts
|
|
6532
|
+
// Separate duplicated points by up to 0.1 kilometres
|
|
6533
|
+
await table.addGeoNoise(0.1, {
|
|
6534
|
+
column: "geom",
|
|
6535
|
+
unit: "km",
|
|
6536
|
+
onlyDuplicates: true,
|
|
6537
|
+
}).log();
|
|
6538
|
+
```
|
|
6539
|
+
|
|
6321
6540
|
#### `reducePrecision`
|
|
6322
6541
|
|
|
6323
6542
|
Reduces the precision of geometries in a specified column to a given number of
|
|
6324
6543
|
decimal places.
|
|
6325
6544
|
|
|
6326
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6327
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6328
|
-
|
|
6329
6545
|
##### Signature
|
|
6330
6546
|
|
|
6331
6547
|
```typescript
|
|
@@ -6361,9 +6577,6 @@ await table.reducePrecision(2, { column: "myGeom" }).log();
|
|
|
6361
6577
|
Reprojects the geometries in a specified column to another Spatial Reference
|
|
6362
6578
|
System (SRS).
|
|
6363
6579
|
|
|
6364
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6365
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6366
|
-
|
|
6367
6580
|
##### Signature
|
|
6368
6581
|
|
|
6369
6582
|
```typescript
|
|
@@ -6399,9 +6612,6 @@ await table.reproject("EPSG:3347", { column: "myGeom" }).log();
|
|
|
6399
6612
|
Computes the area of geometries in square meters (`"m2"`) or optionally square
|
|
6400
6613
|
kilometers (`"km2"`). The input geometry is assumed to be in EPSG:4326 (WGS84).
|
|
6401
6614
|
|
|
6402
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6403
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6404
|
-
|
|
6405
6615
|
##### Signature
|
|
6406
6616
|
|
|
6407
6617
|
```typescript
|
|
@@ -6451,9 +6661,6 @@ await table.area("myGeomArea", { column: "myGeom" }).log();
|
|
|
6451
6661
|
Computes the length of line geometries in meters (`"m"`) or optionally
|
|
6452
6662
|
kilometers (`"km"`). The input geometry is assumed to be in EPSG:4326 (WGS84).
|
|
6453
6663
|
|
|
6454
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6455
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6456
|
-
|
|
6457
6664
|
##### Signature
|
|
6458
6665
|
|
|
6459
6666
|
```typescript
|
|
@@ -6503,9 +6710,6 @@ await table.length("routeLength", { column: "routeGeom" }).log();
|
|
|
6503
6710
|
Computes the perimeter of polygon geometries in meters (`"m"`) or optionally
|
|
6504
6711
|
kilometers (`"km"`). The input geometry is assumed to be in EPSG:4326 (WGS84).
|
|
6505
6712
|
|
|
6506
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6507
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6508
|
-
|
|
6509
6713
|
##### Signature
|
|
6510
6714
|
|
|
6511
6715
|
```typescript
|
|
@@ -6557,9 +6761,6 @@ Computes a buffer (a polygon representing a specified distance around a
|
|
|
6557
6761
|
geometry) for geometries in a specified column. The distance is in the Spatial
|
|
6558
6762
|
Reference System (SRS) unit of the input geometries.
|
|
6559
6763
|
|
|
6560
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6561
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6562
|
-
|
|
6563
6764
|
##### Signature
|
|
6564
6765
|
|
|
6565
6766
|
```typescript
|
|
@@ -6598,10 +6799,7 @@ Merges the data of this table (considered the left table) with another table
|
|
|
6598
6799
|
(the right table) based on a spatial relationship. Note that the order of rows
|
|
6599
6800
|
in the returned data is not guaranteed to be the same as in the original tables.
|
|
6600
6801
|
This operation might create temporary files in a `.tmp` folder; consider adding
|
|
6601
|
-
`.tmp` to your `.gitignore`.
|
|
6602
|
-
async observer method (like `getData()` or `log()`) is awaited, or when `run()`
|
|
6603
|
-
is called. The join uses the other table's state as of this call: operations
|
|
6604
|
-
queued on it afterwards run after the join.
|
|
6802
|
+
`.tmp` to your `.gitignore`.
|
|
6605
6803
|
|
|
6606
6804
|
##### Signature
|
|
6607
6805
|
|
|
@@ -6694,9 +6892,6 @@ const tableC = await tableA.joinGeo(tableB, "intersect", {
|
|
|
6694
6892
|
Computes the intersection of two sets of geometries, creating new geometries
|
|
6695
6893
|
where they overlap.
|
|
6696
6894
|
|
|
6697
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6698
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6699
|
-
|
|
6700
6895
|
##### Signature
|
|
6701
6896
|
|
|
6702
6897
|
```typescript
|
|
@@ -6727,9 +6922,6 @@ await table.intersection("geomA", "geomB", "intersectGeom").log();
|
|
|
6727
6922
|
Computes the geometric difference between two geometries, returning the portion
|
|
6728
6923
|
of the first geometry that does not intersect the second.
|
|
6729
6924
|
|
|
6730
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6731
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6732
|
-
|
|
6733
6925
|
##### Signature
|
|
6734
6926
|
|
|
6735
6927
|
```typescript
|
|
@@ -6760,9 +6952,6 @@ await table.difference("geomA", "geomB", "geomA_minus_geomB").log();
|
|
|
6760
6952
|
|
|
6761
6953
|
Fills holes in polygon geometries.
|
|
6762
6954
|
|
|
6763
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6764
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6765
|
-
|
|
6766
6955
|
##### Signature
|
|
6767
6956
|
|
|
6768
6957
|
```typescript
|
|
@@ -6795,9 +6984,6 @@ await table.fillHoles("polygonGeom").log();
|
|
|
6795
6984
|
Returns `TRUE` if two geometries intersect (overlap in any way), and `FALSE`
|
|
6796
6985
|
otherwise.
|
|
6797
6986
|
|
|
6798
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6799
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6800
|
-
|
|
6801
6987
|
##### Signature
|
|
6802
6988
|
|
|
6803
6989
|
```typescript
|
|
@@ -6828,9 +7014,6 @@ await table.intersects("geomA", "geomB", "doIntersect").log();
|
|
|
6828
7014
|
Returns `TRUE` if every point of a geometry in `column` is covered by a geometry
|
|
6829
7015
|
in `containerColumn`, including their boundaries, and `FALSE` otherwise.
|
|
6830
7016
|
|
|
6831
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6832
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6833
|
-
|
|
6834
7017
|
##### Signature
|
|
6835
7018
|
|
|
6836
7019
|
```typescript
|
|
@@ -6862,9 +7045,6 @@ await table.coveredBy("pointGeom", "polygonGeom", "isCovered").log();
|
|
|
6862
7045
|
Computes the union of two geometries, creating a new geometry that represents
|
|
6863
7046
|
the merged area of both.
|
|
6864
7047
|
|
|
6865
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6866
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6867
|
-
|
|
6868
7048
|
##### Signature
|
|
6869
7049
|
|
|
6870
7050
|
```typescript
|
|
@@ -6895,9 +7075,6 @@ await table.union("geomA", "geomB", "unionGeom").log();
|
|
|
6895
7075
|
Extracts the latitude (y) and longitude (x) coordinates from point geometries.
|
|
6896
7076
|
The input geometry is assumed to be in EPSG:4326 (WGS84).
|
|
6897
7077
|
|
|
6898
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6899
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6900
|
-
|
|
6901
7078
|
##### Signature
|
|
6902
7079
|
|
|
6903
7080
|
```typescript
|
|
@@ -6928,9 +7105,6 @@ await table.extractLatLon("geom", "lat", "lon").log();
|
|
|
6928
7105
|
Simplifies geometries while preserving their overall coverage. A higher
|
|
6929
7106
|
tolerance results in more significant simplification.
|
|
6930
7107
|
|
|
6931
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6932
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6933
|
-
|
|
6934
7108
|
##### Signature
|
|
6935
7109
|
|
|
6936
7110
|
```typescript
|
|
@@ -6969,9 +7143,6 @@ await table.simplify(0.05, { column: "myGeom", simplifyBoundary: false }).log();
|
|
|
6969
7143
|
Computes the centroid of geometries. The values are returned in the SRS unit of
|
|
6970
7144
|
the input geometries.
|
|
6971
7145
|
|
|
6972
|
-
This method queues the operation; it runs when an async observer method (like
|
|
6973
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
6974
|
-
|
|
6975
7146
|
##### Signature
|
|
6976
7147
|
|
|
6977
7148
|
```typescript
|
|
@@ -7006,9 +7177,6 @@ await table.centroid("areaCentroid", { column: "areaGeom" }).log();
|
|
|
7006
7177
|
|
|
7007
7178
|
Generates a random point within the geometries of a specified column.
|
|
7008
7179
|
|
|
7009
|
-
This method queues the operation; it runs when an async observer method (like
|
|
7010
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
7011
|
-
|
|
7012
7180
|
##### Signature
|
|
7013
7181
|
|
|
7014
7182
|
```typescript
|
|
@@ -7054,9 +7222,6 @@ input geometries. You can optionally specify `"spheroid"` or `"haversine"`
|
|
|
7054
7222
|
methods to get results in meters or kilometers. If using `"spheroid"` or
|
|
7055
7223
|
`"haversine"`, the input geometries must be in EPSG:4326 (WGS84).
|
|
7056
7224
|
|
|
7057
|
-
This method queues the operation; it runs when an async observer method (like
|
|
7058
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
7059
|
-
|
|
7060
7225
|
##### Signature
|
|
7061
7226
|
|
|
7062
7227
|
```typescript
|
|
@@ -7121,9 +7286,6 @@ await table.distance("area1", "area2", "distance_spheroid_km", {
|
|
|
7121
7286
|
Unnests geometries recursively, transforming multi-part geometries (e.g.,
|
|
7122
7287
|
MultiPolygon) into individual single-part geometries (e.g., Polygon).
|
|
7123
7288
|
|
|
7124
|
-
This method queues the operation; it runs when an async observer method (like
|
|
7125
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
7126
|
-
|
|
7127
7289
|
##### Signature
|
|
7128
7290
|
|
|
7129
7291
|
```typescript
|
|
@@ -7156,9 +7318,6 @@ await table.unnestGeo("multiGeom").log();
|
|
|
7156
7318
|
Adds the bounding box coordinates of geometries in a specified column as four
|
|
7157
7319
|
new columns: `minLon`, `minLat`, `maxLon`, and `maxLat`.
|
|
7158
7320
|
|
|
7159
|
-
This method queues the operation; it runs when an async observer method (like
|
|
7160
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
7161
|
-
|
|
7162
7321
|
##### Signature
|
|
7163
7322
|
|
|
7164
7323
|
```typescript
|
|
@@ -7197,9 +7356,6 @@ await table.addBoundingBox({ column: "geom", decimals: 2 }).log();
|
|
|
7197
7356
|
Aggregates geometries in a specified column based on a chosen aggregation
|
|
7198
7357
|
method.
|
|
7199
7358
|
|
|
7200
|
-
This method queues the operation; it runs when an async observer method (like
|
|
7201
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
7202
|
-
|
|
7203
7359
|
##### Signature
|
|
7204
7360
|
|
|
7205
7361
|
```typescript
|
|
@@ -7251,9 +7407,6 @@ const intersectionTable = await table.aggregateGeo("intersection", {
|
|
|
7251
7407
|
|
|
7252
7408
|
Transforms closed linestring geometries into polygon geometries.
|
|
7253
7409
|
|
|
7254
|
-
This method queues the operation; it runs when an async observer method (like
|
|
7255
|
-
`getData()` or `log()`) is awaited, or when `run()` is called.
|
|
7256
|
-
|
|
7257
7410
|
##### Signature
|
|
7258
7411
|
|
|
7259
7412
|
```typescript
|
|
@@ -7319,8 +7472,10 @@ console.log(areaBbox);
|
|
|
7319
7472
|
|
|
7320
7473
|
#### `getGeoData`
|
|
7321
7474
|
|
|
7322
|
-
Returns the table's geospatial data as a GeoJSON
|
|
7323
|
-
|
|
7475
|
+
Returns the table's geospatial data as a GeoJSON FeatureCollection. Each row
|
|
7476
|
+
becomes a Feature: the selected geometry column becomes its geometry, and the
|
|
7477
|
+
remaining columns become its properties. If the table has multiple geometry
|
|
7478
|
+
columns, you must specify which one to use.
|
|
7324
7479
|
|
|
7325
7480
|
##### Signature
|
|
7326
7481
|
|
|
@@ -7364,7 +7519,8 @@ console.log(rewoundGeojson);
|
|
|
7364
7519
|
#### `writeData`
|
|
7365
7520
|
|
|
7366
7521
|
Writes the table's data to a file in various formats (CSV, JSON, Parquet,
|
|
7367
|
-
DuckDB, SQLite).
|
|
7522
|
+
DuckDB, SQLite). Tables with geometry columns must use `writeGeoData()` instead.
|
|
7523
|
+
If the specified path does not exist, it will be created.
|
|
7368
7524
|
|
|
7369
7525
|
##### Signature
|
|
7370
7526
|
|
|
@@ -7385,8 +7541,8 @@ async writeData(file: string, options?: { compression?: boolean; dataAsArrays?:
|
|
|
7385
7541
|
`{ "col1": [v1, v2], "col2": [v3, v4] }`) instead of an array of objects. This
|
|
7386
7542
|
can reduce file size for web projects. You can use the `arraysToData` function
|
|
7387
7543
|
from the
|
|
7388
|
-
[journalism library](https://jsr.io/@nshiab/journalism/doc/~/arraysToData)
|
|
7389
|
-
convert it back.
|
|
7544
|
+
[journalism-format library](https://jsr.io/@nshiab/journalism-format/doc/~/arraysToData)
|
|
7545
|
+
to convert it back.
|
|
7390
7546
|
- **`options.formatDates`**: For CSV and JSON files only. If `true`, date and
|
|
7391
7547
|
timestamp columns will be formatted as ISO 8601 strings (e.g.,
|
|
7392
7548
|
`"2025-01-01T01:00:00.000Z"`). Defaults to `false`.
|
|
@@ -7503,28 +7659,28 @@ await table.writeGeoData("./output_high_precision.geojson", {
|
|
|
7503
7659
|
Caches the results of computations in `./.sda-cache`. You should add
|
|
7504
7660
|
`./.sda-cache` to your `.gitignore` file.
|
|
7505
7661
|
|
|
7506
|
-
Cache entries are stored as DuckDB database files. Full-text search (FTS)
|
|
7507
|
-
indexes are persisted in the cache file and restored directly on a cache hit.
|
|
7508
|
-
Vector similarity search (VSS/HNSW) indexes are not persisted in the cache file;
|
|
7509
|
-
their definitions are stored as metadata and used to rebuild the indexes on
|
|
7510
|
-
every cache hit. If loading the entry or restoring its indexes fails, the
|
|
7511
|
-
computation runs again and replaces the cache entry.
|
|
7512
|
-
|
|
7513
7662
|
`cache()` automatically tracks whether earlier SDA operations changed the table.
|
|
7514
7663
|
It also records every other already registered `SimpleTable` read through
|
|
7515
7664
|
`SimpleTable` methods while `compute` runs and invalidates the cached step when
|
|
7516
7665
|
any of their generations change. Tables created inside `compute` are part of the
|
|
7517
7666
|
computation itself and are not dependencies.
|
|
7518
7667
|
|
|
7668
|
+
Inside `compute`, you may modify the table being cached and create or modify
|
|
7669
|
+
temporary tables, provided you remove them before the callback finishes. Other
|
|
7670
|
+
tables that already existed may be read but not modified, renamed, or removed.
|
|
7671
|
+
`customQuery()` bypasses enforcement of these rules.
|
|
7672
|
+
|
|
7519
7673
|
`SimpleDB.customQuery()` bypasses this tracking. Reading or changing a table
|
|
7520
7674
|
with `customQuery()` can therefore return stale cached data. Include a value
|
|
7521
7675
|
that identifies the custom query's dependencies in `options.inputs` (such as a
|
|
7522
7676
|
table content hash), or use tracked `SimpleTable` methods.
|
|
7523
7677
|
|
|
7524
|
-
|
|
7525
|
-
|
|
7526
|
-
|
|
7527
|
-
|
|
7678
|
+
Cache entries are stored as DuckDB database files. Full-text search (FTS)
|
|
7679
|
+
indexes are persisted in the cache file and restored directly on a cache hit.
|
|
7680
|
+
Vector similarity search (VSS/HNSW) indexes are not persisted in the cache file;
|
|
7681
|
+
their definitions are stored as metadata and used to rebuild the indexes on
|
|
7682
|
+
every cache hit. If loading the entry or restoring its indexes fails, the
|
|
7683
|
+
computation runs again and replaces the cache entry.
|
|
7528
7684
|
|
|
7529
7685
|
##### Signature
|
|
7530
7686
|
|
|
@@ -7624,8 +7780,18 @@ await summary.log();
|
|
|
7624
7780
|
|
|
7625
7781
|
Logs a specified number of rows from the table to the console. By default, the
|
|
7626
7782
|
first 10 rows are logged. You can optionally log the column types and filter the
|
|
7627
|
-
data based on conditions.
|
|
7628
|
-
|
|
7783
|
+
data based on conditions. SQL dates and timestamps retain their native precision
|
|
7784
|
+
for display, including temporal infinities. Lists and objects are stringified
|
|
7785
|
+
using the same nested representations as `getData()`, then truncated according
|
|
7786
|
+
to `charsToLog`. Unsafe top-level large integers throw, just as with
|
|
7787
|
+
`getData()`. Type annotations describe the SQL type and the JavaScript
|
|
7788
|
+
extraction type before display formatting; nulls do not determine the column's
|
|
7789
|
+
annotation. Colors reflect the SQL value's meaning, so exact decimal strings use
|
|
7790
|
+
numeric coloring. Column width is independent of the content truncation budget.
|
|
7791
|
+
|
|
7792
|
+
With the default `SimpleDB.expressionSyntax: "js"`, conditions support
|
|
7793
|
+
JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
|
|
7794
|
+
`expressionSyntax: "sql"` for unchanged SQL.
|
|
7629
7795
|
|
|
7630
7796
|
##### Signature
|
|
7631
7797
|
|
|
@@ -7866,6 +8032,10 @@ Logs the bottom `n` rows of the table to the console. By default, the last row
|
|
|
7866
8032
|
will be returned first. To preserve the original order, use the `originalOrder`
|
|
7867
8033
|
option.
|
|
7868
8034
|
|
|
8035
|
+
Uses the same precision-preserving temporal display, nested stringification,
|
|
8036
|
+
numeric coloring, and `charsToLog` truncation as `log()`. Unsafe top-level large
|
|
8037
|
+
integers throw, just as with `getData()`.
|
|
8038
|
+
|
|
7869
8039
|
##### Signature
|
|
7870
8040
|
|
|
7871
8041
|
```typescript
|