@nshiab/simple-data-analysis-core 2.0.4 → 2.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (280) hide show
  1. package/README.md +118 -12
  2. package/esm/class/Simple.d.ts +8 -2
  3. package/esm/class/Simple.d.ts.map +1 -1
  4. package/esm/class/Simple.js +9 -3
  5. package/esm/class/SimpleDB.d.ts +72 -1
  6. package/esm/class/SimpleDB.d.ts.map +1 -1
  7. package/esm/class/SimpleDB.js +81 -1
  8. package/esm/class/SimpleTable.d.ts +434 -263
  9. package/esm/class/SimpleTable.d.ts.map +1 -1
  10. package/esm/class/SimpleTable.js +441 -316
  11. package/esm/helpers/LogValue.d.ts +8 -0
  12. package/esm/helpers/LogValue.d.ts.map +1 -0
  13. package/esm/helpers/LogValue.js +23 -0
  14. package/esm/helpers/cacheTableDependencies.d.ts +9 -0
  15. package/esm/helpers/cacheTableDependencies.d.ts.map +1 -1
  16. package/esm/helpers/cacheTableDependencies.js +131 -21
  17. package/esm/helpers/cleanSQL.d.ts +1 -1
  18. package/esm/helpers/cleanSQL.d.ts.map +1 -1
  19. package/esm/helpers/cleanSQL.js +268 -23
  20. package/esm/helpers/flushAllTables.d.ts.map +1 -1
  21. package/esm/helpers/flushAllTables.js +23 -28
  22. package/esm/helpers/foldIdentifier.d.ts +3 -0
  23. package/esm/helpers/foldIdentifier.d.ts.map +1 -0
  24. package/esm/helpers/foldIdentifier.js +4 -0
  25. package/esm/helpers/geometryFromJSON.d.ts +3 -0
  26. package/esm/helpers/geometryFromJSON.d.ts.map +1 -0
  27. package/esm/helpers/geometryFromJSON.js +4 -0
  28. package/esm/helpers/index.d.ts +1 -0
  29. package/esm/helpers/index.d.ts.map +1 -1
  30. package/esm/helpers/index.js +1 -0
  31. package/esm/helpers/logData.d.ts.map +1 -1
  32. package/esm/helpers/logData.js +45 -44
  33. package/esm/helpers/makeLogConverter.d.ts +5 -0
  34. package/esm/helpers/makeLogConverter.d.ts.map +1 -0
  35. package/esm/helpers/makeLogConverter.js +64 -0
  36. package/esm/helpers/osmFiles.d.ts.map +1 -1
  37. package/esm/helpers/parseTypes.d.ts +1 -1
  38. package/esm/helpers/parseTypes.d.ts.map +1 -1
  39. package/esm/helpers/parseTypes.js +4 -0
  40. package/esm/helpers/pendingOps.d.ts +4 -2
  41. package/esm/helpers/pendingOps.d.ts.map +1 -1
  42. package/esm/helpers/prepareGeometry.d.ts +3 -0
  43. package/esm/helpers/prepareGeometry.d.ts.map +1 -0
  44. package/esm/helpers/prepareGeometry.js +99 -0
  45. package/esm/helpers/prepareJSON.d.ts +3 -0
  46. package/esm/helpers/prepareJSON.d.ts.map +1 -0
  47. package/esm/helpers/prepareJSON.js +50 -0
  48. package/esm/helpers/printTable.d.ts +5 -4
  49. package/esm/helpers/printTable.d.ts.map +1 -1
  50. package/esm/helpers/printTable.js +32 -36
  51. package/esm/helpers/queryDB.js +1 -1
  52. package/esm/helpers/queueAsyncBarrier.d.ts +4 -0
  53. package/esm/helpers/queueAsyncBarrier.d.ts.map +1 -1
  54. package/esm/helpers/queueAsyncBarrier.js +4 -0
  55. package/esm/helpers/queueOp.d.ts.map +1 -1
  56. package/esm/helpers/queueOp.js +5 -3
  57. package/esm/helpers/readMutationRows.d.ts +9 -0
  58. package/esm/helpers/readMutationRows.d.ts.map +1 -0
  59. package/esm/helpers/readMutationRows.js +94 -0
  60. package/esm/helpers/referencedTables.d.ts +9 -0
  61. package/esm/helpers/referencedTables.d.ts.map +1 -0
  62. package/esm/helpers/referencedTables.js +22 -0
  63. package/esm/helpers/runQuery.d.ts.map +1 -1
  64. package/esm/helpers/runQuery.js +11 -13
  65. package/esm/helpers/scanSQL.d.ts +7 -0
  66. package/esm/helpers/scanSQL.d.ts.map +1 -0
  67. package/esm/helpers/scanSQL.js +71 -0
  68. package/esm/helpers/tableRegistry.d.ts +2 -0
  69. package/esm/helpers/tableRegistry.d.ts.map +1 -1
  70. package/esm/helpers/tableRegistry.js +12 -4
  71. package/esm/helpers/updateColumnsWithJS.d.ts +52 -0
  72. package/esm/helpers/updateColumnsWithJS.d.ts.map +1 -0
  73. package/esm/helpers/updateColumnsWithJS.js +139 -0
  74. package/esm/methods/addCharacterCount.d.ts +3 -0
  75. package/esm/methods/addCharacterCount.d.ts.map +1 -0
  76. package/esm/methods/addCharacterCount.js +15 -0
  77. package/esm/methods/addColumn.d.ts +1 -1
  78. package/esm/methods/addColumn.d.ts.map +1 -1
  79. package/esm/methods/addGeoNoise.d.ts +7 -0
  80. package/esm/methods/addGeoNoise.d.ts.map +1 -0
  81. package/esm/methods/addGeoNoise.js +135 -0
  82. package/esm/methods/addNoise.d.ts +5 -0
  83. package/esm/methods/addNoise.d.ts.map +1 -0
  84. package/esm/methods/addNoise.js +53 -0
  85. package/esm/methods/addWordCount.d.ts +3 -0
  86. package/esm/methods/addWordCount.d.ts.map +1 -0
  87. package/esm/methods/addWordCount.js +15 -0
  88. package/esm/methods/bm25.d.ts.map +1 -1
  89. package/esm/methods/bm25.js +1 -2
  90. package/esm/methods/cache.d.ts.map +1 -1
  91. package/esm/methods/cache.js +4 -6
  92. package/esm/methods/columnProportions.d.ts +1 -0
  93. package/esm/methods/columnProportions.d.ts.map +1 -1
  94. package/esm/methods/columnProportions.js +7 -2
  95. package/esm/methods/convert.d.ts +2 -2
  96. package/esm/methods/convert.d.ts.map +1 -1
  97. package/esm/methods/convert.js +5 -0
  98. package/esm/methods/getBottom.d.ts +1 -1
  99. package/esm/methods/getBottom.d.ts.map +1 -1
  100. package/esm/methods/getBottom.js +2 -2
  101. package/esm/methods/getData.d.ts +1 -1
  102. package/esm/methods/getData.d.ts.map +1 -1
  103. package/esm/methods/getData.js +2 -2
  104. package/esm/methods/join.d.ts.map +1 -1
  105. package/esm/methods/join.js +8 -16
  106. package/esm/methods/joinGeo.d.ts.map +1 -1
  107. package/esm/methods/joinGeo.js +10 -19
  108. package/esm/methods/loadArray.d.ts +4 -2
  109. package/esm/methods/loadArray.d.ts.map +1 -1
  110. package/esm/methods/loadArray.js +182 -18
  111. package/esm/methods/loadData.js +3 -0
  112. package/esm/methods/loadYahooFinanceData.d.ts +10 -0
  113. package/esm/methods/loadYahooFinanceData.d.ts.map +1 -0
  114. package/esm/methods/loadYahooFinanceData.js +133 -0
  115. package/esm/methods/log.d.ts +7 -0
  116. package/esm/methods/log.d.ts.map +1 -0
  117. package/esm/methods/log.js +58 -0
  118. package/esm/methods/logBottom.d.ts +5 -0
  119. package/esm/methods/logBottom.d.ts.map +1 -0
  120. package/esm/methods/logBottom.js +9 -0
  121. package/esm/methods/removeTable.d.ts.map +1 -1
  122. package/esm/methods/removeTable.js +2 -0
  123. package/esm/methods/removeTables.d.ts.map +1 -1
  124. package/esm/methods/removeTables.js +7 -0
  125. package/esm/methods/renameTable.d.ts.map +1 -1
  126. package/esm/methods/renameTable.js +4 -0
  127. package/esm/methods/rowProportions.d.ts +1 -0
  128. package/esm/methods/rowProportions.d.ts.map +1 -1
  129. package/esm/methods/rowProportions.js +5 -1
  130. package/esm/methods/rowToText.js +2 -2
  131. package/esm/methods/selectTables.d.ts.map +1 -1
  132. package/esm/methods/selectTables.js +6 -0
  133. package/esm/methods/setTypes.d.ts +1 -1
  134. package/esm/methods/setTypes.d.ts.map +1 -1
  135. package/esm/methods/stream.d.ts.map +1 -1
  136. package/esm/methods/stream.js +2 -1
  137. package/esm/methods/updateWithJS.d.ts +1 -3
  138. package/esm/methods/updateWithJS.d.ts.map +1 -1
  139. package/esm/methods/updateWithJS.js +123 -37
  140. package/llm.md +600 -430
  141. package/llms.txt +1 -1
  142. package/package.json +1 -1
  143. package/script/class/Simple.d.ts +8 -2
  144. package/script/class/Simple.d.ts.map +1 -1
  145. package/script/class/Simple.js +9 -3
  146. package/script/class/SimpleDB.d.ts +72 -1
  147. package/script/class/SimpleDB.d.ts.map +1 -1
  148. package/script/class/SimpleDB.js +81 -1
  149. package/script/class/SimpleTable.d.ts +434 -263
  150. package/script/class/SimpleTable.d.ts.map +1 -1
  151. package/script/class/SimpleTable.js +441 -316
  152. package/script/helpers/LogValue.d.ts +8 -0
  153. package/script/helpers/LogValue.d.ts.map +1 -0
  154. package/script/helpers/LogValue.js +26 -0
  155. package/script/helpers/cacheTableDependencies.d.ts +9 -0
  156. package/script/helpers/cacheTableDependencies.d.ts.map +1 -1
  157. package/script/helpers/cacheTableDependencies.js +170 -20
  158. package/script/helpers/cleanSQL.d.ts +1 -1
  159. package/script/helpers/cleanSQL.d.ts.map +1 -1
  160. package/script/helpers/cleanSQL.js +271 -23
  161. package/script/helpers/flushAllTables.d.ts.map +1 -1
  162. package/script/helpers/flushAllTables.js +24 -29
  163. package/script/helpers/foldIdentifier.d.ts +3 -0
  164. package/script/helpers/foldIdentifier.d.ts.map +1 -0
  165. package/script/helpers/foldIdentifier.js +7 -0
  166. package/script/helpers/geometryFromJSON.d.ts +3 -0
  167. package/script/helpers/geometryFromJSON.d.ts.map +1 -0
  168. package/script/helpers/geometryFromJSON.js +7 -0
  169. package/script/helpers/index.d.ts +1 -0
  170. package/script/helpers/index.d.ts.map +1 -1
  171. package/script/helpers/index.js +3 -1
  172. package/script/helpers/logData.d.ts.map +1 -1
  173. package/script/helpers/logData.js +45 -44
  174. package/script/helpers/makeLogConverter.d.ts +5 -0
  175. package/script/helpers/makeLogConverter.d.ts.map +1 -0
  176. package/script/helpers/makeLogConverter.js +70 -0
  177. package/script/helpers/osmFiles.d.ts.map +1 -1
  178. package/script/helpers/parseTypes.d.ts +1 -1
  179. package/script/helpers/parseTypes.d.ts.map +1 -1
  180. package/script/helpers/parseTypes.js +4 -0
  181. package/script/helpers/pendingOps.d.ts +4 -2
  182. package/script/helpers/pendingOps.d.ts.map +1 -1
  183. package/script/helpers/prepareGeometry.d.ts +3 -0
  184. package/script/helpers/prepareGeometry.d.ts.map +1 -0
  185. package/script/helpers/prepareGeometry.js +102 -0
  186. package/script/helpers/prepareJSON.d.ts +3 -0
  187. package/script/helpers/prepareJSON.d.ts.map +1 -0
  188. package/script/helpers/prepareJSON.js +53 -0
  189. package/script/helpers/printTable.d.ts +5 -4
  190. package/script/helpers/printTable.d.ts.map +1 -1
  191. package/script/helpers/printTable.js +32 -36
  192. package/script/helpers/queryDB.js +1 -1
  193. package/script/helpers/queueAsyncBarrier.d.ts +4 -0
  194. package/script/helpers/queueAsyncBarrier.d.ts.map +1 -1
  195. package/script/helpers/queueAsyncBarrier.js +4 -0
  196. package/script/helpers/queueOp.d.ts.map +1 -1
  197. package/script/helpers/queueOp.js +4 -2
  198. package/script/helpers/readMutationRows.d.ts +9 -0
  199. package/script/helpers/readMutationRows.d.ts.map +1 -0
  200. package/script/helpers/readMutationRows.js +97 -0
  201. package/script/helpers/referencedTables.d.ts +9 -0
  202. package/script/helpers/referencedTables.d.ts.map +1 -0
  203. package/script/helpers/referencedTables.js +28 -0
  204. package/script/helpers/runQuery.d.ts.map +1 -1
  205. package/script/helpers/runQuery.js +11 -13
  206. package/script/helpers/scanSQL.d.ts +7 -0
  207. package/script/helpers/scanSQL.d.ts.map +1 -0
  208. package/script/helpers/scanSQL.js +74 -0
  209. package/script/helpers/tableRegistry.d.ts +2 -0
  210. package/script/helpers/tableRegistry.d.ts.map +1 -1
  211. package/script/helpers/tableRegistry.js +16 -4
  212. package/script/helpers/updateColumnsWithJS.d.ts +52 -0
  213. package/script/helpers/updateColumnsWithJS.d.ts.map +1 -0
  214. package/script/helpers/updateColumnsWithJS.js +145 -0
  215. package/script/methods/addCharacterCount.d.ts +3 -0
  216. package/script/methods/addCharacterCount.d.ts.map +1 -0
  217. package/script/methods/addCharacterCount.js +21 -0
  218. package/script/methods/addColumn.d.ts +1 -1
  219. package/script/methods/addColumn.d.ts.map +1 -1
  220. package/script/methods/addGeoNoise.d.ts +7 -0
  221. package/script/methods/addGeoNoise.d.ts.map +1 -0
  222. package/script/methods/addGeoNoise.js +141 -0
  223. package/script/methods/addNoise.d.ts +5 -0
  224. package/script/methods/addNoise.d.ts.map +1 -0
  225. package/script/methods/addNoise.js +59 -0
  226. package/script/methods/addWordCount.d.ts +3 -0
  227. package/script/methods/addWordCount.d.ts.map +1 -0
  228. package/script/methods/addWordCount.js +21 -0
  229. package/script/methods/bm25.d.ts.map +1 -1
  230. package/script/methods/bm25.js +1 -2
  231. package/script/methods/cache.d.ts.map +1 -1
  232. package/script/methods/cache.js +3 -5
  233. package/script/methods/columnProportions.d.ts +1 -0
  234. package/script/methods/columnProportions.d.ts.map +1 -1
  235. package/script/methods/columnProportions.js +7 -2
  236. package/script/methods/convert.d.ts +2 -2
  237. package/script/methods/convert.d.ts.map +1 -1
  238. package/script/methods/convert.js +5 -0
  239. package/script/methods/getBottom.d.ts +1 -1
  240. package/script/methods/getBottom.d.ts.map +1 -1
  241. package/script/methods/getBottom.js +2 -2
  242. package/script/methods/getData.d.ts +1 -1
  243. package/script/methods/getData.d.ts.map +1 -1
  244. package/script/methods/getData.js +2 -2
  245. package/script/methods/join.d.ts.map +1 -1
  246. package/script/methods/join.js +8 -16
  247. package/script/methods/joinGeo.d.ts.map +1 -1
  248. package/script/methods/joinGeo.js +10 -19
  249. package/script/methods/loadArray.d.ts +4 -2
  250. package/script/methods/loadArray.d.ts.map +1 -1
  251. package/script/methods/loadArray.js +181 -17
  252. package/script/methods/loadData.js +3 -0
  253. package/script/methods/loadYahooFinanceData.d.ts +10 -0
  254. package/script/methods/loadYahooFinanceData.d.ts.map +1 -0
  255. package/script/methods/loadYahooFinanceData.js +139 -0
  256. package/script/methods/log.d.ts +7 -0
  257. package/script/methods/log.d.ts.map +1 -0
  258. package/script/methods/log.js +64 -0
  259. package/script/methods/logBottom.d.ts +5 -0
  260. package/script/methods/logBottom.d.ts.map +1 -0
  261. package/script/methods/logBottom.js +15 -0
  262. package/script/methods/removeTable.d.ts.map +1 -1
  263. package/script/methods/removeTable.js +2 -0
  264. package/script/methods/removeTables.d.ts.map +1 -1
  265. package/script/methods/removeTables.js +7 -0
  266. package/script/methods/renameTable.d.ts.map +1 -1
  267. package/script/methods/renameTable.js +4 -0
  268. package/script/methods/rowProportions.d.ts +1 -0
  269. package/script/methods/rowProportions.d.ts.map +1 -1
  270. package/script/methods/rowProportions.js +5 -1
  271. package/script/methods/rowToText.js +2 -2
  272. package/script/methods/selectTables.d.ts.map +1 -1
  273. package/script/methods/selectTables.js +6 -0
  274. package/script/methods/setTypes.d.ts +1 -1
  275. package/script/methods/setTypes.d.ts.map +1 -1
  276. package/script/methods/stream.d.ts.map +1 -1
  277. package/script/methods/stream.js +2 -1
  278. package/script/methods/updateWithJS.d.ts +1 -3
  279. package/script/methods/updateWithJS.d.ts.map +1 -1
  280. package/script/methods/updateWithJS.js +122 -36
package/llm.md CHANGED
@@ -1,7 +1,7 @@
1
1
  # The Simple Data Analysis (Core) Library
2
2
 
3
3
  - Package: `@nshiab/simple-data-analysis-core`
4
- - Version: `2.0.4`
4
+ - Version: `2.0.6`
5
5
 
6
6
  To install the library with Deno, use:
7
7
 
@@ -91,12 +91,16 @@ schema is reserved for versioned SDA metadata. SQLite files can be imported with
91
91
  - **`options.logDuration`**: A flag indicating whether to log the total
92
92
  execution duration.
93
93
  - **`options.rowsToLog`**: The number of rows to display when logging a table.
94
- - **`options.charsToLog`**: The maximum number of characters to display for
95
- text-based cells.
94
+ - **`options.charsToLog`**: The maximum characters to display for text and
95
+ stringified nested cells, including truncation markers (default: 75).
96
+ Independent of column width. Use Infinity for unlimited text.
96
97
  - **`options.typesToLog`**: A flag indicating whether to include data types when
97
98
  logging a table.
98
99
  - **`options.cacheVerbose`**: Whether to log cache hits and misses, code and
99
100
  input changes, TTL status, and cache read/write timing.
101
+ - **`options.expressionSyntax`**: Operator syntax for expressions and custom
102
+ queries: `"js"` (default) translates JavaScript-style operators in SQL
103
+ expressions; `"sql"` preserves SQL unchanged.
100
104
  - **`options.logSQL`**: A flag indicating whether to log SQL immediately before
101
105
  execution.
102
106
  - **`options.explainSQL`**: A flag indicating whether to log DuckDB query plans
@@ -116,7 +120,8 @@ schema is reserved for versioned SDA metadata. SQLite files can be imported with
116
120
 
117
121
  #### `newTable`
118
122
 
119
- Creates a new SimpleTable instance within the database.
123
+ Creates a new SimpleTable instance within the database. The name must be unique
124
+ among registered tables, ignoring ASCII letter case.
120
125
 
121
126
  ##### Signature
122
127
 
@@ -385,9 +390,11 @@ console.log(extensions); // Output: [{ extension_name: "spatial", loaded: true,
385
390
 
386
391
  #### `customQuery`
387
392
 
388
- Executes a custom SQL query directly against the DuckDB instance. Queries run in
389
- UTC. When data is returned, temporal values use the same JavaScript
390
- representations as `SimpleTable.getData()`.
393
+ Executes a custom SQL query directly against the DuckDB instance. With the
394
+ default `SimpleDB.expressionSyntax: "js"`, queries support JavaScript-style
395
+ operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for
396
+ unchanged SQL. Queries run in UTC. When data is returned, temporal values use
397
+ the same JavaScript representations as `SimpleTable.getData()`.
391
398
 
392
399
  `customQuery()` bypasses the dependency and table-generation tracking used by
393
400
  `SimpleTable.cache()`. Reading or changing a table with `customQuery()` can
@@ -643,8 +650,9 @@ Creates an instance of SimpleTable.
643
650
  - **`simpleDB`**: The SimpleDB instance that this table belongs to.
644
651
  - **`options`**: An optional object with configuration options:
645
652
  - **`options.rowsToLog`**: The number of rows to log when displaying table data.
646
- - **`options.charsToLog`**: The maximum number of characters to log for strings.
647
- Useful to avoid logging large text content.
653
+ - **`options.charsToLog`**: The maximum characters to display for text and
654
+ stringified nested cells, including truncation markers (default: 75).
655
+ Independent of column width. Use Infinity for unlimited text.
648
656
  - **`options.typesToLog`**: A boolean indicating whether to include data types
649
657
  when logging a table.
650
658
 
@@ -701,7 +709,8 @@ await table.log();
701
709
 
702
710
  #### `renameTable`
703
711
 
704
- Renames the current table.
712
+ Renames the current table. The new name must not belong to another registered
713
+ table, ignoring ASCII letter case.
705
714
 
706
715
  ##### Signature
707
716
 
@@ -731,19 +740,17 @@ Sets the data types for columns in a new table. If the table already exists, it
731
740
  will be replaced. To convert the types of an existing table, use the
732
741
  `.convert()` method instead.
733
742
 
734
- This method queues the operation; it runs when an async observer method (like
735
- `getData()` or `log()`) is awaited, or when `run()` is called.
736
-
737
743
  ##### Signature
738
744
 
739
745
  ```typescript
740
- setTypes(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | geometry('${string}') | GEOMETRY('${string}')>): this;
746
+ setTypes(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | float[${number}] | FLOAT[${number}] | geometry('${string}') | GEOMETRY('${string}')>): this;
741
747
  ```
742
748
 
743
749
  ##### Parameters
744
750
 
745
751
  - **`types`**: An object specifying the column names and their target data types
746
- (JavaScript or SQL types).
752
+ (JavaScript or SQL types), including JSON, FLOAT[n] vectors, and GEOMETRY with
753
+ a CRS.
747
754
 
748
755
  ##### Returns
749
756
 
@@ -760,27 +767,44 @@ await table.setTypes({
760
767
  }).log();
761
768
  ```
762
769
 
770
+ ```ts
771
+ // Create an empty table with JSON, embedding, and geometry columns.
772
+ await table.setTypes({
773
+ details: "JSON",
774
+ embedding: "FLOAT[3]",
775
+ geom: "GEOMETRY('EPSG:4326')",
776
+ }).log();
777
+ ```
778
+
763
779
  #### `loadArray`
764
780
 
765
- Loads an array of JavaScript objects into the table. This method queues the
766
- load; it runs when an async observer method (like `getData()` or `log()`) is
767
- awaited, or when `run()` is called.
781
+ Loads an array of JavaScript objects into the table. Types are inferred for
782
+ numbers, bigints, strings, booleans, and Date values. Array and object cells
783
+ require an explicit supported type in columnTypes. Types can also be specified
784
+ for scalar columns instead of inferred from their values.
768
785
 
769
786
  JavaScript `Date` values are inferred as DuckDB `TIMESTAMP` values. Their
770
787
  instant is preserved, but JavaScript `Date` does not retain the timezone or
771
- offset originally used to construct it. String values remain `VARCHAR`; use
772
- `convert()` to parse them as temporal values.
788
+ offset originally used to construct it.
789
+
790
+ Declare `JSON` for general nested data, `FLOAT[n]` for a fixed-size float
791
+ vector, or `GEOMETRY('EPSG:4326')` for GeoJSON geometries in WGS84.
773
792
 
774
793
  ##### Signature
775
794
 
776
795
  ```typescript
777
- loadArray(rows: Record<string, unknown>[]): this;
796
+ loadArray(rows: Record<string, unknown>[], options?: { columnTypes?: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "INTEGER" | "BIGINT" | "DOUBLE" | "VARCHAR" | "BOOLEAN" | "DATE" | "TIME" | "TIMESTAMP" | "TIMESTAMP WITH TIME ZONE" | "GEOMETRY('EPSG:4326')" | "geometry('EPSG:4326')" | "JSON" | "json" | FLOAT[${number}] | float[${number}]> }): this;
778
797
  ```
779
798
 
780
799
  ##### Parameters
781
800
 
782
801
  - **`rows`**: An array of objects, where each object represents a row and its
783
802
  properties represent columns.
803
+ - **`options`**: Options for loading the array, captured when called.
804
+ - **`options.columnTypes`**: Types for specific columns. Required for array and
805
+ object cells; omitted scalar columns are inferred, and all-null columns
806
+ default to VARCHAR. Values must be compatible with the selected type without
807
+ losing information.
784
808
 
785
809
  ##### Returns
786
810
 
@@ -789,7 +813,7 @@ The table, so methods can be chained.
789
813
  ##### Examples
790
814
 
791
815
  ```ts
792
- // Load data from an array of objects
816
+ // Load scalar columns without columnTypes; strings and numbers are inferred.
793
817
  const data = [
794
818
  { letter: "a", number: 1 },
795
819
  { letter: "b", number: 2 },
@@ -798,19 +822,53 @@ await table.loadArray(data).log();
798
822
  ```
799
823
 
800
824
  ```ts
801
- // The offset determines the instant; the loaded TIMESTAMP is returned as
802
- // the equivalent UTC JavaScript Date.
825
+ // Choose a numeric type when all-null values provide nothing to infer.
826
+ await table.loadArray(
827
+ [{ name: "A", value: null }, { name: "B", value: null }],
828
+ { columnTypes: { value: "DOUBLE" } },
829
+ ).log();
830
+ ```
831
+
832
+ ```ts
833
+ // Store the Date as a UTC timestamp with JavaScript's millisecond precision.
834
+ // The original -04:00 offset is not retained.
803
835
  await table.loadArray([{
804
836
  observedAt: new Date("2024-04-07T13:00:00-04:00"),
805
837
  }]).log();
806
838
  ```
807
839
 
840
+ ```ts
841
+ // Declare JSON to load nested objects and variable-length arrays directly.
842
+ await table.loadArray([{
843
+ name: "Station",
844
+ metadata: { active: true, tags: ["weather", "urban"] },
845
+ measurements: [1, 2.5, null],
846
+ }], { columnTypes: { metadata: "JSON", measurements: "JSON" } }).log();
847
+ ```
848
+
849
+ ```ts
850
+ // Declare FLOAT[3] to store each three-element array as a fixed-size vector.
851
+ await table.loadArray([{ vector: [1, 2, 3] }], {
852
+ columnTypes: { vector: "FLOAT[3]" },
853
+ }).log();
854
+ ```
855
+
856
+ ```ts
857
+ // Declare a geometry type to store WGS84 GeoJSON as a spatial column.
858
+ await table.loadArray([{
859
+ name: "Montreal",
860
+ geom: { type: "Point", coordinates: [-73.57, 45.50] },
861
+ }], { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } }).log();
862
+ ```
863
+
808
864
  #### `loadData`
809
865
 
810
866
  Loads data from one or more local or remote files into the table. Supported file
811
- formats include CSV, JSON, Parquet, and Excel. This method queues the load; it
812
- runs when an async observer method (like `getData()` or `log()`) is awaited, or
813
- when `run()` is called.
867
+ formats include CSV, JSON, Parquet, and Excel.
868
+
869
+ With the default `SimpleDB.expressionSyntax: "js"`, conditions support
870
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
871
+ `expressionSyntax: "sql"` for unchanged SQL.
814
872
 
815
873
  ##### Signature
816
874
 
@@ -828,10 +886,8 @@ loadData(files: string | string[], options?: { fileType?: "csv" | "dsv" | "json"
828
886
  - **`options.autoDetect`**: A boolean indicating whether to automatically detect
829
887
  the data format. Defaults to `true`.
830
888
  - **`options.conditions`**: A SQL `WHERE` clause expression, without the `WHERE`
831
- keyword, to filter source rows before applying `limit`. Uses the same syntax
832
- as `filter()`, including JavaScript operators. Can reference source columns
833
- excluded from `columns`. Defaults to no filtering; an empty string behaves the
834
- same as omitting this option.
889
+ keyword, to filter source rows before applying `limit`. Can reference source
890
+ columns excluded from `columns`.
835
891
  - **`options.limit`**: A number indicating the maximum number of matching rows
836
892
  to load, after applying `conditions` if provided. Defaults to all matching
837
893
  rows.
@@ -841,7 +897,8 @@ loadData(files: string | string[], options?: { fileType?: "csv" | "dsv" | "json"
841
897
  across multiple files when their structures differ. Missing columns will be
842
898
  filled with `NULL` values. Defaults to `false`.
843
899
  - **`options.columnTypes`**: An object mapping column names to their expected
844
- data types. By default, types are inferred.
900
+ data types. By default, types are inferred. Geometry types are not supported;
901
+ use loadGeoData() instead.
845
902
  - **`options.columns`**: An array of column names to load. When provided, only
846
903
  the specified columns are loaded, reducing memory usage and improving load
847
904
  times. Not supported for Excel files — combining `columns` with Excel files
@@ -931,9 +988,7 @@ await table
931
988
 
932
989
  #### `loadStatCanData`
933
990
 
934
- Downloads a complete Statistics Canada table and loads it into this table. The
935
- method queues the download and load; they run when an async observer method
936
- (like `getData()` or `log()`) is awaited, or when `run()` is called.
991
+ Downloads a complete Statistics Canada table and loads it into this table.
937
992
 
938
993
  Results are cached as Parquet files in `.sda-cache/statcan` by default. Cached
939
994
  data does not expire unless a TTL is provided.
@@ -979,12 +1034,81 @@ await table
979
1034
  .log();
980
1035
  ```
981
1036
 
982
- #### `loadGeoData`
1037
+ #### `loadYahooFinanceData`
983
1038
 
984
- Loads geospatial data from an external file or URL into the table.
1039
+ Downloads historical market data from Yahoo Finance and loads it into this table
1040
+ as `datetime`, `open`, `high`, `low`, `close`, `adjustedClose`, and `volume`
1041
+ columns. `datetime` contains Yahoo's timestamps as JavaScript `Date` values.
1042
+ Unavailable values are preserved as `null`.
985
1043
 
986
- This method queues the operation; it runs when an async observer method (like
987
- `getData()` or `log()`) is awaited, or when `run()` is called.
1044
+ All range boundaries are evaluated in UTC. Use ISO date-time strings ending in
1045
+ `Z`. Yahoo trading sessions may occur on a different UTC date from their
1046
+ exchange-local date, so choose the range appropriate for the market and
1047
+ interval. Avoid numeric constructors such as `new Date(2025, 2, 15)`, which use
1048
+ the runtime's local timezone.
1049
+
1050
+ This method uses an undocumented Yahoo Finance endpoint and is not affiliated
1051
+ with or endorsed by Yahoo. It is provided for educational, research, and
1052
+ journalistic purposes. Before using it, review Yahoo's terms and any applicable
1053
+ data-provider restrictions.
1054
+
1055
+ ##### Signature
1056
+
1057
+ ```typescript
1058
+ loadYahooFinanceData(symbol: string, startDate: Date, endDate: Date, interval: "1d" | "1h" | "1m"): this;
1059
+ ```
1060
+
1061
+ ##### Parameters
1062
+
1063
+ - **`symbol`**: The stock or index symbol, such as `"AAPL"` or `"^GSPTSE"`.
1064
+ - **`startDate`**: The inclusive UTC start of the requested range.
1065
+ - **`endDate`**: The inclusive UTC end of the requested range. The UTC day,
1066
+ hour, or minute containing this instant is included, according to `interval`.
1067
+ - **`interval`**: The interval between observations: daily, hourly, or every
1068
+ minute.
1069
+
1070
+ ##### Returns
1071
+
1072
+ The table, so methods can be chained.
1073
+
1074
+ ##### Throws
1075
+
1076
+ - **`RangeError`**: If either date is invalid or `endDate` is before
1077
+ `startDate`.
1078
+ - **`Error`**: If Yahoo rejects the request or returns no data.
1079
+
1080
+ ##### Examples
1081
+
1082
+ ```ts
1083
+ // Request daily observations using explicit UTC boundaries.
1084
+ await table
1085
+ .loadYahooFinanceData(
1086
+ "^GSPTSE",
1087
+ new Date("2025-03-01T00:00:00Z"),
1088
+ new Date("2025-03-15T00:00:00Z"),
1089
+ "1d",
1090
+ )
1091
+ .log();
1092
+ ```
1093
+
1094
+ ```ts
1095
+ // Include the UTC hours from 13:00 through 16:00.
1096
+ await table
1097
+ .loadYahooFinanceData(
1098
+ "AAPL",
1099
+ new Date("2025-03-14T13:00:00Z"),
1100
+ new Date("2025-03-14T16:00:00Z"),
1101
+ "1h",
1102
+ )
1103
+ .log();
1104
+ ```
1105
+
1106
+ #### `loadGeoData`
1107
+
1108
+ Loads geospatial data from an external file or URL into the table. With the
1109
+ default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style
1110
+ operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for
1111
+ unchanged SQL.
988
1112
 
989
1113
  ##### Signature
990
1114
 
@@ -1003,11 +1127,9 @@ loadGeoData(file: string, options?: { toEPSG4326?: boolean; columns?: string[];
1003
1127
  should remain in the resulting table, usually `"geom"`. By default, all
1004
1128
  columns are loaded.
1005
1129
  - **`options.conditions`**: A SQL `WHERE` clause expression, without the `WHERE`
1006
- keyword, to filter source rows before materialization and reprojection. Uses
1007
- the same syntax as `filter()`, including JavaScript operators. Can reference
1008
- source columns excluded from `columns`. Geometry conditions use the source
1009
- coordinate system. Defaults to no filtering; an empty string behaves the same
1010
- as omitting this option.
1130
+ keyword, to filter source rows before materialization and reprojection. Can
1131
+ reference source columns excluded from `columns`. Geometry conditions use the
1132
+ source coordinate system.
1011
1133
 
1012
1134
  ##### Returns
1013
1135
 
@@ -1060,8 +1182,7 @@ await table
1060
1182
  Loads OpenStreetMap data into the table from a local `.osm` or `.osm.pbf` file,
1061
1183
  a remote file URL, or an Overpass bounding-box query. Pass a path or URL string
1062
1184
  to load an existing file, or pass a bounding box with `filters` to download
1063
- matching features. The method queues the load; it runs when an async observer
1064
- method (like `getData()` or `log()`) is awaited, or when `run()` is called.
1185
+ matching features.
1065
1186
 
1066
1187
  DuckDB's
1067
1188
  [Osmium community extension](https://duckdb.org/community_extensions/extensions/osmium)
@@ -1194,9 +1315,6 @@ method requires an FTS index and creates one automatically when needed. DuckDB
1194
1315
  FTS indexes do not update automatically when the table changes; use
1195
1316
  `overwrite: true` to rebuild the index after modifying the table.
1196
1317
 
1197
- This method queues the operation; it runs when an async observer method (like
1198
- `getData()` or `log()`) is awaited, or when `run()` is called.
1199
-
1200
1318
  ##### Signature
1201
1319
 
1202
1320
  ```typescript
@@ -1270,9 +1388,6 @@ If a VSS index already exists on the table, this method will skip creation and
1270
1388
  log a message (when verbose is enabled), unless the `overwrite` option is set to
1271
1389
  `true`. The index definition is recorded in {@link indexes}.
1272
1390
 
1273
- This method queues the operation; it runs when an async observer method (like
1274
- `getData()` or `log()`) is awaited, or when `run()` is called.
1275
-
1276
1391
  ##### Signature
1277
1392
 
1278
1393
  ```typescript
@@ -1347,9 +1462,6 @@ It reuses the table's existing FTS index unless `overwriteIndex` is `true`.
1347
1462
  DuckDB FTS indexes do not update automatically when the source table changes;
1348
1463
  use `overwriteIndex: true` to rebuild the index after modifying the table.
1349
1464
 
1350
- This method queues the operation; it runs when an async observer method (like
1351
- `getData()` or `log()`) is awaited, or when `run()` is called.
1352
-
1353
1465
  ##### Signature
1354
1466
 
1355
1467
  ```typescript
@@ -1472,9 +1584,6 @@ await table.bm25("italian sauce", "Dish", "Recipe", 5, {
1472
1584
 
1473
1585
  Inserts rows, provided as an array of JavaScript objects, into the table.
1474
1586
 
1475
- This method queues the operation; it runs when an async observer method (like
1476
- `getData()` or `log()`) is awaited, or when `run()` is called.
1477
-
1478
1587
  ##### Signature
1479
1588
 
1480
1589
  ```typescript
@@ -1505,9 +1614,7 @@ await table.insertRows(newRows).log();
1505
1614
 
1506
1615
  Inserts all rows from one or more other tables into this table. If tables do not
1507
1616
  have the same columns, an error will be thrown unless the `unifyColumns` option
1508
- is set to `true`. This method queues the operation; it runs when an async
1509
- observer method (like `getData()` or `log()`) is awaited, or when `run()` is
1510
- called.
1617
+ is set to `true`.
1511
1618
 
1512
1619
  ##### Signature
1513
1620
 
@@ -1549,9 +1656,6 @@ await tableA.insertTables(["tableB", "tableC"], { unifyColumns: true }).log();
1549
1656
 
1550
1657
  Fetches sample data from the simple-data-analysis-core GitHub repository.
1551
1658
 
1552
- This method queues the operation; it runs when an async observer method (like
1553
- `getData()` or `log()`) is awaited, or when `run()` is called.
1554
-
1555
1659
  ##### Signature
1556
1660
 
1557
1661
  ```typescript
@@ -1583,7 +1687,10 @@ await table.loadSample("fires").log();
1583
1687
  #### `clone`
1584
1688
 
1585
1689
  Returns a new table with the same structure and data as this table. The data can
1586
- be optionally filtered, limited to a specific number of rows, and offset.
1690
+ be optionally filtered, limited to a specific number of rows, and offset. With
1691
+ the default `SimpleDB.expressionSyntax: "js"`, conditions support
1692
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
1693
+ `expressionSyntax: "sql"` for unchanged SQL.
1587
1694
 
1588
1695
  If `conditions`, `limit`, and `offset` are all used, they are applied in this
1589
1696
  order: `conditions` (WHERE clause) first, then `offset`, and finally `limit`
@@ -1591,9 +1698,6 @@ order: `conditions` (WHERE clause) first, then `offset`, and finally `limit`
1591
1698
 
1592
1699
  Note that cloning large tables can be a slow operation.
1593
1700
 
1594
- This method queues the operation; it runs when an async observer method (like
1595
- `getData()` or `log()`) is awaited, or when `run()` is called.
1596
-
1597
1701
  ##### Signature
1598
1702
 
1599
1703
  ```typescript
@@ -1674,9 +1778,6 @@ const tableB = await tableA.clone({
1674
1778
  Clones an existing column in this table, creating a new column with identical
1675
1779
  values.
1676
1780
 
1677
- This method queues the operation; it runs when an async observer method (like
1678
- `getData()` or `log()`) is awaited, or when `run()` is called.
1679
-
1680
1781
  ##### Signature
1681
1782
 
1682
1783
  ```typescript
@@ -1709,9 +1810,6 @@ different time points.
1709
1810
  table. For meaningful results, ensure your data is sorted appropriately (e.g.,
1710
1811
  by date/time for time-series analysis) before calling this method.
1711
1812
 
1712
- This method queues the operation; it runs when an async observer method (like
1713
- `getData()` or `log()`) is awaited, or when `run()` is called.
1714
-
1715
1813
  ##### Signature
1716
1814
 
1717
1815
  ```typescript
@@ -1776,9 +1874,6 @@ use it as the X-axis, so that interpolated values are proportional to the actual
1776
1874
  distances between X-axis values rather than treating every row as equidistant.
1777
1875
  When `interpolateBy` is set, `interpolate` is automatically assumed `true`.
1778
1876
 
1779
- This method queues the operation; it runs when an async observer method (like
1780
- `getData()` or `log()`) is awaited, or when `run()` is called.
1781
-
1782
1877
  ##### Signature
1783
1878
 
1784
1879
  ```typescript
@@ -1852,11 +1947,9 @@ Sorts the rows of the table based on specified column(s) and order(s). If no
1852
1947
  columns are specified, all columns are sorted from left to right in ascending
1853
1948
  order.
1854
1949
 
1855
- This method queues the operation; it runs when an async observer method (like
1856
- `getData()` or `log()`) is awaited, or when `run()` is called. Order-preserving
1857
- transformations queued after a sort retain that order. Operations such as joins,
1858
- grouping, aggregation, and sampling do not guarantee input order; chain `sort()`
1859
- after them when deterministic output order matters.
1950
+ Order-preserving transformations after a sort retain that order. Operations such
1951
+ as joins, grouping, aggregation, and sampling do not guarantee input order;
1952
+ chain `sort()` after them when deterministic output order matters.
1860
1953
 
1861
1954
  ##### Signature
1862
1955
 
@@ -1902,9 +1995,7 @@ await table.sort({ column1: "asc" }, { lang: { column1: "fr" } }).log();
1902
1995
 
1903
1996
  #### `selectColumns`
1904
1997
 
1905
- Selects specific columns in the table, removing all others. This method queues
1906
- the operation; it runs when an async observer method (like `getData()` or
1907
- `log()`) is awaited, or when `run()` is called.
1998
+ Selects specific columns in the table, removing all others.
1908
1999
 
1909
2000
  ##### Signature
1910
2001
 
@@ -1936,9 +2027,6 @@ await table.selectColumns("productName").log();
1936
2027
 
1937
2028
  Skips the first `n` rows of the table, effectively removing them.
1938
2029
 
1939
- This method queues the operation; it runs when an async observer method (like
1940
- `getData()` or `log()`) is awaited, or when `run()` is called.
1941
-
1942
2030
  ##### Signature
1943
2031
 
1944
2032
  ```typescript
@@ -1991,9 +2079,6 @@ console.log(hasAgeColumn); // Output: true or false
1991
2079
  Selects random rows from the table, removing all others. You can optionally
1992
2080
  specify a seed to ensure repeatable sampling.
1993
2081
 
1994
- This method queues the operation; it runs when an async observer method (like
1995
- `getData()` or `log()`) is awaited, or when `run()` is called.
1996
-
1997
2082
  ##### Signature
1998
2083
 
1999
2084
  ```typescript
@@ -2035,9 +2120,6 @@ await table.sample("10%", { seed: 123 }).log();
2035
2120
  Selects a specified number of rows from this table. An offset can be applied to
2036
2121
  skip initial rows, and the results can be output to a new table.
2037
2122
 
2038
- This method queues the operation; it runs when an async observer method (like
2039
- `getData()` or `log()`) is awaited, or when `run()` is called.
2040
-
2041
2123
  ##### Signature
2042
2124
 
2043
2125
  ```typescript
@@ -2089,9 +2171,6 @@ const topCustomersTable = await table.selectRows(75, {
2089
2171
  Removes duplicate rows from this table, keeping only unique rows. Note that the
2090
2172
  resulting data order might differ from the original.
2091
2173
 
2092
- This method queues the operation; it runs when an async observer method (like
2093
- `getData()` or `log()`) is awaited, or when `run()` is called.
2094
-
2095
2174
  ##### Signature
2096
2175
 
2097
2176
  ```typescript
@@ -2130,9 +2209,7 @@ await table.removeDuplicates({ on: ["firstName", "lastName"] }).log();
2130
2209
 
2131
2210
  Removes rows with missing values from this table. By default, missing values
2132
2211
  include SQL `NULL`, as well as string representations like `"NULL"`, `"null"`,
2133
- `"NaN"`, `"undefined"`, and empty strings `""`. This method queues the
2134
- operation; it runs when an async observer method (like `getData()` or `log()`)
2135
- is awaited, or when `run()` is called.
2212
+ `"NaN"`, `"undefined"`, and empty strings `""`.
2136
2213
 
2137
2214
  ##### Signature
2138
2215
 
@@ -2180,9 +2257,7 @@ await table.removeMissing({ columns: "age", missingValues: [-1] }).log();
2180
2257
  #### `trim`
2181
2258
 
2182
2259
  Trims specified characters from the beginning, end, or both sides of string
2183
- values in the given columns. This method queues the operation; it runs when an
2184
- async observer method (like `getData()` or `log()`) is awaited, or when `run()`
2185
- is called.
2260
+ values in the given columns.
2186
2261
 
2187
2262
  ##### Signature
2188
2263
 
@@ -2224,10 +2299,9 @@ await table.trim(["description", "notes"], { side: "right" }).log();
2224
2299
  #### `filter`
2225
2300
 
2226
2301
  Filters rows from this table based on SQL conditions. Note that it's often
2227
- faster to use the `removeRows` method for simple removals. You can also use
2228
- JavaScript syntax for conditions (e.g., `&&`, `||`, `===`, `!==`). This method
2229
- queues the operation; it runs when an async observer method (like `getData()` or
2230
- `log()`) is awaited, or when `run()` is called.
2302
+ faster to use the `removeRows` method for simple removals. With the default
2303
+ `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
2304
+ (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
2231
2305
 
2232
2306
  ##### Signature
2233
2307
 
@@ -2272,9 +2346,6 @@ await table.filter(`lastPurchaseDate >= '2023-01-01'`).log();
2272
2346
  Keeps rows in this table that have specific values in specified columns,
2273
2347
  removing all other rows.
2274
2348
 
2275
- This method queues the operation; it runs when an async observer method (like
2276
- `getData()` or `log()`) is awaited, or when `run()` is called.
2277
-
2278
2349
  ##### Signature
2279
2350
 
2280
2351
  ```typescript
@@ -2313,9 +2384,6 @@ await table.keepValues({ status: null }).log();
2313
2384
 
2314
2385
  Removes rows from this table that have specific values in specified columns.
2315
2386
 
2316
- This method queues the operation; it runs when an async observer method (like
2317
- `getData()` or `log()`) is awaited, or when `run()` is called.
2318
-
2319
2387
  ##### Signature
2320
2388
 
2321
2389
  ```typescript
@@ -2354,11 +2422,9 @@ await table.removeValues({ status: null }).log();
2354
2422
  #### `removeRows`
2355
2423
 
2356
2424
  Removes rows from this table based on SQL conditions. This method is similar to
2357
- `filter()`, but removes rows instead of keeping them. You can also use
2358
- JavaScript syntax for conditions (e.g., `&&`, `||`, `===`, `!==`).
2359
-
2360
- This method queues the operation; it runs when an async observer method (like
2361
- `getData()` or `log()`) is awaited, or when `run()` is called.
2425
+ `filter()`, but removes rows instead of keeping them. With the default
2426
+ `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
2427
+ (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
2362
2428
 
2363
2429
  ##### Signature
2364
2430
 
@@ -2404,9 +2470,6 @@ await table.removeRows(
2404
2470
  Renames one or more columns in the table. Throws if a source column does not
2405
2471
  exist, so a typo fails loudly instead of being silently ignored.
2406
2472
 
2407
- This method queues the operation; it runs when an async observer method (like
2408
- `getData()` or `log()`) is awaited, or when `run()` is called.
2409
-
2410
2473
  ##### Signature
2411
2474
 
2412
2475
  ```typescript
@@ -2450,9 +2513,6 @@ await table.renameColumns({ "product_id": "productId" }, { strict: false })
2450
2513
  Cleans column names by removing non-alphanumeric characters and formatting them
2451
2514
  to camel case.
2452
2515
 
2453
- This method queues the operation; it runs when an async observer method (like
2454
- `getData()` or `log()`) is awaited, or when `run()` is called.
2455
-
2456
2516
  ##### Signature
2457
2517
 
2458
2518
  ```typescript
@@ -2523,9 +2583,6 @@ The table will then look like this:
2523
2583
  | Sales | 2022 | 75 |
2524
2584
  | Sales | 2023 | 98 |
2525
2585
 
2526
- This method queues the operation; it runs when an async observer method (like
2527
- `getData()` or `log()`) is awaited, or when `run()` is called.
2528
-
2529
2586
  #### `wider`
2530
2587
 
2531
2588
  Restructures this table by unstacking (pivoting) values, transforming data from
@@ -2584,9 +2641,6 @@ When multiple rows share the same `namesFrom`/grouping combination, their
2584
2641
  `valuesFrom` values are combined with the `options.stat` function (`"sum"` by
2585
2642
  default).
2586
2643
 
2587
- This method queues the operation; it runs when an async observer method (like
2588
- `getData()` or `log()`) is awaited, or when `run()` is called.
2589
-
2590
2644
  #### `convert`
2591
2645
 
2592
2646
  Converts data types of specified columns to target types (JavaScript or SQL
@@ -2606,20 +2660,16 @@ UTC).
2606
2660
  When converting strings to numbers, commas (often used as thousand separators)
2607
2661
  will be automatically removed before conversion.
2608
2662
 
2609
- This method queues the operation; it runs when an async observer method (like
2610
- `getData()` or `log()`) is awaited, or when `run()` is called. If a column
2611
- doesn't exist, the error is thrown at that point too.
2612
-
2613
2663
  ##### Signature
2614
2664
 
2615
2665
  ```typescript
2616
- convert(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean">, options?: { strict?: boolean; datetimeFormat?: string }): this;
2666
+ convert(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | float[${number}] | FLOAT[${number}] | geometry('${string}') | GEOMETRY('${string}')>, options?: { strict?: boolean; datetimeFormat?: string }): this;
2617
2667
  ```
2618
2668
 
2619
2669
  ##### Parameters
2620
2670
 
2621
2671
  - **`types`**: An object mapping column names to their target data types for
2622
- conversion.
2672
+ conversion, including JSON, FLOAT[n] vectors, and GEOMETRY with a CRS.
2623
2673
  - **`options`**: An optional object with configuration options:
2624
2674
  - **`options.strict`**: If `false`, values that cannot be converted will be
2625
2675
  replaced by `NULL` instead of throwing an error. Defaults to `true`.
@@ -2672,6 +2722,21 @@ await table.convert({ column3: "string" }, {
2672
2722
  await table.convert({ amount: "float" }, { strict: false }).log();
2673
2723
  ```
2674
2724
 
2725
+ ```ts
2726
+ // Parse JSON, a fixed-size float vector, and WKT geometry from text.
2727
+ await table.loadArray([{
2728
+ details: '{"count":3}',
2729
+ embedding: "[0.25,0.5,0.75]",
2730
+ geom: "POINT (-73 45)",
2731
+ }])
2732
+ .convert({
2733
+ details: "JSON",
2734
+ embedding: "FLOAT[3]",
2735
+ geom: "GEOMETRY('EPSG:4326')",
2736
+ })
2737
+ .log();
2738
+ ```
2739
+
2675
2740
  #### `removeTable`
2676
2741
 
2677
2742
  Removes the table from the database. After this operation, invoking methods on
@@ -2696,9 +2761,7 @@ await table.removeTable();
2696
2761
 
2697
2762
  #### `removeColumns`
2698
2763
 
2699
- Removes one or more columns from this table. This method queues the operation;
2700
- it runs when an async observer method (like `getData()` or `log()`) is awaited,
2701
- or when `run()` is called.
2764
+ Removes one or more columns from this table.
2702
2765
 
2703
2766
  ##### Signature
2704
2767
 
@@ -2729,21 +2792,22 @@ await table.removeColumns("tempColumn").log();
2729
2792
  #### `addColumn`
2730
2793
 
2731
2794
  Adds a new column to the table based on a specified data type (JavaScript or SQL
2732
- types) and a SQL definition. This method queues the operation; it runs when an
2733
- async observer method (like `getData()` or `log()`) is awaited, or when `run()`
2734
- is called.
2795
+ types) and a SQL definition. With the default `SimpleDB.expressionSyntax: "js"`,
2796
+ expressions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
2797
+ `expressionSyntax: "sql"` for unchanged SQL.
2735
2798
 
2736
2799
  ##### Signature
2737
2800
 
2738
2801
  ```typescript
2739
- addColumn(newColumn: string, type: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | geometry('${string}') | GEOMETRY('${string}'), definition: string): this;
2802
+ addColumn(newColumn: string, type: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | float[${number}] | FLOAT[${number}] | geometry('${string}') | GEOMETRY('${string}'), definition: string): this;
2740
2803
  ```
2741
2804
 
2742
2805
  ##### Parameters
2743
2806
 
2744
2807
  - **`newColumn`**: The name of the new column to be added.
2745
2808
  - **`type`**: The data type for the new column. Can be a JavaScript type (e.g.,
2746
- `"number"`, `"string"`) or a SQL type (e.g., `"integer"`, `"varchar"`).
2809
+ `"number"`, `"string"`) or a SQL type (e.g., `"integer"`, `"varchar"`,
2810
+ `"JSON"`, `"FLOAT[3]"`, `"GEOMETRY('EPSG:4326')"`).
2747
2811
  - **`definition`**: A SQL expression defining how the values for the new column
2748
2812
  should be computed (e.g., `"column1 + column2"`,
2749
2813
  `"ST_Centroid(geom_column)"`).
@@ -2768,6 +2832,23 @@ await table.addColumn(
2768
2832
  ).log();
2769
2833
  ```
2770
2834
 
2835
+ ```ts
2836
+ // Read a number from a JSON object into a new column using dot notation.
2837
+ await table.loadArray([{ details: { count: 3 } }], {
2838
+ columnTypes: { details: "JSON" },
2839
+ })
2840
+ .addColumn("count", "integer", "details.count")
2841
+ .log();
2842
+ ```
2843
+
2844
+ ```ts
2845
+ // Create JSON and fixed-size vector columns with DuckDB expressions.
2846
+ await table.loadArray([{ count: 3 }])
2847
+ .addColumn("details", "JSON", "json_object('count', count)")
2848
+ .addColumn("embedding", "FLOAT[3]", "[0.25, 0.5, 0.75]")
2849
+ .log();
2850
+ ```
2851
+
2771
2852
  #### `extractDatePart`
2772
2853
 
2773
2854
  Extracts one or more components from a temporal column into new columns. Pass a
@@ -2782,9 +2863,6 @@ time parts apply to times and timestamps. `NULL` input values produce `NULL`
2782
2863
  extracted values. Parts extracted from `TIMESTAMP WITH TIME ZONE` values use
2783
2864
  UTC.
2784
2865
 
2785
- This method queues the operation; it runs when an async observer method (like
2786
- `getData()` or `log()`) is awaited, or when `run()` is called.
2787
-
2788
2866
  ##### Signature
2789
2867
 
2790
2868
  ```typescript
@@ -2821,9 +2899,6 @@ await table.extractDatePart("publishedAt", {
2821
2899
  Adds a new column to the table containing the row number, starting at 0 (like an
2822
2900
  index).
2823
2901
 
2824
- This method queues the operation; it runs when an async observer method (like
2825
- `getData()` or `log()`) is awaited, or when `run()` is called.
2826
-
2827
2902
  ##### Signature
2828
2903
 
2829
2904
  ```typescript
@@ -2860,9 +2935,6 @@ Cartesian product of the rows from both tables, meaning all possible pairs of
2860
2935
  rows will be in the resulting table. This means that if the left table has `n`
2861
2936
  rows and the right table has `m` rows, the result will have `n * m` rows.
2862
2937
 
2863
- This method queues the operation; it runs when an async observer method (like
2864
- `getData()` or `log()`) is awaited, or when `run()` is called.
2865
-
2866
2938
  ##### Signature
2867
2939
 
2868
2940
  ```typescript
@@ -2906,10 +2978,7 @@ Merges the data of this table (considered the left table) with another table
2906
2978
  (the right table) based on a common column or multiple columns. Note that the
2907
2979
  order of rows in the returned data is not guaranteed to be the same as in the
2908
2980
  original tables. This operation might create temporary files in a `.tmp` folder;
2909
- consider adding `.tmp` to your `.gitignore`. This method queues the operation;
2910
- it runs when an async observer method (like `getData()` or `log()`) is awaited,
2911
- or when `run()` is called. The join uses the other table's state as of this
2912
- call: operations queued on it afterwards run after the join.
2981
+ consider adding `.tmp` to your `.gitignore`.
2913
2982
 
2914
2983
  ##### Signature
2915
2984
 
@@ -2972,11 +3041,6 @@ order alphabetically by the left column and then by the right column.
2972
3041
  This operation might create temporary files in a `.tmp` folder; consider adding
2973
3042
  `.tmp` to your `.gitignore`.
2974
3043
 
2975
- This method queues the operation; it runs when an async observer method (like
2976
- `getData()` or `log()`) is awaited, or when `run()` is called. The join uses the
2977
- other table's state as of this call: operations queued on it afterwards run
2978
- after the join.
2979
-
2980
3044
  ##### Signature
2981
3045
 
2982
3046
  ```typescript
@@ -3061,9 +3125,6 @@ Similarity is computed using the
3061
3125
  [rapidfuzz](https://query.farm/duckdb_extension_rapidfuzz) DuckDB community
3062
3126
  extension, which is installed and loaded automatically.
3063
3127
 
3064
- This method queues the operation; it runs when an async observer method (like
3065
- `getData()` or `log()`) is awaited, or when `run()` is called.
3066
-
3067
3128
  ##### Signature
3068
3129
 
3069
3130
  ```typescript
@@ -3135,9 +3196,6 @@ await table.fuzzyClean("category", "category", 80, {
3135
3196
 
3136
3197
  Replaces specified strings in the selected columns.
3137
3198
 
3138
- This method queues the operation; it runs when an async observer method (like
3139
- `getData()` or `log()`) is awaited, or when `run()` is called.
3140
-
3141
3199
  ##### Signature
3142
3200
 
3143
3201
  ```typescript
@@ -3195,9 +3253,7 @@ await table.replace("all", { "%": "" }).log();
3195
3253
 
3196
3254
  #### `lower`
3197
3255
 
3198
- Converts string values in the specified columns to lowercase. This method queues
3199
- the operation; it runs when an async observer method (like `getData()` or
3200
- `log()`) is awaited, or when `run()` is called.
3256
+ Converts string values in the specified columns to lowercase.
3201
3257
 
3202
3258
  ##### Signature
3203
3259
 
@@ -3228,9 +3284,7 @@ await table.lower(["column1", "column2"]).log();
3228
3284
 
3229
3285
  #### `upper`
3230
3286
 
3231
- Converts string values in the specified columns to uppercase. This method queues
3232
- the operation; it runs when an async observer method (like `getData()` or
3233
- `log()`) is awaited, or when `run()` is called.
3287
+ Converts string values in the specified columns to uppercase.
3234
3288
 
3235
3289
  ##### Signature
3236
3290
 
@@ -3262,9 +3316,7 @@ await table.upper(["column1", "column2"]).log();
3262
3316
  #### `capitalize`
3263
3317
 
3264
3318
  Capitalizes the first letter of each string in the specified columns and
3265
- converts the rest of the string to lowercase. This method queues the operation;
3266
- it runs when an async observer method (like `getData()` or `log()`) is awaited,
3267
- or when `run()` is called.
3319
+ converts the rest of the string to lowercase.
3268
3320
 
3269
3321
  ##### Signature
3270
3322
 
@@ -3296,9 +3348,6 @@ await table.capitalize(["column1", "column2"]).log();
3296
3348
 
3297
3349
  Truncates string values in a specified column to a maximum number of characters.
3298
3350
 
3299
- This method queues the operation; it runs when an async observer method (like
3300
- `getData()` or `log()`) is awaited, or when `run()` is called.
3301
-
3302
3351
  ##### Signature
3303
3352
 
3304
3353
  ```typescript
@@ -3334,9 +3383,6 @@ The columns must contain string (VARCHAR) values. An error is thrown if any
3334
3383
  column is of a different type. `null` values remain `null`. If any string
3335
3384
  already exceeds the target length, an error is thrown (no silent truncation).
3336
3385
 
3337
- This method queues the operation; it runs when an async observer method (like
3338
- `getData()` or `log()`) is awaited, or when `run()` is called.
3339
-
3340
3386
  ##### Signature
3341
3387
 
3342
3388
  ```typescript
@@ -3386,9 +3432,6 @@ Splits strings in a specified column by a separator and extracts a substring at
3386
3432
  a given index, storing the result in a new or existing column. If the index is
3387
3433
  out of bounds, an empty string will be returned for that row.
3388
3434
 
3389
- This method queues the operation; it runs when an async observer method (like
3390
- `getData()` or `log()`) is awaited, or when `run()` is called.
3391
-
3392
3435
  ##### Signature
3393
3436
 
3394
3437
  ```typescript
@@ -3435,9 +3478,6 @@ extra columns will contain empty strings (unless `strict` is set to `false`). If
3435
3478
  a row has more parts than the number of new columns, an error will be thrown
3436
3479
  unless `strict` is set to `false`.
3437
3480
 
3438
- This method queues the operation; it runs when an async observer method (like
3439
- `getData()` or `log()`) is awaited, or when `run()` is called.
3440
-
3441
3481
  ##### Signature
3442
3482
 
3443
3483
  ```typescript
@@ -3482,9 +3522,6 @@ await table.splitSpread("data", "|", ["col1", "col2"], { strict: false }).log();
3482
3522
  Extracts a specific number of characters from the beginning (left side) of
3483
3523
  string values in the specified column.
3484
3524
 
3485
- This method queues the operation; it runs when an async observer method (like
3486
- `getData()` or `log()`) is awaited, or when `run()` is called.
3487
-
3488
3525
  ##### Signature
3489
3526
 
3490
3527
  ```typescript
@@ -3514,9 +3551,6 @@ await table.firstChars("productCode", 2).log();
3514
3551
  Extracts a specific number of characters from the end (right side) of string
3515
3552
  values in the specified column.
3516
3553
 
3517
- This method queues the operation; it runs when an async observer method (like
3518
- `getData()` or `log()`) is awaited, or when `run()` is called.
3519
-
3520
3554
  ##### Signature
3521
3555
 
3522
3556
  ```typescript
@@ -3545,9 +3579,6 @@ await table.lastChars("productCode", 2).log();
3545
3579
 
3546
3580
  Replaces `NULL` values in the specified columns with a given value.
3547
3581
 
3548
- This method queues the operation; it runs when an async observer method (like
3549
- `getData()` or `log()`) is awaited, or when `run()` is called.
3550
-
3551
3582
  ##### Signature
3552
3583
 
3553
3584
  ```typescript
@@ -3590,9 +3621,6 @@ await table.replaceNulls("all", 0).log();
3590
3621
 
3591
3622
  Concatenates values from specified columns into a new column.
3592
3623
 
3593
- This method queues the operation; it runs when an async observer method (like
3594
- `getData()` or `log()`) is awaited, or when `run()` is called.
3595
-
3596
3624
  ##### Signature
3597
3625
 
3598
3626
  ```typescript
@@ -3639,9 +3667,6 @@ All values must be string, otherwise an error will be thrown. Use the
3639
3667
  If a column value is `NULL`, it will be replaced by `'Unknown'` in the
3640
3668
  concatenated result.
3641
3669
 
3642
- This method queues the operation; it runs when an async observer method (like
3643
- `getData()` or `log()`) is awaited, or when `run()` is called.
3644
-
3645
3670
  ##### Signature
3646
3671
 
3647
3672
  ```typescript
@@ -3701,9 +3726,6 @@ Each value in the specified column is split using the provided separator, and a
3701
3726
  new row is created for each resulting substring. All other column values are
3702
3727
  duplicated across the newly created rows.
3703
3728
 
3704
- This method queues the operation; it runs when an async observer method (like
3705
- `getData()` or `log()`) is awaited, or when `run()` is called.
3706
-
3707
3729
  ##### Signature
3708
3730
 
3709
3731
  ```typescript
@@ -3745,9 +3767,6 @@ Repeats rows based on the values in a column.
3745
3767
  If a row has a value of 3 in the specified column, it will be repeated 3 times.
3746
3768
  If the value is 0 or negative, the row will be removed.
3747
3769
 
3748
- This method queues the operation; it runs when an async observer method (like
3749
- `getData()` or `log()`) is awaited, or when `run()` is called.
3750
-
3751
3770
  ##### Signature
3752
3771
 
3753
3772
  ```typescript
@@ -3791,9 +3810,6 @@ This is the inverse operation of `unnest()`. Multiple rows are combined into
3791
3810
  fewer rows by grouping on specified category columns and concatenating the
3792
3811
  target column values with a separator.
3793
3812
 
3794
- This method queues the operation; it runs when an async observer method (like
3795
- `getData()` or `log()`) is awaited, or when `run()` is called.
3796
-
3797
3813
  ##### Signature
3798
3814
 
3799
3815
  ```typescript
@@ -3834,9 +3850,6 @@ await table.nest("tags", ",", ["country", "city"]).log();
3834
3850
 
3835
3851
  Rounds numeric values in specified columns.
3836
3852
 
3837
- This method queues the operation; it runs when an async observer method (like
3838
- `getData()` or `log()`) is awaited, or when `run()` is called.
3839
-
3840
3853
  ##### Signature
3841
3854
 
3842
3855
  ```typescript
@@ -3888,12 +3901,57 @@ await table.round(["columnA", "columnB"], { decimals: 1, method: "ceiling" })
3888
3901
  await table.round("column1", 2).log();
3889
3902
  ```
3890
3903
 
3891
- #### `updateColumn`
3904
+ #### `addNoise`
3905
+
3906
+ Adds independently generated uniform random noise to numeric values in one or
3907
+ more columns. Each changed value receives an offset between `-max` and `max`.
3908
+ Selected integer and decimal columns become `DOUBLE` columns so fractional noise
3909
+ is retained. This method adds random jitter; it does not provide anonymization
3910
+ or differential privacy guarantees.
3892
3911
 
3893
- Updates values in a specified column using a SQL expression.
3912
+ ##### Signature
3894
3913
 
3895
- This method queues the operation; it runs when an async observer method (like
3896
- `getData()` or `log()`) is awaited, or when `run()` is called.
3914
+ ```typescript
3915
+ addNoise(columns: string | string[], max: number, options?: { onlyDuplicates?: boolean }): this;
3916
+ ```
3917
+
3918
+ ##### Parameters
3919
+
3920
+ - **`columns`**: The numeric column name or array of numeric column names to
3921
+ which noise will be added. When multiple columns are provided, each value
3922
+ receives an independent random offset.
3923
+ - **`max`**: The maximum absolute offset, expressed in each column's units. Must
3924
+ be a finite number greater than or equal to `0`.
3925
+ - **`options`**: An optional object with configuration options:
3926
+ - **`options.onlyDuplicates`**: If `true`, adds noise only to values that occur
3927
+ more than once in their column. Each selected column is evaluated
3928
+ independently, and every occurrence of a duplicated value is changed. Defaults
3929
+ to `false`.
3930
+
3931
+ ##### Returns
3932
+
3933
+ The table, so methods can be chained.
3934
+
3935
+ ##### Examples
3936
+
3937
+ ```ts
3938
+ // Add an offset between -0.01 and 0.01 to every value
3939
+ await table.addNoise("measurement", 0.01).log();
3940
+ ```
3941
+
3942
+ ```ts
3943
+ // Add noise independently to duplicated x values and duplicated y values
3944
+ await table.addNoise(["x", "y"], 0.1, {
3945
+ onlyDuplicates: true,
3946
+ }).log();
3947
+ ```
3948
+
3949
+ #### `updateColumn`
3950
+
3951
+ Updates values in a specified column using a SQL expression. With the default
3952
+ `SimpleDB.expressionSyntax: "js"`, expressions support JavaScript-style
3953
+ operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for
3954
+ unchanged SQL.
3897
3955
 
3898
3956
  ##### Signature
3899
3957
 
@@ -3931,13 +3989,19 @@ await table.updateColumn(
3931
3989
  ).log();
3932
3990
  ```
3933
3991
 
3992
+ ```ts
3993
+ // Update an existing column with a value from a JSON object using dot notation.
3994
+ await table.loadArray([{ count: 0, details: { count: 3 } }], {
3995
+ columnTypes: { details: "JSON" },
3996
+ })
3997
+ .updateColumn("count", "details.count")
3998
+ .log();
3999
+ ```
4000
+
3934
4001
  #### `ranks`
3935
4002
 
3936
4003
  Assigns ranks to rows in a new column based on the values of a specified column.
3937
4004
 
3938
- This method queues the operation; it runs when an async observer method (like
3939
- `getData()` or `log()`) is awaited, or when `run()` is called.
3940
-
3941
4005
  ##### Signature
3942
4006
 
3943
4007
  ```typescript
@@ -3988,9 +4052,6 @@ await table.ranks("sales", "salesRank", { by: ["department", "city"] }).log();
3988
4052
 
3989
4053
  Assigns quantiles to rows in a new column based on specified column values.
3990
4054
 
3991
- This method queues the operation; it runs when an async observer method (like
3992
- `getData()` or `log()`) is awaited, or when `run()` is called.
3993
-
3994
4055
  ##### Signature
3995
4056
 
3996
4057
  ```typescript
@@ -4034,9 +4095,6 @@ await table.quantiles("sales", 4, "salesQuartile").log();
4034
4095
 
4035
4096
  Assigns bins for specified column values based on an interval size.
4036
4097
 
4037
- This method queues the operation; it runs when an async observer method (like
4038
- `getData()` or `log()`) is awaited, or when `run()` is called.
4039
-
4040
4098
  ##### Signature
4041
4099
 
4042
4100
  ```typescript
@@ -4089,7 +4147,7 @@ each row, adding new columns for these proportions.
4089
4147
  ##### Signature
4090
4148
 
4091
4149
  ```typescript
4092
- rowProportions(columns: string[], options?: { suffix?: string; decimals?: number }): this;
4150
+ rowProportions(columns: string[], options?: { suffix?: string; base?: number; decimals?: number }): this;
4093
4151
  ```
4094
4152
 
4095
4153
  ##### Parameters
@@ -4099,6 +4157,8 @@ rowProportions(columns: string[], options?: { suffix?: string; decimals?: number
4099
4157
  - **`options`**: An optional object with configuration options:
4100
4158
  - **`options.suffix`**: A string suffix to append to the names of the new
4101
4159
  columns storing the computed proportions. Defaults to `"Perc"`.
4160
+ - **`options.base`**: A finite positive value that the proportions in each row
4161
+ sum to before rounding. Defaults to `1`.
4102
4162
  - **`options.decimals`**: The number of decimal places to round the computed
4103
4163
  proportions. Defaults to `undefined` (no rounding).
4104
4164
 
@@ -4141,8 +4201,13 @@ The table will then look like this:
4141
4201
  | 2022 | 354 | 278 | 56 | 0.51 | 0.4 | 0.08 |
4142
4202
  | 2023 | 856 | 321 | 221 | 0.61 | 0.23 | 0.16 |
4143
4203
 
4144
- This method queues the operation; it runs when an async observer method (like
4145
- `getData()` or `log()`) is awaited, or when `run()` is called.
4204
+ ```ts
4205
+ // Compute percentages that sum to 100 on each row before rounding
4206
+ await table.rowProportions(["Men", "Women", "NonBinary"], {
4207
+ base: 100,
4208
+ decimals: 1,
4209
+ }).log();
4210
+ ```
4146
4211
 
4147
4212
  #### `rowRanks`
4148
4213
 
@@ -4156,9 +4221,6 @@ to produce one row for each tied column. The `"all"` option can therefore
4156
4221
  increase the table's row count. If null values leave a row without the requested
4157
4222
  rank, the new columns contain null.
4158
4223
 
4159
- This method queues the operation; it runs when an async observer method (like
4160
- `getData()` or `log()`) is awaited, or when `run()` is called.
4161
-
4162
4224
  ##### Signature
4163
4225
 
4164
4226
  ```typescript
@@ -4210,13 +4272,10 @@ await table.rowRanks(["CAQ", "PLQ", "PQ"], {
4210
4272
  Computes proportions vertically over a column's values, relative to the sum of
4211
4273
  all values in that column or group.
4212
4274
 
4213
- This method queues the operation; it runs when an async observer method (like
4214
- `getData()` or `log()`) is awaited, or when `run()` is called.
4215
-
4216
4275
  ##### Signature
4217
4276
 
4218
4277
  ```typescript
4219
- columnProportions(column: string, newColumn: string, options?: { by?: string | string[]; decimals?: number }): this;
4278
+ columnProportions(column: string, newColumn: string, options?: { by?: string | string[]; base?: number; decimals?: number }): this;
4220
4279
  ```
4221
4280
 
4222
4281
  ##### Parameters
@@ -4229,6 +4288,8 @@ columnProportions(column: string, newColumn: string, options?: { by?: string | s
4229
4288
  - **`options`**: An optional object with configuration options:
4230
4289
  - **`options.by`**: The column name or an array of column names to partition by.
4231
4290
  Proportions are calculated independently within each group.
4291
+ - **`options.base`**: A finite positive value that the proportions in the column
4292
+ or each group sum to before rounding. Defaults to `1`.
4232
4293
  - **`options.decimals`**: The number of decimal places to round the computed
4233
4294
  proportions. Defaults to `undefined` (no rounding).
4234
4295
 
@@ -4249,6 +4310,14 @@ await table.columnProportions("column1", "perc", { by: "column2", decimals: 2 })
4249
4310
  .log();
4250
4311
  ```
4251
4312
 
4313
+ ```ts
4314
+ // Compute percentages that sum to 100 before rounding
4315
+ await table.columnProportions("sales", "sales_percentage", {
4316
+ base: 100,
4317
+ decimals: 1,
4318
+ }).log();
4319
+ ```
4320
+
4252
4321
  ```ts
4253
4322
  // Compute proportions for 'sales' by 'region' and 'product_type'
4254
4323
  await table.columnProportions("sales", "sales_proportion", {
@@ -4262,9 +4331,6 @@ Creates a summary table from selected columns, optionally grouped by other
4262
4331
  columns. This method allows you to aggregate data, calculate statistics (e.g.,
4263
4332
  count, mean, sum), and group results by categorical columns.
4264
4333
 
4265
- This method queues the operation; it runs when an async observer method (like
4266
- `getData()` or `log()`) is awaited, or when `run()` is called.
4267
-
4268
4334
  ##### Signature
4269
4335
 
4270
4336
  ```typescript
@@ -4392,9 +4458,6 @@ summarized nor used for labels contain `NULL` in the added rows. A stat string
4392
4458
  is also used as its row label; pass an object to customize that label. If
4393
4459
  `options.stats` is omitted, every supported stat is added.
4394
4460
 
4395
- This method queues the operation; it runs when an async observer method (like
4396
- `getData()` or `log()`) is awaited, or when `run()` is called.
4397
-
4398
4461
  ##### Signature
4399
4462
 
4400
4463
  ```typescript
@@ -4449,9 +4512,6 @@ await table.addSummaryRows("all", "region", {
4449
4512
  Computes the cumulative sum of values in a column. For this method to work
4450
4513
  properly, ensure your data is sorted first.
4451
4514
 
4452
- This method queues the operation; it runs when an async observer method (like
4453
- `getData()` or `log()`) is awaited, or when `run()` is called.
4454
-
4455
4515
  ##### Signature
4456
4516
 
4457
4517
  ```typescript
@@ -4500,9 +4560,6 @@ column. For rows without enough preceding or following rows to form a complete
4500
4560
  window, `NULL` will be returned. For this method to work properly, ensure your
4501
4561
  data is sorted by the relevant column(s) first.
4502
4562
 
4503
- This method queues the operation; it runs when an async observer method (like
4504
- `getData()` or `log()`) is awaited, or when `run()` is called.
4505
-
4506
4563
  ##### Signature
4507
4564
 
4508
4565
  ```typescript
@@ -4559,9 +4616,6 @@ specified, the method computes the correlations for all numeric column
4559
4616
  combinations. Note that correlation is symmetrical: the correlation of `x` with
4560
4617
  `y` is the same as `y` with `x`.
4561
4618
 
4562
- This method queues the operation; it runs when an async observer method (like
4563
- `getData()` or `log()`) is awaited, or when `run()` is called.
4564
-
4565
4619
  ##### Signature
4566
4620
 
4567
4621
  ```typescript
@@ -4628,9 +4682,6 @@ the method computes linear regression analysis for all numeric column
4628
4682
  permutations. Note that linear regression analysis is asymmetrical: the linear
4629
4683
  regression of `x` over `y` is not the same as `y` over `x`.
4630
4684
 
4631
- This method queues the operation; it runs when an async observer method (like
4632
- `getData()` or `log()`) is awaited, or when `run()` is called.
4633
-
4634
4685
  ##### Signature
4635
4686
 
4636
4687
  ```typescript
@@ -4695,9 +4746,6 @@ await table.linearRegressions({ decimals: 3 }).log();
4695
4746
  Identifies outliers in a specified column using the Interquartile Range (IQR)
4696
4747
  method.
4697
4748
 
4698
- This method queues the operation; it runs when an async observer method (like
4699
- `getData()` or `log()`) is awaited, or when `run()` is called.
4700
-
4701
4749
  ##### Signature
4702
4750
 
4703
4751
  ```typescript
@@ -4733,9 +4781,6 @@ await table.outliersIQR("salary", "salaryOutlier", { by: "gender" }).log();
4733
4781
 
4734
4782
  Computes the Z-score for values in a specified column.
4735
4783
 
4736
- This method queues the operation; it runs when an async observer method (like
4737
- `getData()` or `log()`) is awaited, or when `run()` is called.
4738
-
4739
4784
  ##### Signature
4740
4785
 
4741
4786
  ```typescript
@@ -4778,9 +4823,6 @@ await table.zScore("score", "scoreZScore", { decimals: 2 }).log();
4778
4823
 
4779
4824
  Normalizes the values in a column using min-max normalization.
4780
4825
 
4781
- This method queues the operation; it runs when an async observer method (like
4782
- `getData()` or `log()`) is awaited, or when `run()` is called.
4783
-
4784
4826
  ##### Signature
4785
4827
 
4786
4828
  ```typescript
@@ -4846,9 +4888,6 @@ an instant. Construct them with an explicit timezone, such as
4846
4888
  `new Date("2001-01-01T00:00:00Z")`; date-time strings without `Z` or an offset
4847
4889
  use the user's local timezone.
4848
4890
 
4849
- This method queues the operation; it runs when an async observer method (like
4850
- `getData()` or `log()`) is awaited, or when `run()` is called.
4851
-
4852
4891
  ##### Signature
4853
4892
 
4854
4893
  ```typescript
@@ -4927,16 +4966,19 @@ Updates data in the table using a JavaScript function. The function receives the
4927
4966
  existing rows as an array of objects and must return the modified rows as an
4928
4967
  array of objects. This method offers high flexibility for data manipulation but
4929
4968
  can be slow for large tables as it involves transferring data between DuckDB and
4930
- JavaScript. This method does not work with tables containing geometries.
4969
+ JavaScript. Before writing a JavaScript callback, check for an existing SDA
4970
+ method that performs the same operation; it will usually be faster and more
4971
+ efficient.
4931
4972
 
4932
- This method queues the update; the dataModifier function runs when an async
4933
- observer method (like `getData()` or `log()`) is awaited, or when `run()` is
4934
- called.
4973
+ If the table has geometry columns, the callback can read and modify their
4974
+ GeoJSON geometry objects directly. Extra properties added to these objects are
4975
+ ignored; create new columns on the returned rows to store additional attributes.
4976
+ All geometry columns must use EPSG:4326; use reproject() first if needed.
4935
4977
 
4936
4978
  ##### Signature
4937
4979
 
4938
4980
  ```typescript
4939
- updateWithJS(dataModifier: ((rows: Record<string, unknown>[]) => Promise<Record<string, unknown>[]>) | ((rows: Record<string, unknown>[]) => Record<string, unknown>[]), options?: { batchSize?: number }): this;
4981
+ updateWithJS(dataModifier: ((rows: Record<string, unknown>[]) => Promise<Record<string, unknown>[]>) | ((rows: Record<string, unknown>[]) => Record<string, unknown>[]), options?: { batchSize?: number; columnTypes?: Record<string, string> }): this;
4940
4982
  ```
4941
4983
 
4942
4984
  ##### Parameters
@@ -4947,7 +4989,15 @@ updateWithJS(dataModifier: ((rows: Record<string, unknown>[]) => Promise<Record<
4947
4989
  - **`options`**: An optional object with configuration options:
4948
4990
  - **`options.batchSize`**: If provided, rows are processed in batches of this
4949
4991
  size instead of all at once, so large tables don't have to be materialized
4950
- entirely in memory. The modifier function is called once per batch.
4992
+ entirely in memory. This limits input rows per batch, but does not eliminate
4993
+ conversion work or bound geometry size or callback output expansion. The
4994
+ modifier function is called once per batch. Tables with a column named `rowid`
4995
+ or `__sda_rowid` (case-insensitive) are rejected before the modifier runs;
4996
+ rename the column or omit batchSize.
4997
+ - **`options.columnTypes`**: Explicit types for newly added columns, following
4998
+ loadArray's columnTypes contract. Declare geometry as GEOMETRY('EPSG:4326');
4999
+ geometry is never inferred from object shape. Existing columns cannot be
5000
+ redeclared.
4951
5001
 
4952
5002
  ##### Returns
4953
5003
 
@@ -4956,38 +5006,73 @@ The table, so methods can be chained.
4956
5006
  ##### Examples
4957
5007
 
4958
5008
  ```ts
4959
- // Count words correctly across multilingual article text.
4960
- const segmenter = new Intl.Segmenter(undefined, { granularity: "word" });
4961
- const table = await sdb
4962
- .newTable()
4963
- .loadData("articles.csv")
4964
- .updateWithJS((rows) => {
4965
- return rows.map((row) => ({
5009
+ // Double the value in each row with a JavaScript callback.
5010
+ await table.loadArray([{ value: 1 }, { value: 2 }])
5011
+ .updateWithJS((rows) =>
5012
+ rows.map((row) => ({
5013
+ value: Number(row.value) * 2,
5014
+ }))
5015
+ )
5016
+ .log();
5017
+ ```
5018
+
5019
+ ```ts
5020
+ // Process two rows per callback to limit how many rows are handled at once.
5021
+ await table.loadArray([{ value: 1 }, { value: 2 }, { value: 3 }])
5022
+ .updateWithJS((rows) =>
5023
+ rows.map((row) => ({
5024
+ value: Number(row.value) * 2,
5025
+ })), { batchSize: 2 })
5026
+ .log();
5027
+ ```
5028
+
5029
+ ```ts
5030
+ // Add a JSON object column to keep related attributes together.
5031
+ await table.loadArray([{ name: "Station" }])
5032
+ .updateWithJS((rows) =>
5033
+ rows.map((row) => ({
4966
5034
  ...row,
4967
- wordCount: typeof row.text === "string"
4968
- ? [...segmenter.segment(row.text)].filter((part) => part.isWordLike)
4969
- .length
4970
- : null,
4971
- }));
4972
- })
5035
+ details: { active: true, category: "weather" },
5036
+ })), { columnTypes: { details: "JSON" } })
4973
5037
  .log();
4974
5038
  ```
4975
5039
 
4976
5040
  ```ts
4977
- // Enrich reviews with scores from an external service, 100 at a time.
4978
- const reviews = await table
4979
- .updateWithJS(async (rows) => {
4980
- const response = await fetch("https://api.example.com/score", {
4981
- method: "POST",
4982
- headers: { "content-type": "application/json" },
4983
- body: JSON.stringify(rows.map((row) => row.review)),
4984
- });
4985
- const scores = await response.json() as number[];
4986
- return rows.map((row, index) => ({ ...row, score: scores[index] }));
4987
- }, { batchSize: 100 })
5041
+ // Add a three-element float vector, such as an embedding.
5042
+ await table.loadArray([{ name: "Station" }])
5043
+ .updateWithJS((rows) =>
5044
+ rows.map((row) => ({
5045
+ ...row,
5046
+ embedding: [0.25, 0.5, 0.75],
5047
+ })), { columnTypes: { embedding: "FLOAT[3]" } })
4988
5048
  .log();
4989
5049
  ```
4990
5050
 
5051
+ ```ts
5052
+ // Move each point slightly east by editing its longitude.
5053
+ await sdb.newTable().loadArray([{
5054
+ geom: { type: "Point", coordinates: [-73, 45] },
5055
+ }], { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } })
5056
+ .updateWithJS((rows) =>
5057
+ rows.map((row) => {
5058
+ // This example loads only Points. Narrow the type for coordinate access.
5059
+ const geom = row.geom as { type: "Point"; coordinates: number[] } | null;
5060
+ if (geom !== null) geom.coordinates[0] += 0.01;
5061
+ return row;
5062
+ })
5063
+ ).log();
5064
+ ```
5065
+
5066
+ ```ts
5067
+ // Create a geometry column from longitude and latitude values.
5068
+ await sdb.newTable().loadArray([{ longitude: -73, latitude: 45 }])
5069
+ .updateWithJS((rows) =>
5070
+ rows.map((row) => ({
5071
+ ...row,
5072
+ geom: { type: "Point", coordinates: [row.longitude, row.latitude] },
5073
+ })), { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } }).log();
5074
+ ```
5075
+
4991
5076
  #### `getSchema`
4992
5077
 
4993
5078
  Returns the schema of the table, including column names and their data types.
@@ -5093,9 +5178,6 @@ Normalizes string values in a column by:
5093
5178
  Produces identical output to `journalism-format`'s `normalizeString()` function
5094
5179
  for all common cases including accented Latin characters.
5095
5180
 
5096
- This method queues the operation; it runs when an async observer method (like
5097
- `getData()` or `log()`) is awaited, or when `run()` is called.
5098
-
5099
5181
  ##### Signature
5100
5182
 
5101
5183
  ```typescript
@@ -5155,9 +5237,75 @@ const columnCount = await table.getColumnCount();
5155
5237
  console.log(columnCount); // e.g., 3
5156
5238
  ```
5157
5239
 
5240
+ #### `addCharacterCount`
5241
+
5242
+ Adds a new column containing the number of characters in each string in the
5243
+ specified column. Counts are based on Unicode code points, not grapheme
5244
+ clusters. A user-perceived character composed of multiple code points, such as
5245
+ some emoji or decomposed accented letters, counts as multiple characters. `NULL`
5246
+ input values produce `NULL` counts.
5247
+
5248
+ ##### Signature
5249
+
5250
+ ```typescript
5251
+ addCharacterCount(column: string, newColumn: string): this;
5252
+ ```
5253
+
5254
+ ##### Parameters
5255
+
5256
+ - **`column`**: The name of the column containing the strings to count.
5257
+ - **`newColumn`**: The name of the new column where the character counts will be
5258
+ stored.
5259
+
5260
+ ##### Returns
5261
+
5262
+ The table, so methods can be chained.
5263
+
5264
+ ##### Examples
5265
+
5266
+ ```ts
5267
+ // Add a character count for each value in the 'name' column
5268
+ await table.addCharacterCount("name", "nameCharacterCount").log();
5269
+ ```
5270
+
5271
+ #### `addWordCount`
5272
+
5273
+ Adds a new column containing the word count for each string in the specified
5274
+ column. A word is any contiguous sequence of non-whitespace characters. Spaces,
5275
+ tabs, and line breaks separate words. Punctuation is not removed, so a
5276
+ standalone punctuation sequence counts as a word. Text without whitespace counts
5277
+ as one word, regardless of language. Empty or whitespace-only strings produce
5278
+ `0`, and `NULL` input values produce `NULL` counts.
5279
+
5280
+ ##### Signature
5281
+
5282
+ ```typescript
5283
+ addWordCount(column: string, newColumn: string): this;
5284
+ ```
5285
+
5286
+ ##### Parameters
5287
+
5288
+ - **`column`**: The name of the column containing the strings to count.
5289
+ - **`newColumn`**: The name of the new column where the word counts will be
5290
+ stored.
5291
+
5292
+ ##### Returns
5293
+
5294
+ The table, so methods can be chained.
5295
+
5296
+ ##### Examples
5297
+
5298
+ ```ts
5299
+ // Add a word count for each value in the 'article' column
5300
+ await table.addWordCount("article", "wordCount").log();
5301
+ ```
5302
+
5158
5303
  #### `getCharacterCount`
5159
5304
 
5160
- Returns the total number of characters in a column storing strings.
5305
+ Returns the total number of characters in a column storing strings. Counts are
5306
+ based on Unicode code points, not grapheme clusters. A user-perceived character
5307
+ composed of multiple code points, such as some emoji or decomposed accented
5308
+ letters, counts as multiple characters.
5161
5309
 
5162
5310
  ##### Signature
5163
5311
 
@@ -5184,7 +5332,9 @@ console.log(totalChars); // e.g., 523
5184
5332
 
5185
5333
  #### `getRowCount`
5186
5334
 
5187
- Returns the number of rows in the table.
5335
+ Returns the number of rows in the table. With the default
5336
+ `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
5337
+ (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
5188
5338
 
5189
5339
  ##### Signature
5190
5340
 
@@ -5674,9 +5824,11 @@ console.log(uniqueCategories); // e.g., ["Books", "Clothing", "Electronics"]
5674
5824
 
5675
5825
  #### `getFirstRow`
5676
5826
 
5677
- Returns the first row of the table, optionally filtered by SQL conditions. You
5678
- can also use JavaScript syntax for conditions (e.g., `&&`, `||`, `===`, `!==`).
5679
- Temporal values use the same JavaScript representations as `getData()`.
5827
+ Returns the first row of the table, optionally filtered by SQL conditions. With
5828
+ the default `SimpleDB.expressionSyntax: "js"`, conditions support
5829
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
5830
+ `expressionSyntax: "sql"` for unchanged SQL. Temporal values use the same
5831
+ JavaScript representations as `getData()`.
5680
5832
 
5681
5833
  ##### Signature
5682
5834
 
@@ -5713,9 +5865,11 @@ console.log(firstRowBooks);
5713
5865
 
5714
5866
  #### `getLastRow`
5715
5867
 
5716
- Returns the last row of the table, optionally filtered by SQL conditions. You
5717
- can also use JavaScript syntax for conditions (e.g., `&&`, `||`, `===`, `!==`).
5718
- Temporal values use the same JavaScript representations as `getData()`.
5868
+ Returns the last row of the table, optionally filtered by SQL conditions. With
5869
+ the default `SimpleDB.expressionSyntax: "js"`, conditions support
5870
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
5871
+ `expressionSyntax: "sql"` for unchanged SQL. Temporal values use the same
5872
+ JavaScript representations as `getData()`.
5719
5873
 
5720
5874
  ##### Signature
5721
5875
 
@@ -5753,8 +5907,10 @@ console.log(lastRowBooks);
5753
5907
  #### `getTop`
5754
5908
 
5755
5909
  Returns the top `n` rows of the table, optionally filtered by SQL conditions.
5756
- You can also use JavaScript syntax for conditions (e.g., `&&`, `||`, `===`,
5757
- `!==`). Temporal values use the same JavaScript representations as `getData()`.
5910
+ With the default `SimpleDB.expressionSyntax: "js"`, conditions support
5911
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
5912
+ `expressionSyntax: "sql"` for unchanged SQL. Temporal values use the same
5913
+ JavaScript representations as `getData()`.
5758
5914
 
5759
5915
  ##### Signature
5760
5916
 
@@ -5791,9 +5947,10 @@ console.log(top5Books);
5791
5947
 
5792
5948
  Returns the bottom `n` rows of the table, optionally filtered by SQL conditions.
5793
5949
  By default, the last row will be returned first. To preserve the original order,
5794
- use the `originalOrder` option. You can also use JavaScript syntax for
5795
- conditions (e.g., `&&`, `||`, `===`, `!==`). Temporal values use the same
5796
- JavaScript representations as `getData()`.
5950
+ use the `originalOrder` option. With the default
5951
+ `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
5952
+ (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
5953
+ Temporal values use the same JavaScript representations as `getData()`.
5797
5954
 
5798
5955
  ##### Signature
5799
5956
 
@@ -5842,9 +5999,10 @@ console.log(bottom5Books);
5842
5999
  #### `getRow`
5843
6000
 
5844
6001
  Returns a single row that matches the specified conditions. If no row matches or
5845
- if more than one row matches, an error is thrown by default. You can also use
5846
- JavaScript syntax for conditions (e.g., `AND`, `||`, `===`, `!==`). Temporal
5847
- values use the same JavaScript representations as `getData()`.
6002
+ if more than one row matches, an error is thrown by default. With the default
6003
+ `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
6004
+ (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
6005
+ Temporal values use the same JavaScript representations as `getData()`.
5848
6006
 
5849
6007
  ##### Signature
5850
6008
 
@@ -5893,14 +6051,28 @@ console.log(flexibleRow);
5893
6051
  #### `getData`
5894
6052
 
5895
6053
  Returns the data from the table as an array of objects, optionally filtered by
5896
- SQL conditions. You can also use JavaScript syntax for conditions (e.g., `&&`,
5897
- `||`, `===`, `!==`).
6054
+ SQL conditions. With the default `SimpleDB.expressionSyntax: "js"`, conditions
6055
+ support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
6056
+ `expressionSyntax: "sql"` for unchanged SQL.
5898
6057
 
5899
6058
  Top-level DuckDB `DATE` and `TIMESTAMP` columns are returned as JavaScript
5900
6059
  `Date` objects interpreted in UTC. `TIMESTAMP WITH TIME ZONE` values are
5901
6060
  returned as UTC strings, preserving DuckDB's microsecond precision; JavaScript
5902
6061
  `Date` supports only milliseconds.
5903
6062
 
6063
+ Top-level `BIGINT`, `UBIGINT`, `HUGEINT`, and `UHUGEINT` columns are returned as
6064
+ JavaScript numbers. Values outside `Number.MIN_SAFE_INTEGER` through
6065
+ `Number.MAX_SAFE_INTEGER` throw instead of losing precision. Use
6066
+ `convert({ columnName: "string" })` before `getData()` to retrieve their exact
6067
+ digits as strings.
6068
+
6069
+ SQL arrays and lists, including float vectors, return their actual elements as
6070
+ JavaScript arrays. Null cells and null elements remain null.
6071
+
6072
+ Nested SQL lists and structs become JavaScript arrays and objects. Within them,
6073
+ `BIGINT`, `UBIGINT`, `HUGEINT`, `UHUGEINT`, and `DECIMAL` values remain strings
6074
+ to preserve precision, even for small integers.
6075
+
5904
6076
  ##### Signature
5905
6077
 
5906
6078
  ```typescript
@@ -5950,11 +6122,24 @@ console.log(booksData);
5950
6122
  const preview = await table.getData({ limit: 2 });
5951
6123
  ```
5952
6124
 
6125
+ ```ts
6126
+ // Read nested integers without losing digits. SQL creates the nested types.
6127
+ await sdb.customQuery(`CREATE TABLE nested_example AS SELECT
6128
+ [42::BIGINT, 9007199254740993::BIGINT] AS ids,
6129
+ {'count': 2::INTEGER, 'amount': 123.450::DECIMAL(6,3)} AS details`);
6130
+ const nested = sdb.newTable("nested_example");
6131
+ const rows = await nested.getData();
6132
+ // [{ ids: ["42", "9007199254740993"], details: { count: 2, amount: "123.450" } }]
6133
+ await nested.log();
6134
+ ```
6135
+
5953
6136
  #### `stream`
5954
6137
 
5955
6138
  Streams the table rows one by one as an async iterator, without materializing
5956
6139
  the whole table in memory. Values are converted to JavaScript types the same way
5957
- as `getData()`.
6140
+ as `getData()`. With the default `SimpleDB.expressionSyntax: "js"`, conditions
6141
+ support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
6142
+ `expressionSyntax: "sql"` for unchanged SQL.
5958
6143
 
5959
6144
  The underlying DuckDB result is streamed chunk by chunk, so tables larger than
5960
6145
  the available memory can be iterated. Avoid running other queries on the same
@@ -6001,9 +6186,10 @@ for await (
6001
6186
  #### `getDataAsCSV`
6002
6187
 
6003
6188
  Returns the data from the table as a CSV string, optionally filtered by SQL
6004
- conditions. You can also use JavaScript syntax for conditions (e.g., `&&`, `||`,
6005
- `===`, `!==`). Temporal values are first converted as they are in `getData()`,
6006
- then serialized using UTC date and timestamp text.
6189
+ conditions. With the default `SimpleDB.expressionSyntax: "js"`, conditions
6190
+ support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
6191
+ `expressionSyntax: "sql"` for unchanged SQL. Temporal values are first converted
6192
+ as they are in `getData()`, then serialized using UTC date and timestamp text.
6007
6193
 
6008
6194
  ##### Signature
6009
6195
 
@@ -6053,9 +6239,6 @@ console.log(booksDataCSV);
6053
6239
 
6054
6240
  Creates point geometries from latitude (y) and longitude (x) columns.
6055
6241
 
6056
- This method queues the operation; it runs when an async observer method (like
6057
- `getData()` or `log()`) is awaited, or when `run()` is called.
6058
-
6059
6242
  ##### Signature
6060
6243
 
6061
6244
  ```typescript
@@ -6096,9 +6279,6 @@ await table.createPoints("y", "x", "geom", { projection: "EPSG:3347" }).log();
6096
6279
 
6097
6280
  Adds a column with boolean values indicating the validity of geometries.
6098
6281
 
6099
- This method queues the operation; it runs when an async observer method (like
6100
- `getData()` or `log()`) is awaited, or when `run()` is called.
6101
-
6102
6282
  ##### Signature
6103
6283
 
6104
6284
  ```typescript
@@ -6135,9 +6315,6 @@ await table.addGeoValidity("isValidMyGeom", { column: "myGeom" }).log();
6135
6315
 
6136
6316
  Adds a column with the number of vertices (points) in each geometry.
6137
6317
 
6138
- This method queues the operation; it runs when an async observer method (like
6139
- `getData()` or `log()`) is awaited, or when `run()` is called.
6140
-
6141
6318
  ##### Signature
6142
6319
 
6143
6320
  ```typescript
@@ -6173,9 +6350,6 @@ await table.addVertexCount("myGeomVertices", { column: "myGeom" }).log();
6173
6350
 
6174
6351
  Attempts to make invalid geometries valid without removing any vertices.
6175
6352
 
6176
- This method queues the operation; it runs when an async observer method (like
6177
- `getData()` or `log()`) is awaited, or when `run()` is called.
6178
-
6179
6353
  ##### Signature
6180
6354
 
6181
6355
  ```typescript
@@ -6208,9 +6382,6 @@ await table.fixGeo("myGeom").log();
6208
6382
  Adds a column with boolean values indicating whether geometries are closed
6209
6383
  (e.g., polygons) or open (e.g., linestrings).
6210
6384
 
6211
- This method queues the operation; it runs when an async observer method (like
6212
- `getData()` or `log()`) is awaited, or when `run()` is called.
6213
-
6214
6385
  ##### Signature
6215
6386
 
6216
6387
  ```typescript
@@ -6247,9 +6418,6 @@ await table.addGeoClosedStatus("boundaryClosed", { column: "boundaryGeom" })
6247
6418
  Adds a column with the geometry type (e.g., `"POINT"`, `"LINESTRING"`,
6248
6419
  `"POLYGON"`) for each geometry.
6249
6420
 
6250
- This method queues the operation; it runs when an async observer method (like
6251
- `getData()` or `log()`) is awaited, or when `run()` is called.
6252
-
6253
6421
  ##### Signature
6254
6422
 
6255
6423
  ```typescript
@@ -6288,9 +6456,6 @@ vice-versa). **Warning:** This method should be used with caution as it directly
6288
6456
  manipulates coordinate order and can affect the accuracy of geospatial
6289
6457
  operations if not used correctly.
6290
6458
 
6291
- This method queues the operation; it runs when an async observer method (like
6292
- `getData()` or `log()`) is awaited, or when `run()` is called.
6293
-
6294
6459
  ##### Signature
6295
6460
 
6296
6461
  ```typescript
@@ -6318,14 +6483,65 @@ await table.flipCoordinates().log();
6318
6483
  await table.flipCoordinates("myGeom").log();
6319
6484
  ```
6320
6485
 
6486
+ #### `addGeoNoise`
6487
+
6488
+ Moves EPSG:4326 point geometries to random locations within a maximum
6489
+ great-circle distance of their original positions.
6490
+
6491
+ Points are sampled uniformly within the requested distance using the spherical
6492
+ direct geodesic (destination-point) formula and the same spherical Earth model
6493
+ as DuckDB's `ST_Distance_Sphere()`. This accounts for longitude scale changing
6494
+ with latitude. This method adds random jitter; it does not provide anonymization
6495
+ or differential privacy guarantees.
6496
+
6497
+ This method supports only `POINT` geometries in `EPSG:4326`. Null and empty
6498
+ geometries are preserved.
6499
+
6500
+ ##### Signature
6501
+
6502
+ ```typescript
6503
+ addGeoNoise(maxDistance: number, options?: { column?: string; unit?: "m" | "km"; onlyDuplicates?: boolean }): this;
6504
+ ```
6505
+
6506
+ ##### Parameters
6507
+
6508
+ - **`maxDistance`**: The maximum great-circle displacement in the selected unit.
6509
+ Must be a finite number greater than or equal to `0` and no greater than half
6510
+ Earth's circumference.
6511
+ - **`options`**: An optional object with configuration options:
6512
+ - **`options.column`**: The name of the EPSG:4326 point geometry column. If
6513
+ omitted, the method will automatically attempt to find a geometry column.
6514
+ - **`options.unit`**: The unit of `maxDistance`: `"m"` for metres or `"km"` for
6515
+ kilometres. Defaults to `"m"`.
6516
+ - **`options.onlyDuplicates`**: If `true`, moves only points whose complete
6517
+ original geometry occurs more than once in the selected geometry column. Every
6518
+ point in a duplicated group is moved. Defaults to `false`.
6519
+
6520
+ ##### Returns
6521
+
6522
+ The table, so methods can be chained.
6523
+
6524
+ ##### Examples
6525
+
6526
+ ```ts
6527
+ // Move every point by no more than 500 metres
6528
+ await table.addGeoNoise(500).log();
6529
+ ```
6530
+
6531
+ ```ts
6532
+ // Separate duplicated points by up to 0.1 kilometres
6533
+ await table.addGeoNoise(0.1, {
6534
+ column: "geom",
6535
+ unit: "km",
6536
+ onlyDuplicates: true,
6537
+ }).log();
6538
+ ```
6539
+
6321
6540
  #### `reducePrecision`
6322
6541
 
6323
6542
  Reduces the precision of geometries in a specified column to a given number of
6324
6543
  decimal places.
6325
6544
 
6326
- This method queues the operation; it runs when an async observer method (like
6327
- `getData()` or `log()`) is awaited, or when `run()` is called.
6328
-
6329
6545
  ##### Signature
6330
6546
 
6331
6547
  ```typescript
@@ -6361,9 +6577,6 @@ await table.reducePrecision(2, { column: "myGeom" }).log();
6361
6577
  Reprojects the geometries in a specified column to another Spatial Reference
6362
6578
  System (SRS).
6363
6579
 
6364
- This method queues the operation; it runs when an async observer method (like
6365
- `getData()` or `log()`) is awaited, or when `run()` is called.
6366
-
6367
6580
  ##### Signature
6368
6581
 
6369
6582
  ```typescript
@@ -6399,9 +6612,6 @@ await table.reproject("EPSG:3347", { column: "myGeom" }).log();
6399
6612
  Computes the area of geometries in square meters (`"m2"`) or optionally square
6400
6613
  kilometers (`"km2"`). The input geometry is assumed to be in EPSG:4326 (WGS84).
6401
6614
 
6402
- This method queues the operation; it runs when an async observer method (like
6403
- `getData()` or `log()`) is awaited, or when `run()` is called.
6404
-
6405
6615
  ##### Signature
6406
6616
 
6407
6617
  ```typescript
@@ -6451,9 +6661,6 @@ await table.area("myGeomArea", { column: "myGeom" }).log();
6451
6661
  Computes the length of line geometries in meters (`"m"`) or optionally
6452
6662
  kilometers (`"km"`). The input geometry is assumed to be in EPSG:4326 (WGS84).
6453
6663
 
6454
- This method queues the operation; it runs when an async observer method (like
6455
- `getData()` or `log()`) is awaited, or when `run()` is called.
6456
-
6457
6664
  ##### Signature
6458
6665
 
6459
6666
  ```typescript
@@ -6503,9 +6710,6 @@ await table.length("routeLength", { column: "routeGeom" }).log();
6503
6710
  Computes the perimeter of polygon geometries in meters (`"m"`) or optionally
6504
6711
  kilometers (`"km"`). The input geometry is assumed to be in EPSG:4326 (WGS84).
6505
6712
 
6506
- This method queues the operation; it runs when an async observer method (like
6507
- `getData()` or `log()`) is awaited, or when `run()` is called.
6508
-
6509
6713
  ##### Signature
6510
6714
 
6511
6715
  ```typescript
@@ -6557,9 +6761,6 @@ Computes a buffer (a polygon representing a specified distance around a
6557
6761
  geometry) for geometries in a specified column. The distance is in the Spatial
6558
6762
  Reference System (SRS) unit of the input geometries.
6559
6763
 
6560
- This method queues the operation; it runs when an async observer method (like
6561
- `getData()` or `log()`) is awaited, or when `run()` is called.
6562
-
6563
6764
  ##### Signature
6564
6765
 
6565
6766
  ```typescript
@@ -6598,10 +6799,7 @@ Merges the data of this table (considered the left table) with another table
6598
6799
  (the right table) based on a spatial relationship. Note that the order of rows
6599
6800
  in the returned data is not guaranteed to be the same as in the original tables.
6600
6801
  This operation might create temporary files in a `.tmp` folder; consider adding
6601
- `.tmp` to your `.gitignore`. This method queues the operation; it runs when an
6602
- async observer method (like `getData()` or `log()`) is awaited, or when `run()`
6603
- is called. The join uses the other table's state as of this call: operations
6604
- queued on it afterwards run after the join.
6802
+ `.tmp` to your `.gitignore`.
6605
6803
 
6606
6804
  ##### Signature
6607
6805
 
@@ -6694,9 +6892,6 @@ const tableC = await tableA.joinGeo(tableB, "intersect", {
6694
6892
  Computes the intersection of two sets of geometries, creating new geometries
6695
6893
  where they overlap.
6696
6894
 
6697
- This method queues the operation; it runs when an async observer method (like
6698
- `getData()` or `log()`) is awaited, or when `run()` is called.
6699
-
6700
6895
  ##### Signature
6701
6896
 
6702
6897
  ```typescript
@@ -6727,9 +6922,6 @@ await table.intersection("geomA", "geomB", "intersectGeom").log();
6727
6922
  Computes the geometric difference between two geometries, returning the portion
6728
6923
  of the first geometry that does not intersect the second.
6729
6924
 
6730
- This method queues the operation; it runs when an async observer method (like
6731
- `getData()` or `log()`) is awaited, or when `run()` is called.
6732
-
6733
6925
  ##### Signature
6734
6926
 
6735
6927
  ```typescript
@@ -6760,9 +6952,6 @@ await table.difference("geomA", "geomB", "geomA_minus_geomB").log();
6760
6952
 
6761
6953
  Fills holes in polygon geometries.
6762
6954
 
6763
- This method queues the operation; it runs when an async observer method (like
6764
- `getData()` or `log()`) is awaited, or when `run()` is called.
6765
-
6766
6955
  ##### Signature
6767
6956
 
6768
6957
  ```typescript
@@ -6795,9 +6984,6 @@ await table.fillHoles("polygonGeom").log();
6795
6984
  Returns `TRUE` if two geometries intersect (overlap in any way), and `FALSE`
6796
6985
  otherwise.
6797
6986
 
6798
- This method queues the operation; it runs when an async observer method (like
6799
- `getData()` or `log()`) is awaited, or when `run()` is called.
6800
-
6801
6987
  ##### Signature
6802
6988
 
6803
6989
  ```typescript
@@ -6828,9 +7014,6 @@ await table.intersects("geomA", "geomB", "doIntersect").log();
6828
7014
  Returns `TRUE` if every point of a geometry in `column` is covered by a geometry
6829
7015
  in `containerColumn`, including their boundaries, and `FALSE` otherwise.
6830
7016
 
6831
- This method queues the operation; it runs when an async observer method (like
6832
- `getData()` or `log()`) is awaited, or when `run()` is called.
6833
-
6834
7017
  ##### Signature
6835
7018
 
6836
7019
  ```typescript
@@ -6862,9 +7045,6 @@ await table.coveredBy("pointGeom", "polygonGeom", "isCovered").log();
6862
7045
  Computes the union of two geometries, creating a new geometry that represents
6863
7046
  the merged area of both.
6864
7047
 
6865
- This method queues the operation; it runs when an async observer method (like
6866
- `getData()` or `log()`) is awaited, or when `run()` is called.
6867
-
6868
7048
  ##### Signature
6869
7049
 
6870
7050
  ```typescript
@@ -6895,9 +7075,6 @@ await table.union("geomA", "geomB", "unionGeom").log();
6895
7075
  Extracts the latitude (y) and longitude (x) coordinates from point geometries.
6896
7076
  The input geometry is assumed to be in EPSG:4326 (WGS84).
6897
7077
 
6898
- This method queues the operation; it runs when an async observer method (like
6899
- `getData()` or `log()`) is awaited, or when `run()` is called.
6900
-
6901
7078
  ##### Signature
6902
7079
 
6903
7080
  ```typescript
@@ -6928,9 +7105,6 @@ await table.extractLatLon("geom", "lat", "lon").log();
6928
7105
  Simplifies geometries while preserving their overall coverage. A higher
6929
7106
  tolerance results in more significant simplification.
6930
7107
 
6931
- This method queues the operation; it runs when an async observer method (like
6932
- `getData()` or `log()`) is awaited, or when `run()` is called.
6933
-
6934
7108
  ##### Signature
6935
7109
 
6936
7110
  ```typescript
@@ -6969,9 +7143,6 @@ await table.simplify(0.05, { column: "myGeom", simplifyBoundary: false }).log();
6969
7143
  Computes the centroid of geometries. The values are returned in the SRS unit of
6970
7144
  the input geometries.
6971
7145
 
6972
- This method queues the operation; it runs when an async observer method (like
6973
- `getData()` or `log()`) is awaited, or when `run()` is called.
6974
-
6975
7146
  ##### Signature
6976
7147
 
6977
7148
  ```typescript
@@ -7006,9 +7177,6 @@ await table.centroid("areaCentroid", { column: "areaGeom" }).log();
7006
7177
 
7007
7178
  Generates a random point within the geometries of a specified column.
7008
7179
 
7009
- This method queues the operation; it runs when an async observer method (like
7010
- `getData()` or `log()`) is awaited, or when `run()` is called.
7011
-
7012
7180
  ##### Signature
7013
7181
 
7014
7182
  ```typescript
@@ -7054,9 +7222,6 @@ input geometries. You can optionally specify `"spheroid"` or `"haversine"`
7054
7222
  methods to get results in meters or kilometers. If using `"spheroid"` or
7055
7223
  `"haversine"`, the input geometries must be in EPSG:4326 (WGS84).
7056
7224
 
7057
- This method queues the operation; it runs when an async observer method (like
7058
- `getData()` or `log()`) is awaited, or when `run()` is called.
7059
-
7060
7225
  ##### Signature
7061
7226
 
7062
7227
  ```typescript
@@ -7121,9 +7286,6 @@ await table.distance("area1", "area2", "distance_spheroid_km", {
7121
7286
  Unnests geometries recursively, transforming multi-part geometries (e.g.,
7122
7287
  MultiPolygon) into individual single-part geometries (e.g., Polygon).
7123
7288
 
7124
- This method queues the operation; it runs when an async observer method (like
7125
- `getData()` or `log()`) is awaited, or when `run()` is called.
7126
-
7127
7289
  ##### Signature
7128
7290
 
7129
7291
  ```typescript
@@ -7156,9 +7318,6 @@ await table.unnestGeo("multiGeom").log();
7156
7318
  Adds the bounding box coordinates of geometries in a specified column as four
7157
7319
  new columns: `minLon`, `minLat`, `maxLon`, and `maxLat`.
7158
7320
 
7159
- This method queues the operation; it runs when an async observer method (like
7160
- `getData()` or `log()`) is awaited, or when `run()` is called.
7161
-
7162
7321
  ##### Signature
7163
7322
 
7164
7323
  ```typescript
@@ -7197,9 +7356,6 @@ await table.addBoundingBox({ column: "geom", decimals: 2 }).log();
7197
7356
  Aggregates geometries in a specified column based on a chosen aggregation
7198
7357
  method.
7199
7358
 
7200
- This method queues the operation; it runs when an async observer method (like
7201
- `getData()` or `log()`) is awaited, or when `run()` is called.
7202
-
7203
7359
  ##### Signature
7204
7360
 
7205
7361
  ```typescript
@@ -7251,9 +7407,6 @@ const intersectionTable = await table.aggregateGeo("intersection", {
7251
7407
 
7252
7408
  Transforms closed linestring geometries into polygon geometries.
7253
7409
 
7254
- This method queues the operation; it runs when an async observer method (like
7255
- `getData()` or `log()`) is awaited, or when `run()` is called.
7256
-
7257
7410
  ##### Signature
7258
7411
 
7259
7412
  ```typescript
@@ -7319,8 +7472,10 @@ console.log(areaBbox);
7319
7472
 
7320
7473
  #### `getGeoData`
7321
7474
 
7322
- Returns the table's geospatial data as a GeoJSON object. If the table has
7323
- multiple geometry columns, you must specify which one to use.
7475
+ Returns the table's geospatial data as a GeoJSON FeatureCollection. Each row
7476
+ becomes a Feature: the selected geometry column becomes its geometry, and the
7477
+ remaining columns become its properties. If the table has multiple geometry
7478
+ columns, you must specify which one to use.
7324
7479
 
7325
7480
  ##### Signature
7326
7481
 
@@ -7364,7 +7519,8 @@ console.log(rewoundGeojson);
7364
7519
  #### `writeData`
7365
7520
 
7366
7521
  Writes the table's data to a file in various formats (CSV, JSON, Parquet,
7367
- DuckDB, SQLite). If the specified path does not exist, it will be created.
7522
+ DuckDB, SQLite). Tables with geometry columns must use `writeGeoData()` instead.
7523
+ If the specified path does not exist, it will be created.
7368
7524
 
7369
7525
  ##### Signature
7370
7526
 
@@ -7385,8 +7541,8 @@ async writeData(file: string, options?: { compression?: boolean; dataAsArrays?:
7385
7541
  `{ "col1": [v1, v2], "col2": [v3, v4] }`) instead of an array of objects. This
7386
7542
  can reduce file size for web projects. You can use the `arraysToData` function
7387
7543
  from the
7388
- [journalism library](https://jsr.io/@nshiab/journalism/doc/~/arraysToData) to
7389
- convert it back.
7544
+ [journalism-format library](https://jsr.io/@nshiab/journalism-format/doc/~/arraysToData)
7545
+ to convert it back.
7390
7546
  - **`options.formatDates`**: For CSV and JSON files only. If `true`, date and
7391
7547
  timestamp columns will be formatted as ISO 8601 strings (e.g.,
7392
7548
  `"2025-01-01T01:00:00.000Z"`). Defaults to `false`.
@@ -7503,28 +7659,28 @@ await table.writeGeoData("./output_high_precision.geojson", {
7503
7659
  Caches the results of computations in `./.sda-cache`. You should add
7504
7660
  `./.sda-cache` to your `.gitignore` file.
7505
7661
 
7506
- Cache entries are stored as DuckDB database files. Full-text search (FTS)
7507
- indexes are persisted in the cache file and restored directly on a cache hit.
7508
- Vector similarity search (VSS/HNSW) indexes are not persisted in the cache file;
7509
- their definitions are stored as metadata and used to rebuild the indexes on
7510
- every cache hit. If loading the entry or restoring its indexes fails, the
7511
- computation runs again and replaces the cache entry.
7512
-
7513
7662
  `cache()` automatically tracks whether earlier SDA operations changed the table.
7514
7663
  It also records every other already registered `SimpleTable` read through
7515
7664
  `SimpleTable` methods while `compute` runs and invalidates the cached step when
7516
7665
  any of their generations change. Tables created inside `compute` are part of the
7517
7666
  computation itself and are not dependencies.
7518
7667
 
7668
+ Inside `compute`, you may modify the table being cached and create or modify
7669
+ temporary tables, provided you remove them before the callback finishes. Other
7670
+ tables that already existed may be read but not modified, renamed, or removed.
7671
+ `customQuery()` bypasses enforcement of these rules.
7672
+
7519
7673
  `SimpleDB.customQuery()` bypasses this tracking. Reading or changing a table
7520
7674
  with `customQuery()` can therefore return stale cached data. Include a value
7521
7675
  that identifies the custom query's dependencies in `options.inputs` (such as a
7522
7676
  table content hash), or use tracked `SimpleTable` methods.
7523
7677
 
7524
- `compute` may modify only the table being cached. Other tables that existed
7525
- before `compute` must remain read-only. Temporary tables may be created and
7526
- modified inside `compute`, but they must be removed before it finishes because a
7527
- cache hit does not run `compute` again.
7678
+ Cache entries are stored as DuckDB database files. Full-text search (FTS)
7679
+ indexes are persisted in the cache file and restored directly on a cache hit.
7680
+ Vector similarity search (VSS/HNSW) indexes are not persisted in the cache file;
7681
+ their definitions are stored as metadata and used to rebuild the indexes on
7682
+ every cache hit. If loading the entry or restoring its indexes fails, the
7683
+ computation runs again and replaces the cache entry.
7528
7684
 
7529
7685
  ##### Signature
7530
7686
 
@@ -7624,8 +7780,18 @@ await summary.log();
7624
7780
 
7625
7781
  Logs a specified number of rows from the table to the console. By default, the
7626
7782
  first 10 rows are logged. You can optionally log the column types and filter the
7627
- data based on conditions. You can also use JavaScript syntax for conditions
7628
- (e.g., `&&`, `||`, `===`, `!==`).
7783
+ data based on conditions. SQL dates and timestamps retain their native precision
7784
+ for display, including temporal infinities. Lists and objects are stringified
7785
+ using the same nested representations as `getData()`, then truncated according
7786
+ to `charsToLog`. Unsafe top-level large integers throw, just as with
7787
+ `getData()`. Type annotations describe the SQL type and the JavaScript
7788
+ extraction type before display formatting; nulls do not determine the column's
7789
+ annotation. Colors reflect the SQL value's meaning, so exact decimal strings use
7790
+ numeric coloring. Column width is independent of the content truncation budget.
7791
+
7792
+ With the default `SimpleDB.expressionSyntax: "js"`, conditions support
7793
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
7794
+ `expressionSyntax: "sql"` for unchanged SQL.
7629
7795
 
7630
7796
  ##### Signature
7631
7797
 
@@ -7866,6 +8032,10 @@ Logs the bottom `n` rows of the table to the console. By default, the last row
7866
8032
  will be returned first. To preserve the original order, use the `originalOrder`
7867
8033
  option.
7868
8034
 
8035
+ Uses the same precision-preserving temporal display, nested stringification,
8036
+ numeric coloring, and `charsToLog` truncation as `log()`. Unsafe top-level large
8037
+ integers throw, just as with `getData()`.
8038
+
7869
8039
  ##### Signature
7870
8040
 
7871
8041
  ```typescript