@nshiab/simple-data-analysis-core 2.0.5 → 2.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/README.md +118 -12
  2. package/esm/class/Simple.d.ts +8 -2
  3. package/esm/class/Simple.d.ts.map +1 -1
  4. package/esm/class/Simple.js +9 -3
  5. package/esm/class/SimpleDB.d.ts +72 -1
  6. package/esm/class/SimpleDB.d.ts.map +1 -1
  7. package/esm/class/SimpleDB.js +81 -1
  8. package/esm/class/SimpleTable.d.ts +271 -288
  9. package/esm/class/SimpleTable.d.ts.map +1 -1
  10. package/esm/class/SimpleTable.js +269 -340
  11. package/esm/helpers/LogValue.d.ts +8 -0
  12. package/esm/helpers/LogValue.d.ts.map +1 -0
  13. package/esm/helpers/LogValue.js +23 -0
  14. package/esm/helpers/cacheTableDependencies.d.ts +9 -0
  15. package/esm/helpers/cacheTableDependencies.d.ts.map +1 -1
  16. package/esm/helpers/cacheTableDependencies.js +131 -21
  17. package/esm/helpers/cleanSQL.d.ts +1 -1
  18. package/esm/helpers/cleanSQL.d.ts.map +1 -1
  19. package/esm/helpers/cleanSQL.js +268 -23
  20. package/esm/helpers/flushAllTables.d.ts.map +1 -1
  21. package/esm/helpers/flushAllTables.js +23 -28
  22. package/esm/helpers/foldIdentifier.d.ts +3 -0
  23. package/esm/helpers/foldIdentifier.d.ts.map +1 -0
  24. package/esm/helpers/foldIdentifier.js +4 -0
  25. package/esm/helpers/geometryFromJSON.d.ts +3 -0
  26. package/esm/helpers/geometryFromJSON.d.ts.map +1 -0
  27. package/esm/helpers/geometryFromJSON.js +4 -0
  28. package/esm/helpers/index.d.ts +1 -0
  29. package/esm/helpers/index.d.ts.map +1 -1
  30. package/esm/helpers/index.js +1 -0
  31. package/esm/helpers/logData.d.ts.map +1 -1
  32. package/esm/helpers/logData.js +45 -44
  33. package/esm/helpers/makeLogConverter.d.ts +5 -0
  34. package/esm/helpers/makeLogConverter.d.ts.map +1 -0
  35. package/esm/helpers/makeLogConverter.js +64 -0
  36. package/esm/helpers/osmFiles.d.ts.map +1 -1
  37. package/esm/helpers/parseTypes.d.ts +1 -1
  38. package/esm/helpers/parseTypes.d.ts.map +1 -1
  39. package/esm/helpers/parseTypes.js +4 -0
  40. package/esm/helpers/pendingOps.d.ts +4 -2
  41. package/esm/helpers/pendingOps.d.ts.map +1 -1
  42. package/esm/helpers/prepareGeometry.d.ts +3 -0
  43. package/esm/helpers/prepareGeometry.d.ts.map +1 -0
  44. package/esm/helpers/prepareGeometry.js +99 -0
  45. package/esm/helpers/prepareJSON.d.ts +3 -0
  46. package/esm/helpers/prepareJSON.d.ts.map +1 -0
  47. package/esm/helpers/prepareJSON.js +50 -0
  48. package/esm/helpers/printTable.d.ts +5 -4
  49. package/esm/helpers/printTable.d.ts.map +1 -1
  50. package/esm/helpers/printTable.js +32 -36
  51. package/esm/helpers/queryDB.js +1 -1
  52. package/esm/helpers/queueAsyncBarrier.d.ts +4 -0
  53. package/esm/helpers/queueAsyncBarrier.d.ts.map +1 -1
  54. package/esm/helpers/queueAsyncBarrier.js +4 -0
  55. package/esm/helpers/queueOp.d.ts.map +1 -1
  56. package/esm/helpers/queueOp.js +5 -3
  57. package/esm/helpers/readMutationRows.d.ts +9 -0
  58. package/esm/helpers/readMutationRows.d.ts.map +1 -0
  59. package/esm/helpers/readMutationRows.js +94 -0
  60. package/esm/helpers/referencedTables.d.ts +9 -0
  61. package/esm/helpers/referencedTables.d.ts.map +1 -0
  62. package/esm/helpers/referencedTables.js +22 -0
  63. package/esm/helpers/runQuery.d.ts.map +1 -1
  64. package/esm/helpers/runQuery.js +11 -13
  65. package/esm/helpers/scanSQL.d.ts +7 -0
  66. package/esm/helpers/scanSQL.d.ts.map +1 -0
  67. package/esm/helpers/scanSQL.js +71 -0
  68. package/esm/helpers/tableRegistry.d.ts +2 -0
  69. package/esm/helpers/tableRegistry.d.ts.map +1 -1
  70. package/esm/helpers/tableRegistry.js +12 -4
  71. package/esm/helpers/updateColumnsWithJS.d.ts +52 -0
  72. package/esm/helpers/updateColumnsWithJS.d.ts.map +1 -0
  73. package/esm/helpers/updateColumnsWithJS.js +139 -0
  74. package/esm/methods/addColumn.d.ts +1 -1
  75. package/esm/methods/addColumn.d.ts.map +1 -1
  76. package/esm/methods/bins.d.ts +1 -1
  77. package/esm/methods/bins.d.ts.map +1 -1
  78. package/esm/methods/bins.js +45 -29
  79. package/esm/methods/bm25.d.ts.map +1 -1
  80. package/esm/methods/bm25.js +1 -2
  81. package/esm/methods/cache.d.ts.map +1 -1
  82. package/esm/methods/cache.js +4 -6
  83. package/esm/methods/convert.d.ts +2 -2
  84. package/esm/methods/convert.d.ts.map +1 -1
  85. package/esm/methods/convert.js +5 -0
  86. package/esm/methods/getBottom.d.ts +1 -1
  87. package/esm/methods/getBottom.d.ts.map +1 -1
  88. package/esm/methods/getBottom.js +2 -2
  89. package/esm/methods/getData.d.ts +1 -1
  90. package/esm/methods/getData.d.ts.map +1 -1
  91. package/esm/methods/getData.js +2 -2
  92. package/esm/methods/join.d.ts.map +1 -1
  93. package/esm/methods/join.js +8 -16
  94. package/esm/methods/joinGeo.d.ts.map +1 -1
  95. package/esm/methods/joinGeo.js +10 -19
  96. package/esm/methods/loadArray.d.ts +1 -1
  97. package/esm/methods/loadArray.d.ts.map +1 -1
  98. package/esm/methods/loadArray.js +74 -19
  99. package/esm/methods/loadData.js +3 -0
  100. package/esm/methods/log.d.ts +7 -0
  101. package/esm/methods/log.d.ts.map +1 -0
  102. package/esm/methods/log.js +63 -0
  103. package/esm/methods/logBottom.d.ts +5 -0
  104. package/esm/methods/logBottom.d.ts.map +1 -0
  105. package/esm/methods/logBottom.js +9 -0
  106. package/esm/methods/removeTable.d.ts.map +1 -1
  107. package/esm/methods/removeTable.js +2 -0
  108. package/esm/methods/removeTables.d.ts.map +1 -1
  109. package/esm/methods/removeTables.js +7 -0
  110. package/esm/methods/renameTable.d.ts.map +1 -1
  111. package/esm/methods/renameTable.js +4 -0
  112. package/esm/methods/rowToText.js +2 -2
  113. package/esm/methods/selectTables.d.ts.map +1 -1
  114. package/esm/methods/selectTables.js +6 -0
  115. package/esm/methods/setTypes.d.ts +1 -1
  116. package/esm/methods/setTypes.d.ts.map +1 -1
  117. package/esm/methods/stream.d.ts.map +1 -1
  118. package/esm/methods/stream.js +2 -1
  119. package/esm/methods/updateWithJS.d.ts +1 -3
  120. package/esm/methods/updateWithJS.d.ts.map +1 -1
  121. package/esm/methods/updateWithJS.js +123 -37
  122. package/llm.md +365 -461
  123. package/llms.txt +1 -1
  124. package/package.json +1 -1
  125. package/script/class/Simple.d.ts +8 -2
  126. package/script/class/Simple.d.ts.map +1 -1
  127. package/script/class/Simple.js +9 -3
  128. package/script/class/SimpleDB.d.ts +72 -1
  129. package/script/class/SimpleDB.d.ts.map +1 -1
  130. package/script/class/SimpleDB.js +81 -1
  131. package/script/class/SimpleTable.d.ts +271 -288
  132. package/script/class/SimpleTable.d.ts.map +1 -1
  133. package/script/class/SimpleTable.js +269 -340
  134. package/script/helpers/LogValue.d.ts +8 -0
  135. package/script/helpers/LogValue.d.ts.map +1 -0
  136. package/script/helpers/LogValue.js +26 -0
  137. package/script/helpers/cacheTableDependencies.d.ts +9 -0
  138. package/script/helpers/cacheTableDependencies.d.ts.map +1 -1
  139. package/script/helpers/cacheTableDependencies.js +170 -20
  140. package/script/helpers/cleanSQL.d.ts +1 -1
  141. package/script/helpers/cleanSQL.d.ts.map +1 -1
  142. package/script/helpers/cleanSQL.js +271 -23
  143. package/script/helpers/flushAllTables.d.ts.map +1 -1
  144. package/script/helpers/flushAllTables.js +24 -29
  145. package/script/helpers/foldIdentifier.d.ts +3 -0
  146. package/script/helpers/foldIdentifier.d.ts.map +1 -0
  147. package/script/helpers/foldIdentifier.js +7 -0
  148. package/script/helpers/geometryFromJSON.d.ts +3 -0
  149. package/script/helpers/geometryFromJSON.d.ts.map +1 -0
  150. package/script/helpers/geometryFromJSON.js +7 -0
  151. package/script/helpers/index.d.ts +1 -0
  152. package/script/helpers/index.d.ts.map +1 -1
  153. package/script/helpers/index.js +3 -1
  154. package/script/helpers/logData.d.ts.map +1 -1
  155. package/script/helpers/logData.js +45 -44
  156. package/script/helpers/makeLogConverter.d.ts +5 -0
  157. package/script/helpers/makeLogConverter.d.ts.map +1 -0
  158. package/script/helpers/makeLogConverter.js +70 -0
  159. package/script/helpers/osmFiles.d.ts.map +1 -1
  160. package/script/helpers/parseTypes.d.ts +1 -1
  161. package/script/helpers/parseTypes.d.ts.map +1 -1
  162. package/script/helpers/parseTypes.js +4 -0
  163. package/script/helpers/pendingOps.d.ts +4 -2
  164. package/script/helpers/pendingOps.d.ts.map +1 -1
  165. package/script/helpers/prepareGeometry.d.ts +3 -0
  166. package/script/helpers/prepareGeometry.d.ts.map +1 -0
  167. package/script/helpers/prepareGeometry.js +102 -0
  168. package/script/helpers/prepareJSON.d.ts +3 -0
  169. package/script/helpers/prepareJSON.d.ts.map +1 -0
  170. package/script/helpers/prepareJSON.js +53 -0
  171. package/script/helpers/printTable.d.ts +5 -4
  172. package/script/helpers/printTable.d.ts.map +1 -1
  173. package/script/helpers/printTable.js +32 -36
  174. package/script/helpers/queryDB.js +1 -1
  175. package/script/helpers/queueAsyncBarrier.d.ts +4 -0
  176. package/script/helpers/queueAsyncBarrier.d.ts.map +1 -1
  177. package/script/helpers/queueAsyncBarrier.js +4 -0
  178. package/script/helpers/queueOp.d.ts.map +1 -1
  179. package/script/helpers/queueOp.js +4 -2
  180. package/script/helpers/readMutationRows.d.ts +9 -0
  181. package/script/helpers/readMutationRows.d.ts.map +1 -0
  182. package/script/helpers/readMutationRows.js +97 -0
  183. package/script/helpers/referencedTables.d.ts +9 -0
  184. package/script/helpers/referencedTables.d.ts.map +1 -0
  185. package/script/helpers/referencedTables.js +28 -0
  186. package/script/helpers/runQuery.d.ts.map +1 -1
  187. package/script/helpers/runQuery.js +11 -13
  188. package/script/helpers/scanSQL.d.ts +7 -0
  189. package/script/helpers/scanSQL.d.ts.map +1 -0
  190. package/script/helpers/scanSQL.js +74 -0
  191. package/script/helpers/tableRegistry.d.ts +2 -0
  192. package/script/helpers/tableRegistry.d.ts.map +1 -1
  193. package/script/helpers/tableRegistry.js +16 -4
  194. package/script/helpers/updateColumnsWithJS.d.ts +52 -0
  195. package/script/helpers/updateColumnsWithJS.d.ts.map +1 -0
  196. package/script/helpers/updateColumnsWithJS.js +145 -0
  197. package/script/methods/addColumn.d.ts +1 -1
  198. package/script/methods/addColumn.d.ts.map +1 -1
  199. package/script/methods/bins.d.ts +1 -1
  200. package/script/methods/bins.d.ts.map +1 -1
  201. package/script/methods/bins.js +45 -29
  202. package/script/methods/bm25.d.ts.map +1 -1
  203. package/script/methods/bm25.js +1 -2
  204. package/script/methods/cache.d.ts.map +1 -1
  205. package/script/methods/cache.js +3 -5
  206. package/script/methods/convert.d.ts +2 -2
  207. package/script/methods/convert.d.ts.map +1 -1
  208. package/script/methods/convert.js +5 -0
  209. package/script/methods/getBottom.d.ts +1 -1
  210. package/script/methods/getBottom.d.ts.map +1 -1
  211. package/script/methods/getBottom.js +2 -2
  212. package/script/methods/getData.d.ts +1 -1
  213. package/script/methods/getData.d.ts.map +1 -1
  214. package/script/methods/getData.js +2 -2
  215. package/script/methods/join.d.ts.map +1 -1
  216. package/script/methods/join.js +8 -16
  217. package/script/methods/joinGeo.d.ts.map +1 -1
  218. package/script/methods/joinGeo.js +10 -19
  219. package/script/methods/loadArray.d.ts +1 -1
  220. package/script/methods/loadArray.d.ts.map +1 -1
  221. package/script/methods/loadArray.js +74 -19
  222. package/script/methods/loadData.js +3 -0
  223. package/script/methods/log.d.ts +7 -0
  224. package/script/methods/log.d.ts.map +1 -0
  225. package/script/methods/log.js +69 -0
  226. package/script/methods/logBottom.d.ts +5 -0
  227. package/script/methods/logBottom.d.ts.map +1 -0
  228. package/script/methods/logBottom.js +15 -0
  229. package/script/methods/removeTable.d.ts.map +1 -1
  230. package/script/methods/removeTable.js +2 -0
  231. package/script/methods/removeTables.d.ts.map +1 -1
  232. package/script/methods/removeTables.js +7 -0
  233. package/script/methods/renameTable.d.ts.map +1 -1
  234. package/script/methods/renameTable.js +4 -0
  235. package/script/methods/rowToText.js +2 -2
  236. package/script/methods/selectTables.d.ts.map +1 -1
  237. package/script/methods/selectTables.js +6 -0
  238. package/script/methods/setTypes.d.ts +1 -1
  239. package/script/methods/setTypes.d.ts.map +1 -1
  240. package/script/methods/stream.d.ts.map +1 -1
  241. package/script/methods/stream.js +2 -1
  242. package/script/methods/updateWithJS.d.ts +1 -3
  243. package/script/methods/updateWithJS.d.ts.map +1 -1
  244. package/script/methods/updateWithJS.js +122 -36
package/llm.md CHANGED
@@ -1,7 +1,7 @@
1
1
  # The Simple Data Analysis (Core) Library
2
2
 
3
3
  - Package: `@nshiab/simple-data-analysis-core`
4
- - Version: `2.0.5`
4
+ - Version: `2.0.7`
5
5
 
6
6
  To install the library with Deno, use:
7
7
 
@@ -91,12 +91,16 @@ schema is reserved for versioned SDA metadata. SQLite files can be imported with
91
91
  - **`options.logDuration`**: A flag indicating whether to log the total
92
92
  execution duration.
93
93
  - **`options.rowsToLog`**: The number of rows to display when logging a table.
94
- - **`options.charsToLog`**: The maximum number of characters to display for
95
- text-based cells.
94
+ - **`options.charsToLog`**: The maximum characters to display for text and
95
+ stringified nested cells, including truncation markers (default: 75).
96
+ Independent of column width. Use Infinity for unlimited text.
96
97
  - **`options.typesToLog`**: A flag indicating whether to include data types when
97
98
  logging a table.
98
99
  - **`options.cacheVerbose`**: Whether to log cache hits and misses, code and
99
100
  input changes, TTL status, and cache read/write timing.
101
+ - **`options.expressionSyntax`**: Operator syntax for expressions and custom
102
+ queries: `"js"` (default) translates JavaScript-style operators in SQL
103
+ expressions; `"sql"` preserves SQL unchanged.
100
104
  - **`options.logSQL`**: A flag indicating whether to log SQL immediately before
101
105
  execution.
102
106
  - **`options.explainSQL`**: A flag indicating whether to log DuckDB query plans
@@ -116,7 +120,8 @@ schema is reserved for versioned SDA metadata. SQLite files can be imported with
116
120
 
117
121
  #### `newTable`
118
122
 
119
- Creates a new SimpleTable instance within the database.
123
+ Creates a new SimpleTable instance within the database. The name must be unique
124
+ among registered tables, ignoring ASCII letter case.
120
125
 
121
126
  ##### Signature
122
127
 
@@ -385,9 +390,11 @@ console.log(extensions); // Output: [{ extension_name: "spatial", loaded: true,
385
390
 
386
391
  #### `customQuery`
387
392
 
388
- Executes a custom SQL query directly against the DuckDB instance. Queries run in
389
- UTC. When data is returned, temporal values use the same JavaScript
390
- representations as `SimpleTable.getData()`.
393
+ Executes a custom SQL query directly against the DuckDB instance. With the
394
+ default `SimpleDB.expressionSyntax: "js"`, queries support JavaScript-style
395
+ operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for
396
+ unchanged SQL. Queries run in UTC. When data is returned, temporal values use
397
+ the same JavaScript representations as `SimpleTable.getData()`.
391
398
 
392
399
  `customQuery()` bypasses the dependency and table-generation tracking used by
393
400
  `SimpleTable.cache()`. Reading or changing a table with `customQuery()` can
@@ -643,8 +650,9 @@ Creates an instance of SimpleTable.
643
650
  - **`simpleDB`**: The SimpleDB instance that this table belongs to.
644
651
  - **`options`**: An optional object with configuration options:
645
652
  - **`options.rowsToLog`**: The number of rows to log when displaying table data.
646
- - **`options.charsToLog`**: The maximum number of characters to log for strings.
647
- Useful to avoid logging large text content.
653
+ - **`options.charsToLog`**: The maximum characters to display for text and
654
+ stringified nested cells, including truncation markers (default: 75).
655
+ Independent of column width. Use Infinity for unlimited text.
648
656
  - **`options.typesToLog`**: A boolean indicating whether to include data types
649
657
  when logging a table.
650
658
 
@@ -701,7 +709,8 @@ await table.log();
701
709
 
702
710
  #### `renameTable`
703
711
 
704
- Renames the current table.
712
+ Renames the current table. The new name must not belong to another registered
713
+ table, ignoring ASCII letter case.
705
714
 
706
715
  ##### Signature
707
716
 
@@ -731,19 +740,17 @@ Sets the data types for columns in a new table. If the table already exists, it
731
740
  will be replaced. To convert the types of an existing table, use the
732
741
  `.convert()` method instead.
733
742
 
734
- This method queues the operation; it runs when an async observer method (like
735
- `getData()` or `log()`) is awaited, or when `run()` is called.
736
-
737
743
  ##### Signature
738
744
 
739
745
  ```typescript
740
- setTypes(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | geometry('${string}') | GEOMETRY('${string}')>): this;
746
+ setTypes(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | float[${number}] | FLOAT[${number}] | geometry('${string}') | GEOMETRY('${string}')>): this;
741
747
  ```
742
748
 
743
749
  ##### Parameters
744
750
 
745
751
  - **`types`**: An object specifying the column names and their target data types
746
- (JavaScript or SQL types).
752
+ (JavaScript or SQL types), including JSON, FLOAT[n] vectors, and GEOMETRY with
753
+ a CRS.
747
754
 
748
755
  ##### Returns
749
756
 
@@ -760,22 +767,33 @@ await table.setTypes({
760
767
  }).log();
761
768
  ```
762
769
 
770
+ ```ts
771
+ // Create an empty table with JSON, embedding, and geometry columns.
772
+ await table.setTypes({
773
+ details: "JSON",
774
+ embedding: "FLOAT[3]",
775
+ geom: "GEOMETRY('EPSG:4326')",
776
+ }).log();
777
+ ```
778
+
763
779
  #### `loadArray`
764
780
 
765
- Loads an array of JavaScript objects into the table. Types can also be specified
766
- for individual columns instead of inferred from their values. This method queues
767
- the load; it runs when an async observer method (like `getData()` or `log()`) is
768
- awaited, or when `run()` is called.
781
+ Loads an array of JavaScript objects into the table. Types are inferred for
782
+ numbers, bigints, strings, booleans, and Date values. Array and object cells
783
+ require an explicit supported type in columnTypes. Types can also be specified
784
+ for scalar columns instead of inferred from their values.
769
785
 
770
786
  JavaScript `Date` values are inferred as DuckDB `TIMESTAMP` values. Their
771
787
  instant is preserved, but JavaScript `Date` does not retain the timezone or
772
- offset originally used to construct it. String values remain `VARCHAR`; use
773
- `convert()` to parse them as temporal values.
788
+ offset originally used to construct it.
789
+
790
+ Declare `JSON` for general nested data, `FLOAT[n]` for a fixed-size float
791
+ vector, or `GEOMETRY('EPSG:4326')` for GeoJSON geometries in WGS84.
774
792
 
775
793
  ##### Signature
776
794
 
777
795
  ```typescript
778
- loadArray(rows: Record<string, unknown>[], options?: { columnTypes?: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "INTEGER" | "BIGINT" | "DOUBLE" | "VARCHAR" | "BOOLEAN" | "DATE" | "TIME" | "TIMESTAMP" | "TIMESTAMP WITH TIME ZONE" | FLOAT[${number}] | float[${number}]> }): this;
796
+ loadArray(rows: Record<string, unknown>[], options?: { columnTypes?: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "INTEGER" | "BIGINT" | "DOUBLE" | "VARCHAR" | "BOOLEAN" | "DATE" | "TIME" | "TIMESTAMP" | "TIMESTAMP WITH TIME ZONE" | "GEOMETRY('EPSG:4326')" | "geometry('EPSG:4326')" | "JSON" | "json" | FLOAT[${number}] | float[${number}]> }): this;
779
797
  ```
780
798
 
781
799
  ##### Parameters
@@ -783,9 +801,10 @@ loadArray(rows: Record<string, unknown>[], options?: { columnTypes?: Record<stri
783
801
  - **`rows`**: An array of objects, where each object represents a row and its
784
802
  properties represent columns.
785
803
  - **`options`**: Options for loading the array, captured when called.
786
- - **`options.columnTypes`**: Types for specific columns; omitted columns are
787
- inferred. Values must be compatible with the selected type without losing
788
- information.
804
+ - **`options.columnTypes`**: Types for specific columns. Required for array and
805
+ object cells; omitted scalar columns are inferred, and all-null columns
806
+ default to VARCHAR. Values must be compatible with the selected type without
807
+ losing information.
789
808
 
790
809
  ##### Returns
791
810
 
@@ -794,7 +813,7 @@ The table, so methods can be chained.
794
813
  ##### Examples
795
814
 
796
815
  ```ts
797
- // Load data from an array of objects
816
+ // Load scalar columns without columnTypes; strings and numbers are inferred.
798
817
  const data = [
799
818
  { letter: "a", number: 1 },
800
819
  { letter: "b", number: 2 },
@@ -803,7 +822,7 @@ await table.loadArray(data).log();
803
822
  ```
804
823
 
805
824
  ```ts
806
- // Specify a numeric type for an all-null column
825
+ // Choose a numeric type when all-null values provide nothing to infer.
807
826
  await table.loadArray(
808
827
  [{ name: "A", value: null }, { name: "B", value: null }],
809
828
  { columnTypes: { value: "DOUBLE" } },
@@ -811,19 +830,45 @@ await table.loadArray(
811
830
  ```
812
831
 
813
832
  ```ts
814
- // The offset determines the instant; the loaded TIMESTAMP is returned as
815
- // the equivalent UTC JavaScript Date.
833
+ // Store the Date as a UTC timestamp with JavaScript's millisecond precision.
834
+ // The original -04:00 offset is not retained.
816
835
  await table.loadArray([{
817
836
  observedAt: new Date("2024-04-07T13:00:00-04:00"),
818
837
  }]).log();
819
838
  ```
820
839
 
840
+ ```ts
841
+ // Declare JSON to load nested objects and variable-length arrays directly.
842
+ await table.loadArray([{
843
+ name: "Station",
844
+ metadata: { active: true, tags: ["weather", "urban"] },
845
+ measurements: [1, 2.5, null],
846
+ }], { columnTypes: { metadata: "JSON", measurements: "JSON" } }).log();
847
+ ```
848
+
849
+ ```ts
850
+ // Declare FLOAT[3] to store each three-element array as a fixed-size vector.
851
+ await table.loadArray([{ vector: [1, 2, 3] }], {
852
+ columnTypes: { vector: "FLOAT[3]" },
853
+ }).log();
854
+ ```
855
+
856
+ ```ts
857
+ // Declare a geometry type to store WGS84 GeoJSON as a spatial column.
858
+ await table.loadArray([{
859
+ name: "Montreal",
860
+ geom: { type: "Point", coordinates: [-73.57, 45.50] },
861
+ }], { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } }).log();
862
+ ```
863
+
821
864
  #### `loadData`
822
865
 
823
866
  Loads data from one or more local or remote files into the table. Supported file
824
- formats include CSV, JSON, Parquet, and Excel. This method queues the load; it
825
- runs when an async observer method (like `getData()` or `log()`) is awaited, or
826
- when `run()` is called.
867
+ formats include CSV, JSON, Parquet, and Excel.
868
+
869
+ With the default `SimpleDB.expressionSyntax: "js"`, conditions support
870
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
871
+ `expressionSyntax: "sql"` for unchanged SQL.
827
872
 
828
873
  ##### Signature
829
874
 
@@ -841,10 +886,8 @@ loadData(files: string | string[], options?: { fileType?: "csv" | "dsv" | "json"
841
886
  - **`options.autoDetect`**: A boolean indicating whether to automatically detect
842
887
  the data format. Defaults to `true`.
843
888
  - **`options.conditions`**: A SQL `WHERE` clause expression, without the `WHERE`
844
- keyword, to filter source rows before applying `limit`. Uses the same syntax
845
- as `filter()`, including JavaScript operators. Can reference source columns
846
- excluded from `columns`. Defaults to no filtering; an empty string behaves the
847
- same as omitting this option.
889
+ keyword, to filter source rows before applying `limit`. Can reference source
890
+ columns excluded from `columns`.
848
891
  - **`options.limit`**: A number indicating the maximum number of matching rows
849
892
  to load, after applying `conditions` if provided. Defaults to all matching
850
893
  rows.
@@ -854,7 +897,8 @@ loadData(files: string | string[], options?: { fileType?: "csv" | "dsv" | "json"
854
897
  across multiple files when their structures differ. Missing columns will be
855
898
  filled with `NULL` values. Defaults to `false`.
856
899
  - **`options.columnTypes`**: An object mapping column names to their expected
857
- data types. By default, types are inferred.
900
+ data types. By default, types are inferred. Geometry types are not supported;
901
+ use loadGeoData() instead.
858
902
  - **`options.columns`**: An array of column names to load. When provided, only
859
903
  the specified columns are loaded, reducing memory usage and improving load
860
904
  times. Not supported for Excel files — combining `columns` with Excel files
@@ -944,9 +988,7 @@ await table
944
988
 
945
989
  #### `loadStatCanData`
946
990
 
947
- Downloads a complete Statistics Canada table and loads it into this table. The
948
- method queues the download and load; they run when an async observer method
949
- (like `getData()` or `log()`) is awaited, or when `run()` is called.
991
+ Downloads a complete Statistics Canada table and loads it into this table.
950
992
 
951
993
  Results are cached as Parquet files in `.sda-cache/statcan` by default. Cached
952
994
  data does not expire unless a TTL is provided.
@@ -1010,9 +1052,6 @@ with or endorsed by Yahoo. It is provided for educational, research, and
1010
1052
  journalistic purposes. Before using it, review Yahoo's terms and any applicable
1011
1053
  data-provider restrictions.
1012
1054
 
1013
- The method queues the download and load; they run when an async observer method
1014
- (like `getData()` or `log()`) is awaited, or when `run()` is called.
1015
-
1016
1055
  ##### Signature
1017
1056
 
1018
1057
  ```typescript
@@ -1066,10 +1105,10 @@ await table
1066
1105
 
1067
1106
  #### `loadGeoData`
1068
1107
 
1069
- Loads geospatial data from an external file or URL into the table.
1070
-
1071
- This method queues the operation; it runs when an async observer method (like
1072
- `getData()` or `log()`) is awaited, or when `run()` is called.
1108
+ Loads geospatial data from an external file or URL into the table. With the
1109
+ default `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style
1110
+ operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for
1111
+ unchanged SQL.
1073
1112
 
1074
1113
  ##### Signature
1075
1114
 
@@ -1088,11 +1127,9 @@ loadGeoData(file: string, options?: { toEPSG4326?: boolean; columns?: string[];
1088
1127
  should remain in the resulting table, usually `"geom"`. By default, all
1089
1128
  columns are loaded.
1090
1129
  - **`options.conditions`**: A SQL `WHERE` clause expression, without the `WHERE`
1091
- keyword, to filter source rows before materialization and reprojection. Uses
1092
- the same syntax as `filter()`, including JavaScript operators. Can reference
1093
- source columns excluded from `columns`. Geometry conditions use the source
1094
- coordinate system. Defaults to no filtering; an empty string behaves the same
1095
- as omitting this option.
1130
+ keyword, to filter source rows before materialization and reprojection. Can
1131
+ reference source columns excluded from `columns`. Geometry conditions use the
1132
+ source coordinate system.
1096
1133
 
1097
1134
  ##### Returns
1098
1135
 
@@ -1145,8 +1182,7 @@ await table
1145
1182
  Loads OpenStreetMap data into the table from a local `.osm` or `.osm.pbf` file,
1146
1183
  a remote file URL, or an Overpass bounding-box query. Pass a path or URL string
1147
1184
  to load an existing file, or pass a bounding box with `filters` to download
1148
- matching features. The method queues the load; it runs when an async observer
1149
- method (like `getData()` or `log()`) is awaited, or when `run()` is called.
1185
+ matching features.
1150
1186
 
1151
1187
  DuckDB's
1152
1188
  [Osmium community extension](https://duckdb.org/community_extensions/extensions/osmium)
@@ -1279,9 +1315,6 @@ method requires an FTS index and creates one automatically when needed. DuckDB
1279
1315
  FTS indexes do not update automatically when the table changes; use
1280
1316
  `overwrite: true` to rebuild the index after modifying the table.
1281
1317
 
1282
- This method queues the operation; it runs when an async observer method (like
1283
- `getData()` or `log()`) is awaited, or when `run()` is called.
1284
-
1285
1318
  ##### Signature
1286
1319
 
1287
1320
  ```typescript
@@ -1355,9 +1388,6 @@ If a VSS index already exists on the table, this method will skip creation and
1355
1388
  log a message (when verbose is enabled), unless the `overwrite` option is set to
1356
1389
  `true`. The index definition is recorded in {@link indexes}.
1357
1390
 
1358
- This method queues the operation; it runs when an async observer method (like
1359
- `getData()` or `log()`) is awaited, or when `run()` is called.
1360
-
1361
1391
  ##### Signature
1362
1392
 
1363
1393
  ```typescript
@@ -1432,9 +1462,6 @@ It reuses the table's existing FTS index unless `overwriteIndex` is `true`.
1432
1462
  DuckDB FTS indexes do not update automatically when the source table changes;
1433
1463
  use `overwriteIndex: true` to rebuild the index after modifying the table.
1434
1464
 
1435
- This method queues the operation; it runs when an async observer method (like
1436
- `getData()` or `log()`) is awaited, or when `run()` is called.
1437
-
1438
1465
  ##### Signature
1439
1466
 
1440
1467
  ```typescript
@@ -1557,9 +1584,6 @@ await table.bm25("italian sauce", "Dish", "Recipe", 5, {
1557
1584
 
1558
1585
  Inserts rows, provided as an array of JavaScript objects, into the table.
1559
1586
 
1560
- This method queues the operation; it runs when an async observer method (like
1561
- `getData()` or `log()`) is awaited, or when `run()` is called.
1562
-
1563
1587
  ##### Signature
1564
1588
 
1565
1589
  ```typescript
@@ -1590,9 +1614,7 @@ await table.insertRows(newRows).log();
1590
1614
 
1591
1615
  Inserts all rows from one or more other tables into this table. If tables do not
1592
1616
  have the same columns, an error will be thrown unless the `unifyColumns` option
1593
- is set to `true`. This method queues the operation; it runs when an async
1594
- observer method (like `getData()` or `log()`) is awaited, or when `run()` is
1595
- called.
1617
+ is set to `true`.
1596
1618
 
1597
1619
  ##### Signature
1598
1620
 
@@ -1634,9 +1656,6 @@ await tableA.insertTables(["tableB", "tableC"], { unifyColumns: true }).log();
1634
1656
 
1635
1657
  Fetches sample data from the simple-data-analysis-core GitHub repository.
1636
1658
 
1637
- This method queues the operation; it runs when an async observer method (like
1638
- `getData()` or `log()`) is awaited, or when `run()` is called.
1639
-
1640
1659
  ##### Signature
1641
1660
 
1642
1661
  ```typescript
@@ -1668,7 +1687,10 @@ await table.loadSample("fires").log();
1668
1687
  #### `clone`
1669
1688
 
1670
1689
  Returns a new table with the same structure and data as this table. The data can
1671
- be optionally filtered, limited to a specific number of rows, and offset.
1690
+ be optionally filtered, limited to a specific number of rows, and offset. With
1691
+ the default `SimpleDB.expressionSyntax: "js"`, conditions support
1692
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
1693
+ `expressionSyntax: "sql"` for unchanged SQL.
1672
1694
 
1673
1695
  If `conditions`, `limit`, and `offset` are all used, they are applied in this
1674
1696
  order: `conditions` (WHERE clause) first, then `offset`, and finally `limit`
@@ -1676,9 +1698,6 @@ order: `conditions` (WHERE clause) first, then `offset`, and finally `limit`
1676
1698
 
1677
1699
  Note that cloning large tables can be a slow operation.
1678
1700
 
1679
- This method queues the operation; it runs when an async observer method (like
1680
- `getData()` or `log()`) is awaited, or when `run()` is called.
1681
-
1682
1701
  ##### Signature
1683
1702
 
1684
1703
  ```typescript
@@ -1759,9 +1778,6 @@ const tableB = await tableA.clone({
1759
1778
  Clones an existing column in this table, creating a new column with identical
1760
1779
  values.
1761
1780
 
1762
- This method queues the operation; it runs when an async observer method (like
1763
- `getData()` or `log()`) is awaited, or when `run()` is called.
1764
-
1765
1781
  ##### Signature
1766
1782
 
1767
1783
  ```typescript
@@ -1794,9 +1810,6 @@ different time points.
1794
1810
  table. For meaningful results, ensure your data is sorted appropriately (e.g.,
1795
1811
  by date/time for time-series analysis) before calling this method.
1796
1812
 
1797
- This method queues the operation; it runs when an async observer method (like
1798
- `getData()` or `log()`) is awaited, or when `run()` is called.
1799
-
1800
1813
  ##### Signature
1801
1814
 
1802
1815
  ```typescript
@@ -1861,9 +1874,6 @@ use it as the X-axis, so that interpolated values are proportional to the actual
1861
1874
  distances between X-axis values rather than treating every row as equidistant.
1862
1875
  When `interpolateBy` is set, `interpolate` is automatically assumed `true`.
1863
1876
 
1864
- This method queues the operation; it runs when an async observer method (like
1865
- `getData()` or `log()`) is awaited, or when `run()` is called.
1866
-
1867
1877
  ##### Signature
1868
1878
 
1869
1879
  ```typescript
@@ -1937,11 +1947,9 @@ Sorts the rows of the table based on specified column(s) and order(s). If no
1937
1947
  columns are specified, all columns are sorted from left to right in ascending
1938
1948
  order.
1939
1949
 
1940
- This method queues the operation; it runs when an async observer method (like
1941
- `getData()` or `log()`) is awaited, or when `run()` is called. Order-preserving
1942
- transformations queued after a sort retain that order. Operations such as joins,
1943
- grouping, aggregation, and sampling do not guarantee input order; chain `sort()`
1944
- after them when deterministic output order matters.
1950
+ Order-preserving transformations after a sort retain that order. Operations such
1951
+ as joins, grouping, aggregation, and sampling do not guarantee input order;
1952
+ chain `sort()` after them when deterministic output order matters.
1945
1953
 
1946
1954
  ##### Signature
1947
1955
 
@@ -1987,9 +1995,7 @@ await table.sort({ column1: "asc" }, { lang: { column1: "fr" } }).log();
1987
1995
 
1988
1996
  #### `selectColumns`
1989
1997
 
1990
- Selects specific columns in the table, removing all others. This method queues
1991
- the operation; it runs when an async observer method (like `getData()` or
1992
- `log()`) is awaited, or when `run()` is called.
1998
+ Selects specific columns in the table, removing all others.
1993
1999
 
1994
2000
  ##### Signature
1995
2001
 
@@ -2021,9 +2027,6 @@ await table.selectColumns("productName").log();
2021
2027
 
2022
2028
  Skips the first `n` rows of the table, effectively removing them.
2023
2029
 
2024
- This method queues the operation; it runs when an async observer method (like
2025
- `getData()` or `log()`) is awaited, or when `run()` is called.
2026
-
2027
2030
  ##### Signature
2028
2031
 
2029
2032
  ```typescript
@@ -2076,9 +2079,6 @@ console.log(hasAgeColumn); // Output: true or false
2076
2079
  Selects random rows from the table, removing all others. You can optionally
2077
2080
  specify a seed to ensure repeatable sampling.
2078
2081
 
2079
- This method queues the operation; it runs when an async observer method (like
2080
- `getData()` or `log()`) is awaited, or when `run()` is called.
2081
-
2082
2082
  ##### Signature
2083
2083
 
2084
2084
  ```typescript
@@ -2120,9 +2120,6 @@ await table.sample("10%", { seed: 123 }).log();
2120
2120
  Selects a specified number of rows from this table. An offset can be applied to
2121
2121
  skip initial rows, and the results can be output to a new table.
2122
2122
 
2123
- This method queues the operation; it runs when an async observer method (like
2124
- `getData()` or `log()`) is awaited, or when `run()` is called.
2125
-
2126
2123
  ##### Signature
2127
2124
 
2128
2125
  ```typescript
@@ -2174,9 +2171,6 @@ const topCustomersTable = await table.selectRows(75, {
2174
2171
  Removes duplicate rows from this table, keeping only unique rows. Note that the
2175
2172
  resulting data order might differ from the original.
2176
2173
 
2177
- This method queues the operation; it runs when an async observer method (like
2178
- `getData()` or `log()`) is awaited, or when `run()` is called.
2179
-
2180
2174
  ##### Signature
2181
2175
 
2182
2176
  ```typescript
@@ -2215,9 +2209,7 @@ await table.removeDuplicates({ on: ["firstName", "lastName"] }).log();
2215
2209
 
2216
2210
  Removes rows with missing values from this table. By default, missing values
2217
2211
  include SQL `NULL`, as well as string representations like `"NULL"`, `"null"`,
2218
- `"NaN"`, `"undefined"`, and empty strings `""`. This method queues the
2219
- operation; it runs when an async observer method (like `getData()` or `log()`)
2220
- is awaited, or when `run()` is called.
2212
+ `"NaN"`, `"undefined"`, and empty strings `""`.
2221
2213
 
2222
2214
  ##### Signature
2223
2215
 
@@ -2265,9 +2257,7 @@ await table.removeMissing({ columns: "age", missingValues: [-1] }).log();
2265
2257
  #### `trim`
2266
2258
 
2267
2259
  Trims specified characters from the beginning, end, or both sides of string
2268
- values in the given columns. This method queues the operation; it runs when an
2269
- async observer method (like `getData()` or `log()`) is awaited, or when `run()`
2270
- is called.
2260
+ values in the given columns.
2271
2261
 
2272
2262
  ##### Signature
2273
2263
 
@@ -2309,10 +2299,9 @@ await table.trim(["description", "notes"], { side: "right" }).log();
2309
2299
  #### `filter`
2310
2300
 
2311
2301
  Filters rows from this table based on SQL conditions. Note that it's often
2312
- faster to use the `removeRows` method for simple removals. You can also use
2313
- JavaScript syntax for conditions (e.g., `&&`, `||`, `===`, `!==`). This method
2314
- queues the operation; it runs when an async observer method (like `getData()` or
2315
- `log()`) is awaited, or when `run()` is called.
2302
+ faster to use the `removeRows` method for simple removals. With the default
2303
+ `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
2304
+ (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
2316
2305
 
2317
2306
  ##### Signature
2318
2307
 
@@ -2357,9 +2346,6 @@ await table.filter(`lastPurchaseDate >= '2023-01-01'`).log();
2357
2346
  Keeps rows in this table that have specific values in specified columns,
2358
2347
  removing all other rows.
2359
2348
 
2360
- This method queues the operation; it runs when an async observer method (like
2361
- `getData()` or `log()`) is awaited, or when `run()` is called.
2362
-
2363
2349
  ##### Signature
2364
2350
 
2365
2351
  ```typescript
@@ -2398,9 +2384,6 @@ await table.keepValues({ status: null }).log();
2398
2384
 
2399
2385
  Removes rows from this table that have specific values in specified columns.
2400
2386
 
2401
- This method queues the operation; it runs when an async observer method (like
2402
- `getData()` or `log()`) is awaited, or when `run()` is called.
2403
-
2404
2387
  ##### Signature
2405
2388
 
2406
2389
  ```typescript
@@ -2439,11 +2422,9 @@ await table.removeValues({ status: null }).log();
2439
2422
  #### `removeRows`
2440
2423
 
2441
2424
  Removes rows from this table based on SQL conditions. This method is similar to
2442
- `filter()`, but removes rows instead of keeping them. You can also use
2443
- JavaScript syntax for conditions (e.g., `&&`, `||`, `===`, `!==`).
2444
-
2445
- This method queues the operation; it runs when an async observer method (like
2446
- `getData()` or `log()`) is awaited, or when `run()` is called.
2425
+ `filter()`, but removes rows instead of keeping them. With the default
2426
+ `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
2427
+ (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
2447
2428
 
2448
2429
  ##### Signature
2449
2430
 
@@ -2489,9 +2470,6 @@ await table.removeRows(
2489
2470
  Renames one or more columns in the table. Throws if a source column does not
2490
2471
  exist, so a typo fails loudly instead of being silently ignored.
2491
2472
 
2492
- This method queues the operation; it runs when an async observer method (like
2493
- `getData()` or `log()`) is awaited, or when `run()` is called.
2494
-
2495
2473
  ##### Signature
2496
2474
 
2497
2475
  ```typescript
@@ -2535,9 +2513,6 @@ await table.renameColumns({ "product_id": "productId" }, { strict: false })
2535
2513
  Cleans column names by removing non-alphanumeric characters and formatting them
2536
2514
  to camel case.
2537
2515
 
2538
- This method queues the operation; it runs when an async observer method (like
2539
- `getData()` or `log()`) is awaited, or when `run()` is called.
2540
-
2541
2516
  ##### Signature
2542
2517
 
2543
2518
  ```typescript
@@ -2608,9 +2583,6 @@ The table will then look like this:
2608
2583
  | Sales | 2022 | 75 |
2609
2584
  | Sales | 2023 | 98 |
2610
2585
 
2611
- This method queues the operation; it runs when an async observer method (like
2612
- `getData()` or `log()`) is awaited, or when `run()` is called.
2613
-
2614
2586
  #### `wider`
2615
2587
 
2616
2588
  Restructures this table by unstacking (pivoting) values, transforming data from
@@ -2669,9 +2641,6 @@ When multiple rows share the same `namesFrom`/grouping combination, their
2669
2641
  `valuesFrom` values are combined with the `options.stat` function (`"sum"` by
2670
2642
  default).
2671
2643
 
2672
- This method queues the operation; it runs when an async observer method (like
2673
- `getData()` or `log()`) is awaited, or when `run()` is called.
2674
-
2675
2644
  #### `convert`
2676
2645
 
2677
2646
  Converts data types of specified columns to target types (JavaScript or SQL
@@ -2691,20 +2660,16 @@ UTC).
2691
2660
  When converting strings to numbers, commas (often used as thousand separators)
2692
2661
  will be automatically removed before conversion.
2693
2662
 
2694
- This method queues the operation; it runs when an async observer method (like
2695
- `getData()` or `log()`) is awaited, or when `run()` is called. If a column
2696
- doesn't exist, the error is thrown at that point too.
2697
-
2698
2663
  ##### Signature
2699
2664
 
2700
2665
  ```typescript
2701
- convert(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean">, options?: { strict?: boolean; datetimeFormat?: string }): this;
2666
+ convert(types: Record<string, "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | float[${number}] | FLOAT[${number}] | geometry('${string}') | GEOMETRY('${string}')>, options?: { strict?: boolean; datetimeFormat?: string }): this;
2702
2667
  ```
2703
2668
 
2704
2669
  ##### Parameters
2705
2670
 
2706
2671
  - **`types`**: An object mapping column names to their target data types for
2707
- conversion.
2672
+ conversion, including JSON, FLOAT[n] vectors, and GEOMETRY with a CRS.
2708
2673
  - **`options`**: An optional object with configuration options:
2709
2674
  - **`options.strict`**: If `false`, values that cannot be converted will be
2710
2675
  replaced by `NULL` instead of throwing an error. Defaults to `true`.
@@ -2757,6 +2722,21 @@ await table.convert({ column3: "string" }, {
2757
2722
  await table.convert({ amount: "float" }, { strict: false }).log();
2758
2723
  ```
2759
2724
 
2725
+ ```ts
2726
+ // Parse JSON, a fixed-size float vector, and WKT geometry from text.
2727
+ await table.loadArray([{
2728
+ details: '{"count":3}',
2729
+ embedding: "[0.25,0.5,0.75]",
2730
+ geom: "POINT (-73 45)",
2731
+ }])
2732
+ .convert({
2733
+ details: "JSON",
2734
+ embedding: "FLOAT[3]",
2735
+ geom: "GEOMETRY('EPSG:4326')",
2736
+ })
2737
+ .log();
2738
+ ```
2739
+
2760
2740
  #### `removeTable`
2761
2741
 
2762
2742
  Removes the table from the database. After this operation, invoking methods on
@@ -2781,9 +2761,7 @@ await table.removeTable();
2781
2761
 
2782
2762
  #### `removeColumns`
2783
2763
 
2784
- Removes one or more columns from this table. This method queues the operation;
2785
- it runs when an async observer method (like `getData()` or `log()`) is awaited,
2786
- or when `run()` is called.
2764
+ Removes one or more columns from this table.
2787
2765
 
2788
2766
  ##### Signature
2789
2767
 
@@ -2814,21 +2792,22 @@ await table.removeColumns("tempColumn").log();
2814
2792
  #### `addColumn`
2815
2793
 
2816
2794
  Adds a new column to the table based on a specified data type (JavaScript or SQL
2817
- types) and a SQL definition. This method queues the operation; it runs when an
2818
- async observer method (like `getData()` or `log()`) is awaited, or when `run()`
2819
- is called.
2795
+ types) and a SQL definition. With the default `SimpleDB.expressionSyntax: "js"`,
2796
+ expressions support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
2797
+ `expressionSyntax: "sql"` for unchanged SQL.
2820
2798
 
2821
2799
  ##### Signature
2822
2800
 
2823
2801
  ```typescript
2824
- addColumn(newColumn: string, type: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | geometry('${string}') | GEOMETRY('${string}'), definition: string): this;
2802
+ addColumn(newColumn: string, type: "integer" | "float" | "number" | "string" | "date" | "time" | "datetime" | "datetimeTz" | "bigint" | "double" | "varchar" | "timestamp" | "timestamp with time zone" | "boolean" | "json" | "JSON" | float[${number}] | FLOAT[${number}] | geometry('${string}') | GEOMETRY('${string}'), definition: string): this;
2825
2803
  ```
2826
2804
 
2827
2805
  ##### Parameters
2828
2806
 
2829
2807
  - **`newColumn`**: The name of the new column to be added.
2830
2808
  - **`type`**: The data type for the new column. Can be a JavaScript type (e.g.,
2831
- `"number"`, `"string"`) or a SQL type (e.g., `"integer"`, `"varchar"`).
2809
+ `"number"`, `"string"`) or a SQL type (e.g., `"integer"`, `"varchar"`,
2810
+ `"JSON"`, `"FLOAT[3]"`, `"GEOMETRY('EPSG:4326')"`).
2832
2811
  - **`definition`**: A SQL expression defining how the values for the new column
2833
2812
  should be computed (e.g., `"column1 + column2"`,
2834
2813
  `"ST_Centroid(geom_column)"`).
@@ -2853,6 +2832,23 @@ await table.addColumn(
2853
2832
  ).log();
2854
2833
  ```
2855
2834
 
2835
+ ```ts
2836
+ // Read a number from a JSON object into a new column using dot notation.
2837
+ await table.loadArray([{ details: { count: 3 } }], {
2838
+ columnTypes: { details: "JSON" },
2839
+ })
2840
+ .addColumn("count", "integer", "details.count")
2841
+ .log();
2842
+ ```
2843
+
2844
+ ```ts
2845
+ // Create JSON and fixed-size vector columns with DuckDB expressions.
2846
+ await table.loadArray([{ count: 3 }])
2847
+ .addColumn("details", "JSON", "json_object('count', count)")
2848
+ .addColumn("embedding", "FLOAT[3]", "[0.25, 0.5, 0.75]")
2849
+ .log();
2850
+ ```
2851
+
2856
2852
  #### `extractDatePart`
2857
2853
 
2858
2854
  Extracts one or more components from a temporal column into new columns. Pass a
@@ -2867,9 +2863,6 @@ time parts apply to times and timestamps. `NULL` input values produce `NULL`
2867
2863
  extracted values. Parts extracted from `TIMESTAMP WITH TIME ZONE` values use
2868
2864
  UTC.
2869
2865
 
2870
- This method queues the operation; it runs when an async observer method (like
2871
- `getData()` or `log()`) is awaited, or when `run()` is called.
2872
-
2873
2866
  ##### Signature
2874
2867
 
2875
2868
  ```typescript
@@ -2906,9 +2899,6 @@ await table.extractDatePart("publishedAt", {
2906
2899
  Adds a new column to the table containing the row number, starting at 0 (like an
2907
2900
  index).
2908
2901
 
2909
- This method queues the operation; it runs when an async observer method (like
2910
- `getData()` or `log()`) is awaited, or when `run()` is called.
2911
-
2912
2902
  ##### Signature
2913
2903
 
2914
2904
  ```typescript
@@ -2945,9 +2935,6 @@ Cartesian product of the rows from both tables, meaning all possible pairs of
2945
2935
  rows will be in the resulting table. This means that if the left table has `n`
2946
2936
  rows and the right table has `m` rows, the result will have `n * m` rows.
2947
2937
 
2948
- This method queues the operation; it runs when an async observer method (like
2949
- `getData()` or `log()`) is awaited, or when `run()` is called.
2950
-
2951
2938
  ##### Signature
2952
2939
 
2953
2940
  ```typescript
@@ -2991,10 +2978,7 @@ Merges the data of this table (considered the left table) with another table
2991
2978
  (the right table) based on a common column or multiple columns. Note that the
2992
2979
  order of rows in the returned data is not guaranteed to be the same as in the
2993
2980
  original tables. This operation might create temporary files in a `.tmp` folder;
2994
- consider adding `.tmp` to your `.gitignore`. This method queues the operation;
2995
- it runs when an async observer method (like `getData()` or `log()`) is awaited,
2996
- or when `run()` is called. The join uses the other table's state as of this
2997
- call: operations queued on it afterwards run after the join.
2981
+ consider adding `.tmp` to your `.gitignore`.
2998
2982
 
2999
2983
  ##### Signature
3000
2984
 
@@ -3057,11 +3041,6 @@ order alphabetically by the left column and then by the right column.
3057
3041
  This operation might create temporary files in a `.tmp` folder; consider adding
3058
3042
  `.tmp` to your `.gitignore`.
3059
3043
 
3060
- This method queues the operation; it runs when an async observer method (like
3061
- `getData()` or `log()`) is awaited, or when `run()` is called. The join uses the
3062
- other table's state as of this call: operations queued on it afterwards run
3063
- after the join.
3064
-
3065
3044
  ##### Signature
3066
3045
 
3067
3046
  ```typescript
@@ -3146,9 +3125,6 @@ Similarity is computed using the
3146
3125
  [rapidfuzz](https://query.farm/duckdb_extension_rapidfuzz) DuckDB community
3147
3126
  extension, which is installed and loaded automatically.
3148
3127
 
3149
- This method queues the operation; it runs when an async observer method (like
3150
- `getData()` or `log()`) is awaited, or when `run()` is called.
3151
-
3152
3128
  ##### Signature
3153
3129
 
3154
3130
  ```typescript
@@ -3220,9 +3196,6 @@ await table.fuzzyClean("category", "category", 80, {
3220
3196
 
3221
3197
  Replaces specified strings in the selected columns.
3222
3198
 
3223
- This method queues the operation; it runs when an async observer method (like
3224
- `getData()` or `log()`) is awaited, or when `run()` is called.
3225
-
3226
3199
  ##### Signature
3227
3200
 
3228
3201
  ```typescript
@@ -3280,9 +3253,7 @@ await table.replace("all", { "%": "" }).log();
3280
3253
 
3281
3254
  #### `lower`
3282
3255
 
3283
- Converts string values in the specified columns to lowercase. This method queues
3284
- the operation; it runs when an async observer method (like `getData()` or
3285
- `log()`) is awaited, or when `run()` is called.
3256
+ Converts string values in the specified columns to lowercase.
3286
3257
 
3287
3258
  ##### Signature
3288
3259
 
@@ -3313,9 +3284,7 @@ await table.lower(["column1", "column2"]).log();
3313
3284
 
3314
3285
  #### `upper`
3315
3286
 
3316
- Converts string values in the specified columns to uppercase. This method queues
3317
- the operation; it runs when an async observer method (like `getData()` or
3318
- `log()`) is awaited, or when `run()` is called.
3287
+ Converts string values in the specified columns to uppercase.
3319
3288
 
3320
3289
  ##### Signature
3321
3290
 
@@ -3347,9 +3316,7 @@ await table.upper(["column1", "column2"]).log();
3347
3316
  #### `capitalize`
3348
3317
 
3349
3318
  Capitalizes the first letter of each string in the specified columns and
3350
- converts the rest of the string to lowercase. This method queues the operation;
3351
- it runs when an async observer method (like `getData()` or `log()`) is awaited,
3352
- or when `run()` is called.
3319
+ converts the rest of the string to lowercase.
3353
3320
 
3354
3321
  ##### Signature
3355
3322
 
@@ -3381,9 +3348,6 @@ await table.capitalize(["column1", "column2"]).log();
3381
3348
 
3382
3349
  Truncates string values in a specified column to a maximum number of characters.
3383
3350
 
3384
- This method queues the operation; it runs when an async observer method (like
3385
- `getData()` or `log()`) is awaited, or when `run()` is called.
3386
-
3387
3351
  ##### Signature
3388
3352
 
3389
3353
  ```typescript
@@ -3419,9 +3383,6 @@ The columns must contain string (VARCHAR) values. An error is thrown if any
3419
3383
  column is of a different type. `null` values remain `null`. If any string
3420
3384
  already exceeds the target length, an error is thrown (no silent truncation).
3421
3385
 
3422
- This method queues the operation; it runs when an async observer method (like
3423
- `getData()` or `log()`) is awaited, or when `run()` is called.
3424
-
3425
3386
  ##### Signature
3426
3387
 
3427
3388
  ```typescript
@@ -3471,9 +3432,6 @@ Splits strings in a specified column by a separator and extracts a substring at
3471
3432
  a given index, storing the result in a new or existing column. If the index is
3472
3433
  out of bounds, an empty string will be returned for that row.
3473
3434
 
3474
- This method queues the operation; it runs when an async observer method (like
3475
- `getData()` or `log()`) is awaited, or when `run()` is called.
3476
-
3477
3435
  ##### Signature
3478
3436
 
3479
3437
  ```typescript
@@ -3520,9 +3478,6 @@ extra columns will contain empty strings (unless `strict` is set to `false`). If
3520
3478
  a row has more parts than the number of new columns, an error will be thrown
3521
3479
  unless `strict` is set to `false`.
3522
3480
 
3523
- This method queues the operation; it runs when an async observer method (like
3524
- `getData()` or `log()`) is awaited, or when `run()` is called.
3525
-
3526
3481
  ##### Signature
3527
3482
 
3528
3483
  ```typescript
@@ -3567,9 +3522,6 @@ await table.splitSpread("data", "|", ["col1", "col2"], { strict: false }).log();
3567
3522
  Extracts a specific number of characters from the beginning (left side) of
3568
3523
  string values in the specified column.
3569
3524
 
3570
- This method queues the operation; it runs when an async observer method (like
3571
- `getData()` or `log()`) is awaited, or when `run()` is called.
3572
-
3573
3525
  ##### Signature
3574
3526
 
3575
3527
  ```typescript
@@ -3599,9 +3551,6 @@ await table.firstChars("productCode", 2).log();
3599
3551
  Extracts a specific number of characters from the end (right side) of string
3600
3552
  values in the specified column.
3601
3553
 
3602
- This method queues the operation; it runs when an async observer method (like
3603
- `getData()` or `log()`) is awaited, or when `run()` is called.
3604
-
3605
3554
  ##### Signature
3606
3555
 
3607
3556
  ```typescript
@@ -3630,9 +3579,6 @@ await table.lastChars("productCode", 2).log();
3630
3579
 
3631
3580
  Replaces `NULL` values in the specified columns with a given value.
3632
3581
 
3633
- This method queues the operation; it runs when an async observer method (like
3634
- `getData()` or `log()`) is awaited, or when `run()` is called.
3635
-
3636
3582
  ##### Signature
3637
3583
 
3638
3584
  ```typescript
@@ -3675,9 +3621,6 @@ await table.replaceNulls("all", 0).log();
3675
3621
 
3676
3622
  Concatenates values from specified columns into a new column.
3677
3623
 
3678
- This method queues the operation; it runs when an async observer method (like
3679
- `getData()` or `log()`) is awaited, or when `run()` is called.
3680
-
3681
3624
  ##### Signature
3682
3625
 
3683
3626
  ```typescript
@@ -3724,9 +3667,6 @@ All values must be string, otherwise an error will be thrown. Use the
3724
3667
  If a column value is `NULL`, it will be replaced by `'Unknown'` in the
3725
3668
  concatenated result.
3726
3669
 
3727
- This method queues the operation; it runs when an async observer method (like
3728
- `getData()` or `log()`) is awaited, or when `run()` is called.
3729
-
3730
3670
  ##### Signature
3731
3671
 
3732
3672
  ```typescript
@@ -3786,9 +3726,6 @@ Each value in the specified column is split using the provided separator, and a
3786
3726
  new row is created for each resulting substring. All other column values are
3787
3727
  duplicated across the newly created rows.
3788
3728
 
3789
- This method queues the operation; it runs when an async observer method (like
3790
- `getData()` or `log()`) is awaited, or when `run()` is called.
3791
-
3792
3729
  ##### Signature
3793
3730
 
3794
3731
  ```typescript
@@ -3830,9 +3767,6 @@ Repeats rows based on the values in a column.
3830
3767
  If a row has a value of 3 in the specified column, it will be repeated 3 times.
3831
3768
  If the value is 0 or negative, the row will be removed.
3832
3769
 
3833
- This method queues the operation; it runs when an async observer method (like
3834
- `getData()` or `log()`) is awaited, or when `run()` is called.
3835
-
3836
3770
  ##### Signature
3837
3771
 
3838
3772
  ```typescript
@@ -3876,9 +3810,6 @@ This is the inverse operation of `unnest()`. Multiple rows are combined into
3876
3810
  fewer rows by grouping on specified category columns and concatenating the
3877
3811
  target column values with a separator.
3878
3812
 
3879
- This method queues the operation; it runs when an async observer method (like
3880
- `getData()` or `log()`) is awaited, or when `run()` is called.
3881
-
3882
3813
  ##### Signature
3883
3814
 
3884
3815
  ```typescript
@@ -3919,9 +3850,6 @@ await table.nest("tags", ",", ["country", "city"]).log();
3919
3850
 
3920
3851
  Rounds numeric values in specified columns.
3921
3852
 
3922
- This method queues the operation; it runs when an async observer method (like
3923
- `getData()` or `log()`) is awaited, or when `run()` is called.
3924
-
3925
3853
  ##### Signature
3926
3854
 
3927
3855
  ```typescript
@@ -3981,9 +3909,6 @@ Selected integer and decimal columns become `DOUBLE` columns so fractional noise
3981
3909
  is retained. This method adds random jitter; it does not provide anonymization
3982
3910
  or differential privacy guarantees.
3983
3911
 
3984
- This method queues the operation; it runs when an async observer method (like
3985
- `getData()` or `log()`) is awaited, or when `run()` is called.
3986
-
3987
3912
  ##### Signature
3988
3913
 
3989
3914
  ```typescript
@@ -4023,10 +3948,10 @@ await table.addNoise(["x", "y"], 0.1, {
4023
3948
 
4024
3949
  #### `updateColumn`
4025
3950
 
4026
- Updates values in a specified column using a SQL expression.
4027
-
4028
- This method queues the operation; it runs when an async observer method (like
4029
- `getData()` or `log()`) is awaited, or when `run()` is called.
3951
+ Updates values in a specified column using a SQL expression. With the default
3952
+ `SimpleDB.expressionSyntax: "js"`, expressions support JavaScript-style
3953
+ operators (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for
3954
+ unchanged SQL.
4030
3955
 
4031
3956
  ##### Signature
4032
3957
 
@@ -4064,13 +3989,19 @@ await table.updateColumn(
4064
3989
  ).log();
4065
3990
  ```
4066
3991
 
3992
+ ```ts
3993
+ // Update an existing column with a value from a JSON object using dot notation.
3994
+ await table.loadArray([{ count: 0, details: { count: 3 } }], {
3995
+ columnTypes: { details: "JSON" },
3996
+ })
3997
+ .updateColumn("count", "details.count")
3998
+ .log();
3999
+ ```
4000
+
4067
4001
  #### `ranks`
4068
4002
 
4069
4003
  Assigns ranks to rows in a new column based on the values of a specified column.
4070
4004
 
4071
- This method queues the operation; it runs when an async observer method (like
4072
- `getData()` or `log()`) is awaited, or when `run()` is called.
4073
-
4074
4005
  ##### Signature
4075
4006
 
4076
4007
  ```typescript
@@ -4121,9 +4052,6 @@ await table.ranks("sales", "salesRank", { by: ["department", "city"] }).log();
4121
4052
 
4122
4053
  Assigns quantiles to rows in a new column based on specified column values.
4123
4054
 
4124
- This method queues the operation; it runs when an async observer method (like
4125
- `getData()` or `log()`) is awaited, or when `run()` is called.
4126
-
4127
4055
  ##### Signature
4128
4056
 
4129
4057
  ```typescript
@@ -4165,25 +4093,30 @@ await table.quantiles("sales", 4, "salesQuartile").log();
4165
4093
 
4166
4094
  #### `bins`
4167
4095
 
4168
- Assigns bins for specified column values based on an interval size.
4169
-
4170
- This method queues the operation; it runs when an async observer method (like
4171
- `getData()` or `log()`) is awaited, or when `run()` is called.
4096
+ Adds numeric start and end columns for bins of the specified interval size. Each
4097
+ bin includes its start and excludes its end: `start <= value < end`. A value
4098
+ exactly on an end boundary belongs to the next bin. Null source values produce
4099
+ null in both output columns.
4172
4100
 
4173
4101
  ##### Signature
4174
4102
 
4175
4103
  ```typescript
4176
- bins(column: string, interval: number, newColumn: string, options?: { startValue?: number }): this;
4104
+ bins(column: string, interval: number, startColumn: string, endColumn: string, options?: { startValue?: number }): this;
4177
4105
  ```
4178
4106
 
4179
4107
  ##### Parameters
4180
4108
 
4181
- - **`column`**: The column containing values from which bins will be computed.
4182
- - **`interval`**: The interval size for binning the values.
4183
- - **`newColumn`**: The name of the new column where the bins will be stored.
4109
+ - **`column`**: The numeric column containing values from which bins will be
4110
+ computed.
4111
+ - **`interval`**: The finite, positive interval size for binning the values.
4112
+ - **`startColumn`**: The required name of the new numeric column containing
4113
+ inclusive bin starts.
4114
+ - **`endColumn`**: The required name of the new numeric column containing
4115
+ exclusive bin ends. Must differ from startColumn.
4184
4116
  - **`options`**: An optional object with configuration options:
4185
- - **`options.startValue`**: The starting value for binning. Defaults to the
4186
- minimum value in the specified column.
4117
+ - **`options.startValue`**: The finite starting value for binning, no greater
4118
+ than the minimum source value. Defaults to the minimum value in the specified
4119
+ column.
4187
4120
 
4188
4121
  ##### Returns
4189
4122
 
@@ -4192,15 +4125,15 @@ The table, so methods can be chained.
4192
4125
  ##### Examples
4193
4126
 
4194
4127
  ```ts
4195
- // Assigns a bin for each row in a new 'bins' column based on 'column1' values, with an interval of 10.
4196
- // If the minimum value in 'column1' is 5, the bins will follow this pattern: "[5-14]", "[15-24]", etc.
4197
- await table.bins("column1", 10, "bins").log();
4128
+ // If the minimum is 5, bins have boundaries 5 and 15, 15 and 25, etc.
4129
+ await table.bins("column1", 10, "binStart", "binEnd").log();
4198
4130
  ```
4199
4131
 
4200
4132
  ```ts
4201
- // Assigns bins starting at a specific value (0) with an interval of 10.
4202
- // The bins will follow this pattern: "[0-9]", "[10-19]", "[20-29]", etc.
4203
- await table.bins("column1", 10, "bins", { startValue: 0 }).log();
4133
+ // Bins start at 0: a value of 10 has binStart 10 and binEnd 20.
4134
+ await table
4135
+ .bins("column1", 10, "binStart", "binEnd", { startValue: 0 })
4136
+ .log();
4204
4137
  ```
4205
4138
 
4206
4139
  #### `rowProportions`
@@ -4284,9 +4217,6 @@ await table.rowProportions(["Men", "Women", "NonBinary"], {
4284
4217
  }).log();
4285
4218
  ```
4286
4219
 
4287
- This method queues the operation; it runs when an async observer method (like
4288
- `getData()` or `log()`) is awaited, or when `run()` is called.
4289
-
4290
4220
  #### `rowRanks`
4291
4221
 
4292
4222
  Selects a ranked numeric value within each row and adds its source column name,
@@ -4299,9 +4229,6 @@ to produce one row for each tied column. The `"all"` option can therefore
4299
4229
  increase the table's row count. If null values leave a row without the requested
4300
4230
  rank, the new columns contain null.
4301
4231
 
4302
- This method queues the operation; it runs when an async observer method (like
4303
- `getData()` or `log()`) is awaited, or when `run()` is called.
4304
-
4305
4232
  ##### Signature
4306
4233
 
4307
4234
  ```typescript
@@ -4353,9 +4280,6 @@ await table.rowRanks(["CAQ", "PLQ", "PQ"], {
4353
4280
  Computes proportions vertically over a column's values, relative to the sum of
4354
4281
  all values in that column or group.
4355
4282
 
4356
- This method queues the operation; it runs when an async observer method (like
4357
- `getData()` or `log()`) is awaited, or when `run()` is called.
4358
-
4359
4283
  ##### Signature
4360
4284
 
4361
4285
  ```typescript
@@ -4415,9 +4339,6 @@ Creates a summary table from selected columns, optionally grouped by other
4415
4339
  columns. This method allows you to aggregate data, calculate statistics (e.g.,
4416
4340
  count, mean, sum), and group results by categorical columns.
4417
4341
 
4418
- This method queues the operation; it runs when an async observer method (like
4419
- `getData()` or `log()`) is awaited, or when `run()` is called.
4420
-
4421
4342
  ##### Signature
4422
4343
 
4423
4344
  ```typescript
@@ -4545,9 +4466,6 @@ summarized nor used for labels contain `NULL` in the added rows. A stat string
4545
4466
  is also used as its row label; pass an object to customize that label. If
4546
4467
  `options.stats` is omitted, every supported stat is added.
4547
4468
 
4548
- This method queues the operation; it runs when an async observer method (like
4549
- `getData()` or `log()`) is awaited, or when `run()` is called.
4550
-
4551
4469
  ##### Signature
4552
4470
 
4553
4471
  ```typescript
@@ -4602,9 +4520,6 @@ await table.addSummaryRows("all", "region", {
4602
4520
  Computes the cumulative sum of values in a column. For this method to work
4603
4521
  properly, ensure your data is sorted first.
4604
4522
 
4605
- This method queues the operation; it runs when an async observer method (like
4606
- `getData()` or `log()`) is awaited, or when `run()` is called.
4607
-
4608
4523
  ##### Signature
4609
4524
 
4610
4525
  ```typescript
@@ -4653,9 +4568,6 @@ column. For rows without enough preceding or following rows to form a complete
4653
4568
  window, `NULL` will be returned. For this method to work properly, ensure your
4654
4569
  data is sorted by the relevant column(s) first.
4655
4570
 
4656
- This method queues the operation; it runs when an async observer method (like
4657
- `getData()` or `log()`) is awaited, or when `run()` is called.
4658
-
4659
4571
  ##### Signature
4660
4572
 
4661
4573
  ```typescript
@@ -4712,9 +4624,6 @@ specified, the method computes the correlations for all numeric column
4712
4624
  combinations. Note that correlation is symmetrical: the correlation of `x` with
4713
4625
  `y` is the same as `y` with `x`.
4714
4626
 
4715
- This method queues the operation; it runs when an async observer method (like
4716
- `getData()` or `log()`) is awaited, or when `run()` is called.
4717
-
4718
4627
  ##### Signature
4719
4628
 
4720
4629
  ```typescript
@@ -4781,9 +4690,6 @@ the method computes linear regression analysis for all numeric column
4781
4690
  permutations. Note that linear regression analysis is asymmetrical: the linear
4782
4691
  regression of `x` over `y` is not the same as `y` over `x`.
4783
4692
 
4784
- This method queues the operation; it runs when an async observer method (like
4785
- `getData()` or `log()`) is awaited, or when `run()` is called.
4786
-
4787
4693
  ##### Signature
4788
4694
 
4789
4695
  ```typescript
@@ -4848,9 +4754,6 @@ await table.linearRegressions({ decimals: 3 }).log();
4848
4754
  Identifies outliers in a specified column using the Interquartile Range (IQR)
4849
4755
  method.
4850
4756
 
4851
- This method queues the operation; it runs when an async observer method (like
4852
- `getData()` or `log()`) is awaited, or when `run()` is called.
4853
-
4854
4757
  ##### Signature
4855
4758
 
4856
4759
  ```typescript
@@ -4886,9 +4789,6 @@ await table.outliersIQR("salary", "salaryOutlier", { by: "gender" }).log();
4886
4789
 
4887
4790
  Computes the Z-score for values in a specified column.
4888
4791
 
4889
- This method queues the operation; it runs when an async observer method (like
4890
- `getData()` or `log()`) is awaited, or when `run()` is called.
4891
-
4892
4792
  ##### Signature
4893
4793
 
4894
4794
  ```typescript
@@ -4931,9 +4831,6 @@ await table.zScore("score", "scoreZScore", { decimals: 2 }).log();
4931
4831
 
4932
4832
  Normalizes the values in a column using min-max normalization.
4933
4833
 
4934
- This method queues the operation; it runs when an async observer method (like
4935
- `getData()` or `log()`) is awaited, or when `run()` is called.
4936
-
4937
4834
  ##### Signature
4938
4835
 
4939
4836
  ```typescript
@@ -4999,9 +4896,6 @@ an instant. Construct them with an explicit timezone, such as
4999
4896
  `new Date("2001-01-01T00:00:00Z")`; date-time strings without `Z` or an offset
5000
4897
  use the user's local timezone.
5001
4898
 
5002
- This method queues the operation; it runs when an async observer method (like
5003
- `getData()` or `log()`) is awaited, or when `run()` is called.
5004
-
5005
4899
  ##### Signature
5006
4900
 
5007
4901
  ```typescript
@@ -5080,16 +4974,19 @@ Updates data in the table using a JavaScript function. The function receives the
5080
4974
  existing rows as an array of objects and must return the modified rows as an
5081
4975
  array of objects. This method offers high flexibility for data manipulation but
5082
4976
  can be slow for large tables as it involves transferring data between DuckDB and
5083
- JavaScript. This method does not work with tables containing geometries.
4977
+ JavaScript. Before writing a JavaScript callback, check for an existing SDA
4978
+ method that performs the same operation; it will usually be faster and more
4979
+ efficient.
5084
4980
 
5085
- This method queues the update; the dataModifier function runs when an async
5086
- observer method (like `getData()` or `log()`) is awaited, or when `run()` is
5087
- called.
4981
+ If the table has geometry columns, the callback can read and modify their
4982
+ GeoJSON geometry objects directly. Extra properties added to these objects are
4983
+ ignored; create new columns on the returned rows to store additional attributes.
4984
+ All geometry columns must use EPSG:4326; use reproject() first if needed.
5088
4985
 
5089
4986
  ##### Signature
5090
4987
 
5091
4988
  ```typescript
5092
- updateWithJS(dataModifier: ((rows: Record<string, unknown>[]) => Promise<Record<string, unknown>[]>) | ((rows: Record<string, unknown>[]) => Record<string, unknown>[]), options?: { batchSize?: number }): this;
4989
+ updateWithJS(dataModifier: ((rows: Record<string, unknown>[]) => Promise<Record<string, unknown>[]>) | ((rows: Record<string, unknown>[]) => Record<string, unknown>[]), options?: { batchSize?: number; columnTypes?: Record<string, string> }): this;
5093
4990
  ```
5094
4991
 
5095
4992
  ##### Parameters
@@ -5100,7 +4997,15 @@ updateWithJS(dataModifier: ((rows: Record<string, unknown>[]) => Promise<Record<
5100
4997
  - **`options`**: An optional object with configuration options:
5101
4998
  - **`options.batchSize`**: If provided, rows are processed in batches of this
5102
4999
  size instead of all at once, so large tables don't have to be materialized
5103
- entirely in memory. The modifier function is called once per batch.
5000
+ entirely in memory. This limits input rows per batch, but does not eliminate
5001
+ conversion work or bound geometry size or callback output expansion. The
5002
+ modifier function is called once per batch. Tables with a column named `rowid`
5003
+ or `__sda_rowid` (case-insensitive) are rejected before the modifier runs;
5004
+ rename the column or omit batchSize.
5005
+ - **`options.columnTypes`**: Explicit types for newly added columns, following
5006
+ loadArray's columnTypes contract. Declare geometry as GEOMETRY('EPSG:4326');
5007
+ geometry is never inferred from object shape. Existing columns cannot be
5008
+ redeclared.
5104
5009
 
5105
5010
  ##### Returns
5106
5011
 
@@ -5109,34 +5014,73 @@ The table, so methods can be chained.
5109
5014
  ##### Examples
5110
5015
 
5111
5016
  ```ts
5112
- // Extract hostnames with JavaScript's URL parser.
5113
- const table = await sdb
5114
- .newTable()
5115
- .loadData("websites.csv")
5116
- .updateWithJS((rows) => {
5117
- return rows.map((row) => ({
5017
+ // Double the value in each row with a JavaScript callback.
5018
+ await table.loadArray([{ value: 1 }, { value: 2 }])
5019
+ .updateWithJS((rows) =>
5020
+ rows.map((row) => ({
5021
+ value: Number(row.value) * 2,
5022
+ }))
5023
+ )
5024
+ .log();
5025
+ ```
5026
+
5027
+ ```ts
5028
+ // Process two rows per callback to limit how many rows are handled at once.
5029
+ await table.loadArray([{ value: 1 }, { value: 2 }, { value: 3 }])
5030
+ .updateWithJS((rows) =>
5031
+ rows.map((row) => ({
5032
+ value: Number(row.value) * 2,
5033
+ })), { batchSize: 2 })
5034
+ .log();
5035
+ ```
5036
+
5037
+ ```ts
5038
+ // Add a JSON object column to keep related attributes together.
5039
+ await table.loadArray([{ name: "Station" }])
5040
+ .updateWithJS((rows) =>
5041
+ rows.map((row) => ({
5118
5042
  ...row,
5119
- hostname: typeof row.url === "string" ? new URL(row.url).hostname : null,
5120
- }));
5121
- })
5043
+ details: { active: true, category: "weather" },
5044
+ })), { columnTypes: { details: "JSON" } })
5122
5045
  .log();
5123
5046
  ```
5124
5047
 
5125
5048
  ```ts
5126
- // Enrich reviews with scores from an external service, 100 at a time.
5127
- const reviews = await table
5128
- .updateWithJS(async (rows) => {
5129
- const response = await fetch("https://api.example.com/score", {
5130
- method: "POST",
5131
- headers: { "content-type": "application/json" },
5132
- body: JSON.stringify(rows.map((row) => row.review)),
5133
- });
5134
- const scores = await response.json() as number[];
5135
- return rows.map((row, index) => ({ ...row, score: scores[index] }));
5136
- }, { batchSize: 100 })
5049
+ // Add a three-element float vector, such as an embedding.
5050
+ await table.loadArray([{ name: "Station" }])
5051
+ .updateWithJS((rows) =>
5052
+ rows.map((row) => ({
5053
+ ...row,
5054
+ embedding: [0.25, 0.5, 0.75],
5055
+ })), { columnTypes: { embedding: "FLOAT[3]" } })
5137
5056
  .log();
5138
5057
  ```
5139
5058
 
5059
+ ```ts
5060
+ // Move each point slightly east by editing its longitude.
5061
+ await sdb.newTable().loadArray([{
5062
+ geom: { type: "Point", coordinates: [-73, 45] },
5063
+ }], { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } })
5064
+ .updateWithJS((rows) =>
5065
+ rows.map((row) => {
5066
+ // This example loads only Points. Narrow the type for coordinate access.
5067
+ const geom = row.geom as { type: "Point"; coordinates: number[] } | null;
5068
+ if (geom !== null) geom.coordinates[0] += 0.01;
5069
+ return row;
5070
+ })
5071
+ ).log();
5072
+ ```
5073
+
5074
+ ```ts
5075
+ // Create a geometry column from longitude and latitude values.
5076
+ await sdb.newTable().loadArray([{ longitude: -73, latitude: 45 }])
5077
+ .updateWithJS((rows) =>
5078
+ rows.map((row) => ({
5079
+ ...row,
5080
+ geom: { type: "Point", coordinates: [row.longitude, row.latitude] },
5081
+ })), { columnTypes: { geom: "GEOMETRY('EPSG:4326')" } }).log();
5082
+ ```
5083
+
5140
5084
  #### `getSchema`
5141
5085
 
5142
5086
  Returns the schema of the table, including column names and their data types.
@@ -5242,9 +5186,6 @@ Normalizes string values in a column by:
5242
5186
  Produces identical output to `journalism-format`'s `normalizeString()` function
5243
5187
  for all common cases including accented Latin characters.
5244
5188
 
5245
- This method queues the operation; it runs when an async observer method (like
5246
- `getData()` or `log()`) is awaited, or when `run()` is called.
5247
-
5248
5189
  ##### Signature
5249
5190
 
5250
5191
  ```typescript
@@ -5312,9 +5253,6 @@ clusters. A user-perceived character composed of multiple code points, such as
5312
5253
  some emoji or decomposed accented letters, counts as multiple characters. `NULL`
5313
5254
  input values produce `NULL` counts.
5314
5255
 
5315
- This method queues the operation; it runs when an async observer method (like
5316
- `getData()` or `log()`) is awaited, or when `run()` is called.
5317
-
5318
5256
  ##### Signature
5319
5257
 
5320
5258
  ```typescript
@@ -5347,9 +5285,6 @@ standalone punctuation sequence counts as a word. Text without whitespace counts
5347
5285
  as one word, regardless of language. Empty or whitespace-only strings produce
5348
5286
  `0`, and `NULL` input values produce `NULL` counts.
5349
5287
 
5350
- This method queues the operation; it runs when an async observer method (like
5351
- `getData()` or `log()`) is awaited, or when `run()` is called.
5352
-
5353
5288
  ##### Signature
5354
5289
 
5355
5290
  ```typescript
@@ -5405,7 +5340,9 @@ console.log(totalChars); // e.g., 523
5405
5340
 
5406
5341
  #### `getRowCount`
5407
5342
 
5408
- Returns the number of rows in the table.
5343
+ Returns the number of rows in the table. With the default
5344
+ `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
5345
+ (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
5409
5346
 
5410
5347
  ##### Signature
5411
5348
 
@@ -5895,9 +5832,11 @@ console.log(uniqueCategories); // e.g., ["Books", "Clothing", "Electronics"]
5895
5832
 
5896
5833
  #### `getFirstRow`
5897
5834
 
5898
- Returns the first row of the table, optionally filtered by SQL conditions. You
5899
- can also use JavaScript syntax for conditions (e.g., `&&`, `||`, `===`, `!==`).
5900
- Temporal values use the same JavaScript representations as `getData()`.
5835
+ Returns the first row of the table, optionally filtered by SQL conditions. With
5836
+ the default `SimpleDB.expressionSyntax: "js"`, conditions support
5837
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
5838
+ `expressionSyntax: "sql"` for unchanged SQL. Temporal values use the same
5839
+ JavaScript representations as `getData()`.
5901
5840
 
5902
5841
  ##### Signature
5903
5842
 
@@ -5934,9 +5873,11 @@ console.log(firstRowBooks);
5934
5873
 
5935
5874
  #### `getLastRow`
5936
5875
 
5937
- Returns the last row of the table, optionally filtered by SQL conditions. You
5938
- can also use JavaScript syntax for conditions (e.g., `&&`, `||`, `===`, `!==`).
5939
- Temporal values use the same JavaScript representations as `getData()`.
5876
+ Returns the last row of the table, optionally filtered by SQL conditions. With
5877
+ the default `SimpleDB.expressionSyntax: "js"`, conditions support
5878
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
5879
+ `expressionSyntax: "sql"` for unchanged SQL. Temporal values use the same
5880
+ JavaScript representations as `getData()`.
5940
5881
 
5941
5882
  ##### Signature
5942
5883
 
@@ -5974,8 +5915,10 @@ console.log(lastRowBooks);
5974
5915
  #### `getTop`
5975
5916
 
5976
5917
  Returns the top `n` rows of the table, optionally filtered by SQL conditions.
5977
- You can also use JavaScript syntax for conditions (e.g., `&&`, `||`, `===`,
5978
- `!==`). Temporal values use the same JavaScript representations as `getData()`.
5918
+ With the default `SimpleDB.expressionSyntax: "js"`, conditions support
5919
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
5920
+ `expressionSyntax: "sql"` for unchanged SQL. Temporal values use the same
5921
+ JavaScript representations as `getData()`.
5979
5922
 
5980
5923
  ##### Signature
5981
5924
 
@@ -6012,9 +5955,10 @@ console.log(top5Books);
6012
5955
 
6013
5956
  Returns the bottom `n` rows of the table, optionally filtered by SQL conditions.
6014
5957
  By default, the last row will be returned first. To preserve the original order,
6015
- use the `originalOrder` option. You can also use JavaScript syntax for
6016
- conditions (e.g., `&&`, `||`, `===`, `!==`). Temporal values use the same
6017
- JavaScript representations as `getData()`.
5958
+ use the `originalOrder` option. With the default
5959
+ `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
5960
+ (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
5961
+ Temporal values use the same JavaScript representations as `getData()`.
6018
5962
 
6019
5963
  ##### Signature
6020
5964
 
@@ -6063,9 +6007,10 @@ console.log(bottom5Books);
6063
6007
  #### `getRow`
6064
6008
 
6065
6009
  Returns a single row that matches the specified conditions. If no row matches or
6066
- if more than one row matches, an error is thrown by default. You can also use
6067
- JavaScript syntax for conditions (e.g., `AND`, `||`, `===`, `!==`). Temporal
6068
- values use the same JavaScript representations as `getData()`.
6010
+ if more than one row matches, an error is thrown by default. With the default
6011
+ `SimpleDB.expressionSyntax: "js"`, conditions support JavaScript-style operators
6012
+ (`&&`, `||`, `===`, `!==`). Set `expressionSyntax: "sql"` for unchanged SQL.
6013
+ Temporal values use the same JavaScript representations as `getData()`.
6069
6014
 
6070
6015
  ##### Signature
6071
6016
 
@@ -6114,14 +6059,28 @@ console.log(flexibleRow);
6114
6059
  #### `getData`
6115
6060
 
6116
6061
  Returns the data from the table as an array of objects, optionally filtered by
6117
- SQL conditions. You can also use JavaScript syntax for conditions (e.g., `&&`,
6118
- `||`, `===`, `!==`).
6062
+ SQL conditions. With the default `SimpleDB.expressionSyntax: "js"`, conditions
6063
+ support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
6064
+ `expressionSyntax: "sql"` for unchanged SQL.
6119
6065
 
6120
6066
  Top-level DuckDB `DATE` and `TIMESTAMP` columns are returned as JavaScript
6121
6067
  `Date` objects interpreted in UTC. `TIMESTAMP WITH TIME ZONE` values are
6122
6068
  returned as UTC strings, preserving DuckDB's microsecond precision; JavaScript
6123
6069
  `Date` supports only milliseconds.
6124
6070
 
6071
+ Top-level `BIGINT`, `UBIGINT`, `HUGEINT`, and `UHUGEINT` columns are returned as
6072
+ JavaScript numbers. Values outside `Number.MIN_SAFE_INTEGER` through
6073
+ `Number.MAX_SAFE_INTEGER` throw instead of losing precision. Use
6074
+ `convert({ columnName: "string" })` before `getData()` to retrieve their exact
6075
+ digits as strings.
6076
+
6077
+ SQL arrays and lists, including float vectors, return their actual elements as
6078
+ JavaScript arrays. Null cells and null elements remain null.
6079
+
6080
+ Nested SQL lists and structs become JavaScript arrays and objects. Within them,
6081
+ `BIGINT`, `UBIGINT`, `HUGEINT`, `UHUGEINT`, and `DECIMAL` values remain strings
6082
+ to preserve precision, even for small integers.
6083
+
6125
6084
  ##### Signature
6126
6085
 
6127
6086
  ```typescript
@@ -6171,11 +6130,24 @@ console.log(booksData);
6171
6130
  const preview = await table.getData({ limit: 2 });
6172
6131
  ```
6173
6132
 
6133
+ ```ts
6134
+ // Read nested integers without losing digits. SQL creates the nested types.
6135
+ await sdb.customQuery(`CREATE TABLE nested_example AS SELECT
6136
+ [42::BIGINT, 9007199254740993::BIGINT] AS ids,
6137
+ {'count': 2::INTEGER, 'amount': 123.450::DECIMAL(6,3)} AS details`);
6138
+ const nested = sdb.newTable("nested_example");
6139
+ const rows = await nested.getData();
6140
+ // [{ ids: ["42", "9007199254740993"], details: { count: 2, amount: "123.450" } }]
6141
+ await nested.log();
6142
+ ```
6143
+
6174
6144
  #### `stream`
6175
6145
 
6176
6146
  Streams the table rows one by one as an async iterator, without materializing
6177
6147
  the whole table in memory. Values are converted to JavaScript types the same way
6178
- as `getData()`.
6148
+ as `getData()`. With the default `SimpleDB.expressionSyntax: "js"`, conditions
6149
+ support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
6150
+ `expressionSyntax: "sql"` for unchanged SQL.
6179
6151
 
6180
6152
  The underlying DuckDB result is streamed chunk by chunk, so tables larger than
6181
6153
  the available memory can be iterated. Avoid running other queries on the same
@@ -6222,9 +6194,10 @@ for await (
6222
6194
  #### `getDataAsCSV`
6223
6195
 
6224
6196
  Returns the data from the table as a CSV string, optionally filtered by SQL
6225
- conditions. You can also use JavaScript syntax for conditions (e.g., `&&`, `||`,
6226
- `===`, `!==`). Temporal values are first converted as they are in `getData()`,
6227
- then serialized using UTC date and timestamp text.
6197
+ conditions. With the default `SimpleDB.expressionSyntax: "js"`, conditions
6198
+ support JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
6199
+ `expressionSyntax: "sql"` for unchanged SQL. Temporal values are first converted
6200
+ as they are in `getData()`, then serialized using UTC date and timestamp text.
6228
6201
 
6229
6202
  ##### Signature
6230
6203
 
@@ -6274,9 +6247,6 @@ console.log(booksDataCSV);
6274
6247
 
6275
6248
  Creates point geometries from latitude (y) and longitude (x) columns.
6276
6249
 
6277
- This method queues the operation; it runs when an async observer method (like
6278
- `getData()` or `log()`) is awaited, or when `run()` is called.
6279
-
6280
6250
  ##### Signature
6281
6251
 
6282
6252
  ```typescript
@@ -6317,9 +6287,6 @@ await table.createPoints("y", "x", "geom", { projection: "EPSG:3347" }).log();
6317
6287
 
6318
6288
  Adds a column with boolean values indicating the validity of geometries.
6319
6289
 
6320
- This method queues the operation; it runs when an async observer method (like
6321
- `getData()` or `log()`) is awaited, or when `run()` is called.
6322
-
6323
6290
  ##### Signature
6324
6291
 
6325
6292
  ```typescript
@@ -6356,9 +6323,6 @@ await table.addGeoValidity("isValidMyGeom", { column: "myGeom" }).log();
6356
6323
 
6357
6324
  Adds a column with the number of vertices (points) in each geometry.
6358
6325
 
6359
- This method queues the operation; it runs when an async observer method (like
6360
- `getData()` or `log()`) is awaited, or when `run()` is called.
6361
-
6362
6326
  ##### Signature
6363
6327
 
6364
6328
  ```typescript
@@ -6394,9 +6358,6 @@ await table.addVertexCount("myGeomVertices", { column: "myGeom" }).log();
6394
6358
 
6395
6359
  Attempts to make invalid geometries valid without removing any vertices.
6396
6360
 
6397
- This method queues the operation; it runs when an async observer method (like
6398
- `getData()` or `log()`) is awaited, or when `run()` is called.
6399
-
6400
6361
  ##### Signature
6401
6362
 
6402
6363
  ```typescript
@@ -6429,9 +6390,6 @@ await table.fixGeo("myGeom").log();
6429
6390
  Adds a column with boolean values indicating whether geometries are closed
6430
6391
  (e.g., polygons) or open (e.g., linestrings).
6431
6392
 
6432
- This method queues the operation; it runs when an async observer method (like
6433
- `getData()` or `log()`) is awaited, or when `run()` is called.
6434
-
6435
6393
  ##### Signature
6436
6394
 
6437
6395
  ```typescript
@@ -6468,9 +6426,6 @@ await table.addGeoClosedStatus("boundaryClosed", { column: "boundaryGeom" })
6468
6426
  Adds a column with the geometry type (e.g., `"POINT"`, `"LINESTRING"`,
6469
6427
  `"POLYGON"`) for each geometry.
6470
6428
 
6471
- This method queues the operation; it runs when an async observer method (like
6472
- `getData()` or `log()`) is awaited, or when `run()` is called.
6473
-
6474
6429
  ##### Signature
6475
6430
 
6476
6431
  ```typescript
@@ -6509,9 +6464,6 @@ vice-versa). **Warning:** This method should be used with caution as it directly
6509
6464
  manipulates coordinate order and can affect the accuracy of geospatial
6510
6465
  operations if not used correctly.
6511
6466
 
6512
- This method queues the operation; it runs when an async observer method (like
6513
- `getData()` or `log()`) is awaited, or when `run()` is called.
6514
-
6515
6467
  ##### Signature
6516
6468
 
6517
6469
  ```typescript
@@ -6551,9 +6503,7 @@ with latitude. This method adds random jitter; it does not provide anonymization
6551
6503
  or differential privacy guarantees.
6552
6504
 
6553
6505
  This method supports only `POINT` geometries in `EPSG:4326`. Null and empty
6554
- geometries are preserved. It queues the operation; the operation runs when an
6555
- async observer method (like `getData()` or `log()`) is awaited, or when `run()`
6556
- is called.
6506
+ geometries are preserved.
6557
6507
 
6558
6508
  ##### Signature
6559
6509
 
@@ -6600,9 +6550,6 @@ await table.addGeoNoise(0.1, {
6600
6550
  Reduces the precision of geometries in a specified column to a given number of
6601
6551
  decimal places.
6602
6552
 
6603
- This method queues the operation; it runs when an async observer method (like
6604
- `getData()` or `log()`) is awaited, or when `run()` is called.
6605
-
6606
6553
  ##### Signature
6607
6554
 
6608
6555
  ```typescript
@@ -6638,9 +6585,6 @@ await table.reducePrecision(2, { column: "myGeom" }).log();
6638
6585
  Reprojects the geometries in a specified column to another Spatial Reference
6639
6586
  System (SRS).
6640
6587
 
6641
- This method queues the operation; it runs when an async observer method (like
6642
- `getData()` or `log()`) is awaited, or when `run()` is called.
6643
-
6644
6588
  ##### Signature
6645
6589
 
6646
6590
  ```typescript
@@ -6676,9 +6620,6 @@ await table.reproject("EPSG:3347", { column: "myGeom" }).log();
6676
6620
  Computes the area of geometries in square meters (`"m2"`) or optionally square
6677
6621
  kilometers (`"km2"`). The input geometry is assumed to be in EPSG:4326 (WGS84).
6678
6622
 
6679
- This method queues the operation; it runs when an async observer method (like
6680
- `getData()` or `log()`) is awaited, or when `run()` is called.
6681
-
6682
6623
  ##### Signature
6683
6624
 
6684
6625
  ```typescript
@@ -6728,9 +6669,6 @@ await table.area("myGeomArea", { column: "myGeom" }).log();
6728
6669
  Computes the length of line geometries in meters (`"m"`) or optionally
6729
6670
  kilometers (`"km"`). The input geometry is assumed to be in EPSG:4326 (WGS84).
6730
6671
 
6731
- This method queues the operation; it runs when an async observer method (like
6732
- `getData()` or `log()`) is awaited, or when `run()` is called.
6733
-
6734
6672
  ##### Signature
6735
6673
 
6736
6674
  ```typescript
@@ -6780,9 +6718,6 @@ await table.length("routeLength", { column: "routeGeom" }).log();
6780
6718
  Computes the perimeter of polygon geometries in meters (`"m"`) or optionally
6781
6719
  kilometers (`"km"`). The input geometry is assumed to be in EPSG:4326 (WGS84).
6782
6720
 
6783
- This method queues the operation; it runs when an async observer method (like
6784
- `getData()` or `log()`) is awaited, or when `run()` is called.
6785
-
6786
6721
  ##### Signature
6787
6722
 
6788
6723
  ```typescript
@@ -6834,9 +6769,6 @@ Computes a buffer (a polygon representing a specified distance around a
6834
6769
  geometry) for geometries in a specified column. The distance is in the Spatial
6835
6770
  Reference System (SRS) unit of the input geometries.
6836
6771
 
6837
- This method queues the operation; it runs when an async observer method (like
6838
- `getData()` or `log()`) is awaited, or when `run()` is called.
6839
-
6840
6772
  ##### Signature
6841
6773
 
6842
6774
  ```typescript
@@ -6875,10 +6807,7 @@ Merges the data of this table (considered the left table) with another table
6875
6807
  (the right table) based on a spatial relationship. Note that the order of rows
6876
6808
  in the returned data is not guaranteed to be the same as in the original tables.
6877
6809
  This operation might create temporary files in a `.tmp` folder; consider adding
6878
- `.tmp` to your `.gitignore`. This method queues the operation; it runs when an
6879
- async observer method (like `getData()` or `log()`) is awaited, or when `run()`
6880
- is called. The join uses the other table's state as of this call: operations
6881
- queued on it afterwards run after the join.
6810
+ `.tmp` to your `.gitignore`.
6882
6811
 
6883
6812
  ##### Signature
6884
6813
 
@@ -6971,9 +6900,6 @@ const tableC = await tableA.joinGeo(tableB, "intersect", {
6971
6900
  Computes the intersection of two sets of geometries, creating new geometries
6972
6901
  where they overlap.
6973
6902
 
6974
- This method queues the operation; it runs when an async observer method (like
6975
- `getData()` or `log()`) is awaited, or when `run()` is called.
6976
-
6977
6903
  ##### Signature
6978
6904
 
6979
6905
  ```typescript
@@ -7004,9 +6930,6 @@ await table.intersection("geomA", "geomB", "intersectGeom").log();
7004
6930
  Computes the geometric difference between two geometries, returning the portion
7005
6931
  of the first geometry that does not intersect the second.
7006
6932
 
7007
- This method queues the operation; it runs when an async observer method (like
7008
- `getData()` or `log()`) is awaited, or when `run()` is called.
7009
-
7010
6933
  ##### Signature
7011
6934
 
7012
6935
  ```typescript
@@ -7037,9 +6960,6 @@ await table.difference("geomA", "geomB", "geomA_minus_geomB").log();
7037
6960
 
7038
6961
  Fills holes in polygon geometries.
7039
6962
 
7040
- This method queues the operation; it runs when an async observer method (like
7041
- `getData()` or `log()`) is awaited, or when `run()` is called.
7042
-
7043
6963
  ##### Signature
7044
6964
 
7045
6965
  ```typescript
@@ -7072,9 +6992,6 @@ await table.fillHoles("polygonGeom").log();
7072
6992
  Returns `TRUE` if two geometries intersect (overlap in any way), and `FALSE`
7073
6993
  otherwise.
7074
6994
 
7075
- This method queues the operation; it runs when an async observer method (like
7076
- `getData()` or `log()`) is awaited, or when `run()` is called.
7077
-
7078
6995
  ##### Signature
7079
6996
 
7080
6997
  ```typescript
@@ -7105,9 +7022,6 @@ await table.intersects("geomA", "geomB", "doIntersect").log();
7105
7022
  Returns `TRUE` if every point of a geometry in `column` is covered by a geometry
7106
7023
  in `containerColumn`, including their boundaries, and `FALSE` otherwise.
7107
7024
 
7108
- This method queues the operation; it runs when an async observer method (like
7109
- `getData()` or `log()`) is awaited, or when `run()` is called.
7110
-
7111
7025
  ##### Signature
7112
7026
 
7113
7027
  ```typescript
@@ -7139,9 +7053,6 @@ await table.coveredBy("pointGeom", "polygonGeom", "isCovered").log();
7139
7053
  Computes the union of two geometries, creating a new geometry that represents
7140
7054
  the merged area of both.
7141
7055
 
7142
- This method queues the operation; it runs when an async observer method (like
7143
- `getData()` or `log()`) is awaited, or when `run()` is called.
7144
-
7145
7056
  ##### Signature
7146
7057
 
7147
7058
  ```typescript
@@ -7172,9 +7083,6 @@ await table.union("geomA", "geomB", "unionGeom").log();
7172
7083
  Extracts the latitude (y) and longitude (x) coordinates from point geometries.
7173
7084
  The input geometry is assumed to be in EPSG:4326 (WGS84).
7174
7085
 
7175
- This method queues the operation; it runs when an async observer method (like
7176
- `getData()` or `log()`) is awaited, or when `run()` is called.
7177
-
7178
7086
  ##### Signature
7179
7087
 
7180
7088
  ```typescript
@@ -7205,9 +7113,6 @@ await table.extractLatLon("geom", "lat", "lon").log();
7205
7113
  Simplifies geometries while preserving their overall coverage. A higher
7206
7114
  tolerance results in more significant simplification.
7207
7115
 
7208
- This method queues the operation; it runs when an async observer method (like
7209
- `getData()` or `log()`) is awaited, or when `run()` is called.
7210
-
7211
7116
  ##### Signature
7212
7117
 
7213
7118
  ```typescript
@@ -7246,9 +7151,6 @@ await table.simplify(0.05, { column: "myGeom", simplifyBoundary: false }).log();
7246
7151
  Computes the centroid of geometries. The values are returned in the SRS unit of
7247
7152
  the input geometries.
7248
7153
 
7249
- This method queues the operation; it runs when an async observer method (like
7250
- `getData()` or `log()`) is awaited, or when `run()` is called.
7251
-
7252
7154
  ##### Signature
7253
7155
 
7254
7156
  ```typescript
@@ -7283,9 +7185,6 @@ await table.centroid("areaCentroid", { column: "areaGeom" }).log();
7283
7185
 
7284
7186
  Generates a random point within the geometries of a specified column.
7285
7187
 
7286
- This method queues the operation; it runs when an async observer method (like
7287
- `getData()` or `log()`) is awaited, or when `run()` is called.
7288
-
7289
7188
  ##### Signature
7290
7189
 
7291
7190
  ```typescript
@@ -7331,9 +7230,6 @@ input geometries. You can optionally specify `"spheroid"` or `"haversine"`
7331
7230
  methods to get results in meters or kilometers. If using `"spheroid"` or
7332
7231
  `"haversine"`, the input geometries must be in EPSG:4326 (WGS84).
7333
7232
 
7334
- This method queues the operation; it runs when an async observer method (like
7335
- `getData()` or `log()`) is awaited, or when `run()` is called.
7336
-
7337
7233
  ##### Signature
7338
7234
 
7339
7235
  ```typescript
@@ -7398,9 +7294,6 @@ await table.distance("area1", "area2", "distance_spheroid_km", {
7398
7294
  Unnests geometries recursively, transforming multi-part geometries (e.g.,
7399
7295
  MultiPolygon) into individual single-part geometries (e.g., Polygon).
7400
7296
 
7401
- This method queues the operation; it runs when an async observer method (like
7402
- `getData()` or `log()`) is awaited, or when `run()` is called.
7403
-
7404
7297
  ##### Signature
7405
7298
 
7406
7299
  ```typescript
@@ -7433,9 +7326,6 @@ await table.unnestGeo("multiGeom").log();
7433
7326
  Adds the bounding box coordinates of geometries in a specified column as four
7434
7327
  new columns: `minLon`, `minLat`, `maxLon`, and `maxLat`.
7435
7328
 
7436
- This method queues the operation; it runs when an async observer method (like
7437
- `getData()` or `log()`) is awaited, or when `run()` is called.
7438
-
7439
7329
  ##### Signature
7440
7330
 
7441
7331
  ```typescript
@@ -7474,9 +7364,6 @@ await table.addBoundingBox({ column: "geom", decimals: 2 }).log();
7474
7364
  Aggregates geometries in a specified column based on a chosen aggregation
7475
7365
  method.
7476
7366
 
7477
- This method queues the operation; it runs when an async observer method (like
7478
- `getData()` or `log()`) is awaited, or when `run()` is called.
7479
-
7480
7367
  ##### Signature
7481
7368
 
7482
7369
  ```typescript
@@ -7528,9 +7415,6 @@ const intersectionTable = await table.aggregateGeo("intersection", {
7528
7415
 
7529
7416
  Transforms closed linestring geometries into polygon geometries.
7530
7417
 
7531
- This method queues the operation; it runs when an async observer method (like
7532
- `getData()` or `log()`) is awaited, or when `run()` is called.
7533
-
7534
7418
  ##### Signature
7535
7419
 
7536
7420
  ```typescript
@@ -7596,8 +7480,10 @@ console.log(areaBbox);
7596
7480
 
7597
7481
  #### `getGeoData`
7598
7482
 
7599
- Returns the table's geospatial data as a GeoJSON object. If the table has
7600
- multiple geometry columns, you must specify which one to use.
7483
+ Returns the table's geospatial data as a GeoJSON FeatureCollection. Each row
7484
+ becomes a Feature: the selected geometry column becomes its geometry, and the
7485
+ remaining columns become its properties. If the table has multiple geometry
7486
+ columns, you must specify which one to use.
7601
7487
 
7602
7488
  ##### Signature
7603
7489
 
@@ -7641,7 +7527,8 @@ console.log(rewoundGeojson);
7641
7527
  #### `writeData`
7642
7528
 
7643
7529
  Writes the table's data to a file in various formats (CSV, JSON, Parquet,
7644
- DuckDB, SQLite). If the specified path does not exist, it will be created.
7530
+ DuckDB, SQLite). Tables with geometry columns must use `writeGeoData()` instead.
7531
+ If the specified path does not exist, it will be created.
7645
7532
 
7646
7533
  ##### Signature
7647
7534
 
@@ -7662,8 +7549,8 @@ async writeData(file: string, options?: { compression?: boolean; dataAsArrays?:
7662
7549
  `{ "col1": [v1, v2], "col2": [v3, v4] }`) instead of an array of objects. This
7663
7550
  can reduce file size for web projects. You can use the `arraysToData` function
7664
7551
  from the
7665
- [journalism library](https://jsr.io/@nshiab/journalism/doc/~/arraysToData) to
7666
- convert it back.
7552
+ [journalism-format library](https://jsr.io/@nshiab/journalism-format/doc/~/arraysToData)
7553
+ to convert it back.
7667
7554
  - **`options.formatDates`**: For CSV and JSON files only. If `true`, date and
7668
7555
  timestamp columns will be formatted as ISO 8601 strings (e.g.,
7669
7556
  `"2025-01-01T01:00:00.000Z"`). Defaults to `false`.
@@ -7780,28 +7667,28 @@ await table.writeGeoData("./output_high_precision.geojson", {
7780
7667
  Caches the results of computations in `./.sda-cache`. You should add
7781
7668
  `./.sda-cache` to your `.gitignore` file.
7782
7669
 
7783
- Cache entries are stored as DuckDB database files. Full-text search (FTS)
7784
- indexes are persisted in the cache file and restored directly on a cache hit.
7785
- Vector similarity search (VSS/HNSW) indexes are not persisted in the cache file;
7786
- their definitions are stored as metadata and used to rebuild the indexes on
7787
- every cache hit. If loading the entry or restoring its indexes fails, the
7788
- computation runs again and replaces the cache entry.
7789
-
7790
7670
  `cache()` automatically tracks whether earlier SDA operations changed the table.
7791
7671
  It also records every other already registered `SimpleTable` read through
7792
7672
  `SimpleTable` methods while `compute` runs and invalidates the cached step when
7793
7673
  any of their generations change. Tables created inside `compute` are part of the
7794
7674
  computation itself and are not dependencies.
7795
7675
 
7676
+ Inside `compute`, you may modify the table being cached and create or modify
7677
+ temporary tables, provided you remove them before the callback finishes. Other
7678
+ tables that already existed may be read but not modified, renamed, or removed.
7679
+ `customQuery()` bypasses enforcement of these rules.
7680
+
7796
7681
  `SimpleDB.customQuery()` bypasses this tracking. Reading or changing a table
7797
7682
  with `customQuery()` can therefore return stale cached data. Include a value
7798
7683
  that identifies the custom query's dependencies in `options.inputs` (such as a
7799
7684
  table content hash), or use tracked `SimpleTable` methods.
7800
7685
 
7801
- `compute` may modify only the table being cached. Other tables that existed
7802
- before `compute` must remain read-only. Temporary tables may be created and
7803
- modified inside `compute`, but they must be removed before it finishes because a
7804
- cache hit does not run `compute` again.
7686
+ Cache entries are stored as DuckDB database files. Full-text search (FTS)
7687
+ indexes are persisted in the cache file and restored directly on a cache hit.
7688
+ Vector similarity search (VSS/HNSW) indexes are not persisted in the cache file;
7689
+ their definitions are stored as metadata and used to rebuild the indexes on
7690
+ every cache hit. If loading the entry or restoring its indexes fails, the
7691
+ computation runs again and replaces the cache entry.
7805
7692
 
7806
7693
  ##### Signature
7807
7694
 
@@ -7899,10 +7786,23 @@ await summary.log();
7899
7786
 
7900
7787
  #### `log`
7901
7788
 
7902
- Logs a specified number of rows from the table to the console. By default, the
7903
- first 10 rows are logged. You can optionally log the column types and filter the
7904
- data based on conditions. You can also use JavaScript syntax for conditions
7905
- (e.g., `&&`, `||`, `===`, `!==`).
7789
+ Logs up to a specified number of rows from the table to the console. By default,
7790
+ the first 10 rows are logged. You can optionally log the column types and filter
7791
+ the data based on conditions. The footer reports the total number of matching
7792
+ rows and, only when some rows are omitted, the number actually shown (for
7793
+ example, `100 rows in total / showing 15 rows`). SQL dates and timestamps retain
7794
+ their native precision for display, including temporal infinities. Lists and
7795
+ objects are stringified using the same nested representations as `getData()`,
7796
+ then truncated according to `charsToLog`. Unsafe top-level large integers throw,
7797
+ just as with `getData()`. Type annotations describe the SQL type and the
7798
+ JavaScript extraction type before display formatting; nulls do not determine the
7799
+ column's annotation. Colors reflect the SQL value's meaning, so exact decimal
7800
+ strings use numeric coloring. Column width is independent of the content
7801
+ truncation budget.
7802
+
7803
+ With the default `SimpleDB.expressionSyntax: "js"`, conditions support
7804
+ JavaScript-style operators (`&&`, `||`, `===`, `!==`). Set
7805
+ `expressionSyntax: "sql"` for unchanged SQL.
7906
7806
 
7907
7807
  ##### Signature
7908
7808
 
@@ -7914,8 +7814,8 @@ async log(options?: "all" | number | { count?: number | "all"; types?: boolean;
7914
7814
 
7915
7815
  - **`options`**: Either the number of rows to log (a specific number or `"all"`)
7916
7816
  or an object with configuration options:
7917
- - **`options.count`**: The number of rows to log. Defaults to 10 or the value
7918
- set in the SimpleDB instance. Use `"all"` to log all rows.
7817
+ - **`options.count`**: The maximum number of rows to log. Defaults to 10 or the
7818
+ value set in the SimpleDB instance. Use `"all"` to log all rows.
7919
7819
  - **`options.types`**: Whether to log the column types along with the data.
7920
7820
  Defaults to the value set in the SimpleDB instance.
7921
7821
  - **`options.conditions`**: A SQL `WHERE` clause condition to filter the data
@@ -8143,6 +8043,10 @@ Logs the bottom `n` rows of the table to the console. By default, the last row
8143
8043
  will be returned first. To preserve the original order, use the `originalOrder`
8144
8044
  option.
8145
8045
 
8046
+ Uses the same precision-preserving temporal display, nested stringification,
8047
+ numeric coloring, and `charsToLog` truncation as `log()`. Unsafe top-level large
8048
+ integers throw, just as with `getData()`.
8049
+
8146
8050
  ##### Signature
8147
8051
 
8148
8052
  ```typescript