@math.gl/sql 0.0.0-stage → 5.0.0-beta.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (353) hide show
  1. package/LICENSE +249 -0
  2. package/README.md +26 -2
  3. package/THIRD-PARTY-NOTICES.md +5 -0
  4. package/dist/arrow/aggregates.d.ts +46 -0
  5. package/dist/arrow/aggregates.d.ts.map +1 -0
  6. package/dist/arrow/aggregates.js +289 -0
  7. package/dist/arrow/aggregates.js.map +1 -0
  8. package/dist/arrow/buffers.d.ts +30 -0
  9. package/dist/arrow/buffers.d.ts.map +1 -0
  10. package/dist/arrow/buffers.js +177 -0
  11. package/dist/arrow/buffers.js.map +1 -0
  12. package/dist/arrow/cast.d.ts +19 -0
  13. package/dist/arrow/cast.d.ts.map +1 -0
  14. package/dist/arrow/cast.js +138 -0
  15. package/dist/arrow/cast.js.map +1 -0
  16. package/dist/arrow/compiler.d.ts +10 -0
  17. package/dist/arrow/compiler.d.ts.map +1 -0
  18. package/dist/arrow/compiler.js +1538 -0
  19. package/dist/arrow/compiler.js.map +1 -0
  20. package/dist/arrow/float16.d.ts +7 -0
  21. package/dist/arrow/float16.d.ts.map +1 -0
  22. package/dist/arrow/float16.js +26 -0
  23. package/dist/arrow/float16.js.map +1 -0
  24. package/dist/arrow/index.cjs +3321 -0
  25. package/dist/arrow/index.cjs.map +6 -0
  26. package/dist/arrow/index.d.ts +5 -0
  27. package/dist/arrow/index.d.ts.map +1 -0
  28. package/dist/arrow/index.js +7 -0
  29. package/dist/arrow/index.js.map +1 -0
  30. package/dist/arrow/like.d.ts +3 -0
  31. package/dist/arrow/like.d.ts.map +1 -0
  32. package/dist/arrow/like.js +75 -0
  33. package/dist/arrow/like.js.map +1 -0
  34. package/dist/arrow/list-values.d.ts +14 -0
  35. package/dist/arrow/list-values.d.ts.map +1 -0
  36. package/dist/arrow/list-values.js +201 -0
  37. package/dist/arrow/list-values.js.map +1 -0
  38. package/dist/arrow/predicate.d.ts +7 -0
  39. package/dist/arrow/predicate.d.ts.map +1 -0
  40. package/dist/arrow/predicate.js +10 -0
  41. package/dist/arrow/predicate.js.map +1 -0
  42. package/dist/arrow/query-arrow-aggregate-snapshots.d.ts +24 -0
  43. package/dist/arrow/query-arrow-aggregate-snapshots.d.ts.map +1 -0
  44. package/dist/arrow/query-arrow-aggregate-snapshots.js +64 -0
  45. package/dist/arrow/query-arrow-aggregate-snapshots.js.map +1 -0
  46. package/dist/arrow/query-arrow-sql.d.ts +8 -0
  47. package/dist/arrow/query-arrow-sql.d.ts.map +1 -0
  48. package/dist/arrow/query-arrow-sql.js +223 -0
  49. package/dist/arrow/query-arrow-sql.js.map +1 -0
  50. package/dist/arrow/query-arrow-table.d.ts +36 -0
  51. package/dist/arrow/query-arrow-table.d.ts.map +1 -0
  52. package/dist/arrow/query-arrow-table.js +208 -0
  53. package/dist/arrow/query-arrow-table.js.map +1 -0
  54. package/dist/arrow/query-worker-entry.d.ts +2 -0
  55. package/dist/arrow/query-worker-entry.d.ts.map +1 -0
  56. package/dist/arrow/query-worker-entry.js +7 -0
  57. package/dist/arrow/query-worker-entry.js.map +1 -0
  58. package/dist/arrow/query-worker.d.ts +65 -0
  59. package/dist/arrow/query-worker.d.ts.map +1 -0
  60. package/dist/arrow/query-worker.js +135 -0
  61. package/dist/arrow/query-worker.js.map +1 -0
  62. package/dist/arrow/registry.d.ts +4 -0
  63. package/dist/arrow/registry.d.ts.map +1 -0
  64. package/dist/arrow/registry.js +66 -0
  65. package/dist/arrow/registry.js.map +1 -0
  66. package/dist/arrow/relational-query.d.ts +36 -0
  67. package/dist/arrow/relational-query.d.ts.map +1 -0
  68. package/dist/arrow/relational-query.js +174 -0
  69. package/dist/arrow/relational-query.js.map +1 -0
  70. package/dist/arrow/replace-arrow-columns.d.ts +8 -0
  71. package/dist/arrow/replace-arrow-columns.d.ts.map +1 -0
  72. package/dist/arrow/replace-arrow-columns.js +63 -0
  73. package/dist/arrow/replace-arrow-columns.js.map +1 -0
  74. package/dist/arrow/signature.d.ts +6 -0
  75. package/dist/arrow/signature.d.ts.map +1 -0
  76. package/dist/arrow/signature.js +51 -0
  77. package/dist/arrow/signature.js.map +1 -0
  78. package/dist/arrow/temporal-cast.d.ts +9 -0
  79. package/dist/arrow/temporal-cast.d.ts.map +1 -0
  80. package/dist/arrow/temporal-cast.js +60 -0
  81. package/dist/arrow/temporal-cast.js.map +1 -0
  82. package/dist/arrow/types.d.ts +62 -0
  83. package/dist/arrow/types.d.ts.map +1 -0
  84. package/dist/arrow/types.js +5 -0
  85. package/dist/arrow/types.js.map +1 -0
  86. package/dist/arrow/unnest-arrow-column.d.ts +22 -0
  87. package/dist/arrow/unnest-arrow-column.d.ts.map +1 -0
  88. package/dist/arrow/unnest-arrow-column.js +112 -0
  89. package/dist/arrow/unnest-arrow-column.js.map +1 -0
  90. package/dist/arrow/worker-entry.d.ts +2 -0
  91. package/dist/arrow/worker-entry.d.ts.map +1 -0
  92. package/dist/arrow/worker-entry.js +9 -0
  93. package/dist/arrow/worker-entry.js.map +1 -0
  94. package/dist/arrow/worker.cjs +4074 -0
  95. package/dist/arrow/worker.cjs.map +6 -0
  96. package/dist/arrow/worker.d.ts +58 -0
  97. package/dist/arrow/worker.d.ts.map +1 -0
  98. package/dist/arrow/worker.js +84 -0
  99. package/dist/arrow/worker.js.map +1 -0
  100. package/dist/color-functions/index.cjs +382 -0
  101. package/dist/color-functions/index.cjs.map +6 -0
  102. package/dist/color-functions/index.d.ts +26 -0
  103. package/dist/color-functions/index.d.ts.map +1 -0
  104. package/dist/color-functions/index.js +306 -0
  105. package/dist/color-functions/index.js.map +1 -0
  106. package/dist/diagnostic-log.d.ts +8 -0
  107. package/dist/diagnostic-log.d.ts.map +1 -0
  108. package/dist/diagnostic-log.js +15 -0
  109. package/dist/diagnostic-log.js.map +1 -0
  110. package/dist/expressions/ast.d.ts +12 -0
  111. package/dist/expressions/ast.d.ts.map +1 -0
  112. package/dist/expressions/ast.js +114 -0
  113. package/dist/expressions/ast.js.map +1 -0
  114. package/dist/expressions/columnar-predicate.d.ts +56 -0
  115. package/dist/expressions/columnar-predicate.d.ts.map +1 -0
  116. package/dist/expressions/columnar-predicate.js +176 -0
  117. package/dist/expressions/columnar-predicate.js.map +1 -0
  118. package/dist/expressions/evaluation-error.d.ts +14 -0
  119. package/dist/expressions/evaluation-error.d.ts.map +1 -0
  120. package/dist/expressions/evaluation-error.js +26 -0
  121. package/dist/expressions/evaluation-error.js.map +1 -0
  122. package/dist/expressions/expression-eval.d.ts +105 -0
  123. package/dist/expressions/expression-eval.d.ts.map +1 -0
  124. package/dist/expressions/expression-eval.js +330 -0
  125. package/dist/expressions/expression-eval.js.map +1 -0
  126. package/dist/expressions/function-libraries.d.ts +63 -0
  127. package/dist/expressions/function-libraries.d.ts.map +1 -0
  128. package/dist/expressions/function-libraries.js +69 -0
  129. package/dist/expressions/function-libraries.js.map +1 -0
  130. package/dist/expressions/function-registry.d.ts +93 -0
  131. package/dist/expressions/function-registry.d.ts.map +1 -0
  132. package/dist/expressions/function-registry.js +139 -0
  133. package/dist/expressions/function-registry.js.map +1 -0
  134. package/dist/expressions/get.d.ts +9 -0
  135. package/dist/expressions/get.d.ts.map +1 -0
  136. package/dist/expressions/get.js +32 -0
  137. package/dist/expressions/get.js.map +1 -0
  138. package/dist/expressions/index.cjs +1888 -0
  139. package/dist/expressions/index.cjs.map +6 -0
  140. package/dist/expressions/index.d.ts +13 -0
  141. package/dist/expressions/index.d.ts.map +1 -0
  142. package/dist/expressions/index.js +14 -0
  143. package/dist/expressions/index.js.map +1 -0
  144. package/dist/expressions/numeric-functions.d.ts +277 -0
  145. package/dist/expressions/numeric-functions.d.ts.map +1 -0
  146. package/dist/expressions/numeric-functions.js +52 -0
  147. package/dist/expressions/numeric-functions.js.map +1 -0
  148. package/dist/expressions/parse-expression-string.d.ts +21 -0
  149. package/dist/expressions/parse-expression-string.d.ts.map +1 -0
  150. package/dist/expressions/parse-expression-string.js +61 -0
  151. package/dist/expressions/parse-expression-string.js.map +1 -0
  152. package/dist/expressions/predicate-expression.d.ts +7 -0
  153. package/dist/expressions/predicate-expression.d.ts.map +1 -0
  154. package/dist/expressions/predicate-expression.js +112 -0
  155. package/dist/expressions/predicate-expression.js.map +1 -0
  156. package/dist/expressions/scalar-type.d.ts +6 -0
  157. package/dist/expressions/scalar-type.d.ts.map +1 -0
  158. package/dist/expressions/scalar-type.js +23 -0
  159. package/dist/expressions/scalar-type.js.map +1 -0
  160. package/dist/expressions/vendor/jsep/runtime.d.ts +3 -0
  161. package/dist/expressions/vendor/jsep/runtime.d.ts.map +1 -0
  162. package/dist/expressions/vendor/jsep/runtime.js +972 -0
  163. package/dist/expressions/vendor/jsep/runtime.js.map +1 -0
  164. package/dist/expressions/vendor/jsep/types.d.ts +148 -0
  165. package/dist/expressions/vendor/jsep/types.d.ts.map +1 -0
  166. package/dist/expressions/vendor/jsep/types.js +6 -0
  167. package/dist/expressions/vendor/jsep/types.js.map +1 -0
  168. package/dist/index.cjs +5294 -0
  169. package/dist/index.cjs.map +6 -0
  170. package/dist/index.d.ts +11 -0
  171. package/dist/index.d.ts.map +1 -0
  172. package/dist/index.js +11 -0
  173. package/dist/index.js.map +1 -0
  174. package/dist/math-functions/index.cjs +444 -0
  175. package/dist/math-functions/index.cjs.map +6 -0
  176. package/dist/math-functions/index.d.ts +44 -0
  177. package/dist/math-functions/index.d.ts.map +1 -0
  178. package/dist/math-functions/index.js +182 -0
  179. package/dist/math-functions/index.js.map +1 -0
  180. package/dist/parser/bind-sql-predicate.d.ts +4 -0
  181. package/dist/parser/bind-sql-predicate.d.ts.map +1 -0
  182. package/dist/parser/bind-sql-predicate.js +11 -0
  183. package/dist/parser/bind-sql-predicate.js.map +1 -0
  184. package/dist/parser/index.cjs +1807 -0
  185. package/dist/parser/index.cjs.map +6 -0
  186. package/dist/parser/index.d.ts +12 -0
  187. package/dist/parser/index.d.ts.map +1 -0
  188. package/dist/parser/index.js +11 -0
  189. package/dist/parser/index.js.map +1 -0
  190. package/dist/parser/parameter-snapshots.d.ts +21 -0
  191. package/dist/parser/parameter-snapshots.d.ts.map +1 -0
  192. package/dist/parser/parameter-snapshots.js +85 -0
  193. package/dist/parser/parameter-snapshots.js.map +1 -0
  194. package/dist/parser/parse-sql-expression.d.ts +50 -0
  195. package/dist/parser/parse-sql-expression.d.ts.map +1 -0
  196. package/dist/parser/parse-sql-expression.js +501 -0
  197. package/dist/parser/parse-sql-expression.js.map +1 -0
  198. package/dist/parser/parse-sql-predicate.d.ts +13 -0
  199. package/dist/parser/parse-sql-predicate.d.ts.map +1 -0
  200. package/dist/parser/parse-sql-predicate.js +365 -0
  201. package/dist/parser/parse-sql-predicate.js.map +1 -0
  202. package/dist/parser/parse-sql-query.d.ts +9 -0
  203. package/dist/parser/parse-sql-query.d.ts.map +1 -0
  204. package/dist/parser/parse-sql-query.js +81 -0
  205. package/dist/parser/parse-sql-query.js.map +1 -0
  206. package/dist/parser/parse-sql-select.d.ts +57 -0
  207. package/dist/parser/parse-sql-select.d.ts.map +1 -0
  208. package/dist/parser/parse-sql-select.js +247 -0
  209. package/dist/parser/parse-sql-select.js.map +1 -0
  210. package/dist/parser/parse-sql-statement.d.ts +54 -0
  211. package/dist/parser/parse-sql-statement.d.ts.map +1 -0
  212. package/dist/parser/parse-sql-statement.js +207 -0
  213. package/dist/parser/parse-sql-statement.js.map +1 -0
  214. package/dist/parser/query-types.d.ts +56 -0
  215. package/dist/parser/query-types.d.ts.map +1 -0
  216. package/dist/parser/query-types.js +5 -0
  217. package/dist/parser/query-types.js.map +1 -0
  218. package/dist/parser/sql-parameters.d.ts +29 -0
  219. package/dist/parser/sql-parameters.d.ts.map +1 -0
  220. package/dist/parser/sql-parameters.js +110 -0
  221. package/dist/parser/sql-parameters.js.map +1 -0
  222. package/dist/parser/sql-parse-error.d.ts +13 -0
  223. package/dist/parser/sql-parse-error.d.ts.map +1 -0
  224. package/dist/parser/sql-parse-error.js +20 -0
  225. package/dist/parser/sql-parse-error.js.map +1 -0
  226. package/dist/parser/sql-predicate-schema.d.ts +164 -0
  227. package/dist/parser/sql-predicate-schema.d.ts.map +1 -0
  228. package/dist/parser/sql-predicate-schema.js +249 -0
  229. package/dist/parser/sql-predicate-schema.js.map +1 -0
  230. package/dist/parser/sql-predicate-types.d.ts +48 -0
  231. package/dist/parser/sql-predicate-types.d.ts.map +1 -0
  232. package/dist/parser/sql-predicate-types.js +9 -0
  233. package/dist/parser/sql-predicate-types.js.map +1 -0
  234. package/dist/parser/sql-predicate.d.ts +6 -0
  235. package/dist/parser/sql-predicate.d.ts.map +1 -0
  236. package/dist/parser/sql-predicate.js +8 -0
  237. package/dist/parser/sql-predicate.js.map +1 -0
  238. package/dist/scales/arrow-domains.d.ts +22 -0
  239. package/dist/scales/arrow-domains.d.ts.map +1 -0
  240. package/dist/scales/arrow-domains.js +166 -0
  241. package/dist/scales/arrow-domains.js.map +1 -0
  242. package/dist/scales/arrow-scales.d.ts +44 -0
  243. package/dist/scales/arrow-scales.d.ts.map +1 -0
  244. package/dist/scales/arrow-scales.js +255 -0
  245. package/dist/scales/arrow-scales.js.map +1 -0
  246. package/dist/scales/compile-scale.d.ts +2 -0
  247. package/dist/scales/compile-scale.d.ts.map +1 -0
  248. package/dist/scales/compile-scale.js +5 -0
  249. package/dist/scales/compile-scale.js.map +1 -0
  250. package/dist/scales/conformance.cjs +180 -0
  251. package/dist/scales/conformance.cjs.map +6 -0
  252. package/dist/scales/conformance.d.ts +26 -0
  253. package/dist/scales/conformance.d.ts.map +1 -0
  254. package/dist/scales/conformance.js +164 -0
  255. package/dist/scales/conformance.js.map +1 -0
  256. package/dist/scales/hdr-colors.d.ts +2 -0
  257. package/dist/scales/hdr-colors.d.ts.map +1 -0
  258. package/dist/scales/hdr-colors.js +5 -0
  259. package/dist/scales/hdr-colors.js.map +1 -0
  260. package/dist/scales/index.cjs +4334 -0
  261. package/dist/scales/index.cjs.map +6 -0
  262. package/dist/scales/index.d.ts +10 -0
  263. package/dist/scales/index.d.ts.map +1 -0
  264. package/dist/scales/index.js +11 -0
  265. package/dist/scales/index.js.map +1 -0
  266. package/dist/scales/palettes.d.ts +2 -0
  267. package/dist/scales/palettes.d.ts.map +1 -0
  268. package/dist/scales/palettes.js +5 -0
  269. package/dist/scales/palettes.js.map +1 -0
  270. package/dist/scales/parse-sql-scale.d.ts +27 -0
  271. package/dist/scales/parse-sql-scale.d.ts.map +1 -0
  272. package/dist/scales/parse-sql-scale.js +257 -0
  273. package/dist/scales/parse-sql-scale.js.map +1 -0
  274. package/dist/scales/quantile-sketch.d.ts +2 -0
  275. package/dist/scales/quantile-sketch.d.ts.map +1 -0
  276. package/dist/scales/quantile-sketch.js +5 -0
  277. package/dist/scales/quantile-sketch.js.map +1 -0
  278. package/dist/scales/scale-types.d.ts +2 -0
  279. package/dist/scales/scale-types.d.ts.map +1 -0
  280. package/dist/scales/scale-types.js +5 -0
  281. package/dist/scales/scale-types.js.map +1 -0
  282. package/dist/scales/streaming-domains.d.ts +18 -0
  283. package/dist/scales/streaming-domains.d.ts.map +1 -0
  284. package/dist/scales/streaming-domains.js +115 -0
  285. package/dist/scales/streaming-domains.js.map +1 -0
  286. package/package.json +92 -4
  287. package/src/arrow/aggregates.ts +386 -0
  288. package/src/arrow/buffers.ts +177 -0
  289. package/src/arrow/cast.ts +160 -0
  290. package/src/arrow/compiler.ts +1778 -0
  291. package/src/arrow/float16.ts +25 -0
  292. package/src/arrow/index.ts +22 -0
  293. package/src/arrow/like.ts +72 -0
  294. package/src/arrow/list-values.ts +221 -0
  295. package/src/arrow/predicate.ts +19 -0
  296. package/src/arrow/query-arrow-aggregate-snapshots.ts +107 -0
  297. package/src/arrow/query-arrow-sql.ts +255 -0
  298. package/src/arrow/query-arrow-table.ts +298 -0
  299. package/src/arrow/query-worker-entry.ts +8 -0
  300. package/src/arrow/query-worker.ts +194 -0
  301. package/src/arrow/registry.ts +87 -0
  302. package/src/arrow/relational-query.ts +247 -0
  303. package/src/arrow/replace-arrow-columns.ts +71 -0
  304. package/src/arrow/signature.ts +50 -0
  305. package/src/arrow/temporal-cast.ts +66 -0
  306. package/src/arrow/types.ts +76 -0
  307. package/src/arrow/unnest-arrow-column.ts +145 -0
  308. package/src/arrow/worker-entry.ts +10 -0
  309. package/src/arrow/worker.ts +143 -0
  310. package/src/color-functions/index.ts +405 -0
  311. package/src/diagnostic-log.ts +20 -0
  312. package/src/expressions/ast.ts +143 -0
  313. package/src/expressions/columnar-predicate.ts +293 -0
  314. package/src/expressions/evaluation-error.ts +29 -0
  315. package/src/expressions/expression-eval.ts +489 -0
  316. package/src/expressions/function-libraries.ts +126 -0
  317. package/src/expressions/function-registry.ts +171 -0
  318. package/src/expressions/get.ts +37 -0
  319. package/src/expressions/index.ts +40 -0
  320. package/src/expressions/numeric-functions.ts +99 -0
  321. package/src/expressions/parse-expression-string.ts +80 -0
  322. package/src/expressions/predicate-expression.ts +128 -0
  323. package/src/expressions/scalar-type.ts +39 -0
  324. package/src/expressions/vendor/jsep/LICENSE.txt +23 -0
  325. package/src/expressions/vendor/jsep/runtime.ts +949 -0
  326. package/src/expressions/vendor/jsep/types.ts +191 -0
  327. package/src/index.ts +54 -0
  328. package/src/math-functions/index.ts +235 -0
  329. package/src/parser/bind-sql-predicate.ts +21 -0
  330. package/src/parser/index.ts +43 -0
  331. package/src/parser/parameter-snapshots.ts +103 -0
  332. package/src/parser/parse-sql-expression.ts +500 -0
  333. package/src/parser/parse-sql-predicate.ts +454 -0
  334. package/src/parser/parse-sql-query.ts +90 -0
  335. package/src/parser/parse-sql-select.ts +272 -0
  336. package/src/parser/parse-sql-statement.ts +260 -0
  337. package/src/parser/query-types.ts +71 -0
  338. package/src/parser/sql-parameters.ts +174 -0
  339. package/src/parser/sql-parse-error.ts +33 -0
  340. package/src/parser/sql-predicate-schema.ts +279 -0
  341. package/src/parser/sql-predicate-types.ts +91 -0
  342. package/src/parser/sql-predicate.ts +28 -0
  343. package/src/scales/arrow-domains.ts +190 -0
  344. package/src/scales/arrow-scales.ts +363 -0
  345. package/src/scales/compile-scale.ts +5 -0
  346. package/src/scales/conformance.ts +186 -0
  347. package/src/scales/hdr-colors.ts +5 -0
  348. package/src/scales/index.ts +37 -0
  349. package/src/scales/palettes.ts +5 -0
  350. package/src/scales/parse-sql-scale.ts +300 -0
  351. package/src/scales/quantile-sketch.ts +5 -0
  352. package/src/scales/scale-types.ts +5 -0
  353. package/src/scales/streaming-domains.ts +139 -0
@@ -0,0 +1,255 @@
1
+ // math.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
4
+
5
+ import {DataType, type Table} from 'apache-arrow';
6
+ import {unnestArrowColumn} from './unnest-arrow-column';
7
+ import type {Expression} from '../expressions';
8
+ import {
9
+ parseSQLSelectCompound,
10
+ type SQLSelectStatement,
11
+ type SQLSelectStatementOptions,
12
+ type SQLSort
13
+ } from '../parser/parse-sql-statement';
14
+ import {queryArrowTable, type ArrowTableQuery} from './query-arrow-table';
15
+ import type {ArrowAggregate} from './aggregates';
16
+
17
+ type Execution = Pick<ArrowTableQuery, 'signal' | 'functions'>;
18
+
19
+ /** Executes relational SQL over a caller-supplied catalog of in-memory Arrow tables. */
20
+ export function queryArrowSQL(
21
+ sql: string,
22
+ tables: ReadonlyMap<string, Table>,
23
+ options: Execution & SQLSelectStatementOptions = {}
24
+ ): Table {
25
+ const {inputs, orderBy, offset, limit} = parseSQLSelectCompound(sql, options);
26
+ const execution = {signal: options.signal, functions: options.functions};
27
+ const [first, ...rest] = inputs.map(input => executeSelect(input, tables, execution)) as [
28
+ Table,
29
+ ...Table[]
30
+ ];
31
+ // Plan the final order against the projected names, including SELECT aliases.
32
+ return queryArrowTable(first, {
33
+ ...execution,
34
+ union: rest.map(table => {
35
+ if (table.numCols !== first.numCols)
36
+ throw new Error('Arrow SQL UNION ALL requires the same number of columns in each input');
37
+ return queryArrowTable(table, {
38
+ select: table.schema.fields.map((field, index) => ({
39
+ column: field.name,
40
+ as: first.schema.fields[index]!.name
41
+ })),
42
+ signal: execution.signal
43
+ });
44
+ }),
45
+ orderBy: planOrder(orderBy),
46
+ offset,
47
+ limit
48
+ });
49
+ }
50
+
51
+ function planOrder(orderBy?: readonly SQLSort[]) {
52
+ return orderBy?.map(({expression, direction, nulls}) => {
53
+ /** Qualified projected output name, including UNNEST aliases. */
54
+ const column = qualifiedColumn(expression);
55
+ return {
56
+ column: column?.qualifier ? column.qualifier + '.' + column.name : requireColumn(expression),
57
+ direction,
58
+ nulls
59
+ };
60
+ });
61
+ }
62
+
63
+ function executeSelect(
64
+ statement: SQLSelectStatement,
65
+ tables: ReadonlyMap<string, Table>,
66
+ execution: Execution
67
+ ): Table {
68
+ let source = getTable(tables, statement.from.name);
69
+ const leftAlias = statement.from.alias ?? statement.from.name;
70
+ const [right] = statement.joins;
71
+ let join: ArrowTableQuery['join'];
72
+ let rightAlias: string | undefined;
73
+ let rightTable: Table | undefined;
74
+ if (right?.unnest) {
75
+ if (statement.joins.length !== 1)
76
+ throw new Error('UNNEST currently supports one expansion per SELECT');
77
+ const {expression, columns, ordinality} = right.unnest;
78
+ const column = qualifiedColumn(expression);
79
+ if (!column || (column.qualifier && column.qualifier !== leftAlias))
80
+ throw new Error('UNNEST requires a column of its parent table');
81
+ const field = source.schema.fields.find(field => field.name === column.name);
82
+ if (!field) throw new Error(`Unknown UNNEST column: ${column.name}`);
83
+ const map = DataType.isMap(field.type);
84
+ if (!map && !DataType.isList(field.type) && !DataType.isFixedSizeList(field.type))
85
+ throw new Error('SQL UNNEST requires List, FixedSizeList or Map');
86
+ if (columns.length !== (map ? 2 : 1) + (ordinality ? 1 : 0))
87
+ throw new Error('UNNEST column alias count does not match its output');
88
+ rightAlias = right.table.alias!;
89
+ if (rightAlias === leftAlias) throw new Error('Duplicate Arrow SQL table alias');
90
+ source = unnestArrowColumn(source, column.name, {
91
+ valueColumn: rightAlias + '.' + columns[map ? 1 : 0],
92
+ keyColumn: map ? rightAlias + '.' + columns[0] : undefined,
93
+ ordinalColumn: ordinality ? rightAlias + '.' + columns[map ? 2 : 1] : undefined,
94
+ outer: right.type === 'left',
95
+ signal: execution.signal
96
+ });
97
+ } else if (right) {
98
+ if (statement.joins.length > 1 || right.type !== 'inner')
99
+ throw new Error('Arrow SQL supports one INNER JOIN per SELECT');
100
+ rightAlias = right.table.alias ?? right.table.name;
101
+ rightTable = getTable(tables, right.table.name);
102
+ if (leftAlias === rightAlias) throw new Error('Duplicate Arrow SQL table alias');
103
+ const {on} = right;
104
+ if (on.type !== 'BinaryExpression' || on.operator !== '==')
105
+ throw new Error('Arrow SQL JOIN requires equality between two columns');
106
+ const sides = [on.left, on.right].map(node => qualifiedColumn(node as Expression));
107
+ const left = sides.find(column => column?.qualifier === leftAlias);
108
+ const rightColumn = sides.find(column => column?.qualifier === rightAlias);
109
+ if (!left || !rightColumn) throw new Error('Qualify both Arrow SQL JOIN keys');
110
+ join = {
111
+ table: rightTable,
112
+ left: left.name,
113
+ right: rightColumn.name,
114
+ rightPrefix: rightAlias + '.'
115
+ };
116
+ }
117
+
118
+ const bind = (node: Expression): Expression =>
119
+ bindColumns(node, column => {
120
+ if (column.qualifier === leftAlias) return column.name;
121
+ if (column.qualifier && column.qualifier === rightAlias)
122
+ return column.qualifier + '.' + column.name;
123
+ if (column.qualifier) {
124
+ if (
125
+ [source, rightTable].some(table =>
126
+ table?.schema.fields.some(field => field.name === column.qualifier)
127
+ )
128
+ )
129
+ return undefined;
130
+ throw new Error(`Unknown Arrow SQL table alias: ${column.qualifier}`);
131
+ }
132
+ const onRight = rightTable?.schema.fields.some(field => field.name === column.name);
133
+ if (!onRight) return column.name;
134
+ if (source.schema.fields.some(field => field.name === column.name))
135
+ throw new Error(`Ambiguous Arrow SQL column: ${column.name}`);
136
+ return rightAlias + '.' + column.name;
137
+ });
138
+ const groupBy = statement.groupBy?.map(expression => requireColumn(bind(expression)));
139
+ const aggregates = new Map<string, ArrowAggregate>();
140
+ const select = statement.select?.map(({expression, alias}, index) => {
141
+ const aggregate = planAggregate(expression, bind);
142
+ if (aggregate) {
143
+ const name = alias ?? `aggregate_${index + 1}`;
144
+ if (aggregates.has(name) || groupBy?.includes(name))
145
+ throw new Error(`Duplicate Arrow SQL aggregate name: ${name}`);
146
+ aggregates.set(name, aggregate);
147
+ return name;
148
+ }
149
+ const bound = bind(expression);
150
+ if (bound.type === 'Identifier') return {column: bound.name, as: alias};
151
+ return {expression: bound, as: alias};
152
+ });
153
+ return queryArrowTable(source, {
154
+ ...execution,
155
+ join,
156
+ select,
157
+ where: statement.where && {expression: bind(statement.where)},
158
+ groupBy,
159
+ aggregates: Object.fromEntries(aggregates)
160
+ });
161
+ }
162
+
163
+ function getTable(tables: ReadonlyMap<string, Table>, name: string): Table {
164
+ const table = tables.get(name);
165
+ if (!table) throw new Error(`Unknown Arrow SQL table: ${name}`);
166
+ return table;
167
+ }
168
+
169
+ type Column = {name: string; qualifier?: string};
170
+
171
+ function qualifiedColumn(node: Expression): Column | undefined {
172
+ if (node.type === 'Identifier') return {name: node.name};
173
+ if (
174
+ node.type === 'MemberExpression' &&
175
+ !node.computed &&
176
+ node.object.type === 'Identifier' &&
177
+ node.property.type === 'Identifier'
178
+ )
179
+ return {name: String(node.property['name']), qualifier: String(node.object['name'])};
180
+ return undefined;
181
+ }
182
+
183
+ function requireColumn(node: Expression): string {
184
+ if (node.type !== 'Identifier')
185
+ throw new Error('Arrow SQL requires a column name in this clause');
186
+ return node.name;
187
+ }
188
+
189
+ /** Resolves SQL column references while leaving function names and literals intact. */
190
+ function bindColumns(
191
+ node: Expression,
192
+ resolve: (column: Column) => string | undefined
193
+ ): Expression {
194
+ const column = qualifiedColumn(node);
195
+ const name = column && resolve(column);
196
+ if (name !== undefined) return {type: 'Identifier', name};
197
+ const bind = (child: unknown) => bindColumns(child as Expression, resolve);
198
+ switch (node.type) {
199
+ case 'BinaryExpression':
200
+ case 'LogicalExpression':
201
+ return {...node, left: bind(node.left), right: bind(node.right)};
202
+ case 'UnaryExpression':
203
+ return {...node, argument: bind(node.argument)};
204
+ case 'CallExpression':
205
+ return {...node, arguments: node.arguments.map(bind)};
206
+ case 'MemberExpression': {
207
+ if (node.computed || node.property.type !== 'Identifier')
208
+ throw new Error('Arrow SQL struct access requires a field identifier');
209
+ const name = String(node.property['name']);
210
+ return {
211
+ type: 'CallExpression',
212
+ callee: {type: 'Identifier', name: 'STRUCT_EXTRACT'},
213
+ arguments: [bind(node.object), {type: 'Literal', value: name, raw: JSON.stringify(name)}]
214
+ };
215
+ }
216
+ case 'ConditionalExpression':
217
+ return {
218
+ ...node,
219
+ test: bind(node.test),
220
+ consequent: bind(node.consequent),
221
+ alternate: bind(node.alternate)
222
+ };
223
+ default:
224
+ return node;
225
+ }
226
+ }
227
+
228
+ const aggregateOperations = new Set<ArrowAggregate['operation']>([
229
+ 'COUNT',
230
+ 'SUM',
231
+ 'MIN',
232
+ 'MAX',
233
+ 'AVG',
234
+ 'EVERY',
235
+ 'BOOL_AND',
236
+ 'BOOL_OR',
237
+ 'BIT_AND',
238
+ 'BIT_OR',
239
+ 'BIT_XOR'
240
+ ]);
241
+
242
+ function planAggregate(
243
+ node: Expression,
244
+ bind: (expression: Expression) => Expression
245
+ ): ArrowAggregate | undefined {
246
+ if (node.type !== 'CallExpression' || node.callee.type !== 'Identifier' || !node.sqlFunction)
247
+ return undefined;
248
+ const operation = String(node.callee['name']).toUpperCase() as ArrowAggregate['operation'];
249
+ if (!aggregateOperations.has(operation)) return undefined;
250
+ if (node.arguments.length !== 1) throw new Error(`Arrow SQL ${operation} requires one argument`);
251
+ const argument = node.arguments[0] as Expression & {sqlWildcard?: boolean};
252
+ return operation === 'COUNT' && argument.sqlWildcard
253
+ ? {operation}
254
+ : {operation, expression: bind(argument)};
255
+ }
@@ -0,0 +1,298 @@
1
+ // math.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
4
+
5
+ import {
6
+ Data,
7
+ DataType,
8
+ makeData,
9
+ type RecordBatch,
10
+ Schema,
11
+ Struct,
12
+ Table,
13
+ type Field
14
+ } from 'apache-arrow';
15
+ import {
16
+ getRecordBatchMetadata,
17
+ createRecordBatchWithMetadata,
18
+ validateArrowTableSchema,
19
+ takeArrowTableRows
20
+ } from '@math.gl/core/arrow';
21
+ import type {
22
+ ColumnarPredicateParameterValues,
23
+ ParameterizedColumnarPredicate
24
+ } from '../expressions';
25
+ import type {TableQuery} from '../parser/query-types';
26
+ import {compileArrowArray, compileArrowExpression} from './compiler';
27
+ import {compileArrowPredicate} from './predicate';
28
+ import type {ArrowAggregate} from './aggregates';
29
+ import {
30
+ groupArrowTable,
31
+ joinArrowTables,
32
+ orderArrowTable,
33
+ unionArrowTables,
34
+ type ArrowQueryJoin,
35
+ type ArrowQueryOrder
36
+ } from './relational-query';
37
+ import type {ArrowExpressionOptions, CompiledArrowExpression} from './types';
38
+
39
+ /** A linear Arrow query that can execute incrementally over batches. */
40
+ export type ArrowBatchQuery = TableQuery<ParameterizedColumnarPredicate> &
41
+ Omit<ArrowExpressionOptions, 'inputSchema'> & {
42
+ /** Named values for a parameterized predicate. */
43
+ parameters?: ColumnarPredicateParameterValues;
44
+ /** Checked between batches and during expression evaluation. */
45
+ signal?: AbortSignal;
46
+ };
47
+
48
+ /** Options specific to streaming Arrow queries. */
49
+ export type ArrowStreamingQuery = ArrowBatchQuery & {
50
+ /** Emit a schema-bearing empty batch when an input batch produces no rows. */
51
+ emitEmptyBatches?: boolean;
52
+ };
53
+
54
+ /** An in-memory SELECT with optional relational operators. */
55
+ export type ArrowTableQuery = ArrowBatchQuery & {
56
+ /** Append schema-compatible tables before filtering. */
57
+ union?: readonly Table[];
58
+ /** Join a second table before filtering. */
59
+ join?: ArrowQueryJoin;
60
+ /** Scalar source columns that define groups. */
61
+ groupBy?: readonly string[];
62
+ /** Aggregate reductions indexed by output name. */
63
+ aggregates?: Readonly<Record<string, ArrowAggregate>>;
64
+ /** Stable ordering applied after grouping and before pagination. */
65
+ orderBy?: readonly ArrowQueryOrder[];
66
+ };
67
+
68
+ /** A bound output field and either a source column index or a computed expression. */
69
+ type Projection = {field: Field} & ({index: number} | {expression: CompiledArrowExpression});
70
+
71
+ /** Applies WHERE, projection, and pagination directly to a native Apache Arrow table. */
72
+ export function queryArrowTable(source: Table, query: ArrowTableQuery = {}): Table {
73
+ const {select, where, offset = 0, limit, functions, parameters, signal} = query;
74
+ signal?.throwIfAborted();
75
+ validatePagination(offset, limit);
76
+
77
+ const {union, join, groupBy, aggregates, orderBy} = query;
78
+ if (union?.length) source = unionArrowTables(source, union);
79
+ if (join) source = joinArrowTables(source, join, signal);
80
+
81
+ if (groupBy?.length || Object.keys(aggregates ?? {}).length || orderBy?.length) {
82
+ source = queryArrowTable(source, {where, functions, parameters, signal});
83
+ if (groupBy?.length || Object.keys(aggregates ?? {}).length) {
84
+ source = groupArrowTable(source, groupBy ?? [], aggregates ?? {}, {
85
+ inputSchema: source.schema,
86
+ functions,
87
+ signal
88
+ });
89
+ }
90
+ if (orderBy?.length) {
91
+ source = orderArrowTable(source, orderBy, offset, limit, signal);
92
+ return queryArrowTable(source, {select, functions, signal});
93
+ }
94
+ return queryArrowTable(source, {select, offset, limit, functions, signal});
95
+ }
96
+
97
+ const plan = compileArrowQuery(source.schema, query);
98
+ const result: RecordBatch[] = [];
99
+ for (const batch of source.batches) {
100
+ if (plan.done()) break;
101
+ for (const output of plan.execute(batch)) result.push(output);
102
+ }
103
+ return new Table(plan.schema, result);
104
+ }
105
+
106
+ /** Streams a linear Arrow query with global pagination and source backpressure. */
107
+ export async function* queryArrowTableInBatches(
108
+ input: AsyncIterable<RecordBatch> | Iterable<RecordBatch>,
109
+ inputSchema: Schema,
110
+ query: ArrowStreamingQuery = {}
111
+ ): AsyncIterable<RecordBatch> {
112
+ query.signal?.throwIfAborted();
113
+ validatePagination(query.offset ?? 0, query.limit);
114
+ const plan = compileArrowQuery(inputSchema, query);
115
+ if (plan.done()) return;
116
+ for await (const batch of input) {
117
+ query.signal?.throwIfAborted();
118
+ if (batch.schema !== inputSchema) {
119
+ validateArrowTableSchema(new Table(batch.schema, []), inputSchema, {
120
+ rejectExtraFields: true,
121
+ schemaName: 'query input schema'
122
+ });
123
+ }
124
+ yield* plan.execute(batch, query.emitEmptyBatches);
125
+ if (plan.done()) return;
126
+ }
127
+ }
128
+
129
+ /** Validates pagination before the query reads or transforms any rows. */
130
+ function validatePagination(offset: number, limit?: number): void {
131
+ if (!Number.isSafeInteger(offset) || offset < 0) throw new Error('Invalid Arrow query offset');
132
+ if (limit !== undefined && (!Number.isSafeInteger(limit) || limit < 0))
133
+ throw new Error('Invalid Arrow query limit');
134
+ }
135
+
136
+ /** Binds predicates and projection to a schema once per execution. */
137
+ function compileArrowQuery(inputSchema: Schema, query: ArrowBatchQuery) {
138
+ const {select, where, offset = 0, limit, functions, parameters, signal} = query;
139
+ const options = {inputSchema, functions};
140
+ const predicate = where
141
+ ? 'expression' in where
142
+ ? compileArrowExpression(where.expression, options)
143
+ : compileArrowPredicate(where, {...options, parameters})
144
+ : undefined;
145
+ if (predicate && !DataType.isBool(predicate.outputField.type))
146
+ throw new Error('Arrow query WHERE requires a Boolean expression');
147
+
148
+ const names = new Set<string>();
149
+ for (const selection of select ?? []) {
150
+ const name =
151
+ typeof selection === 'string'
152
+ ? selection
153
+ : (selection.as ?? ('column' in selection ? selection.column : undefined));
154
+ if (name === undefined) continue;
155
+ if (names.has(name) || (typeof selection !== 'string' && selection.as === ''))
156
+ throw new Error(`Invalid or duplicate Arrow query output name: ${name}`);
157
+ names.add(name);
158
+ }
159
+ const projection: Projection[] = select
160
+ ? select.map((selection, ordinal) => {
161
+ if (typeof selection !== 'string' && ('array' in selection || 'expression' in selection)) {
162
+ const expression =
163
+ 'array' in selection
164
+ ? compileArrowArray(selection.array, {...options, type: selection.type})
165
+ : compileArrowExpression(selection.expression, options);
166
+ let name = selection.as;
167
+ if (name === undefined) {
168
+ name = `expression_${ordinal + 1}`;
169
+ while (names.has(name)) name += '_';
170
+ names.add(name);
171
+ }
172
+ return {expression, field: expression.outputField.clone({name})};
173
+ }
174
+ const column = typeof selection === 'string' ? selection : selection.column;
175
+ const index = inputSchema.fields.findIndex(field => field.name === column);
176
+ if (index < 0) throw new Error(`Arrow query column not found: ${column}`);
177
+ const field = inputSchema.fields[index]!;
178
+ return {
179
+ index,
180
+ field: typeof selection === 'string' ? field : field.clone({name: selection.as ?? column})
181
+ };
182
+ })
183
+ : inputSchema.fields.map((field, index) => ({field, index}));
184
+
185
+ const schema = select
186
+ ? new Schema(
187
+ projection.map(({field}) => field),
188
+ inputSchema.metadata,
189
+ undefined,
190
+ inputSchema.metadataVersion
191
+ )
192
+ : inputSchema;
193
+ let skip = offset;
194
+ let remaining = limit ?? Infinity;
195
+ /** Evaluates one batch using the remaining pagination budget. */
196
+ function* execute(batch: RecordBatch, emitEmpty = false): Iterable<RecordBatch> {
197
+ signal?.throwIfAborted();
198
+ const metadata = getRecordBatchMetadata(batch);
199
+ const mask = predicate?.evaluate(batch, {signal});
200
+ let selected: RecordBatch | undefined;
201
+ if (mask) {
202
+ const indexes: number[] = [];
203
+ for (let row = 0; row < batch.numRows && indexes.length < remaining; row++) {
204
+ if ((row & 1023) === 0) signal?.throwIfAborted();
205
+ if (mask.get(row) !== true) continue;
206
+ if (skip > 0) skip--;
207
+ else indexes.push(row);
208
+ }
209
+ if (indexes.length === batch.numRows) selected = batch;
210
+ else if (
211
+ indexes.length &&
212
+ indexes[indexes.length - 1]! - indexes[0]! + 1 === indexes.length
213
+ ) {
214
+ selected = createRecordBatchWithMetadata(
215
+ batch.schema,
216
+ batch.data.slice(indexes[0]!, indexes.length),
217
+ metadata
218
+ );
219
+ } else if (indexes.length) {
220
+ const gathered = takeArrowTableRows(
221
+ new Table(inputSchema, [batch]),
222
+ Int32Array.from(indexes)
223
+ );
224
+ selected = makeQueryBatch(
225
+ inputSchema,
226
+ gathered.batches[0]!.data.children,
227
+ indexes.length,
228
+ metadata
229
+ );
230
+ }
231
+ } else {
232
+ const start = Math.min(skip, batch.numRows);
233
+ skip -= start;
234
+ const end = start + Math.min(batch.numRows - start, remaining);
235
+ if (end > start)
236
+ selected =
237
+ start === 0 && end === batch.numRows
238
+ ? batch
239
+ : createRecordBatchWithMetadata(
240
+ batch.schema,
241
+ batch.data.slice(start, end - start),
242
+ metadata
243
+ );
244
+ }
245
+ if (!selected?.numRows) {
246
+ if (emitEmpty) {
247
+ const children = schema.fields.map(field => makeData({type: field.type, nullCount: 0}));
248
+ yield makeQueryBatch(schema, children, 0, metadata);
249
+ }
250
+ return;
251
+ }
252
+ remaining -= selected.numRows;
253
+ if (!select) {
254
+ yield selected;
255
+ return;
256
+ }
257
+
258
+ const columns = projection.map(item =>
259
+ 'index' in item
260
+ ? selected.data.children[item.index]!
261
+ : item.expression.evaluate(selected, {signal})
262
+ );
263
+ // Computed vectors can introduce chunk boundaries inside the selected source batch.
264
+ const boundaries = new Set([selected.numRows]);
265
+ for (const column of columns) {
266
+ if ('children' in column) continue;
267
+ let end = 0;
268
+ for (const chunk of column.data) {
269
+ end += chunk.length;
270
+ if (end > 0) boundaries.add(end);
271
+ }
272
+ }
273
+ let start = 0;
274
+ for (const end of [...boundaries].sort((left, right) => left - right)) {
275
+ const children = columns.map(column =>
276
+ 'children' in column
277
+ ? start === 0 && end === column.length
278
+ ? column
279
+ : column.slice(start, end - start)
280
+ : column.slice(start, end).data[0]!
281
+ );
282
+ yield makeQueryBatch(schema, children, end - start, metadata);
283
+ start = end;
284
+ }
285
+ }
286
+ return {schema, done: () => remaining === 0, execute};
287
+ }
288
+
289
+ /** Creates a batch while preserving its row count even for an empty projection. */
290
+ function makeQueryBatch(
291
+ schema: Schema,
292
+ children: Data[],
293
+ length: number,
294
+ metadata: Map<string, string> | undefined
295
+ ): RecordBatch {
296
+ const data = new Data(new Struct(schema.fields), 0, length, 0, undefined, children);
297
+ return createRecordBatchWithMetadata(schema, data, metadata);
298
+ }
@@ -0,0 +1,8 @@
1
+ // math.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
4
+
5
+ import {initializeTaskWorker} from '@math.gl/tasks';
6
+ import {createArrowQueryTask} from './query-worker';
7
+
8
+ void initializeTaskWorker(createArrowQueryTask());