@math.gl/sql 5.0.0-beta.11 → 5.0.0-beta.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/dist/arrow/aggregates.js +1 -1
  2. package/dist/arrow/aggregates.js.map +1 -1
  3. package/dist/arrow/buffers.js.map +1 -1
  4. package/dist/arrow/cast.d.ts +1 -1
  5. package/dist/arrow/cast.js +1 -1
  6. package/dist/arrow/cast.js.map +1 -1
  7. package/dist/arrow/compiler.d.ts.map +1 -1
  8. package/dist/arrow/compiler.js +146 -29
  9. package/dist/arrow/compiler.js.map +1 -1
  10. package/dist/arrow/index.cjs +124 -23
  11. package/dist/arrow/index.cjs.map +1 -1
  12. package/dist/arrow/index.d.ts.map +1 -1
  13. package/dist/arrow/index.js.map +1 -1
  14. package/dist/arrow/list-values.d.ts.map +1 -1
  15. package/dist/arrow/list-values.js +27 -6
  16. package/dist/arrow/list-values.js.map +1 -1
  17. package/dist/arrow/predicate.d.ts.map +1 -1
  18. package/dist/arrow/predicate.js.map +1 -1
  19. package/dist/arrow/query-arrow-sql.d.ts +2 -2
  20. package/dist/arrow/query-arrow-sql.d.ts.map +1 -1
  21. package/dist/arrow/query-arrow-sql.js +40 -8
  22. package/dist/arrow/query-arrow-sql.js.map +1 -1
  23. package/dist/arrow/query-arrow-table.d.ts +1 -1
  24. package/dist/arrow/query-arrow-table.d.ts.map +1 -1
  25. package/dist/arrow/query-arrow-table.js +1 -1
  26. package/dist/arrow/query-arrow-table.js.map +1 -1
  27. package/dist/arrow/query-worker.d.ts +1 -1
  28. package/dist/arrow/query-worker.d.ts.map +1 -1
  29. package/dist/arrow/query-worker.js +1 -1
  30. package/dist/arrow/query-worker.js.map +1 -1
  31. package/dist/arrow/registry.d.ts.map +1 -1
  32. package/dist/arrow/registry.js +1 -0
  33. package/dist/arrow/registry.js.map +1 -1
  34. package/dist/arrow/relational-query.js +2 -2
  35. package/dist/arrow/relational-query.js.map +1 -1
  36. package/dist/arrow/temporal-cast.js.map +1 -1
  37. package/dist/arrow/types.d.ts.map +1 -1
  38. package/dist/arrow/unnest-arrow-column.d.ts +22 -0
  39. package/dist/arrow/unnest-arrow-column.d.ts.map +1 -0
  40. package/dist/arrow/unnest-arrow-column.js +112 -0
  41. package/dist/arrow/unnest-arrow-column.js.map +1 -0
  42. package/dist/arrow/worker.cjs +130 -29
  43. package/dist/arrow/worker.cjs.map +1 -1
  44. package/dist/arrow/worker.d.ts +1 -1
  45. package/dist/arrow/worker.d.ts.map +1 -1
  46. package/dist/arrow/worker.js +1 -1
  47. package/dist/arrow/worker.js.map +1 -1
  48. package/dist/{color → color-functions}/index.cjs +2 -2
  49. package/dist/{color → color-functions}/index.cjs.map +1 -1
  50. package/dist/color-functions/index.d.ts.map +1 -0
  51. package/dist/color-functions/index.js.map +1 -0
  52. package/dist/expressions/ast.js +3 -1
  53. package/dist/expressions/ast.js.map +1 -1
  54. package/dist/expressions/expression-eval.d.ts +10 -1
  55. package/dist/expressions/expression-eval.d.ts.map +1 -1
  56. package/dist/expressions/expression-eval.js.map +1 -1
  57. package/dist/expressions/index.cjs +1 -1
  58. package/dist/expressions/index.cjs.map +1 -1
  59. package/dist/index.cjs +2792 -2363
  60. package/dist/index.cjs.map +3 -3
  61. package/dist/index.d.ts +2 -0
  62. package/dist/index.d.ts.map +1 -1
  63. package/dist/index.js +1 -0
  64. package/dist/index.js.map +1 -1
  65. package/dist/{math → math-functions}/index.cjs +2 -2
  66. package/dist/{math → math-functions}/index.cjs.map +1 -1
  67. package/dist/math-functions/index.d.ts.map +1 -0
  68. package/dist/math-functions/index.js.map +1 -0
  69. package/dist/parser/bind-sql-predicate.d.ts.map +1 -0
  70. package/dist/parser/bind-sql-predicate.js.map +1 -0
  71. package/dist/{sql → parser}/index.cjs +348 -129
  72. package/dist/parser/index.cjs.map +6 -0
  73. package/dist/{sql → parser}/index.d.ts +2 -0
  74. package/dist/parser/index.d.ts.map +1 -0
  75. package/dist/{sql → parser}/index.js +1 -0
  76. package/dist/parser/index.js.map +1 -0
  77. package/dist/parser/parameter-snapshots.d.ts +21 -0
  78. package/dist/parser/parameter-snapshots.d.ts.map +1 -0
  79. package/dist/parser/parameter-snapshots.js +85 -0
  80. package/dist/parser/parameter-snapshots.js.map +1 -0
  81. package/dist/{sql → parser}/parse-sql-expression.d.ts +8 -2
  82. package/dist/parser/parse-sql-expression.d.ts.map +1 -0
  83. package/dist/{sql → parser}/parse-sql-expression.js +51 -5
  84. package/dist/parser/parse-sql-expression.js.map +1 -0
  85. package/dist/parser/parse-sql-predicate.d.ts.map +1 -0
  86. package/dist/parser/parse-sql-predicate.js.map +1 -0
  87. package/dist/{sql → parser}/parse-sql-query.d.ts +2 -1
  88. package/dist/parser/parse-sql-query.d.ts.map +1 -0
  89. package/dist/{sql → parser}/parse-sql-query.js +21 -6
  90. package/dist/parser/parse-sql-query.js.map +1 -0
  91. package/dist/{sql → parser}/parse-sql-select.d.ts +4 -2
  92. package/dist/parser/parse-sql-select.d.ts.map +1 -0
  93. package/dist/{sql → parser}/parse-sql-select.js +6 -3
  94. package/dist/parser/parse-sql-select.js.map +1 -0
  95. package/dist/{sql → parser}/parse-sql-statement.d.ts +9 -5
  96. package/dist/parser/parse-sql-statement.d.ts.map +1 -0
  97. package/dist/{sql → parser}/parse-sql-statement.js +39 -14
  98. package/dist/parser/parse-sql-statement.js.map +1 -0
  99. package/dist/parser/query-types.d.ts.map +1 -0
  100. package/dist/parser/query-types.js.map +1 -0
  101. package/dist/parser/sql-parameters.d.ts +29 -0
  102. package/dist/parser/sql-parameters.d.ts.map +1 -0
  103. package/dist/parser/sql-parameters.js +110 -0
  104. package/dist/parser/sql-parameters.js.map +1 -0
  105. package/dist/parser/sql-parse-error.d.ts.map +1 -0
  106. package/dist/parser/sql-parse-error.js.map +1 -0
  107. package/dist/parser/sql-predicate-schema.d.ts.map +1 -0
  108. package/dist/parser/sql-predicate-schema.js.map +1 -0
  109. package/dist/parser/sql-predicate-types.d.ts.map +1 -0
  110. package/dist/parser/sql-predicate-types.js.map +1 -0
  111. package/dist/parser/sql-predicate.d.ts.map +1 -0
  112. package/dist/parser/sql-predicate.js.map +1 -0
  113. package/dist/scales/arrow-domains.d.ts +8 -1
  114. package/dist/scales/arrow-domains.d.ts.map +1 -1
  115. package/dist/scales/arrow-domains.js +48 -6
  116. package/dist/scales/arrow-domains.js.map +1 -1
  117. package/dist/scales/arrow-scales.d.ts +28 -4
  118. package/dist/scales/arrow-scales.d.ts.map +1 -1
  119. package/dist/scales/arrow-scales.js +158 -12
  120. package/dist/scales/arrow-scales.js.map +1 -1
  121. package/dist/scales/compile-scale.d.ts +1 -6
  122. package/dist/scales/compile-scale.d.ts.map +1 -1
  123. package/dist/scales/compile-scale.js +1 -191
  124. package/dist/scales/compile-scale.js.map +1 -1
  125. package/dist/scales/conformance.cjs +180 -0
  126. package/dist/scales/conformance.cjs.map +6 -0
  127. package/dist/scales/conformance.d.ts +26 -0
  128. package/dist/scales/conformance.d.ts.map +1 -0
  129. package/dist/scales/conformance.js +164 -0
  130. package/dist/scales/conformance.js.map +1 -0
  131. package/dist/scales/hdr-colors.d.ts +1 -12
  132. package/dist/scales/hdr-colors.d.ts.map +1 -1
  133. package/dist/scales/hdr-colors.js +1 -111
  134. package/dist/scales/hdr-colors.js.map +1 -1
  135. package/dist/scales/index.cjs +3738 -618
  136. package/dist/scales/index.cjs.map +3 -3
  137. package/dist/scales/index.d.ts +2 -1
  138. package/dist/scales/index.d.ts.map +1 -1
  139. package/dist/scales/index.js +2 -1
  140. package/dist/scales/index.js.map +1 -1
  141. package/dist/scales/palettes.d.ts +1 -4
  142. package/dist/scales/palettes.d.ts.map +1 -1
  143. package/dist/scales/palettes.js +1 -81
  144. package/dist/scales/palettes.js.map +1 -1
  145. package/dist/scales/parse-sql-scale.d.ts +14 -1
  146. package/dist/scales/parse-sql-scale.d.ts.map +1 -1
  147. package/dist/scales/parse-sql-scale.js +110 -18
  148. package/dist/scales/parse-sql-scale.js.map +1 -1
  149. package/dist/scales/quantile-sketch.d.ts +1 -21
  150. package/dist/scales/quantile-sketch.d.ts.map +1 -1
  151. package/dist/scales/quantile-sketch.js +1 -104
  152. package/dist/scales/quantile-sketch.js.map +1 -1
  153. package/dist/scales/scale-types.d.ts +1 -91
  154. package/dist/scales/scale-types.d.ts.map +1 -1
  155. package/dist/scales/scale-types.js +1 -115
  156. package/dist/scales/scale-types.js.map +1 -1
  157. package/dist/scales/streaming-domains.d.ts.map +1 -1
  158. package/dist/scales/streaming-domains.js +8 -2
  159. package/dist/scales/streaming-domains.js.map +1 -1
  160. package/package.json +19 -19
  161. package/src/arrow/aggregates.ts +1 -1
  162. package/src/arrow/buffers.ts +1 -1
  163. package/src/arrow/cast.ts +3 -3
  164. package/src/arrow/compiler.ts +155 -36
  165. package/src/arrow/index.ts +1 -1
  166. package/src/arrow/list-values.ts +28 -7
  167. package/src/arrow/predicate.ts +1 -1
  168. package/src/arrow/query-arrow-sql.ts +38 -10
  169. package/src/arrow/query-arrow-table.ts +3 -3
  170. package/src/arrow/query-worker.ts +5 -1
  171. package/src/arrow/registry.ts +1 -0
  172. package/src/arrow/relational-query.ts +2 -2
  173. package/src/arrow/temporal-cast.ts +1 -1
  174. package/src/arrow/types.ts +1 -1
  175. package/src/arrow/unnest-arrow-column.ts +145 -0
  176. package/src/arrow/worker.ts +6 -2
  177. package/src/expressions/ast.ts +3 -1
  178. package/src/expressions/expression-eval.ts +10 -1
  179. package/src/index.ts +14 -0
  180. package/src/{math → math-functions}/index.ts +2 -2
  181. package/src/{sql → parser}/index.ts +15 -0
  182. package/src/parser/parameter-snapshots.ts +103 -0
  183. package/src/{sql → parser}/parse-sql-expression.ts +49 -6
  184. package/src/{sql → parser}/parse-sql-query.ts +24 -6
  185. package/src/{sql → parser}/parse-sql-select.ts +14 -8
  186. package/src/{sql → parser}/parse-sql-statement.ts +42 -24
  187. package/src/{sql → parser}/query-types.ts +1 -1
  188. package/src/parser/sql-parameters.ts +174 -0
  189. package/src/scales/arrow-domains.ts +66 -7
  190. package/src/scales/arrow-scales.ts +221 -14
  191. package/src/scales/compile-scale.ts +1 -213
  192. package/src/scales/conformance.ts +186 -0
  193. package/src/scales/hdr-colors.ts +1 -122
  194. package/src/scales/index.ts +3 -1
  195. package/src/scales/palettes.ts +1 -82
  196. package/src/scales/parse-sql-scale.ts +126 -21
  197. package/src/scales/quantile-sketch.ts +1 -102
  198. package/src/scales/scale-types.ts +1 -220
  199. package/src/scales/streaming-domains.ts +8 -2
  200. package/dist/color/index.d.ts.map +0 -1
  201. package/dist/color/index.js.map +0 -1
  202. package/dist/math/index.d.ts.map +0 -1
  203. package/dist/math/index.js.map +0 -1
  204. package/dist/scales/arrow-scales.cjs +0 -3837
  205. package/dist/scales/arrow-scales.cjs.map +0 -6
  206. package/dist/sql/bind-sql-predicate.d.ts.map +0 -1
  207. package/dist/sql/bind-sql-predicate.js.map +0 -1
  208. package/dist/sql/index.cjs.map +0 -6
  209. package/dist/sql/index.d.ts.map +0 -1
  210. package/dist/sql/index.js.map +0 -1
  211. package/dist/sql/parse-sql-expression.d.ts.map +0 -1
  212. package/dist/sql/parse-sql-expression.js.map +0 -1
  213. package/dist/sql/parse-sql-predicate.d.ts.map +0 -1
  214. package/dist/sql/parse-sql-predicate.js.map +0 -1
  215. package/dist/sql/parse-sql-query.d.ts.map +0 -1
  216. package/dist/sql/parse-sql-query.js.map +0 -1
  217. package/dist/sql/parse-sql-select.d.ts.map +0 -1
  218. package/dist/sql/parse-sql-select.js.map +0 -1
  219. package/dist/sql/parse-sql-statement.d.ts.map +0 -1
  220. package/dist/sql/parse-sql-statement.js.map +0 -1
  221. package/dist/sql/query-types.d.ts.map +0 -1
  222. package/dist/sql/query-types.js.map +0 -1
  223. package/dist/sql/sql-parse-error.d.ts.map +0 -1
  224. package/dist/sql/sql-parse-error.js.map +0 -1
  225. package/dist/sql/sql-predicate-schema.d.ts.map +0 -1
  226. package/dist/sql/sql-predicate-schema.js.map +0 -1
  227. package/dist/sql/sql-predicate-types.d.ts.map +0 -1
  228. package/dist/sql/sql-predicate-types.js.map +0 -1
  229. package/dist/sql/sql-predicate.d.ts.map +0 -1
  230. package/dist/sql/sql-predicate.js.map +0 -1
  231. package/src/expressions/vendor/jsep/README.md +0 -22
  232. /package/dist/{color → color-functions}/index.d.ts +0 -0
  233. /package/dist/{color → color-functions}/index.js +0 -0
  234. /package/dist/{math → math-functions}/index.d.ts +0 -0
  235. /package/dist/{math → math-functions}/index.js +0 -0
  236. /package/dist/{sql → parser}/bind-sql-predicate.d.ts +0 -0
  237. /package/dist/{sql → parser}/bind-sql-predicate.js +0 -0
  238. /package/dist/{sql → parser}/parse-sql-predicate.d.ts +0 -0
  239. /package/dist/{sql → parser}/parse-sql-predicate.js +0 -0
  240. /package/dist/{sql → parser}/query-types.d.ts +0 -0
  241. /package/dist/{sql → parser}/query-types.js +0 -0
  242. /package/dist/{sql → parser}/sql-parse-error.d.ts +0 -0
  243. /package/dist/{sql → parser}/sql-parse-error.js +0 -0
  244. /package/dist/{sql → parser}/sql-predicate-schema.d.ts +0 -0
  245. /package/dist/{sql → parser}/sql-predicate-schema.js +0 -0
  246. /package/dist/{sql → parser}/sql-predicate-types.d.ts +0 -0
  247. /package/dist/{sql → parser}/sql-predicate-types.js +0 -0
  248. /package/dist/{sql → parser}/sql-predicate.d.ts +0 -0
  249. /package/dist/{sql → parser}/sql-predicate.js +0 -0
  250. /package/src/{color → color-functions}/index.ts +0 -0
  251. /package/src/{sql → parser}/bind-sql-predicate.ts +0 -0
  252. /package/src/{sql → parser}/parse-sql-predicate.ts +0 -0
  253. /package/src/{sql → parser}/sql-parse-error.ts +0 -0
  254. /package/src/{sql → parser}/sql-predicate-schema.ts +0 -0
  255. /package/src/{sql → parser}/sql-predicate-types.ts +0 -0
  256. /package/src/{sql → parser}/sql-predicate.ts +0 -0
@@ -3,7 +3,7 @@
3
3
  // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
4
4
 
5
5
  import type {Field, Schema, RecordBatch, Vector, Table} from 'apache-arrow';
6
- import type {Expression} from '../expressions/index';
6
+ import type {Expression} from '../expressions';
7
7
 
8
8
  /**
9
9
  * Scalar values exposed to user callbacks. Date32 uses epoch days and microsecond
@@ -0,0 +1,145 @@
1
+ // math.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
4
+
5
+ import {DataType, Field, Int32, Schema, Table, Utf8, Vector, vectorFromArray} from 'apache-arrow';
6
+ import {takeArrowTableRows} from '@math.gl/core/arrow';
7
+ import {physicalKey, extensionKey} from './signature';
8
+
9
+ /** Names and cancellation for expanding a nested column into relational rows. */
10
+ export type UnnestArrowColumnOptions = {
11
+ /** Name of the appended child-value column. Must not replace an existing column. */
12
+ valueColumn: string;
13
+ /** Optional map/struct key or one-based list position column. */
14
+ keyColumn?: string;
15
+ /** Optional one-based element position, including for maps. */
16
+ ordinalColumn?: string;
17
+ /** Retain null and empty containers as one row with null key/value. Defaults to false. */
18
+ outer?: boolean;
19
+ /** Synchronous cancellation, checked per source row. */
20
+ signal?: AbortSignal;
21
+ };
22
+
23
+ /**
24
+ * Expands a List, FixedSizeList, Map, or homogeneous Struct column.
25
+ * Parent columns and metadata are retained. Lists preserve element order, maps
26
+ * preserve entry order, and structs use schema field order. Null elements remain
27
+ * rows; null/empty containers produce no rows unless outer is true.
28
+ */
29
+ export function unnestArrowColumn(
30
+ table: Table,
31
+ column: string,
32
+ options: UnnestArrowColumnOptions
33
+ ): Table {
34
+ const input = table.getChild(column);
35
+ if (!input) throw new Error(`Unknown nested column: ${column}`);
36
+ const type = input.type;
37
+ const list = DataType.isList(type) || DataType.isFixedSizeList(type);
38
+ const map = DataType.isMap(type);
39
+ const struct = DataType.isStruct(type);
40
+ if (!list && !map && !struct) throw new Error('UNNEST requires a list, map, or struct column');
41
+ const children = map ? type.children[0]!.type.children : type.children;
42
+ const valueField = children[map ? 1 : 0];
43
+ if (!valueField) throw new Error('UNNEST requires a declared child type');
44
+ if (
45
+ struct &&
46
+ children.some(
47
+ field =>
48
+ physicalKey(field.type) !== physicalKey(valueField.type) ||
49
+ extensionKey(field) !== extensionKey(valueField)
50
+ )
51
+ ) {
52
+ throw new Error('UNNEST struct fields must share one type');
53
+ }
54
+ const keyType = list ? new Int32() : map ? children[0]!.type : new Utf8();
55
+ const fields = [
56
+ ...table.schema.fields,
57
+ valueField.clone({name: options.valueColumn, nullable: true}),
58
+ ...(options.keyColumn ? [new Field(options.keyColumn, keyType, true)] : []),
59
+ ...(options.ordinalColumn ? [new Field(options.ordinalColumn, new Int32(), true)] : [])
60
+ ];
61
+ if (new Set(fields.map(field => field.name)).size !== fields.length) {
62
+ throw new Error('Duplicate UNNEST output name');
63
+ }
64
+ const schema = new Schema(fields, table.schema.metadata);
65
+ const parentRows: number[] = [];
66
+ /** Physical child slices preserve timestamps, dictionaries and nested values. */
67
+ const values: Vector[] = [];
68
+ /** Physical map keys or generated struct/list positions. */
69
+ const keys: Vector[] = [];
70
+ /** One-based position within each parent collection. */
71
+ const positions: (number | null)[] = [];
72
+ /** Current collection position, reset for each parent. */
73
+ let position = 0;
74
+ /** Appends one expanded row while preserving its parent index. */
75
+ function append(row: number, key: Vector, value: Vector, ordinal: number | null): void {
76
+ parentRows.push(row);
77
+ keys.push(key);
78
+ values.push(value);
79
+ positions.push(ordinal);
80
+ }
81
+ /** Tracks logical parent positions across Arrow chunks. */
82
+ let row = 0;
83
+ for (const data of input.data) {
84
+ /** Chunk-local parent validity and offsets. */
85
+ const chunk = new Vector([data]);
86
+ for (let local = 0; local < chunk.length; local++, row++) {
87
+ options.signal?.throwIfAborted();
88
+ const before = parentRows.length;
89
+ position = 0;
90
+ if (chunk.isValid(local)) {
91
+ if (struct) {
92
+ for (let index = 0; index < children.length; index++) {
93
+ append(
94
+ row,
95
+ vectorFromArray([children[index]!.name], keyType),
96
+ new Vector([data.children[index]!]).slice(local, local + 1),
97
+ ++position
98
+ );
99
+ }
100
+ } else {
101
+ /** Collection offsets already account for sliced variable-length parents. */
102
+ const begin =
103
+ list && DataType.isFixedSizeList(type)
104
+ ? (data.offset + local) * type.listSize
105
+ : Number(data.valueOffsets[local]);
106
+ const end =
107
+ list && DataType.isFixedSizeList(type)
108
+ ? begin + type.listSize
109
+ : Number(data.valueOffsets[local + 1]);
110
+ /** Physical collection values, including map entry structs. */
111
+ const child = new Vector([data.children[0]!]);
112
+ for (let index = begin; index < end; index++) {
113
+ options.signal?.throwIfAborted();
114
+ const value = map
115
+ ? child.getChildAt(1)!.slice(index, index + 1)
116
+ : child.slice(index, index + 1);
117
+ const key = map
118
+ ? child.getChildAt(0)!.slice(index, index + 1)
119
+ : vectorFromArray([index - begin + 1], keyType);
120
+ append(row, key, value, ++position);
121
+ }
122
+ }
123
+ }
124
+ if (options.outer && before === parentRows.length)
125
+ append(
126
+ row,
127
+ vectorFromArray([null], keyType),
128
+ vectorFromArray([null], valueField.type),
129
+ null
130
+ );
131
+ }
132
+ }
133
+ if (!parentRows.length) return new Table(schema, []);
134
+ const parents = takeArrowTableRows(table, Int32Array.from(parentRows));
135
+ return new Table(schema, {
136
+ ...Object.fromEntries(
137
+ table.schema.fields.map(field => [field.name, parents.getChild(field.name)!])
138
+ ),
139
+ [options.valueColumn]: new Vector(values.flatMap(value => value.data)),
140
+ ...(options.keyColumn ? {[options.keyColumn]: new Vector(keys.flatMap(key => key.data))} : {}),
141
+ ...(options.ordinalColumn
142
+ ? {[options.ordinalColumn]: vectorFromArray(positions, new Int32())}
143
+ : {})
144
+ });
145
+ }
@@ -7,8 +7,12 @@ import type {Task, TaskWorker} from '@math.gl/tasks';
7
7
  import type {
8
8
  ParameterizedColumnarPredicate,
9
9
  ColumnarPredicateParameterValues
10
- } from '../expressions/index';
11
- import {dehydrateArrowTable, hydrateArrowTable, type DehydratedArrowTable} from '@math.gl/arrow';
10
+ } from '../expressions';
11
+ import {
12
+ dehydrateArrowTable,
13
+ hydrateArrowTable,
14
+ type DehydratedArrowTable
15
+ } from '@math.gl/core/arrow';
12
16
  import {compileArrowExpression} from './compiler';
13
17
  import {compileArrowPredicate} from './predicate';
14
18
  import type {ArrowExpression, CompiledArrowExpression} from './types';
@@ -79,7 +79,9 @@ function visitExpressionAST(
79
79
  case 'Literal':
80
80
  if (
81
81
  node.value !== null &&
82
- !['number', 'bigint', 'boolean', 'string'].includes(typeof node.value)
82
+ !['number', 'bigint', 'boolean', 'string'].includes(typeof node.value) &&
83
+ !(node.value instanceof Uint8Array) &&
84
+ !(node.value instanceof Date && Number.isFinite(node.value.getTime()))
83
85
  )
84
86
  throw new Error('Unsupported expression literal');
85
87
  if (node.raw !== undefined && typeof node.raw !== 'string')
@@ -36,7 +36,16 @@ export type Expression =
36
36
  })
37
37
  | JsepTypes.ConditionalExpression
38
38
  | JsepTypes.Identifier
39
- | JsepTypes.Literal
39
+ | {
40
+ /** Literal node discriminator. */
41
+ type: 'Literal';
42
+ /** Original spelling when the literal came from source text. */
43
+ raw?: string;
44
+ /** Scalar value, including structured-cloneable binary and date parameters. */
45
+ value: JsepTypes.Literal['value'] | Date | Uint8Array;
46
+ /** Optional pre-encoded UTF-8 storage corresponding to a string value. */
47
+ utf8?: Uint8Array;
48
+ }
40
49
  | JsepTypes.MemberExpression
41
50
  | JsepTypes.ThisExpression
42
51
  | JsepTypes.UnaryExpression;
package/src/index.ts CHANGED
@@ -38,3 +38,17 @@ export {
38
38
  isExpressionEvaluationError,
39
39
  type ExpressionEvaluationErrorCode
40
40
  } from './expressions/evaluation-error';
41
+
42
+ export type {
43
+ SQLParameterValue,
44
+ SQLParameters,
45
+ SQLParameterOptions,
46
+ SQLNumericArray,
47
+ SQLBinaryParameter
48
+ } from './parser/sql-parameters';
49
+ export {
50
+ prepareSQLParameters,
51
+ areSQLParametersEqual,
52
+ getSQLParameterNames,
53
+ type SQLParameterSnapshot
54
+ } from './parser/parameter-snapshots';
@@ -20,9 +20,9 @@ import {
20
20
  makeData,
21
21
  makeVector
22
22
  } from 'apache-arrow';
23
- import {NUMERIC_FUNCTION_DEFINITIONS, type ExpressionNumericType} from '../expressions/index';
23
+ import {NUMERIC_FUNCTION_DEFINITIONS, type ExpressionNumericType} from '../expressions';
24
24
  import {ColumnReader, createValueValidator} from '../arrow/buffers';
25
- import {ExpressionEvaluationError, isExpressionEvaluationError} from '../expressions/index';
25
+ import {ExpressionEvaluationError, isExpressionEvaluationError} from '../expressions';
26
26
  import {float16Values, roundFloat16} from '../arrow/float16';
27
27
  import type {ArrowTableFunction, ArrowExpressionValue} from '../arrow/types';
28
28
 
@@ -26,3 +26,18 @@ export type {
26
26
  SQLJoin,
27
27
  SQLSort
28
28
  } from './parse-sql-statement';
29
+
30
+ export type {
31
+ SQLParameterValue,
32
+ SQLParameters,
33
+ SQLParameterOptions,
34
+ SQLNumericArray,
35
+ SQLBinaryParameter
36
+ } from './sql-parameters';
37
+
38
+ export {
39
+ prepareSQLParameters,
40
+ areSQLParametersEqual,
41
+ getSQLParameterNames,
42
+ type SQLParameterSnapshot
43
+ } from './parameter-snapshots';
@@ -0,0 +1,103 @@
1
+ // math.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
4
+
5
+ import type {Expression} from '../expressions';
6
+ import {bindSQLParameter, type SQLParameters} from './sql-parameters';
7
+
8
+ /** Opaque immutable handle for validated, detached parameter values. */
9
+ export type SQLParameterSnapshot = Readonly<{kind: 'sql-parameter-snapshot'}>;
10
+ /** Private storage prevents consumers from mutating snapshot buffers. */
11
+ const snapshots = new WeakMap<SQLParameterSnapshot, ReadonlyMap<string, Expression>>();
12
+
13
+ /** Prepares values once, sharing unchanged entries with an optional previous snapshot. */
14
+ export function prepareSQLParameters(
15
+ parameters: SQLParameters,
16
+ options: {previous?: SQLParameterSnapshot} = {}
17
+ ): SQLParameterSnapshot {
18
+ const previous = options.previous && storage(options.previous);
19
+ const entries = new Map<string, Expression>();
20
+ for (const name of Object.keys(parameters)) {
21
+ const expression = bindSQLParameter(name, {parameters});
22
+ encodeStrings(expression);
23
+ const old = previous?.get(name);
24
+ entries.set(name, old && equalValue(old, expression) ? old : expression);
25
+ }
26
+ if (
27
+ options.previous &&
28
+ previous?.size === entries.size &&
29
+ [...entries].every(([name, value]) => previous.get(name) === value)
30
+ )
31
+ return options.previous;
32
+ const snapshot = Object.freeze({kind: 'sql-parameter-snapshot' as const});
33
+ snapshots.set(snapshot, entries);
34
+ return snapshot;
35
+ }
36
+
37
+ /** Compares all values, or only the names used by one query. Missing differs from present. */
38
+ export function areSQLParametersEqual(
39
+ a: SQLParameterSnapshot,
40
+ b: SQLParameterSnapshot,
41
+ options: {names?: readonly string[]} = {}
42
+ ): boolean {
43
+ const left = storage(a);
44
+ const right = storage(b);
45
+ if (a === b) return true;
46
+ if (!options.names && left.size !== right.size) return false;
47
+ return (options.names ?? [...left.keys()]).every(
48
+ name => left.has(name) === right.has(name) && equalValue(left.get(name), right.get(name))
49
+ );
50
+ }
51
+
52
+ /** Recognizes locally prepared handles without reserving any caller parameter names. */
53
+ export function isSQLParameterSnapshot(value: unknown): value is SQLParameterSnapshot {
54
+ return Boolean(
55
+ value && typeof value === 'object' && snapshots.has(value as SQLParameterSnapshot)
56
+ );
57
+ }
58
+
59
+ /** Returns a detached AST for binding while retaining exclusive ownership of cached buffers. */
60
+ export function snapshotParameter(snapshot: SQLParameterSnapshot, name: string): Expression {
61
+ const expression = storage(snapshot).get(name);
62
+ if (!expression) throw new Error(`SQL parameter ":${name}" requires a value`);
63
+ return structuredClone(expression);
64
+ }
65
+
66
+ /** Validates opaque handles, including handles accidentally serialized across a worker boundary. */
67
+ function storage(snapshot: SQLParameterSnapshot): ReadonlyMap<string, Expression> {
68
+ const values = snapshots.get(snapshot);
69
+ if (!values) throw new Error('Invalid SQL parameter snapshot');
70
+ return values;
71
+ }
72
+
73
+ /** Compares detached scalar/array AST contents and physical storage types. */
74
+ function equalValue(a: unknown, b: unknown): boolean {
75
+ if (Object.is(a, b)) return true;
76
+ if (!a || !b || typeof a !== 'object' || typeof b !== 'object' || a.constructor !== b.constructor)
77
+ return false;
78
+ if (a instanceof Date && b instanceof Date) return a.getTime() === b.getTime();
79
+ const left = Object.keys(a);
80
+ const right = Object.keys(b);
81
+ return (
82
+ left.length === right.length &&
83
+ left.every(key => Object.prototype.hasOwnProperty.call(b, key) && equalValue(a[key], b[key]))
84
+ );
85
+ }
86
+
87
+ /** Lists case-sensitive placeholders in first-use order, excluding quotes, comments and casts. */
88
+ export function getSQLParameterNames(sql: string): readonly string[] {
89
+ const names = new Set<string>();
90
+ const tokens =
91
+ /'(?:''|[^'])*'|"(?:""|[^"])*"|--[^\r\n]*|\/\*[\s\S]*?\*\/|::|:([A-Za-z_][A-Za-z0-9_]*)/g;
92
+ for (const match of sql.matchAll(tokens)) if (match[1]) names.add(match[1]);
93
+ return Object.freeze([...names]);
94
+ }
95
+
96
+ /** Pre-encodes text once when a snapshot is prepared for reuse by Arrow compilation. */
97
+ function encodeStrings(expression: Expression): void {
98
+ if (expression.type === 'Literal' && typeof expression.value === 'string' && !expression.utf8) {
99
+ expression.utf8 = new TextEncoder().encode(expression.value);
100
+ } else if (expression.type === 'ArrayExpression') {
101
+ for (const element of expression.elements) encodeStrings(element as Expression);
102
+ }
103
+ }
@@ -2,8 +2,13 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
4
4
 
5
+ import {
6
+ bindSQLParameter,
7
+ validateSQLParameterOptions,
8
+ type SQLParameterOptions
9
+ } from './sql-parameters';
5
10
  import {SQLParseError} from './sql-parse-error';
6
- import type {Expression} from '../expressions/index';
11
+ import type {Expression} from '../expressions';
7
12
  import type {ColumnarScalarType} from '../expressions/scalar-type';
8
13
 
9
14
  export const SQL_RESERVED_WORDS = new Set([
@@ -61,8 +66,8 @@ export function sqlCastType(name: string, source = name, offset = 0): ColumnarSc
61
66
  }
62
67
 
63
68
  /** Parses bounded SQL expressions into the shared expression AST. */
64
- export function parseSQLExpression(source: string): Expression {
65
- const reader = new SQLExpressionReader(source);
69
+ export function parseSQLExpression(source: string, options: SQLParameterOptions = {}): Expression {
70
+ const reader = new SQLExpressionReader(source, 0, options);
66
71
  const expression = reader.expression();
67
72
  reader.space();
68
73
  if (reader.position !== source.length)
@@ -77,8 +82,11 @@ export class SQLExpressionReader {
77
82
  private readonly depths = new WeakMap<object, number>();
78
83
  constructor(
79
84
  readonly source: string,
80
- start = 0
85
+ start = 0,
86
+ /** Binding configuration shared by standalone and structural SQL parsers. */
87
+ protected readonly parameterOptions: SQLParameterOptions = {}
81
88
  ) {
89
+ validateSQLParameterOptions(parameterOptions);
82
90
  this.position = start;
83
91
  if (start === 0 && (typeof source !== 'string' || !source.trim() || source.length > 65536))
84
92
  throw new SQLParseError(
@@ -167,6 +175,27 @@ export class SQLExpressionReader {
167
175
  }
168
176
  if (minimum <= 3) {
169
177
  const negate = this.word('NOT');
178
+ if (this.word('IN')) {
179
+ this.symbol('(');
180
+ const values: Expression[] = [];
181
+ do {
182
+ const item = this.expression(depth + 1);
183
+ if (item.type === 'ArrayExpression') {
184
+ for (const element of item.elements) {
185
+ if (element.type !== 'Literal') throw this.error('SQL IN requires scalar values');
186
+ values.push(element as Expression);
187
+ }
188
+ } else values.push(item);
189
+ if (values.length > 4096) throw this.error('SQL IN exceeds 4096 values');
190
+ } while (this.take(','));
191
+ this.symbol(')');
192
+ left = values.length
193
+ ? this.call('in', [left, ...values])
194
+ : this.node({type: 'Literal', value: false, raw: 'false'});
195
+ if (negate)
196
+ left = this.node({type: 'UnaryExpression', operator: '!', argument: left}, [left]);
197
+ continue;
198
+ }
170
199
  const insensitive = this.word('ILIKE');
171
200
  if (insensitive || this.word('LIKE')) {
172
201
  const pattern = this.expression(depth + 1, 4);
@@ -244,7 +273,10 @@ export class SQLExpressionReader {
244
273
  this.position += width[0].length;
245
274
  this.token();
246
275
  this.symbol(']');
247
- const array = argument as any;
276
+ const array =
277
+ argument.type === 'ArrayExpression'
278
+ ? {type: 'CallExpression', callee: {name: 'array'}, arguments: argument.elements}
279
+ : (argument as any);
248
280
  if (
249
281
  array.type !== 'CallExpression' ||
250
282
  !/^array(?:_(?:float(?:16|32|64)|uint8))?$/.test(array.callee.name) ||
@@ -440,8 +472,19 @@ export class SQLExpressionReader {
440
472
  }
441
473
  return this.call(name, args, fold);
442
474
  }
475
+ /** Reads named value placeholders; quotes and identifiers use separate productions. */
443
476
  protected extendedPrimary(): Expression | undefined {
444
- return undefined;
477
+ if (this.source[this.position] !== ':') return undefined;
478
+ const start = this.position;
479
+ const parameter = /^:([A-Za-z_][A-Za-z0-9_]*)/.exec(this.source.slice(start));
480
+ if (!parameter) throw this.error('SQL named parameters require a name');
481
+ this.position += parameter[0].length;
482
+ this.token();
483
+ try {
484
+ return this.node(bindSQLParameter(parameter[1]!, this.parameterOptions));
485
+ } catch (error) {
486
+ throw this.error(error instanceof Error ? error.message : String(error), start);
487
+ }
445
488
  }
446
489
  /** Lowers field access to a portable expression call for Arrow binding. */
447
490
  protected memberAccess(object: Expression, name: string): Expression {
@@ -2,6 +2,9 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
4
4
 
5
+ import {isSQLParameterSnapshot} from './parameter-snapshots';
6
+ import type {SQLParameterOptions} from './sql-parameters';
7
+ import type {SQLPredicateParameterValues} from './sql-predicate-types';
5
8
  import {reportDiagnosticError, type DiagnosticLog} from '../diagnostic-log';
6
9
  import type {TableQuery} from './query-types';
7
10
  import {parseSQLExpression} from './parse-sql-expression';
@@ -13,10 +16,10 @@ import {parseSQLSelect, SQLQueryReader} from './parse-sql-select';
13
16
  /** Parses the supported SELECT subset or SHOW SOURCES, SHOW TABLES and DESCRIBE commands. */
14
17
  export function parseSQLQuery(
15
18
  sql: string,
16
- options?: {log?: DiagnosticLog}
19
+ options: SQLParameterOptions & {log?: DiagnosticLog} = {}
17
20
  ): TableQuery | SQLCatalogQuery {
18
21
  try {
19
- return parseQuery(sql);
22
+ return parseQuery(sql, options);
20
23
  } catch (error) {
21
24
  reportDiagnosticError(options?.log, error);
22
25
  throw error;
@@ -24,8 +27,8 @@ export function parseSQLQuery(
24
27
  }
25
28
 
26
29
  /** Parses a SQL request, including internal grammar fallbacks. */
27
- function parseQuery(sql: string): TableQuery | SQLCatalogQuery {
28
- const reader = new SQLQueryReader(sql);
30
+ function parseQuery(sql: string, options: SQLParameterOptions): TableQuery | SQLCatalogQuery {
31
+ const reader = new SQLQueryReader(sql, options);
29
32
  if (reader.takeKeyword('SHOW')) {
30
33
  let request: SQLCatalogQuery;
31
34
  if (reader.takeKeyword('SOURCES')) request = {command: 'showSources'};
@@ -45,13 +48,20 @@ function parseQuery(sql: string): TableQuery | SQLCatalogQuery {
45
48
  return {command: 'describe', from};
46
49
  }
47
50
  return parseSQLSelect(sql, {
51
+ ...options,
48
52
  parseWhere: expression => {
49
53
  try {
50
- const predicate = parseSQLPredicate(expression);
54
+ // The portable predicate fast path accepts scalars. Arrays and custom bindings
55
+ // are handled by the shared expression parser below.
56
+ if (options.parseParameter || isSQLParameterSnapshot(options.parameters))
57
+ throw new Error('Custom parameter AST requires expression parsing');
58
+ const predicate = parseSQLPredicate(expression, {
59
+ parameters: options.parameters as SQLPredicateParameterValues
60
+ });
51
61
  validateColumnReferences(predicate);
52
62
  return predicate;
53
63
  } catch {
54
- return {expression: parseSQLExpression(expression)};
64
+ return {expression: parseSQLExpression(expression, options)};
55
65
  }
56
66
  }
57
67
  });
@@ -63,6 +73,14 @@ function validateColumnReferences(predicate: BoundSQLPredicate): void {
63
73
  predicate.args.forEach(validateColumnReferences);
64
74
  } else if (predicate.op === 'not') validateColumnReferences(predicate.args[0]);
65
75
  else {
76
+ const values =
77
+ predicate.op === 'in'
78
+ ? predicate.args[1]
79
+ : predicate.op === 'isNull'
80
+ ? []
81
+ : [predicate.args[1]];
82
+ if (values.some(value => value instanceof Date || value instanceof Uint8Array))
83
+ throw new Error('Date and binary values require expression evaluation');
66
84
  const property = predicate.args[0] as SQLPredicateProperty;
67
85
  if (!property.quoted && property.property.includes('.'))
68
86
  throw new Error(
@@ -2,8 +2,9 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
4
4
 
5
+ import {validateSQLParameterOptions, type SQLParameterOptions} from './sql-parameters';
5
6
  import {SQLParseError} from './sql-parse-error';
6
- import type {Expression} from '../expressions/index';
7
+ import type {Expression} from '../expressions';
7
8
  import {SQLExpressionReader} from './parse-sql-expression';
8
9
  import type {ColumnarPredicate} from '../expressions/columnar-predicate';
9
10
  import type {TableQuery, TableSelection, TableReference} from './query-types';
@@ -12,14 +13,15 @@ const MAXIMUM_SQL_LENGTH = 65536;
12
13
  const MAXIMUM_SELECT_TOKENS = 4096;
13
14
 
14
15
  /** Optional WHERE support for the independently tree-shakeable structural parser. */
15
- export type SQLSelectParserOptions = Readonly<{
16
- /** Converts a WHERE expression to the portable predicate vocabulary. Omission rejects WHERE. */
17
- parseWhere?: (expression: string) => ColumnarPredicate | Readonly<{expression: Expression}>;
18
- }>;
16
+ export type SQLSelectParserOptions = SQLParameterOptions &
17
+ Readonly<{
18
+ /** Converts a WHERE expression to the portable predicate vocabulary. Omission rejects WHERE. */
19
+ parseWhere?: (expression: string) => ColumnarPredicate | Readonly<{expression: Expression}>;
20
+ }>;
19
21
 
20
22
  /** Parses SELECT columns/AS aliases, optional FROM, LIMIT and OFFSET; WHERE requires a supplied parser. */
21
23
  export function parseSQLSelect(sql: string, options: SQLSelectParserOptions = {}): TableQuery {
22
- const reader = new SQLQueryReader(sql);
24
+ const reader = new SQLQueryReader(sql, options);
23
25
  reader.expectKeyword('SELECT');
24
26
  let select: TableSelection[] | undefined;
25
27
  if (!reader.takeSymbol('*')) {
@@ -81,7 +83,11 @@ export class SQLQueryReader {
81
83
  private tokens = 0;
82
84
 
83
85
  /** Rejects empty or oversized statements before scanning. */
84
- constructor(sql: string) {
86
+ constructor(
87
+ sql: string,
88
+ protected readonly parameterOptions: SQLParameterOptions = {}
89
+ ) {
90
+ validateSQLParameterOptions(parameterOptions);
85
91
  if (typeof sql !== 'string' || !sql.trim() || sql.length > MAXIMUM_SQL_LENGTH)
86
92
  throw new SQLParseError(
87
93
  'SQL query requires a non-empty statement of at most 65536 characters.',
@@ -255,7 +261,7 @@ export class SQLQueryReader {
255
261
  }
256
262
 
257
263
  protected expressionReader(): SQLExpressionReader {
258
- return new SQLExpressionReader(this.sql, this.position);
264
+ return new SQLExpressionReader(this.sql, this.position, this.parameterOptions);
259
265
  }
260
266
 
261
267
  /** Bounds large projection lists independently of predicate parsing. */