@omgbase/oqx 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +424 -0
- package/package.json +31 -0
- package/src/adapters/indexed.ts +61 -0
- package/src/adapters/sqlite.ts +120 -0
- package/src/ast.ts +87 -0
- package/src/context.ts +79 -0
- package/src/engine.ts +406 -0
- package/src/errors.ts +15 -0
- package/src/index.ts +90 -0
- package/src/lexer.ts +175 -0
- package/src/parser.ts +517 -0
- package/src/plan.ts +58 -0
- package/src/planner.ts +47 -0
- package/src/semantics.ts +131 -0
package/README.md
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
# oqx
|
|
2
|
+
|
|
3
|
+
**Generic Object Query eXpression engine for JavaScript.**
|
|
4
|
+
|
|
5
|
+
OQX is a small query language for querying ordinary in-memory JavaScript objects
|
|
6
|
+
and collections — arrays of records, nested relations, recursive trees — with a
|
|
7
|
+
readable, declarative syntax. This package is the **generic collection kernel**:
|
|
8
|
+
the OQX language semantics separated from any particular data model, exposed as a
|
|
9
|
+
JavaScript tagged template.
|
|
10
|
+
|
|
11
|
+
```js
|
|
12
|
+
import { oqx } from "oqx";
|
|
13
|
+
|
|
14
|
+
const people = [
|
|
15
|
+
{ name: "Bob", id: 124, title: "Engineer",
|
|
16
|
+
jobs: [{ employer: "Globocorp", start_date: "1984/05/01", end_date: "1990/01/01" },
|
|
17
|
+
{ employer: "Globocorp", start_date: "2001/03/01" }] },
|
|
18
|
+
// …
|
|
19
|
+
];
|
|
20
|
+
const company = "Globocorp";
|
|
21
|
+
|
|
22
|
+
const employees = oqx`
|
|
23
|
+
name, id, title
|
|
24
|
+
from ${people}
|
|
25
|
+
where jobs exists { employer == ${company} && !end_date }
|
|
26
|
+
`;
|
|
27
|
+
// → [{ name: "Bob", id: 124, title: "Engineer" }, …] (current Globocorp employees)
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Why a tagged template
|
|
31
|
+
|
|
32
|
+
Interpolations cross the host/OQX boundary as **typed value bindings, never as
|
|
33
|
+
source text** — prepared-statement semantics. A `${…}` in `from` position is the
|
|
34
|
+
collection being queried; a `${…}` in a predicate is an ordinary host value.
|
|
35
|
+
Because values are never spliced into the query text, they cannot alter the
|
|
36
|
+
grammar and there is no injection surface. The compiled query is cached by the
|
|
37
|
+
template's identity and re-runs with fresh bindings each call.
|
|
38
|
+
|
|
39
|
+
## Language tutorial
|
|
40
|
+
|
|
41
|
+
A query has, in spirit, the shape below — but at the top level the clauses are
|
|
42
|
+
**order-flexible**, so you can lead with the projection (SQL-style) or with
|
|
43
|
+
`from`, whichever reads better:
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
[ [select] projection ] name, id, title: label
|
|
47
|
+
from <collection> from ${people}
|
|
48
|
+
[ where <predicate> ] where age >= 18 && jobs exists { !end }
|
|
49
|
+
[ order by <expr> … ] order by age desc, name
|
|
50
|
+
[ follow <relation> … ] follow children { depth 4 }
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The examples below all use this dataset:
|
|
54
|
+
|
|
55
|
+
```js
|
|
56
|
+
const people = [
|
|
57
|
+
{ name: "Bob", id: 124, title: "Engineer", active: true, age: 41, city: "NYC",
|
|
58
|
+
jobs: [{ employer: "Globocorp", start: "1984", end: "1990" },
|
|
59
|
+
{ employer: "Globocorp", start: "2001" }] },
|
|
60
|
+
{ name: "Alice", id: 7, title: "Director", active: true, age: 52, city: "SF",
|
|
61
|
+
jobs: [{ employer: "Initech", start: "1999", end: "2005" },
|
|
62
|
+
{ employer: "Globocorp", start: "2010", end: "2015" }] },
|
|
63
|
+
{ name: "Carol", id: 55, title: "Analyst", active: false, age: 29, city: "NYC",
|
|
64
|
+
jobs: [{ employer: "Globocorp", start: "2020" }] },
|
|
65
|
+
];
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### 1. Source: `from`
|
|
69
|
+
|
|
70
|
+
Every query reads from a source collection. In the tagged template the source is
|
|
71
|
+
normally an interpolated value; it may also be a named root or a navigation (see
|
|
72
|
+
[data context](#data-context-string-queries-and-named-roots)).
|
|
73
|
+
|
|
74
|
+
```js
|
|
75
|
+
oqx`name from ${people}`;
|
|
76
|
+
// [{ name: "Bob" }, { name: "Alice" }, { name: "Carol" }]
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### 2. Projection (select)
|
|
80
|
+
|
|
81
|
+
List the fields to keep. With **no** projection you get the raw rows unchanged.
|
|
82
|
+
|
|
83
|
+
```js
|
|
84
|
+
oqx`name, id from ${people}`;
|
|
85
|
+
// [{ name: "Bob", id: 124 }, { name: "Alice", id: 7 }, { name: "Carol", id: 55 }]
|
|
86
|
+
|
|
87
|
+
oqx`from ${people} where active`; // no projection → whole objects
|
|
88
|
+
// [ <Bob>, <Alice> ]
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
A projection item can be:
|
|
92
|
+
|
|
93
|
+
- a **bare field** — `name`;
|
|
94
|
+
- a **dotted navigation**, keyed by its last segment — `meta.slug` produces
|
|
95
|
+
`{ slug: … }`;
|
|
96
|
+
- an **alias / value expression** — `label: name`, `decade: age / 10`;
|
|
97
|
+
- a **nested collection** — `current: jobs collect { … }` (see §6).
|
|
98
|
+
|
|
99
|
+
```js
|
|
100
|
+
oqx`label: name, decade: age / 10 from ${people} where name == "Bob"`;
|
|
101
|
+
// [{ label: "Bob", decade: 4.1 }]
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
The `select` keyword is optional and works in any position — `select name from …`
|
|
105
|
+
is identical to `name from …`. (It's the hook for a future `select distinct`.)
|
|
106
|
+
|
|
107
|
+
### 3. Predicates (where)
|
|
108
|
+
|
|
109
|
+
`where` filters rows. The predicate language has comparisons (`== != < <= > >=`),
|
|
110
|
+
boolean operators (`&& || !`) with grouping `( )`, membership (`in`), arithmetic
|
|
111
|
+
(`+ - * / %`), and bare truthiness. The `where` keyword is optional when the
|
|
112
|
+
leading expression is clearly a predicate.
|
|
113
|
+
|
|
114
|
+
```js
|
|
115
|
+
const min = 40;
|
|
116
|
+
oqx`name from ${people} where age >= ${min}`; // → Bob, Alice
|
|
117
|
+
oqx`name from ${people} where city in ${["SF", "LA"]}`; // → Alice
|
|
118
|
+
oqx`name from ${people} where !active`; // → Carol
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Equality is **typed and strict** (`5 == "5"` is false); a comparison against an
|
|
122
|
+
absent (`null`/`undefined`) field is simply false rather than an error.
|
|
123
|
+
|
|
124
|
+
**Interpolations are always values, never syntax.** `where name == ${x}` compares
|
|
125
|
+
against the value of `x`; a string in `x` can't inject operators or identifiers.
|
|
126
|
+
|
|
127
|
+
**Bindings & scoping.** A bare identifier resolves against the current row, and if
|
|
128
|
+
absent it **climbs to enclosing rows** — lexical outer references, for free:
|
|
129
|
+
|
|
130
|
+
```js
|
|
131
|
+
const accounts = [
|
|
132
|
+
{ owner: "x", budget: 100, orders: [{ amount: 50 }, { amount: 150 }] },
|
|
133
|
+
{ owner: "y", budget: 200, orders: [{ amount: 250 }] },
|
|
134
|
+
];
|
|
135
|
+
oqx`owner from ${accounts} where orders exists { amount > budget }`;
|
|
136
|
+
// [{ owner: "x" }, { owner: "y" }] — `budget` climbs from the order to the account
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
A `.member` access does **not** climb — it always navigates the value on its left.
|
|
140
|
+
|
|
141
|
+
**Explicit outer references (`^`).** Implicit climbing only reaches an outer field
|
|
142
|
+
when the inner row *doesn't* have that name. When both scopes share a name, the
|
|
143
|
+
inner one shadows the outer, and you reach past it with `^name` ("one scope out",
|
|
144
|
+
`^^name` for two). This is what makes correlated subqueries work — e.g. each
|
|
145
|
+
person's siblings, where both the person and the candidates have a `parent`:
|
|
146
|
+
|
|
147
|
+
```js
|
|
148
|
+
const family = [
|
|
149
|
+
{ name: "Ada", parent: "Pat" },
|
|
150
|
+
{ name: "Ben", parent: "Pat" },
|
|
151
|
+
{ name: "Cy", parent: "Sam" },
|
|
152
|
+
];
|
|
153
|
+
oqx`
|
|
154
|
+
name,
|
|
155
|
+
siblings: ${family} collect { name where parent == ^parent && name != ^name }
|
|
156
|
+
from ${family}
|
|
157
|
+
`;
|
|
158
|
+
// [{ name: "Ada", siblings: [{ name: "Ben" }] },
|
|
159
|
+
// { name: "Ben", siblings: [{ name: "Ada" }] },
|
|
160
|
+
// { name: "Cy", siblings: [] }]
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
Here `parent` is the inner candidate's parent while `^parent` is the outer
|
|
164
|
+
person's. (`^` in an expression *reads* one scope out; the same `^` as a
|
|
165
|
+
select-item prefix — `^name: …` in §7 — *binds* one scope out. Both mean "one
|
|
166
|
+
scope out.")
|
|
167
|
+
|
|
168
|
+
### 4. Built-in functions
|
|
169
|
+
|
|
170
|
+
Methods on a value: `contains`, `startsWith`, `endsWith`, `matches` (regex),
|
|
171
|
+
`size`, `lower`, `upper`. Free functions: `list(x)` (coerce to an array), `size(x)`,
|
|
172
|
+
`has(x)`.
|
|
173
|
+
|
|
174
|
+
```js
|
|
175
|
+
oqx`name from ${people} where title.startsWith("Eng")`; // → Bob
|
|
176
|
+
oqx`name from ${people} where title.lower() == "director"`; // → Alice
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
### 5. Consumers
|
|
180
|
+
|
|
181
|
+
A consumer shapes a result set. There are five:
|
|
182
|
+
|
|
183
|
+
| Consumer | Returns |
|
|
184
|
+
| --------- | ------------------------------ |
|
|
185
|
+
| `collect` | an array (the default) |
|
|
186
|
+
| `exists` | a boolean |
|
|
187
|
+
| `count` | a number |
|
|
188
|
+
| `first` | one record, or `null` |
|
|
189
|
+
| `single` | one record, or `null`; throws if more than one matches |
|
|
190
|
+
|
|
191
|
+
The bare `from … ` form is always `collect`. To reduce the **whole** query with a
|
|
192
|
+
different consumer, use the directive form `<source> <consumer> { <body> }` — note
|
|
193
|
+
this is *not* SQL: `count from people` would project a field called `count`, whereas
|
|
194
|
+
a real reduction is a directive:
|
|
195
|
+
|
|
196
|
+
```js
|
|
197
|
+
oqx`${people} exists { where active }`; // true
|
|
198
|
+
oqx`${people} count { where active }`; // 2
|
|
199
|
+
oqx`${people} first { name where age > 50 }`; // { name: "Alice" }
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
(Inside a consumer block a bare identifier **projects** — `count { active }` selects
|
|
203
|
+
a field named `active`; write `count { where active }` to filter.)
|
|
204
|
+
|
|
205
|
+
### 6. Nested collections and relations
|
|
206
|
+
|
|
207
|
+
The same consumers work as **postfix directives** over a relation of the current
|
|
208
|
+
row — `<relation> <consumer> { <body> }` — both in `where` and in a projection.
|
|
209
|
+
|
|
210
|
+
In `where`, an `exists { … }` tests non-emptiness and `count { … } <op> N` compares
|
|
211
|
+
cardinality:
|
|
212
|
+
|
|
213
|
+
```js
|
|
214
|
+
oqx`name from ${people} where jobs exists { !end }`; // has a current job → Bob, Carol
|
|
215
|
+
oqx`name from ${people} where jobs count {} >= 2`; // ≥2 jobs → Bob, Alice
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
In a projection, `collect` yields a nested array; `first` / `single` yield one
|
|
219
|
+
nested record:
|
|
220
|
+
|
|
221
|
+
```js
|
|
222
|
+
oqx`name, current: jobs collect { employer where !end } from ${people} where name == "Bob"`;
|
|
223
|
+
// [{ name: "Bob", current: [{ employer: "Globocorp" }] }]
|
|
224
|
+
|
|
225
|
+
oqx`name, firstJob: jobs first { employer } from ${people} where name == "Alice"`;
|
|
226
|
+
// [{ name: "Alice", firstJob: { employer: "Initech" } }]
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
A relation is just an expression evaluated on the row and coerced to a collection,
|
|
230
|
+
so nested blocks compose to any depth and can navigate dotted paths
|
|
231
|
+
(`author.books collect { … }`).
|
|
232
|
+
|
|
233
|
+
### 7. Lifts (`^`)
|
|
234
|
+
|
|
235
|
+
Sometimes you want to filter by a nested collection *and* keep a value from it.
|
|
236
|
+
A `^name:` item inside a `collect { … }` that sits directly in the top-level
|
|
237
|
+
`where` does both: it filters (non-empty) and binds `name` into the outer
|
|
238
|
+
projection as a per-row array.
|
|
239
|
+
|
|
240
|
+
```js
|
|
241
|
+
oqx`
|
|
242
|
+
name, currentEmployers
|
|
243
|
+
from ${people}
|
|
244
|
+
where jobs collect { ^currentEmployers: employer where !end }
|
|
245
|
+
`;
|
|
246
|
+
// [{ name: "Bob", currentEmployers: ["Globocorp"] },
|
|
247
|
+
// { name: "Carol", currentEmployers: ["Globocorp"] }]
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
**Multi-level lifts (`^^`, `^^^`).** The caret count is how many scopes the value
|
|
251
|
+
binds *out* — `^` to the immediate enclosing projection, `^^` two out, and so on
|
|
252
|
+
(the mirror image of the `^`-read in §3). When a deeper lift fires repeatedly as
|
|
253
|
+
an intermediate collection fans out, its values **flatten-append** into one flat
|
|
254
|
+
list at the target scope — "every matching value from the subtree, N scopes out":
|
|
255
|
+
|
|
256
|
+
```js
|
|
257
|
+
const departments = [
|
|
258
|
+
{ name: "Eng", teams: [{ id: "t1", members: [{ name: "Ada" }, { name: "Ben" }] },
|
|
259
|
+
{ id: "t2", members: [{ name: "Cy" }] }] },
|
|
260
|
+
{ name: "Sales", teams: [{ id: "t3", members: [{ name: "Dee" }] }] },
|
|
261
|
+
];
|
|
262
|
+
oqx`
|
|
263
|
+
name, teamIds, allMembers
|
|
264
|
+
from ${departments}
|
|
265
|
+
where teams collect { ^teamIds: id where members collect { ^^allMembers: name } }
|
|
266
|
+
`;
|
|
267
|
+
// [{ name: "Eng", teamIds: ["t1", "t2"], allMembers: ["Ada", "Ben", "Cy"] },
|
|
268
|
+
// { name: "Sales", teamIds: ["t3"], allMembers: ["Dee"] }]
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
`^teamIds` (one out) and `^^allMembers` (two out) bind to the same department row
|
|
272
|
+
at once. Because accumulation happens as each intermediate collection is
|
|
273
|
+
iterated, the intermediate scopes must be `collect`/`count` bodies (which iterate
|
|
274
|
+
fully), not a short-circuiting `exists`.
|
|
275
|
+
|
|
276
|
+
### 8. Ordering
|
|
277
|
+
|
|
278
|
+
`order by <expr> [asc|desc]`, comma-separated for tie-breaks. Absent values sort
|
|
279
|
+
last.
|
|
280
|
+
|
|
281
|
+
```js
|
|
282
|
+
oqx`name from ${people} where city == "NYC" order by age desc`;
|
|
283
|
+
// [{ name: "Bob" }, { name: "Carol" }]
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
### 9. Recursion: `follow`
|
|
287
|
+
|
|
288
|
+
`follow <relation>` turns a query into a bounded recursive traversal: the `where`
|
|
289
|
+
selects the seed rows, and `follow` walks a relation from each reached row. It's
|
|
290
|
+
fully duck-typed — the relation is any expression yielding successors; a row that
|
|
291
|
+
lacks it is simply a leaf.
|
|
292
|
+
|
|
293
|
+
```js
|
|
294
|
+
const tree = [{ id: "root", children: [
|
|
295
|
+
{ id: "a", children: [{ id: "a1", children: [] }] },
|
|
296
|
+
{ id: "b", children: [] },
|
|
297
|
+
]}];
|
|
298
|
+
|
|
299
|
+
oqx`id, depth: $depth from ${tree} follow children order by $depth, id`;
|
|
300
|
+
// [{ id: "root", depth: 1 }, { id: "a", depth: 2 }, { id: "b", depth: 2 }, { id: "a1", depth: 3 }]
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
Reached rows expose recursion **intrinsics** in `select` / `order by`:
|
|
304
|
+
`$depth` (1-based), `$leaf` (no successors), `$frontier` (hit a boundary), and
|
|
305
|
+
`$stop` (`"continue"` / `"leaf"` / `"depth"` / `"frontier"`). Options go in a
|
|
306
|
+
trailing block:
|
|
307
|
+
|
|
308
|
+
```js
|
|
309
|
+
oqx`id, stop: $stop from ${tree} follow children { depth 2 } order by id`;
|
|
310
|
+
// a1 is never reached; a and b report stop:"depth"
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
The block accepts: `where <succ>` (which successors keep participating),
|
|
314
|
+
`frontier <pred>` (cut a relation that could continue), `depth <n>` (1–8), and
|
|
315
|
+
`by <expr>` (the identity used for cycle/duplicate detection — default `.id` or the
|
|
316
|
+
object reference). Reached rows are de-duplicated by that identity, so cycles
|
|
317
|
+
terminate. Give `follow` a stable identity (`.id` or `by`) when your relation
|
|
318
|
+
returns fresh objects rather than shared references.
|
|
319
|
+
|
|
320
|
+
### Cheat-sheet
|
|
321
|
+
|
|
322
|
+
```
|
|
323
|
+
name, alias: expr, nested: rel collect { … } projection (select optional)
|
|
324
|
+
from ${source} source collection
|
|
325
|
+
where a == b && rel exists { … } || !c predicate tree + nested ops
|
|
326
|
+
^name / ^^name read an outer row's field (N scopes out)
|
|
327
|
+
^name: expr / ^^name: expr lift/export a value N scopes out (flatten-append)
|
|
328
|
+
order by expr desc, expr2 ordering
|
|
329
|
+
follow rel { where … frontier … depth n by … } recursion ($depth/$stop/$leaf/$frontier)
|
|
330
|
+
${source} <collect|exists|count|first|single> { … } whole-query consumer
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
## Data context: string queries and named roots
|
|
334
|
+
|
|
335
|
+
When you don't need interpolation, `execute` runs a plain string query against a
|
|
336
|
+
**data context** of named roots:
|
|
337
|
+
|
|
338
|
+
```js
|
|
339
|
+
import { execute } from "oqx";
|
|
340
|
+
|
|
341
|
+
execute("name from people where age >= 18", { people });
|
|
342
|
+
// `from people` resolves the `people` root
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
`parse(source)` returns a reusable AST and `run(query, { values, roots })` returns
|
|
346
|
+
the full discriminated result (`{ consumer, … }`).
|
|
347
|
+
|
|
348
|
+
## Architecture: adapting to other storage & query systems
|
|
349
|
+
|
|
350
|
+
OQX is layered so it can be the front-end for query systems far beyond in-memory
|
|
351
|
+
objects. The parsed `Query` AST is the host-agnostic IR; execution is pluggable.
|
|
352
|
+
|
|
353
|
+
```
|
|
354
|
+
Query AST ─┬─ InMemoryEngine(DataContext) tier 1/2 — drive any data model
|
|
355
|
+
└─ PlannedEngine(QueryPlanner) tier 3 — push work into a store
|
|
356
|
+
└─ finishes the residual on the in-memory engine
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
**Everything obeys one scalar-semantics contract** (`oqx.semantics`): typed/strict
|
|
360
|
+
equality (`5 == "5"` is false), absent operands make ordering comparisons false,
|
|
361
|
+
CEL-style `in`, absent-last sort order. Any backend that can't reproduce a rule
|
|
362
|
+
in its native language must leave that fragment as an in-memory *residual* rather
|
|
363
|
+
than approximate it. The conformance suite verifies this.
|
|
364
|
+
|
|
365
|
+
### Tier 2 — a custom `DataContext` (bind any data model)
|
|
366
|
+
|
|
367
|
+
The engine never touches host objects directly; it asks a `DataContext` to
|
|
368
|
+
resolve named roots, read properties/relations, coerce results to rows, and
|
|
369
|
+
compute identity. Implement it to query an ORM graph, a remote API, or lazily
|
|
370
|
+
loaded relations — the query *semantics* stay in OQX:
|
|
371
|
+
|
|
372
|
+
```js
|
|
373
|
+
import { parse, run } from "oqx";
|
|
374
|
+
|
|
375
|
+
const graph = {
|
|
376
|
+
root: (name) => name === "tree" ? [nodes.get(1)] : undefined,
|
|
377
|
+
get: (row, key) => key === "children" ? row.childIds.map(id => nodes.get(id)) : row[key],
|
|
378
|
+
has: (row, key) => key === "children" || key in row, // declare computed relations
|
|
379
|
+
toRows: (v) => v == null ? [] : Array.isArray(v) ? v : [v],
|
|
380
|
+
identity: (row) => row.id, // for follow dedup
|
|
381
|
+
};
|
|
382
|
+
run(parse("id, depth: $depth from tree follow children"), { context: graph });
|
|
383
|
+
```
|
|
384
|
+
|
|
385
|
+
### Tier 3 — a `QueryPlanner` (pushdown + planning)
|
|
386
|
+
|
|
387
|
+
A planner translates as much of a query as it can into its store's native query
|
|
388
|
+
and returns the produced rows plus a **residual** `Query` for the rest. The
|
|
389
|
+
in-memory engine finishes the residual, so a planner can be as partial as it
|
|
390
|
+
likes and stay correct. Two adapters ship:
|
|
391
|
+
|
|
392
|
+
- `IndexedCollection` — hash-indexes a collection and answers equality predicates
|
|
393
|
+
from the index instead of scanning, leaving other predicates as residual.
|
|
394
|
+
- `oqx/sqlite` — real pushdown to a `node:sqlite` database: the flat query core
|
|
395
|
+
(scan + translatable conjunctive predicates, `LIMIT` for unordered
|
|
396
|
+
`first`/`single`) becomes SQL; `matches()`, nested consumer ops, `follow`, etc.
|
|
397
|
+
fall back to the in-memory residual.
|
|
398
|
+
|
|
399
|
+
```js
|
|
400
|
+
import { parse, PlannedEngine } from "oqx";
|
|
401
|
+
import { SqliteTable } from "oqx/sqlite";
|
|
402
|
+
|
|
403
|
+
const planner = new SqliteTable(db, "emp", { columns: ["id", "name", "dept", "level"] });
|
|
404
|
+
new PlannedEngine(planner).run(parse('name from emp where dept == "eng" && level >= 5'), []);
|
|
405
|
+
// → `dept`/`level` pushed to SQL; anything untranslatable finishes in-memory
|
|
406
|
+
```
|
|
407
|
+
|
|
408
|
+
This is the seam an omgbase adapter uses: its existing OQX→SQL compiler (docs/
|
|
409
|
+
blocks/nodes, the relations table, `$` intrinsics, `WITH RECURSIVE` for `follow`)
|
|
410
|
+
becomes a `QueryPlanner`, while oqx-js contributes the parser, IR, semantics
|
|
411
|
+
contract, and residual executor.
|
|
412
|
+
|
|
413
|
+
## Requirements
|
|
414
|
+
|
|
415
|
+
Node 22.6+ (the sources are TypeScript, run natively via type-stripping — the
|
|
416
|
+
package has **no runtime dependencies**). `npm test` runs the suite; `npm run
|
|
417
|
+
typecheck` typechecks.
|
|
418
|
+
|
|
419
|
+
## Relationship to omgbase
|
|
420
|
+
|
|
421
|
+
This is tier 1 (the in-memory object/collection interpreter) of the OQX
|
|
422
|
+
implementation tiers. The language kernel here is host-agnostic; richer hosts
|
|
423
|
+
(e.g. omgbase's docs/blocks/nodes with index pushdown) layer data-model
|
|
424
|
+
vocabulary and execution capabilities on top of the same surface syntax.
|
package/package.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@omgbase/oqx",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Generic Object Query eXpression engine for JavaScript",
|
|
5
|
+
"license": "ISC",
|
|
6
|
+
"author": "",
|
|
7
|
+
"type": "module",
|
|
8
|
+
"main": "./src/index.ts",
|
|
9
|
+
"types": "./src/index.ts",
|
|
10
|
+
"publishConfig": {
|
|
11
|
+
"access": "public"
|
|
12
|
+
},
|
|
13
|
+
"exports": {
|
|
14
|
+
".": "./src/index.ts",
|
|
15
|
+
"./sqlite": "./src/adapters/sqlite.ts"
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"src"
|
|
19
|
+
],
|
|
20
|
+
"engines": {
|
|
21
|
+
"node": ">=22.6.0"
|
|
22
|
+
},
|
|
23
|
+
"scripts": {
|
|
24
|
+
"test": "node --test test/*.test.ts",
|
|
25
|
+
"typecheck": "tsc --noEmit"
|
|
26
|
+
},
|
|
27
|
+
"devDependencies": {
|
|
28
|
+
"@types/node": "^22.20.2",
|
|
29
|
+
"typescript": "^5.7.0"
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// An in-memory optimizing planner: it hash-indexes a named root collection on
|
|
2
|
+
// chosen fields and, for a query whose where-clause contains equality predicates
|
|
3
|
+
// on those fields, answers from the index (candidate intersection) instead of a
|
|
4
|
+
// full scan. Everything it can't turn into an index probe is left as a residual
|
|
5
|
+
// query the in-memory engine finishes over the candidate rows.
|
|
6
|
+
//
|
|
7
|
+
// This is the smallest honest demonstration of plan optimization: same answers
|
|
8
|
+
// as a naive scan, far fewer rows examined, and partial-pushdown correctness via
|
|
9
|
+
// the residual.
|
|
10
|
+
|
|
11
|
+
import type { Query } from "../ast.ts";
|
|
12
|
+
import type { Plan, QueryPlanner } from "../planner.ts";
|
|
13
|
+
import { partitionPushable, residualQuery, asEquality, constValue } from "../plan.ts";
|
|
14
|
+
|
|
15
|
+
export class IndexedCollection implements QueryPlanner {
|
|
16
|
+
private name: string;
|
|
17
|
+
private rows: readonly unknown[];
|
|
18
|
+
private indexes = new Map<string, Map<unknown, unknown[]>>();
|
|
19
|
+
|
|
20
|
+
/** Index `rows` (exposed as root `name`) on each field in `indexFields`. */
|
|
21
|
+
constructor(name: string, rows: readonly unknown[], indexFields: readonly string[]) {
|
|
22
|
+
this.name = name;
|
|
23
|
+
this.rows = rows;
|
|
24
|
+
for (const field of indexFields) {
|
|
25
|
+
const idx = new Map<unknown, unknown[]>();
|
|
26
|
+
for (const row of rows) {
|
|
27
|
+
const key = (row as Record<string, unknown>)?.[field];
|
|
28
|
+
let bucket = idx.get(key);
|
|
29
|
+
if (!bucket) idx.set(key, (bucket = []));
|
|
30
|
+
bucket.push(row);
|
|
31
|
+
}
|
|
32
|
+
this.indexes.set(field, idx);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
plan(query: Query, params: readonly unknown[]): Plan | null {
|
|
37
|
+
if (query.source.kind !== "ident" || query.source.name !== this.name) return null;
|
|
38
|
+
if (query.from.length > 0 || query.follow) return null;
|
|
39
|
+
|
|
40
|
+
const { pushed, residual } = partitionPushable(query.where, (e) => {
|
|
41
|
+
const eq = asEquality(e);
|
|
42
|
+
return eq != null && this.indexes.has(eq.field);
|
|
43
|
+
});
|
|
44
|
+
if (pushed.length === 0) return null; // no index probe available — let the scan handle it
|
|
45
|
+
|
|
46
|
+
// Intersect the candidate sets from each indexed equality (smallest first).
|
|
47
|
+
let candidate: unknown[] | null = null;
|
|
48
|
+
for (const e of pushed) {
|
|
49
|
+
const eq = asEquality(e)!;
|
|
50
|
+
const bucket = this.indexes.get(eq.field)!.get(constValue(eq.value, params)) ?? [];
|
|
51
|
+
candidate = candidate === null ? bucket.slice() : intersect(candidate, bucket);
|
|
52
|
+
}
|
|
53
|
+
const rows = candidate ?? [];
|
|
54
|
+
return { rows: () => rows, residual: residualQuery(query, residual) };
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function intersect(a: unknown[], b: unknown[]): unknown[] {
|
|
59
|
+
const set = new Set(b);
|
|
60
|
+
return a.filter((x) => set.has(x));
|
|
61
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
// A real storage adapter: pushes the flat core of an OQX query (source scan +
|
|
2
|
+
// translatable conjunctive predicates, and a LIMIT for unordered first/single)
|
|
3
|
+
// into SQL over Node's built-in `node:sqlite`, and leaves everything it can't
|
|
4
|
+
// translate — nested consumer ops, `follow`, negation/disjunction, method calls
|
|
5
|
+
// like matches(), custom functions — as a residual the in-memory engine finishes
|
|
6
|
+
// over the rows SQL returned.
|
|
7
|
+
//
|
|
8
|
+
// This is imported on its own subpath (`oqx/sqlite`) so that plain consumers of
|
|
9
|
+
// `oqx` never pull in the experimental `node:sqlite` module.
|
|
10
|
+
//
|
|
11
|
+
// Semantics note: OQX equality is typed and strict (see semantics.ts). SQL `=`
|
|
12
|
+
// uses column affinity, so this adapter assumes a well-typed schema for the
|
|
13
|
+
// columns it pushes; anything it cannot translate faithfully stays residual.
|
|
14
|
+
|
|
15
|
+
import { DatabaseSync } from "node:sqlite";
|
|
16
|
+
import type { Query, Expr } from "../ast.ts";
|
|
17
|
+
import type { Plan, QueryPlanner } from "../planner.ts";
|
|
18
|
+
import { partitionPushable, residualQuery } from "../plan.ts";
|
|
19
|
+
|
|
20
|
+
export interface SqliteTableOptions {
|
|
21
|
+
/** Columns that map to bare OQX fields (only these are pushable). */
|
|
22
|
+
columns: readonly string[];
|
|
23
|
+
/** Columns whose stored text should be JSON.parse'd back into row values. */
|
|
24
|
+
jsonColumns?: readonly string[];
|
|
25
|
+
/** Custom mapper from a raw SQL row to a query row (overrides jsonColumns). */
|
|
26
|
+
map?: (raw: Record<string, unknown>) => unknown;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const RELOP_SQL: Record<string, string> = {
|
|
30
|
+
"==": "=", "!=": "<>", "<": "<", "<=": "<=", ">": ">", ">=": ">=",
|
|
31
|
+
};
|
|
32
|
+
const ARITH_SQL = new Set(["+", "-", "*", "/", "%"]);
|
|
33
|
+
|
|
34
|
+
export class SqliteTable implements QueryPlanner {
|
|
35
|
+
private db: DatabaseSync;
|
|
36
|
+
private table: string;
|
|
37
|
+
private columns: Set<string>;
|
|
38
|
+
private opts: SqliteTableOptions;
|
|
39
|
+
|
|
40
|
+
constructor(db: DatabaseSync, table: string, opts: SqliteTableOptions) {
|
|
41
|
+
this.db = db;
|
|
42
|
+
this.table = table;
|
|
43
|
+
this.columns = new Set(opts.columns);
|
|
44
|
+
this.opts = opts;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
plan(query: Query, params: readonly unknown[]): Plan | null {
|
|
48
|
+
if (query.source.kind !== "ident" || query.source.name !== this.table) return null;
|
|
49
|
+
if (query.from.length > 0 || query.follow) return null;
|
|
50
|
+
|
|
51
|
+
const { pushed, residual } = partitionPushable(query.where, (e) => this.translatable(e));
|
|
52
|
+
const sqlParams: unknown[] = [];
|
|
53
|
+
const whereSql = pushed.map((e) => this.translate(e, params, sqlParams)).join(" AND ");
|
|
54
|
+
|
|
55
|
+
// A LIMIT is only safe when nothing is left to filter in-memory and the
|
|
56
|
+
// result is unordered (first/single are "some row" without an order by).
|
|
57
|
+
let tail = "";
|
|
58
|
+
if (!residual && !query.orderBy) {
|
|
59
|
+
if (query.consumer === "first") tail = " LIMIT 1";
|
|
60
|
+
else if (query.consumer === "single") tail = " LIMIT 2";
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const sql = `SELECT * FROM "${this.table}"${whereSql ? ` WHERE ${whereSql}` : ""}${tail}`;
|
|
64
|
+
const raw = this.db.prepare(sql).all(...sqlParams.map(toSqlParam)) as Record<string, unknown>[];
|
|
65
|
+
const rows = raw.map((r) => this.mapRow(r));
|
|
66
|
+
return { rows: () => rows, residual: residualQuery(query, residual) };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
private translatable(e: Expr): boolean {
|
|
70
|
+
switch (e.kind) {
|
|
71
|
+
case "lit": case "binding": return true;
|
|
72
|
+
case "ident": return this.columns.has(e.name);
|
|
73
|
+
case "unary": return (e.op === "!" || e.op === "-") && this.translatable(e.expr);
|
|
74
|
+
case "logical": return this.translatable(e.left) && this.translatable(e.right);
|
|
75
|
+
case "binary":
|
|
76
|
+
return (e.op in RELOP_SQL || ARITH_SQL.has(e.op)) && this.translatable(e.left) && this.translatable(e.right);
|
|
77
|
+
default: return false; // member/index/call/in → residual
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
private translate(e: Expr, params: readonly unknown[], out: unknown[]): string {
|
|
82
|
+
switch (e.kind) {
|
|
83
|
+
case "lit": out.push(e.value); return "?";
|
|
84
|
+
case "binding": out.push(params[e.index]); return "?";
|
|
85
|
+
case "ident": return `"${e.name}"`;
|
|
86
|
+
case "unary": return e.op === "!" ? `(NOT ${this.translate(e.expr, params, out)})` : `(-${this.translate(e.expr, params, out)})`;
|
|
87
|
+
case "logical": {
|
|
88
|
+
const op = e.op === "&&" ? "AND" : "OR";
|
|
89
|
+
return `(${this.translate(e.left, params, out)} ${op} ${this.translate(e.right, params, out)})`;
|
|
90
|
+
}
|
|
91
|
+
case "binary": {
|
|
92
|
+
const op = e.op in RELOP_SQL ? RELOP_SQL[e.op]! : e.op;
|
|
93
|
+
return `(${this.translate(e.left, params, out)} ${op} ${this.translate(e.right, params, out)})`;
|
|
94
|
+
}
|
|
95
|
+
default: throw new Error(`sqlite: not translatable: ${e.kind}`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
private mapRow(raw: Record<string, unknown>): unknown {
|
|
100
|
+
if (this.opts.map) return this.opts.map(raw);
|
|
101
|
+
if (this.opts.jsonColumns) {
|
|
102
|
+
const out: Record<string, unknown> = { ...raw };
|
|
103
|
+
for (const c of this.opts.jsonColumns) {
|
|
104
|
+
if (typeof out[c] === "string") out[c] = JSON.parse(out[c] as string);
|
|
105
|
+
}
|
|
106
|
+
return out;
|
|
107
|
+
}
|
|
108
|
+
return raw;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// node:sqlite accepts null | number | bigint | string | Uint8Array. Map the
|
|
113
|
+
// common host values that appear as bound params.
|
|
114
|
+
function toSqlParam(v: unknown): null | number | bigint | string | Uint8Array {
|
|
115
|
+
if (v === undefined || v === null) return null;
|
|
116
|
+
if (typeof v === "boolean") return v ? 1 : 0;
|
|
117
|
+
if (typeof v === "number" || typeof v === "bigint" || typeof v === "string") return v;
|
|
118
|
+
if (v instanceof Uint8Array) return v;
|
|
119
|
+
return String(v);
|
|
120
|
+
}
|