@osqd/jql 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +220 -0
- package/LICENSE +102 -0
- package/README.md +84 -0
- package/bin/jql.mjs +5 -0
- package/conformance/cases.json +290 -0
- package/dist/array.d.ts +11 -0
- package/dist/async.d.ts +44 -0
- package/dist/canonical.d.ts +18 -0
- package/dist/cjs/array.d.ts +11 -0
- package/dist/cjs/async.d.ts +44 -0
- package/dist/cjs/canonical.d.ts +18 -0
- package/dist/cjs/cli.d.ts +15 -0
- package/dist/cjs/collections.d.ts +34 -0
- package/dist/cjs/core.d.ts +117 -0
- package/dist/cjs/errors.d.ts +17 -0
- package/dist/cjs/explain.d.ts +30 -0
- package/dist/cjs/global.d.ts +90 -0
- package/dist/cjs/group.d.ts +47 -0
- package/dist/cjs/index.d.ts +25 -0
- package/dist/cjs/internal/closest.d.ts +9 -0
- package/dist/cjs/internal/duration.d.ts +15 -0
- package/dist/cjs/internal/equal.d.ts +34 -0
- package/dist/cjs/internal/glob.d.ts +30 -0
- package/dist/cjs/internal/order.d.ts +24 -0
- package/dist/cjs/internal/path.d.ts +109 -0
- package/dist/cjs/internal/record.d.ts +18 -0
- package/dist/cjs/internal/values.d.ts +41 -0
- package/dist/cjs/limits.d.ts +64 -0
- package/dist/cjs/operators.d.ts +81 -0
- package/dist/cjs/package.json +3 -0
- package/dist/cjs/plan.d.ts +79 -0
- package/dist/cjs/search.d.ts +42 -0
- package/dist/cjs/targets/mongo.d.ts +55 -0
- package/dist/cjs/text/index.d.ts +12 -0
- package/dist/cjs/text/parse.d.ts +91 -0
- package/dist/cjs/text/suggest.d.ts +16 -0
- package/dist/cjs/text/write.d.ts +34 -0
- package/dist/cjs/types.d.ts +236 -0
- package/dist/cjs/vocabulary.d.ts +106 -0
- package/dist/cli.d.ts +15 -0
- package/dist/cli.js +2729 -0
- package/dist/cli.js.map +1 -0
- package/dist/collections.d.ts +34 -0
- package/dist/core.d.ts +117 -0
- package/dist/errors.d.ts +17 -0
- package/dist/explain.d.ts +30 -0
- package/dist/global.cjs +1953 -0
- package/dist/global.cjs.map +1 -0
- package/dist/global.d.ts +90 -0
- package/dist/global.js +1950 -0
- package/dist/global.js.map +1 -0
- package/dist/group.d.ts +47 -0
- package/dist/index.cjs +2529 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +25 -0
- package/dist/index.js +2495 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/closest.d.ts +9 -0
- package/dist/internal/duration.d.ts +15 -0
- package/dist/internal/equal.d.ts +34 -0
- package/dist/internal/glob.d.ts +30 -0
- package/dist/internal/order.d.ts +24 -0
- package/dist/internal/path.d.ts +109 -0
- package/dist/internal/record.d.ts +18 -0
- package/dist/internal/values.d.ts +41 -0
- package/dist/limits.d.ts +64 -0
- package/dist/mongo.cjs +357 -0
- package/dist/mongo.cjs.map +1 -0
- package/dist/mongo.js +354 -0
- package/dist/mongo.js.map +1 -0
- package/dist/operators.d.ts +81 -0
- package/dist/plan.d.ts +79 -0
- package/dist/search.d.ts +42 -0
- package/dist/targets/mongo.d.ts +55 -0
- package/dist/text/index.d.ts +12 -0
- package/dist/text/parse.d.ts +91 -0
- package/dist/text/suggest.d.ts +16 -0
- package/dist/text/write.d.ts +34 -0
- package/dist/text.cjs +674 -0
- package/dist/text.cjs.map +1 -0
- package/dist/text.js +667 -0
- package/dist/text.js.map +1 -0
- package/dist/types.d.ts +236 -0
- package/dist/vocabulary.d.ts +106 -0
- package/docs/course/01-first-query.md +217 -0
- package/docs/course/02-operators.md +285 -0
- package/docs/course/03-arrays-and-paths.md +239 -0
- package/docs/course/04-combining.md +221 -0
- package/docs/course/05-dates.md +214 -0
- package/docs/course/06-typed-queries.md +240 -0
- package/docs/course/07-requests.md +261 -0
- package/docs/course/08-grouping.md +210 -0
- package/docs/course/09-explaining.md +171 -0
- package/docs/course/10-vocabulary.md +276 -0
- package/docs/course/11-the-search-box.md +349 -0
- package/docs/course/12-untrusted.md +257 -0
- package/docs/course/13-saved-filters.md +199 -0
- package/docs/course/14-streams-and-cli.md +276 -0
- package/docs/course/15-pushdown.md +240 -0
- package/docs/course/16-extending.md +199 -0
- package/docs/course/index.md +185 -0
- package/docs/design/decisions.md +198 -0
- package/docs/design/performance.md +102 -0
- package/docs/guides/adopting.md +81 -0
- package/docs/guides/pushdown.md +147 -0
- package/docs/guides/typescript.md +115 -0
- package/docs/guides/untrusted-input.md +86 -0
- package/docs/index.md +102 -0
- package/docs/reference/api.md +266 -0
- package/docs/reference/cli.md +103 -0
- package/docs/reference/index.md +12 -0
- package/docs/reference/specification.md +549 -0
- package/docs/reference/text-syntax.md +152 -0
- package/docs/start/quick-start.md +84 -0
- package/package.json +136 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# Queries from outside
|
|
2
|
+
|
|
3
|
+
Accepting a query from a URL, a request body or somebody else's saved filter.
|
|
4
|
+
|
|
5
|
+
← [Documentation](../index.md)
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
A JQL query is data, which is why it can travel. It is also why a query can arrive from
|
|
10
|
+
somebody you do not trust. Five things to do with one.
|
|
11
|
+
|
|
12
|
+
## 1. Compile it with the untrusted limits
|
|
13
|
+
|
|
14
|
+
```ts
|
|
15
|
+
import { UNTRUSTED_LIMITS, validate } from "@osqd/jql";
|
|
16
|
+
|
|
17
|
+
const checked = validate<Order>(JSON.parse(body), { limits: UNTRUSTED_LIMITS });
|
|
18
|
+
if (!checked.valid) return respond(400, { error: checked.error.message });
|
|
19
|
+
const matches = orders.filter(checked.test);
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
`validate` returns the predicate it compiled, rather than only a verdict, so the query that
|
|
23
|
+
was checked is the query that runs. Calling `compile` again afterwards works, but it is the
|
|
24
|
+
one place where the limits can quietly differ between the check and the run.
|
|
25
|
+
|
|
26
|
+
`UNTRUSTED_LIMITS` caps nesting at 16 levels and the query at 512 fields and operators, and
|
|
27
|
+
**turns `$regex` off**. No engine can tell a pattern that backtracks for a minute — `(a+)+$`
|
|
28
|
+
against a long run of `a`s — from one that does not without running it. The string operators
|
|
29
|
+
(`$contains`, `$startsWith`, `$endsWith`, `$word`) answer almost everything a pattern would,
|
|
30
|
+
in linear time, and stay available.
|
|
31
|
+
|
|
32
|
+
Every limit refuses rather than truncates: a query cut short answers a different question from
|
|
33
|
+
the one that was asked, and nobody could tell.
|
|
34
|
+
|
|
35
|
+
## 2. Answer the refusal, do not crash on it
|
|
36
|
+
|
|
37
|
+
`validate` never throws for an invalid query, and its error names the place:
|
|
38
|
+
`at $or[1].age.$gtt: "$gtt" is not an operator; did you mean "$gt"?`. That sentence is safe to
|
|
39
|
+
return to the caller — it quotes only the query they sent.
|
|
40
|
+
|
|
41
|
+
## 3. Bound what arrives before it is parsed
|
|
42
|
+
|
|
43
|
+
The limits apply to the parsed query. The body itself is yours to bound: a `$in` of a million
|
|
44
|
+
ids is a legitimate query and costs a million set entries, so cap the request size where you
|
|
45
|
+
read it. `$in` lists are deliberately not counted against `maxNodes`, because looking up an
|
|
46
|
+
item in one costs the same however long it is.
|
|
47
|
+
|
|
48
|
+
## 4. Say which operators are on the menu
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
const limits = { ...UNTRUSTED_LIMITS, allowOperators: ["$eq", "$ne", "$in", "$gt", "$gte", "$lt", "$lte", "$and", "$or"] };
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
`UNTRUSTED_LIMITS` turns `$regex` off, because no engine can tell a pattern that backtracks
|
|
55
|
+
for a minute from one that does not. Which of the *rest* an endpoint wants is a decision only
|
|
56
|
+
that endpoint can make: `$text` walks a whole document, `$elemMatch` carries a query of its
|
|
57
|
+
own, and an endpoint that needs neither should say so once, here, rather than find out later
|
|
58
|
+
which of them somebody used.
|
|
59
|
+
|
|
60
|
+
An operator outside the list is refused by name. A name in the list that is not an operator
|
|
61
|
+
is refused too — an allowlist with a typo would allow nothing while looking like it allowed
|
|
62
|
+
something.
|
|
63
|
+
|
|
64
|
+
## 5. Give it a strict vocabulary for a public API
|
|
65
|
+
|
|
66
|
+
```ts
|
|
67
|
+
const vocabulary = defineVocabulary<Order>()({ fields: { status: {}, total: {}, placed: {} }, strict: true });
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
With `strict`, a query may name only those fields. Documents tend to grow fields nobody promised
|
|
71
|
+
to keep, and a public query language that can reach every one of them turns every internal
|
|
72
|
+
field into part of the API.
|
|
73
|
+
|
|
74
|
+
## What a query cannot do
|
|
75
|
+
|
|
76
|
+
A JQL query is JSON, so it cannot carry code: there is no `$where`, no `$function`, nothing
|
|
77
|
+
that is evaluated. Paths read only an item's own fields, so `constructor`, `__proto__` and
|
|
78
|
+
inherited members are unreachable, and nothing a query does writes to anything. The text
|
|
79
|
+
syntax never throws for anything anybody types, and caps its input at 8 KB and 16 levels
|
|
80
|
+
of brackets — no deeper than the tightest limit the engine is asked to compile under, so the
|
|
81
|
+
box cannot produce a query the engine would refuse.
|
|
82
|
+
|
|
83
|
+
## Related
|
|
84
|
+
|
|
85
|
+
- [Library API](../reference/api.md#limits) — the limits in full
|
|
86
|
+
- [The specification](../reference/specification.md#10-limits) — what the standard says about caps
|
package/docs/index.md
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Documentation
|
|
2
|
+
|
|
3
|
+
Everything about JQL, one page per question.
|
|
4
|
+
|
|
5
|
+
← [README](../README.md)
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## What JQL is
|
|
10
|
+
|
|
11
|
+
JQL is a query language whose queries are **JSON documents**:
|
|
12
|
+
|
|
13
|
+
```json
|
|
14
|
+
{ "status": "open", "customer.country": "GB", "total": { "$gte": 100 } }
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
That single decision is the whole idea. A filter that is *data* rather than code can be saved
|
|
18
|
+
in a database, put in a URL, sent to another service, shown to a person, checked before it is
|
|
19
|
+
run, and split so that a store answers the half it is able to. A function can only be called.
|
|
20
|
+
|
|
21
|
+
`@osqd/jql` is the engine that runs them: zero dependencies, ESM and CommonJS, Node 20 or
|
|
22
|
+
newer.
|
|
23
|
+
|
|
24
|
+
## What it does
|
|
25
|
+
|
|
26
|
+
| | |
|
|
27
|
+
| --- | --- |
|
|
28
|
+
| **Filters a collection** | arrays, `Set`s, `Map`s, iterables, async streams, plain objects |
|
|
29
|
+
| **Answers a whole request** | filter, sort, page and choose fields, in one document |
|
|
30
|
+
| **Reads a search box** | what people type compiles into the same JQL, and back out again |
|
|
31
|
+
| **Accepts queries from strangers** | caps on cost, an operator allowlist, and a field allowlist |
|
|
32
|
+
| **Pushes into a store** | splits a query so a database answers the part it can |
|
|
33
|
+
| **Explains itself** | which clause decided a row, and the values it read |
|
|
34
|
+
| **Refuses what it does not understand** | a misspelt operator throws, rather than quietly matching nothing |
|
|
35
|
+
|
|
36
|
+
That last row is the rule the rest is built on: **a query that is not understood is refused,
|
|
37
|
+
never run as something that looks like it worked.**
|
|
38
|
+
|
|
39
|
+
## Quick start
|
|
40
|
+
|
|
41
|
+
```sh
|
|
42
|
+
npm install @osqd/jql
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
```js
|
|
46
|
+
import { filter, search } from "@osqd/jql";
|
|
47
|
+
|
|
48
|
+
const orders = [
|
|
49
|
+
{ id: "o-1", status: "open", total: 170, tags: ["fragile"], customer: { country: "GB" } },
|
|
50
|
+
{ id: "o-2", status: "paid", total: 22.5, tags: ["gift"], customer: { country: "GB" } },
|
|
51
|
+
{ id: "o-3", status: "open", total: 30, tags: [], customer: { country: "US" } },
|
|
52
|
+
];
|
|
53
|
+
|
|
54
|
+
// Two keys mean "and", and a dot reaches inside a nested object.
|
|
55
|
+
filter(orders, { status: "open", "customer.country": "GB" }); // [o-1]
|
|
56
|
+
|
|
57
|
+
// A value matches any element of an array; operators go where a value would.
|
|
58
|
+
filter(orders, { tags: "gift", total: { $lt: 100 } }); // [o-2]
|
|
59
|
+
|
|
60
|
+
// Filter, sort, page and pick fields — one document, one call.
|
|
61
|
+
search(orders, { where: { status: "open" }, sort: { total: -1 }, fields: ["id", "total"] });
|
|
62
|
+
// [ { id: "o-1", total: 170 }, { id: "o-3", total: 30 } ]
|
|
63
|
+
|
|
64
|
+
// And a query that is wrong says so.
|
|
65
|
+
filter(orders, { total: { $gtt: 100 } });
|
|
66
|
+
// JqlError: at total.$gtt: "$gtt" is not an operator; did you mean "$gt"?
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Where to go from here:
|
|
70
|
+
|
|
71
|
+
- **Another five minutes** — [Quick start](start/quick-start.md): the array methods,
|
|
72
|
+
`compile`, the search box, and queries from outside.
|
|
73
|
+
- **About three hours** — [the course](course/index.md): sixteen lessons that build one real
|
|
74
|
+
integration, teaching every capability in the order that makes each one make sense, with
|
|
75
|
+
something to run at every step.
|
|
76
|
+
- **A specific question** — the reference pages below, which answer "how does X work?"
|
|
77
|
+
rather than "what do I do next?".
|
|
78
|
+
|
|
79
|
+
## Reference
|
|
80
|
+
|
|
81
|
+
| Page | Answers |
|
|
82
|
+
| --- | --- |
|
|
83
|
+
| [The specification](reference/specification.md) | What exactly does a query mean? The standard, independent of any implementation |
|
|
84
|
+
| [Text syntax](reference/text-syntax.md) | What can people type into a search box, and what JSON does it become? |
|
|
85
|
+
| [Library API](reference/api.md) | Which functions does the TypeScript package export? |
|
|
86
|
+
| [The command](reference/cli.md) | How do I filter JSON lines from a shell? |
|
|
87
|
+
|
|
88
|
+
## Guides
|
|
89
|
+
|
|
90
|
+
| Page | Answers |
|
|
91
|
+
| --- | --- |
|
|
92
|
+
| [TypeScript](guides/typescript.md) | How do typed queries, the array methods and vocabularies fit together? |
|
|
93
|
+
| [Queries from outside](guides/untrusted-input.md) | How do I accept a query from a URL or a request body safely? |
|
|
94
|
+
| [Pushing a query into a store](guides/pushdown.md) | How does a backend answer the part of a query it can? |
|
|
95
|
+
| [Adopting JQL in a project](guides/adopting.md) | How does a project in this family move its filtering onto JQL? |
|
|
96
|
+
|
|
97
|
+
## Design
|
|
98
|
+
|
|
99
|
+
| Page | Answers |
|
|
100
|
+
| --- | --- |
|
|
101
|
+
| [Decisions](design/decisions.md) | Why is it shaped this way, and what did that cost? |
|
|
102
|
+
| [Performance](design/performance.md) | How fast is it, how is that measured, and what makes it so? |
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
# Library API
|
|
2
|
+
|
|
3
|
+
Every export of the TypeScript implementation.
|
|
4
|
+
|
|
5
|
+
← [Documentation](../index.md) · [Reference](index.md)
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
Three entry points, so nothing is loaded that is not used:
|
|
10
|
+
|
|
11
|
+
| Import | Holds | Side effects |
|
|
12
|
+
| --- | --- | --- |
|
|
13
|
+
| `@osqd/jql` | the engine, the collection helpers, `search`, vocabularies, types | none |
|
|
14
|
+
| `@osqd/jql/text` | `parseText`, `suggest` — the search-box syntax | none; does not load the engine |
|
|
15
|
+
| `@osqd/jql/global` | the methods on `Array`, `Map` and `Set`, and their types | installs them on import |
|
|
16
|
+
|
|
17
|
+
Every function that takes a query also takes an already compiled predicate or any function
|
|
18
|
+
`(item) => boolean`, and an options object `{ vocabulary?, limits? }`.
|
|
19
|
+
|
|
20
|
+
## `@osqd/jql`
|
|
21
|
+
|
|
22
|
+
### Compiling
|
|
23
|
+
|
|
24
|
+
| Export | Does |
|
|
25
|
+
| --- | --- |
|
|
26
|
+
| `compile(query, options?)` | Validates the query and returns a predicate. Throws `JqlError` for anything that is not valid JQL |
|
|
27
|
+
| `matches(item, query, options?)` | One item, one query. For a one-off; compile once for a loop |
|
|
28
|
+
| `validate(query, options?)` | `{ valid: true, test }` — the compiled predicate — or `{ valid: false, error }`, without throwing |
|
|
29
|
+
| `untyped(json)` | Marks a query from outside the type system so typed helpers accept it. Returns the same object |
|
|
30
|
+
| `explain(query, item, options?)` | Why one item does or does not match, clause by clause |
|
|
31
|
+
| `canonical(query, options?)` | The query in one shape per meaning |
|
|
32
|
+
| `fingerprint(query, options?)` | Sixteen hex characters naming that shape |
|
|
33
|
+
| `plan(query, capabilities, options?)` | Splits a query into what a store can answer and what stays here — see [Pushing a query into a store](../guides/pushdown.md) |
|
|
34
|
+
| `FIELD_OPERATORS`, `QUERY_OPERATORS` | The operator names, for editors and completions |
|
|
35
|
+
|
|
36
|
+
`compile` and everything built on it take `{ vocabulary, limits, now, operators }`:
|
|
37
|
+
`now` supplies the instant relative dates resolve against (read once, when the query
|
|
38
|
+
compiles), and `operators` adds operators of your own — see [below](#operators-of-your-own).
|
|
39
|
+
|
|
40
|
+
### Collections
|
|
41
|
+
|
|
42
|
+
Each takes an array, an array-like, any iterable, or a `Map` (whose **values** are the items).
|
|
43
|
+
|
|
44
|
+
| Export | Returns |
|
|
45
|
+
| --- | --- |
|
|
46
|
+
| `find(source, query)` | the first match, or `undefined`; stops at it |
|
|
47
|
+
| `findIndex(source, query)` | its position, or -1 |
|
|
48
|
+
| `filter(source, query)` | every match, as a new array |
|
|
49
|
+
| `count(source, query)` | how many match, without building the list |
|
|
50
|
+
| `some(source, query)` / `every(source, query)` | whether any / all match; both stop early |
|
|
51
|
+
| `partition(source, query)` | `[matches, rest]`, in one pass |
|
|
52
|
+
| `filterMap(map, query)` | a new `Map` of the entries whose value matches |
|
|
53
|
+
| `filterRecord(object, query)` | a new object of the properties whose value matches |
|
|
54
|
+
| `findEntry(mapOrObject, query)` | the first `[key, value]` whose value matches |
|
|
55
|
+
| `group(source, by, options?)` | the matches counted by one field, largest group first |
|
|
56
|
+
|
|
57
|
+
`group` takes `{ where, limit, items, sort, vocabulary }`. An item counts once in each group
|
|
58
|
+
it belongs to, so a path reaching an array puts it in one group per distinct value; everything
|
|
59
|
+
that cannot name a group — a missing field, a `null`, an object, an **empty list** — shares
|
|
60
|
+
the key `null`, so the rows counted always add up to the rows there are. Ask
|
|
61
|
+
for `items: true` (or a number) to keep the items as well as the count.
|
|
62
|
+
|
|
63
|
+
### Requests
|
|
64
|
+
|
|
65
|
+
`search(source, request, options?)` runs a whole [request](specification.md#8-requests):
|
|
66
|
+
`where`, `sort`, `skip`, `limit`, `fields`, `omit`. Without a sort it stops as soon as the page is
|
|
67
|
+
full; with a sort and a limit it keeps only the best `skip + limit` in a bounded heap. With
|
|
68
|
+
`fields` each result is a new object holding only those paths.
|
|
69
|
+
|
|
70
|
+
`omit` drops paths from each result after `fields` has chosen them, which is how a server
|
|
71
|
+
redacts: the values never leave the process, and the request that says so is data like the
|
|
72
|
+
rest. Results are copies, and anything not on the way to a dropped field is shared with the
|
|
73
|
+
item rather than duplicated.
|
|
74
|
+
|
|
75
|
+
`where` takes a compiled predicate as well as a query, which is how a page and its total are
|
|
76
|
+
asked for without compiling — or writing — the query twice:
|
|
77
|
+
|
|
78
|
+
```ts
|
|
79
|
+
const matching = compile<Order>(query);
|
|
80
|
+
const page = search(orders, { where: matching, sort: { placed: -1 }, skip, limit });
|
|
81
|
+
const total = count(orders, matching);
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Sources that arrive over time
|
|
85
|
+
|
|
86
|
+
For a log read line by line, a cursor, a stream of events. Each takes an `AsyncIterable` — or
|
|
87
|
+
any ordinary collection, so one call site serves both.
|
|
88
|
+
|
|
89
|
+
| Export | Returns |
|
|
90
|
+
| --- | --- |
|
|
91
|
+
| `findAsync(source, query)` | the first match; stops reading at it |
|
|
92
|
+
| `filterAsync(source, query)` | every match |
|
|
93
|
+
| `countAsync`, `someAsync`, `everyAsync` | the same answers as their synchronous namesakes |
|
|
94
|
+
| `filterStream(source, query)` | an async generator of the matches; stops when the caller does |
|
|
95
|
+
| `searchAsync(source, request)` | a whole request |
|
|
96
|
+
|
|
97
|
+
Without a sort, `searchAsync` stops as soon as the page is full, so an endless source still
|
|
98
|
+
answers. With a sort it must read everything, and holds only `skip + limit` items while it
|
|
99
|
+
does.
|
|
100
|
+
|
|
101
|
+
### Why one item matched
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
const why = explain({ verdict: "bot", score: { $gt: 80 } }, row);
|
|
105
|
+
// { matched: false, at: "", because: "1 of 2 parts hold", parts: [
|
|
106
|
+
// { matched: true, at: "verdict", because: 'verdict is "bot"' },
|
|
107
|
+
// { matched: false, at: "score", because: "score is 70" } ] }
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Each clause is compiled and run by the real engine, so an explanation cannot disagree with
|
|
111
|
+
the filter it explains. That costs a small compile per clause, which is why it is for one
|
|
112
|
+
item — a row somebody clicked — rather than for a collection.
|
|
113
|
+
|
|
114
|
+
### Saved filters: one shape, one name
|
|
115
|
+
|
|
116
|
+
```ts
|
|
117
|
+
canonical({ b: 2, a: 1, $comment: "note" }); // { a: { $eq: 1 }, b: { $eq: 2 } }
|
|
118
|
+
fingerprint({ a: 1, b: 2 }) === fingerprint({ b: { $eq: 2 }, a: 1 }); // true
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Equal fingerprints mean the same query. Different ones mean only that the queries are written
|
|
122
|
+
differently — `{ $gt: 3 }` and `{ $gte: 4 }` agree on every integer, and nothing here notices.
|
|
123
|
+
The rules are in the [specification](specification.md#13-canonical-form-optional), so another
|
|
124
|
+
implementation can agree about which saved filters are duplicates.
|
|
125
|
+
|
|
126
|
+
A fingerprint is **stable across versions**, so it is safe as a cache key and as the way a
|
|
127
|
+
stored filter is recognised. A set of them is written down in the tests: changing what a query
|
|
128
|
+
is named would miss every cache and duplicate every saved filter in silence, so it takes a
|
|
129
|
+
deliberate change to those values and a line in the changelog.
|
|
130
|
+
|
|
131
|
+
### Operators of your own
|
|
132
|
+
|
|
133
|
+
```ts
|
|
134
|
+
const cidr: OperatorDefinition = {
|
|
135
|
+
name: "$xCidr", // the $x prefix is required
|
|
136
|
+
compile: (operand, at) => {
|
|
137
|
+
if (typeof operand !== "string") throw new JqlError("takes a network as a string", at);
|
|
138
|
+
return (value) => typeof value === "string" && inNetwork(value, operand);
|
|
139
|
+
},
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
filter(requests, { ip: { $xCidr: "203.0.113.0/24" } }, { operators: [cidr] });
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
**A query using one is not portable JQL.** The `$x` prefix is what says so at a glance, and
|
|
146
|
+
an engine that was not given the operator refuses the query by name rather than ignoring the
|
|
147
|
+
clause. `cost` orders it against the built-in operators, and `elementwise: false` stops it
|
|
148
|
+
being tried against the elements of an array.
|
|
149
|
+
|
|
150
|
+
### Vocabularies
|
|
151
|
+
|
|
152
|
+
`defineVocabulary<T>()({ fields, text?, strict? })` names the fields a query may use:
|
|
153
|
+
|
|
154
|
+
```ts
|
|
155
|
+
const vocabulary = defineVocabulary<Request>()({
|
|
156
|
+
fields: {
|
|
157
|
+
ip: { path: "client.address", aliases: ["actor"], kind: "word" },
|
|
158
|
+
ua: { path: "headers.user-agent" },
|
|
159
|
+
outcome: { get: (request) => outcomeOf(request), kind: "exact", values: ["allow", "deny"] },
|
|
160
|
+
},
|
|
161
|
+
text: ["ip", "ua"],
|
|
162
|
+
});
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
| Field option | Meaning |
|
|
166
|
+
| --- | --- |
|
|
167
|
+
| `path` | where the value lives; default the field's own name |
|
|
168
|
+
| `get` | computes the value instead; it must not throw |
|
|
169
|
+
| `aliases` | other names for the field |
|
|
170
|
+
| `kind` | how the text syntax reads a value: `text`, `word`, `exact`, `number`, `date`, `boolean` |
|
|
171
|
+
| `values` | the closed set of values, for completion |
|
|
172
|
+
|
|
173
|
+
In a projection the two kinds of field differ, because only one of them has somewhere of its
|
|
174
|
+
own to go: a field with a `path` is projected into the document's shape
|
|
175
|
+
(`ip` gives `{ client: { address } }`), while a computed field comes back under its own name
|
|
176
|
+
(`outcome` gives `{ outcome }`).
|
|
177
|
+
|
|
178
|
+
`text` lists the fields a bare search word looks in. `strict: true` refuses any name that is
|
|
179
|
+
not in the vocabulary. Names are matched without regard to case. Two names for one field, a
|
|
180
|
+
field with both `path` and `get`, and a `text` entry that names nothing are refused when the
|
|
181
|
+
vocabulary is defined.
|
|
182
|
+
|
|
183
|
+
### Limits
|
|
184
|
+
|
|
185
|
+
`DEFAULT_LIMITS` and `UNTRUSTED_LIMITS` — see [Queries from outside](../guides/untrusted-input.md).
|
|
186
|
+
|
|
187
|
+
| Limit | Default | Untrusted |
|
|
188
|
+
| --- | --- | --- |
|
|
189
|
+
| `maxDepth` | 32 | 16 |
|
|
190
|
+
| `maxNodes` | 10 000 | 512 |
|
|
191
|
+
| `maxPatternLength` | 1 024 | 0 |
|
|
192
|
+
| `maxGlobLength` | 1 024 | 256 |
|
|
193
|
+
| `maxTextDepth` | 16 | 8 |
|
|
194
|
+
| `allowRegex` | `true` | `false` |
|
|
195
|
+
| `allowOperators` | `"all"` | `"all"` |
|
|
196
|
+
|
|
197
|
+
`allowOperators` is an allowlist: give it the operator names a query may use and the rest are
|
|
198
|
+
refused by name. `$options` and `$comment` are exempt, and `$field` counts as an operator,
|
|
199
|
+
because comparing two fields is a capability of its own. A name that is not an operator is
|
|
200
|
+
itself refused — an allowlist with a typo in it would allow nothing and look like it allowed
|
|
201
|
+
something.
|
|
202
|
+
|
|
203
|
+
```ts
|
|
204
|
+
const publicLimits = { ...UNTRUSTED_LIMITS, allowOperators: ["$eq", "$ne", "$in", "$gt", "$gte", "$lt", "$lte", "$and", "$or"] };
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
### Errors
|
|
208
|
+
|
|
209
|
+
`JqlError` is the one error the engine throws. `error.at` is where in the query the problem is
|
|
210
|
+
(`$or[1].age.$gt`), and the message starts with it: `at $or[1].age.$gt: "$gtt" is not an
|
|
211
|
+
operator; did you mean "$gt"?`.
|
|
212
|
+
|
|
213
|
+
## `@osqd/jql/text`
|
|
214
|
+
|
|
215
|
+
| Export | Does |
|
|
216
|
+
| --- | --- |
|
|
217
|
+
| `parseText(input, { vocabulary?, fields? })` | the search-box syntax to a JQL query; never throws |
|
|
218
|
+
| `toText(query, { vocabulary?, fields? })` | the reverse: `{ text, complete, unexpressed }` |
|
|
219
|
+
| `suggest(input, caret, vocabulary?)` | completions for the token under the caret, and the span they replace |
|
|
220
|
+
| `MAX_TEXT_CHARS`, `MAX_TEXT_DEPTH`, `TEXT_OPERATORS` | the caps and the operator words |
|
|
221
|
+
|
|
222
|
+
`toText` is how a stored JSON filter gets back into a search box. It is deliberately not
|
|
223
|
+
total — `$elemMatch`, `$glob`, `$size`, `$length` and `{ "$field": … }` have no spelling —
|
|
224
|
+
so it returns the text *and* what it left out. The text is exact for what it says; a caller
|
|
225
|
+
that ignores `unexpressed` is showing somebody a filter wider than the one that is running,
|
|
226
|
+
and `complete` makes that check one line.
|
|
227
|
+
|
|
228
|
+
## `@osqd/jql/mongo`
|
|
229
|
+
|
|
230
|
+
| Export | Does |
|
|
231
|
+
| --- | --- |
|
|
232
|
+
| `MONGO_CAPABILITIES` | what a MongoDB `find` filter can answer, for `plan` |
|
|
233
|
+
| `toMongoFilter(query, options?)` | translates the pushed part into a MongoDB filter |
|
|
234
|
+
|
|
235
|
+
What `plan` pushes already carries the names the store knows and instants rather than
|
|
236
|
+
durations, so `toMongoFilter(split.pushed)` needs no vocabulary. Its `vocabulary` option is
|
|
237
|
+
for translating a query that did not come from `plan`.
|
|
238
|
+
|
|
239
|
+
See [Pushing a query into a store](../guides/pushdown.md).
|
|
240
|
+
|
|
241
|
+
## `@osqd/jql/cli`
|
|
242
|
+
|
|
243
|
+
`main(argv, streams)` returns the exit code, and `run()` wires it to the process. The command
|
|
244
|
+
itself is [`jql`](cli.md).
|
|
245
|
+
|
|
246
|
+
## `@osqd/jql/global`
|
|
247
|
+
|
|
248
|
+
Importing it adds, non-enumerably:
|
|
249
|
+
|
|
250
|
+
| On | Methods |
|
|
251
|
+
| --- | --- |
|
|
252
|
+
| every array | `jqlSearch`, `jqlFilter`, `jqlCount`, `jqlSome`, `jqlEvery`, `jqlFindIndex`, `jqlPartition`, `jqlQuery` |
|
|
253
|
+
| `Array` itself | the same, taking the source first: `Array.jqlSearch(iterable, query)` |
|
|
254
|
+
| every `Map` | `jqlSearch` (a value), `jqlFilter` (a `Map`), `jqlCount` |
|
|
255
|
+
| every `Set` | `jqlSearch`, `jqlFilter` (a `Set`), `jqlCount` |
|
|
256
|
+
|
|
257
|
+
It also exports `install()` and `uninstall()`. `install` refuses to replace a method of the
|
|
258
|
+
same name that something else defined, and is a no-op when the methods are already there —
|
|
259
|
+
including when another copy of JQL in the same process installed them, which npm produces
|
|
260
|
+
whenever two dependencies want different versions. The copy that got there first keeps the
|
|
261
|
+
field. `uninstall` removes JQL's methods and nothing else's.
|
|
262
|
+
|
|
263
|
+
## Related
|
|
264
|
+
|
|
265
|
+
- [The specification](specification.md) — what the queries mean
|
|
266
|
+
- [TypeScript](../guides/typescript.md) — the types behind these signatures
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# The command
|
|
2
|
+
|
|
3
|
+
`jql` — filtering JSON lines from a shell.
|
|
4
|
+
|
|
5
|
+
← [Documentation](../index.md) · [Reference](index.md)
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
The shape this library is for arrives as one JSON value per line — a log, an export, a
|
|
10
|
+
`kubectl` dump — more often than it arrives as an array in a program. The command is the
|
|
11
|
+
same engine over a stream, and everything it does is a public export used the way the
|
|
12
|
+
documentation says to use it.
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
npx jql '{"status":"open"}' orders.jsonl
|
|
16
|
+
jql --text 'status:open total:>100' orders.jsonl
|
|
17
|
+
kubectl logs -f app | jql --text 'level:error' --fields ts,msg
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Usage
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
jql [options] <query> [file...]
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
`<query>` is a JQL query as JSON, or the [search-box syntax](text-syntax.md) with `--text`.
|
|
27
|
+
Files hold one JSON value per line; with none, or with `-`, it reads standard input.
|
|
28
|
+
|
|
29
|
+
| Option | Does |
|
|
30
|
+
| --- | --- |
|
|
31
|
+
| `--text` | read the query as search-box text rather than JSON |
|
|
32
|
+
| `--sort <field:order>` | order by a field, `asc` or `desc`; repeatable, the first key breaks ties last |
|
|
33
|
+
| `--skip <n>` / `--limit <n>` | pass over the first n matches / stop after n |
|
|
34
|
+
| `--fields <a,b>` | keep only these paths in each result |
|
|
35
|
+
| `--omit <a,b>` | drop these paths from each result |
|
|
36
|
+
| `--count` | print how many matched instead of the matches |
|
|
37
|
+
| `--group <field>` | print how many matched by that field, largest first; `--limit` then says how many groups, and `--sort`, `--skip`, `--fields`, `--omit` and `--count` are refused rather than ignored |
|
|
38
|
+
| `--explain` | say on stderr why the first line did or did not match |
|
|
39
|
+
| `--canonical` | print the query in its canonical form and stop |
|
|
40
|
+
| `--untrusted` | compile with the untrusted limits: `$regex` off, tighter caps |
|
|
41
|
+
| `--pretty` | print each result over several lines |
|
|
42
|
+
| `-h, --help` / `-v, --version` | |
|
|
43
|
+
|
|
44
|
+
Both `--name value` and `--name=value` work.
|
|
45
|
+
|
|
46
|
+
## The two rules it is built on
|
|
47
|
+
|
|
48
|
+
**stdout is the artifact.** Matching lines and nothing else, so `jql … > kept.jsonl`
|
|
49
|
+
produces a file worth having. Counts, warnings and explanations go to stderr.
|
|
50
|
+
|
|
51
|
+
A matching line is written back **byte for byte as it arrived** — spacing, key order and all
|
|
52
|
+
— so filtering a file cannot change the data in it. That matters for numbers JavaScript
|
|
53
|
+
cannot hold exactly: a thirty-digit identifier read and written again comes back as
|
|
54
|
+
`1.2345678901234568e+29`, and `1e400` comes back as `null`. Asking for the result to be
|
|
55
|
+
reshaped (`--fields`, `--omit`, `--group`) or laid out again (`--pretty`) is asking for
|
|
56
|
+
something other than the line, and then it is written out as JSON. A leading byte-order mark
|
|
57
|
+
is dropped rather than treated as data.
|
|
58
|
+
|
|
59
|
+
**A value is validated, never coerced.** `--limit all` is an error, not a limit that quietly
|
|
60
|
+
stops limiting — `Number("all")` is `NaN`, every comparison against it is false, and the
|
|
61
|
+
failure would be silent in both directions. An option that cannot apply is refused for the
|
|
62
|
+
same reason: `--group` prints groups, so `--fields` beside it is an error rather than a flag
|
|
63
|
+
that goes nowhere.
|
|
64
|
+
|
|
65
|
+
**Everything streams except `--group`.** A filter, a sort with a limit and a count all read
|
|
66
|
+
one line at a time and hold only the page they are building, so a file larger than memory is
|
|
67
|
+
fine. Counting by a field is the exception: a group's total is not known until the last line,
|
|
68
|
+
so the matching items are held while the tally is built.
|
|
69
|
+
|
|
70
|
+
## Exit codes
|
|
71
|
+
|
|
72
|
+
| Code | Means |
|
|
73
|
+
| --- | --- |
|
|
74
|
+
| 0 | it ran; no matches is an answer, not a failure |
|
|
75
|
+
| 1 | a file could not be read, or nothing in the input was JSON |
|
|
76
|
+
| 2 | bad arguments, or a query the engine refuses |
|
|
77
|
+
|
|
78
|
+
A line that is not JSON is reported on stderr and skipped; the lines around it are still
|
|
79
|
+
answered. If *every* line was unreadable, that is a failure rather than an empty result.
|
|
80
|
+
|
|
81
|
+
## Examples
|
|
82
|
+
|
|
83
|
+
```sh
|
|
84
|
+
# the ten slowest requests, as a table of two fields
|
|
85
|
+
jql --text 'path:/api' --sort durationMs:desc --limit 10 --fields path,durationMs access.jsonl
|
|
86
|
+
|
|
87
|
+
# how many of each verdict, for what the query matched
|
|
88
|
+
jql --group verdict '{"score":{"$gte":70}}' feed.jsonl
|
|
89
|
+
|
|
90
|
+
# why a line is not matching the filter you thought it would
|
|
91
|
+
jql --explain --text 'status:open total:>500' orders.jsonl > /dev/null
|
|
92
|
+
|
|
93
|
+
# a filter from somewhere you do not trust
|
|
94
|
+
jql --untrusted "$UNTRUSTED_QUERY" events.jsonl
|
|
95
|
+
|
|
96
|
+
# a redacted copy
|
|
97
|
+
jql --omit customer.email,customer.phone '{}' orders.jsonl > shareable.jsonl
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Related
|
|
101
|
+
|
|
102
|
+
- [Text syntax](text-syntax.md) — what `--text` takes
|
|
103
|
+
- [Library API](api.md) — the exports the command is built from
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Reference
|
|
2
|
+
|
|
3
|
+
The language, its text form, and the library.
|
|
4
|
+
|
|
5
|
+
← [Documentation](../index.md)
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
- [The specification](specification.md) — what a query means; the standard every implementation follows
|
|
10
|
+
- [Text syntax](text-syntax.md) — the search-box form, and the JSON each piece becomes
|
|
11
|
+
- [Library API](api.md) — every export of `@osqd/jql` and its entry points
|
|
12
|
+
- [The command](cli.md) — `jql`, for filtering JSON lines from a shell
|