@osqd/jql 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/CHANGELOG.md +220 -0
  2. package/LICENSE +102 -0
  3. package/README.md +84 -0
  4. package/bin/jql.mjs +5 -0
  5. package/conformance/cases.json +290 -0
  6. package/dist/array.d.ts +11 -0
  7. package/dist/async.d.ts +44 -0
  8. package/dist/canonical.d.ts +18 -0
  9. package/dist/cjs/array.d.ts +11 -0
  10. package/dist/cjs/async.d.ts +44 -0
  11. package/dist/cjs/canonical.d.ts +18 -0
  12. package/dist/cjs/cli.d.ts +15 -0
  13. package/dist/cjs/collections.d.ts +34 -0
  14. package/dist/cjs/core.d.ts +117 -0
  15. package/dist/cjs/errors.d.ts +17 -0
  16. package/dist/cjs/explain.d.ts +30 -0
  17. package/dist/cjs/global.d.ts +90 -0
  18. package/dist/cjs/group.d.ts +47 -0
  19. package/dist/cjs/index.d.ts +25 -0
  20. package/dist/cjs/internal/closest.d.ts +9 -0
  21. package/dist/cjs/internal/duration.d.ts +15 -0
  22. package/dist/cjs/internal/equal.d.ts +34 -0
  23. package/dist/cjs/internal/glob.d.ts +30 -0
  24. package/dist/cjs/internal/order.d.ts +24 -0
  25. package/dist/cjs/internal/path.d.ts +109 -0
  26. package/dist/cjs/internal/record.d.ts +18 -0
  27. package/dist/cjs/internal/values.d.ts +41 -0
  28. package/dist/cjs/limits.d.ts +64 -0
  29. package/dist/cjs/operators.d.ts +81 -0
  30. package/dist/cjs/package.json +3 -0
  31. package/dist/cjs/plan.d.ts +79 -0
  32. package/dist/cjs/search.d.ts +42 -0
  33. package/dist/cjs/targets/mongo.d.ts +55 -0
  34. package/dist/cjs/text/index.d.ts +12 -0
  35. package/dist/cjs/text/parse.d.ts +91 -0
  36. package/dist/cjs/text/suggest.d.ts +16 -0
  37. package/dist/cjs/text/write.d.ts +34 -0
  38. package/dist/cjs/types.d.ts +236 -0
  39. package/dist/cjs/vocabulary.d.ts +106 -0
  40. package/dist/cli.d.ts +15 -0
  41. package/dist/cli.js +2729 -0
  42. package/dist/cli.js.map +1 -0
  43. package/dist/collections.d.ts +34 -0
  44. package/dist/core.d.ts +117 -0
  45. package/dist/errors.d.ts +17 -0
  46. package/dist/explain.d.ts +30 -0
  47. package/dist/global.cjs +1953 -0
  48. package/dist/global.cjs.map +1 -0
  49. package/dist/global.d.ts +90 -0
  50. package/dist/global.js +1950 -0
  51. package/dist/global.js.map +1 -0
  52. package/dist/group.d.ts +47 -0
  53. package/dist/index.cjs +2529 -0
  54. package/dist/index.cjs.map +1 -0
  55. package/dist/index.d.ts +25 -0
  56. package/dist/index.js +2495 -0
  57. package/dist/index.js.map +1 -0
  58. package/dist/internal/closest.d.ts +9 -0
  59. package/dist/internal/duration.d.ts +15 -0
  60. package/dist/internal/equal.d.ts +34 -0
  61. package/dist/internal/glob.d.ts +30 -0
  62. package/dist/internal/order.d.ts +24 -0
  63. package/dist/internal/path.d.ts +109 -0
  64. package/dist/internal/record.d.ts +18 -0
  65. package/dist/internal/values.d.ts +41 -0
  66. package/dist/limits.d.ts +64 -0
  67. package/dist/mongo.cjs +357 -0
  68. package/dist/mongo.cjs.map +1 -0
  69. package/dist/mongo.js +354 -0
  70. package/dist/mongo.js.map +1 -0
  71. package/dist/operators.d.ts +81 -0
  72. package/dist/plan.d.ts +79 -0
  73. package/dist/search.d.ts +42 -0
  74. package/dist/targets/mongo.d.ts +55 -0
  75. package/dist/text/index.d.ts +12 -0
  76. package/dist/text/parse.d.ts +91 -0
  77. package/dist/text/suggest.d.ts +16 -0
  78. package/dist/text/write.d.ts +34 -0
  79. package/dist/text.cjs +674 -0
  80. package/dist/text.cjs.map +1 -0
  81. package/dist/text.js +667 -0
  82. package/dist/text.js.map +1 -0
  83. package/dist/types.d.ts +236 -0
  84. package/dist/vocabulary.d.ts +106 -0
  85. package/docs/course/01-first-query.md +217 -0
  86. package/docs/course/02-operators.md +285 -0
  87. package/docs/course/03-arrays-and-paths.md +239 -0
  88. package/docs/course/04-combining.md +221 -0
  89. package/docs/course/05-dates.md +214 -0
  90. package/docs/course/06-typed-queries.md +240 -0
  91. package/docs/course/07-requests.md +261 -0
  92. package/docs/course/08-grouping.md +210 -0
  93. package/docs/course/09-explaining.md +171 -0
  94. package/docs/course/10-vocabulary.md +276 -0
  95. package/docs/course/11-the-search-box.md +349 -0
  96. package/docs/course/12-untrusted.md +257 -0
  97. package/docs/course/13-saved-filters.md +199 -0
  98. package/docs/course/14-streams-and-cli.md +276 -0
  99. package/docs/course/15-pushdown.md +240 -0
  100. package/docs/course/16-extending.md +199 -0
  101. package/docs/course/index.md +185 -0
  102. package/docs/design/decisions.md +198 -0
  103. package/docs/design/performance.md +102 -0
  104. package/docs/guides/adopting.md +81 -0
  105. package/docs/guides/pushdown.md +147 -0
  106. package/docs/guides/typescript.md +115 -0
  107. package/docs/guides/untrusted-input.md +86 -0
  108. package/docs/index.md +102 -0
  109. package/docs/reference/api.md +266 -0
  110. package/docs/reference/cli.md +103 -0
  111. package/docs/reference/index.md +12 -0
  112. package/docs/reference/specification.md +549 -0
  113. package/docs/reference/text-syntax.md +152 -0
  114. package/docs/start/quick-start.md +84 -0
  115. package/package.json +136 -0
@@ -0,0 +1,86 @@
1
+ # Queries from outside
2
+
3
+ Accepting a query from a URL, a request body or somebody else's saved filter.
4
+
5
+ ← [Documentation](../index.md)
6
+
7
+ ---
8
+
9
+ A JQL query is data, which is why it can travel. It is also why a query can arrive from
10
+ somebody you do not trust. Five things to do with one.
11
+
12
+ ## 1. Compile it with the untrusted limits
13
+
14
+ ```ts
15
+ import { UNTRUSTED_LIMITS, validate } from "@osqd/jql";
16
+
17
+ const checked = validate<Order>(JSON.parse(body), { limits: UNTRUSTED_LIMITS });
18
+ if (!checked.valid) return respond(400, { error: checked.error.message });
19
+ const matches = orders.filter(checked.test);
20
+ ```
21
+
22
+ `validate` returns the predicate it compiled, rather than only a verdict, so the query that
23
+ was checked is the query that runs. Calling `compile` again afterwards works, but it is the
24
+ one place where the limits can quietly differ between the check and the run.
25
+
26
+ `UNTRUSTED_LIMITS` caps nesting at 16 levels and the query at 512 fields and operators, and
27
+ **turns `$regex` off**. No engine can tell a pattern that backtracks for a minute — `(a+)+$`
28
+ against a long run of `a`s — from one that does not without running it. The string operators
29
+ (`$contains`, `$startsWith`, `$endsWith`, `$word`) answer almost everything a pattern would,
30
+ in linear time, and stay available.
31
+
32
+ Every limit refuses rather than truncates: a query cut short answers a different question from
33
+ the one that was asked, and nobody could tell.
34
+
35
+ ## 2. Answer the refusal, do not crash on it
36
+
37
+ `validate` never throws for an invalid query, and its error names the place:
38
+ `at $or[1].age.$gtt: "$gtt" is not an operator; did you mean "$gt"?`. That sentence is safe to
39
+ return to the caller — it quotes only the query they sent.
40
+
41
+ ## 3. Bound what arrives before it is parsed
42
+
43
+ The limits apply to the parsed query. The body itself is yours to bound: a `$in` of a million
44
+ ids is a legitimate query and costs a million set entries, so cap the request size where you
45
+ read it. `$in` lists are deliberately not counted against `maxNodes`, because looking up an
46
+ item in one costs the same however long it is.
47
+
48
+ ## 4. Say which operators are on the menu
49
+
50
+ ```ts
51
+ const limits = { ...UNTRUSTED_LIMITS, allowOperators: ["$eq", "$ne", "$in", "$gt", "$gte", "$lt", "$lte", "$and", "$or"] };
52
+ ```
53
+
54
+ `UNTRUSTED_LIMITS` turns `$regex` off, because no engine can tell a pattern that backtracks
55
+ for a minute from one that does not. Which of the *rest* an endpoint wants is a decision only
56
+ that endpoint can make: `$text` walks a whole document, `$elemMatch` carries a query of its
57
+ own, and an endpoint that needs neither should say so once, here, rather than find out later
58
+ which of them somebody used.
59
+
60
+ An operator outside the list is refused by name. A name in the list that is not an operator
61
+ is refused too — an allowlist with a typo would allow nothing while looking like it allowed
62
+ something.
63
+
64
+ ## 5. Give it a strict vocabulary for a public API
65
+
66
+ ```ts
67
+ const vocabulary = defineVocabulary<Order>()({ fields: { status: {}, total: {}, placed: {} }, strict: true });
68
+ ```
69
+
70
+ With `strict`, a query may name only those fields. Documents tend to grow fields nobody promised
71
+ to keep, and a public query language that can reach every one of them turns every internal
72
+ field into part of the API.
73
+
74
+ ## What a query cannot do
75
+
76
+ A JQL query is JSON, so it cannot carry code: there is no `$where`, no `$function`, nothing
77
+ that is evaluated. Paths read only an item's own fields, so `constructor`, `__proto__` and
78
+ inherited members are unreachable, and nothing a query does writes to anything. The text
79
+ syntax never throws for anything anybody types, and caps its input at 8 KB and 16 levels
80
+ of brackets — no deeper than the tightest limit the engine is asked to compile under, so the
81
+ box cannot produce a query the engine would refuse.
82
+
83
+ ## Related
84
+
85
+ - [Library API](../reference/api.md#limits) — the limits in full
86
+ - [The specification](../reference/specification.md#10-limits) — what the standard says about caps
package/docs/index.md ADDED
@@ -0,0 +1,102 @@
1
+ # Documentation
2
+
3
+ Everything about JQL, one page per question.
4
+
5
+ ← [README](../README.md)
6
+
7
+ ---
8
+
9
+ ## What JQL is
10
+
11
+ JQL is a query language whose queries are **JSON documents**:
12
+
13
+ ```json
14
+ { "status": "open", "customer.country": "GB", "total": { "$gte": 100 } }
15
+ ```
16
+
17
+ That single decision is the whole idea. A filter that is *data* rather than code can be saved
18
+ in a database, put in a URL, sent to another service, shown to a person, checked before it is
19
+ run, and split so that a store answers the half it is able to. A function can only be called.
20
+
21
+ `@osqd/jql` is the engine that runs them: zero dependencies, ESM and CommonJS, Node 20 or
22
+ newer.
23
+
24
+ ## What it does
25
+
26
+ | | |
27
+ | --- | --- |
28
+ | **Filters a collection** | arrays, `Set`s, `Map`s, iterables, async streams, plain objects |
29
+ | **Answers a whole request** | filter, sort, page and choose fields, in one document |
30
+ | **Reads a search box** | what people type compiles into the same JQL, and back out again |
31
+ | **Accepts queries from strangers** | caps on cost, an operator allowlist, and a field allowlist |
32
+ | **Pushes into a store** | splits a query so a database answers the part it can |
33
+ | **Explains itself** | which clause decided a row, and the values it read |
34
+ | **Refuses what it does not understand** | a misspelt operator throws, rather than quietly matching nothing |
35
+
36
+ That last row is the rule the rest is built on: **a query that is not understood is refused,
37
+ never run as something that looks like it worked.**
38
+
39
+ ## Quick start
40
+
41
+ ```sh
42
+ npm install @osqd/jql
43
+ ```
44
+
45
+ ```js
46
+ import { filter, search } from "@osqd/jql";
47
+
48
+ const orders = [
49
+ { id: "o-1", status: "open", total: 170, tags: ["fragile"], customer: { country: "GB" } },
50
+ { id: "o-2", status: "paid", total: 22.5, tags: ["gift"], customer: { country: "GB" } },
51
+ { id: "o-3", status: "open", total: 30, tags: [], customer: { country: "US" } },
52
+ ];
53
+
54
+ // Two keys mean "and", and a dot reaches inside a nested object.
55
+ filter(orders, { status: "open", "customer.country": "GB" }); // [o-1]
56
+
57
+ // A value matches any element of an array; operators go where a value would.
58
+ filter(orders, { tags: "gift", total: { $lt: 100 } }); // [o-2]
59
+
60
+ // Filter, sort, page and pick fields — one document, one call.
61
+ search(orders, { where: { status: "open" }, sort: { total: -1 }, fields: ["id", "total"] });
62
+ // [ { id: "o-1", total: 170 }, { id: "o-3", total: 30 } ]
63
+
64
+ // And a query that is wrong says so.
65
+ filter(orders, { total: { $gtt: 100 } });
66
+ // JqlError: at total.$gtt: "$gtt" is not an operator; did you mean "$gt"?
67
+ ```
68
+
69
+ Where to go from here:
70
+
71
+ - **Another five minutes** — [Quick start](start/quick-start.md): the array methods,
72
+ `compile`, the search box, and queries from outside.
73
+ - **About three hours** — [the course](course/index.md): sixteen lessons that build one real
74
+ integration, teaching every capability in the order that makes each one make sense, with
75
+ something to run at every step.
76
+ - **A specific question** — the reference pages below, which answer "how does X work?"
77
+ rather than "what do I do next?".
78
+
79
+ ## Reference
80
+
81
+ | Page | Answers |
82
+ | --- | --- |
83
+ | [The specification](reference/specification.md) | What exactly does a query mean? The standard, independent of any implementation |
84
+ | [Text syntax](reference/text-syntax.md) | What can people type into a search box, and what JSON does it become? |
85
+ | [Library API](reference/api.md) | Which functions does the TypeScript package export? |
86
+ | [The command](reference/cli.md) | How do I filter JSON lines from a shell? |
87
+
88
+ ## Guides
89
+
90
+ | Page | Answers |
91
+ | --- | --- |
92
+ | [TypeScript](guides/typescript.md) | How do typed queries, the array methods and vocabularies fit together? |
93
+ | [Queries from outside](guides/untrusted-input.md) | How do I accept a query from a URL or a request body safely? |
94
+ | [Pushing a query into a store](guides/pushdown.md) | How does a backend answer the part of a query it can? |
95
+ | [Adopting JQL in a project](guides/adopting.md) | How does a project in this family move its filtering onto JQL? |
96
+
97
+ ## Design
98
+
99
+ | Page | Answers |
100
+ | --- | --- |
101
+ | [Decisions](design/decisions.md) | Why is it shaped this way, and what did that cost? |
102
+ | [Performance](design/performance.md) | How fast is it, how is that measured, and what makes it so? |
@@ -0,0 +1,266 @@
1
+ # Library API
2
+
3
+ Every export of the TypeScript implementation.
4
+
5
+ ← [Documentation](../index.md) · [Reference](index.md)
6
+
7
+ ---
8
+
9
+ Three entry points, so nothing is loaded that is not used:
10
+
11
+ | Import | Holds | Side effects |
12
+ | --- | --- | --- |
13
+ | `@osqd/jql` | the engine, the collection helpers, `search`, vocabularies, types | none |
14
+ | `@osqd/jql/text` | `parseText`, `suggest` — the search-box syntax | none; does not load the engine |
15
+ | `@osqd/jql/global` | the methods on `Array`, `Map` and `Set`, and their types | installs them on import |
16
+
17
+ Every function that takes a query also takes an already compiled predicate or any function
18
+ `(item) => boolean`, and an options object `{ vocabulary?, limits? }`.
19
+
20
+ ## `@osqd/jql`
21
+
22
+ ### Compiling
23
+
24
+ | Export | Does |
25
+ | --- | --- |
26
+ | `compile(query, options?)` | Validates the query and returns a predicate. Throws `JqlError` for anything that is not valid JQL |
27
+ | `matches(item, query, options?)` | One item, one query. For a one-off; compile once for a loop |
28
+ | `validate(query, options?)` | `{ valid: true, test }` — the compiled predicate — or `{ valid: false, error }`, without throwing |
29
+ | `untyped(json)` | Marks a query from outside the type system so typed helpers accept it. Returns the same object |
30
+ | `explain(query, item, options?)` | Why one item does or does not match, clause by clause |
31
+ | `canonical(query, options?)` | The query in one shape per meaning |
32
+ | `fingerprint(query, options?)` | Sixteen hex characters naming that shape |
33
+ | `plan(query, capabilities, options?)` | Splits a query into what a store can answer and what stays here — see [Pushing a query into a store](../guides/pushdown.md) |
34
+ | `FIELD_OPERATORS`, `QUERY_OPERATORS` | The operator names, for editors and completions |
35
+
36
+ `compile` and everything built on it take `{ vocabulary, limits, now, operators }`:
37
+ `now` supplies the instant relative dates resolve against (read once, when the query
38
+ compiles), and `operators` adds operators of your own — see [below](#operators-of-your-own).
39
+
40
+ ### Collections
41
+
42
+ Each takes an array, an array-like, any iterable, or a `Map` (whose **values** are the items).
43
+
44
+ | Export | Returns |
45
+ | --- | --- |
46
+ | `find(source, query)` | the first match, or `undefined`; stops at it |
47
+ | `findIndex(source, query)` | its position, or -1 |
48
+ | `filter(source, query)` | every match, as a new array |
49
+ | `count(source, query)` | how many match, without building the list |
50
+ | `some(source, query)` / `every(source, query)` | whether any / all match; both stop early |
51
+ | `partition(source, query)` | `[matches, rest]`, in one pass |
52
+ | `filterMap(map, query)` | a new `Map` of the entries whose value matches |
53
+ | `filterRecord(object, query)` | a new object of the properties whose value matches |
54
+ | `findEntry(mapOrObject, query)` | the first `[key, value]` whose value matches |
55
+ | `group(source, by, options?)` | the matches counted by one field, largest group first |
56
+
57
+ `group` takes `{ where, limit, items, sort, vocabulary }`. An item counts once in each group
58
+ it belongs to, so a path reaching an array puts it in one group per distinct value; everything
59
+ that cannot name a group — a missing field, a `null`, an object, an **empty list** — shares
60
+ the key `null`, so the rows counted always add up to the rows there are. Ask
61
+ for `items: true` (or a number) to keep the items as well as the count.
62
+
63
+ ### Requests
64
+
65
+ `search(source, request, options?)` runs a whole [request](specification.md#8-requests):
66
+ `where`, `sort`, `skip`, `limit`, `fields`, `omit`. Without a sort it stops as soon as the page is
67
+ full; with a sort and a limit it keeps only the best `skip + limit` in a bounded heap. With
68
+ `fields` each result is a new object holding only those paths.
69
+
70
+ `omit` drops paths from each result after `fields` has chosen them, which is how a server
71
+ redacts: the values never leave the process, and the request that says so is data like the
72
+ rest. Results are copies, and anything not on the way to a dropped field is shared with the
73
+ item rather than duplicated.
74
+
75
+ `where` takes a compiled predicate as well as a query, which is how a page and its total are
76
+ asked for without compiling — or writing — the query twice:
77
+
78
+ ```ts
79
+ const matching = compile<Order>(query);
80
+ const page = search(orders, { where: matching, sort: { placed: -1 }, skip, limit });
81
+ const total = count(orders, matching);
82
+ ```
83
+
84
+ ### Sources that arrive over time
85
+
86
+ For a log read line by line, a cursor, a stream of events. Each takes an `AsyncIterable` — or
87
+ any ordinary collection, so one call site serves both.
88
+
89
+ | Export | Returns |
90
+ | --- | --- |
91
+ | `findAsync(source, query)` | the first match; stops reading at it |
92
+ | `filterAsync(source, query)` | every match |
93
+ | `countAsync`, `someAsync`, `everyAsync` | the same answers as their synchronous namesakes |
94
+ | `filterStream(source, query)` | an async generator of the matches; stops when the caller does |
95
+ | `searchAsync(source, request)` | a whole request |
96
+
97
+ Without a sort, `searchAsync` stops as soon as the page is full, so an endless source still
98
+ answers. With a sort it must read everything, and holds only `skip + limit` items while it
99
+ does.
100
+
101
+ ### Why one item matched
102
+
103
+ ```ts
104
+ const why = explain({ verdict: "bot", score: { $gt: 80 } }, row);
105
+ // { matched: false, at: "", because: "1 of 2 parts hold", parts: [
106
+ // { matched: true, at: "verdict", because: 'verdict is "bot"' },
107
+ // { matched: false, at: "score", because: "score is 70" } ] }
108
+ ```
109
+
110
+ Each clause is compiled and run by the real engine, so an explanation cannot disagree with
111
+ the filter it explains. That costs a small compile per clause, which is why it is for one
112
+ item — a row somebody clicked — rather than for a collection.
113
+
114
+ ### Saved filters: one shape, one name
115
+
116
+ ```ts
117
+ canonical({ b: 2, a: 1, $comment: "note" }); // { a: { $eq: 1 }, b: { $eq: 2 } }
118
+ fingerprint({ a: 1, b: 2 }) === fingerprint({ b: { $eq: 2 }, a: 1 }); // true
119
+ ```
120
+
121
+ Equal fingerprints mean the same query. Different ones mean only that the queries are written
122
+ differently — `{ $gt: 3 }` and `{ $gte: 4 }` agree on every integer, and nothing here notices.
123
+ The rules are in the [specification](specification.md#13-canonical-form-optional), so another
124
+ implementation can agree about which saved filters are duplicates.
125
+
126
+ A fingerprint is **stable across versions**, so it is safe as a cache key and as the way a
127
+ stored filter is recognised. A set of them is written down in the tests: changing what a query
128
+ is named would miss every cache and duplicate every saved filter in silence, so it takes a
129
+ deliberate change to those values and a line in the changelog.
130
+
131
+ ### Operators of your own
132
+
133
+ ```ts
134
+ const cidr: OperatorDefinition = {
135
+ name: "$xCidr", // the $x prefix is required
136
+ compile: (operand, at) => {
137
+ if (typeof operand !== "string") throw new JqlError("takes a network as a string", at);
138
+ return (value) => typeof value === "string" && inNetwork(value, operand);
139
+ },
140
+ };
141
+
142
+ filter(requests, { ip: { $xCidr: "203.0.113.0/24" } }, { operators: [cidr] });
143
+ ```
144
+
145
+ **A query using one is not portable JQL.** The `$x` prefix is what says so at a glance, and
146
+ an engine that was not given the operator refuses the query by name rather than ignoring the
147
+ clause. `cost` orders it against the built-in operators, and `elementwise: false` stops it
148
+ being tried against the elements of an array.
149
+
150
+ ### Vocabularies
151
+
152
+ `defineVocabulary<T>()({ fields, text?, strict? })` names the fields a query may use:
153
+
154
+ ```ts
155
+ const vocabulary = defineVocabulary<Request>()({
156
+ fields: {
157
+ ip: { path: "client.address", aliases: ["actor"], kind: "word" },
158
+ ua: { path: "headers.user-agent" },
159
+ outcome: { get: (request) => outcomeOf(request), kind: "exact", values: ["allow", "deny"] },
160
+ },
161
+ text: ["ip", "ua"],
162
+ });
163
+ ```
164
+
165
+ | Field option | Meaning |
166
+ | --- | --- |
167
+ | `path` | where the value lives; default the field's own name |
168
+ | `get` | computes the value instead; it must not throw |
169
+ | `aliases` | other names for the field |
170
+ | `kind` | how the text syntax reads a value: `text`, `word`, `exact`, `number`, `date`, `boolean` |
171
+ | `values` | the closed set of values, for completion |
172
+
173
+ In a projection the two kinds of field differ, because only one of them has somewhere of its
174
+ own to go: a field with a `path` is projected into the document's shape
175
+ (`ip` gives `{ client: { address } }`), while a computed field comes back under its own name
176
+ (`outcome` gives `{ outcome }`).
177
+
178
+ `text` lists the fields a bare search word looks in. `strict: true` refuses any name that is
179
+ not in the vocabulary. Names are matched without regard to case. Two names for one field, a
180
+ field with both `path` and `get`, and a `text` entry that names nothing are refused when the
181
+ vocabulary is defined.
182
+
183
+ ### Limits
184
+
185
+ `DEFAULT_LIMITS` and `UNTRUSTED_LIMITS` — see [Queries from outside](../guides/untrusted-input.md).
186
+
187
+ | Limit | Default | Untrusted |
188
+ | --- | --- | --- |
189
+ | `maxDepth` | 32 | 16 |
190
+ | `maxNodes` | 10 000 | 512 |
191
+ | `maxPatternLength` | 1 024 | 0 |
192
+ | `maxGlobLength` | 1 024 | 256 |
193
+ | `maxTextDepth` | 16 | 8 |
194
+ | `allowRegex` | `true` | `false` |
195
+ | `allowOperators` | `"all"` | `"all"` |
196
+
197
+ `allowOperators` is an allowlist: give it the operator names a query may use and the rest are
198
+ refused by name. `$options` and `$comment` are exempt, and `$field` counts as an operator,
199
+ because comparing two fields is a capability of its own. A name that is not an operator is
200
+ itself refused — an allowlist with a typo in it would allow nothing and look like it allowed
201
+ something.
202
+
203
+ ```ts
204
+ const publicLimits = { ...UNTRUSTED_LIMITS, allowOperators: ["$eq", "$ne", "$in", "$gt", "$gte", "$lt", "$lte", "$and", "$or"] };
205
+ ```
206
+
207
+ ### Errors
208
+
209
+ `JqlError` is the one error the engine throws. `error.at` is where in the query the problem is
210
+ (`$or[1].age.$gt`), and the message starts with it: `at $or[1].age.$gt: "$gtt" is not an
211
+ operator; did you mean "$gt"?`.
212
+
213
+ ## `@osqd/jql/text`
214
+
215
+ | Export | Does |
216
+ | --- | --- |
217
+ | `parseText(input, { vocabulary?, fields? })` | the search-box syntax to a JQL query; never throws |
218
+ | `toText(query, { vocabulary?, fields? })` | the reverse: `{ text, complete, unexpressed }` |
219
+ | `suggest(input, caret, vocabulary?)` | completions for the token under the caret, and the span they replace |
220
+ | `MAX_TEXT_CHARS`, `MAX_TEXT_DEPTH`, `TEXT_OPERATORS` | the caps and the operator words |
221
+
222
+ `toText` is how a stored JSON filter gets back into a search box. It is deliberately not
223
+ total — `$elemMatch`, `$glob`, `$size`, `$length` and `{ "$field": … }` have no spelling —
224
+ so it returns the text *and* what it left out. The text is exact for what it says; a caller
225
+ that ignores `unexpressed` is showing somebody a filter wider than the one that is running,
226
+ and `complete` makes that check one line.
227
+
228
+ ## `@osqd/jql/mongo`
229
+
230
+ | Export | Does |
231
+ | --- | --- |
232
+ | `MONGO_CAPABILITIES` | what a MongoDB `find` filter can answer, for `plan` |
233
+ | `toMongoFilter(query, options?)` | translates the pushed part into a MongoDB filter |
234
+
235
+ What `plan` pushes already carries the names the store knows and instants rather than
236
+ durations, so `toMongoFilter(split.pushed)` needs no vocabulary. Its `vocabulary` option is
237
+ for translating a query that did not come from `plan`.
238
+
239
+ See [Pushing a query into a store](../guides/pushdown.md).
240
+
241
+ ## `@osqd/jql/cli`
242
+
243
+ `main(argv, streams)` returns the exit code, and `run()` wires it to the process. The command
244
+ itself is [`jql`](cli.md).
245
+
246
+ ## `@osqd/jql/global`
247
+
248
+ Importing it adds, non-enumerably:
249
+
250
+ | On | Methods |
251
+ | --- | --- |
252
+ | every array | `jqlSearch`, `jqlFilter`, `jqlCount`, `jqlSome`, `jqlEvery`, `jqlFindIndex`, `jqlPartition`, `jqlQuery` |
253
+ | `Array` itself | the same, taking the source first: `Array.jqlSearch(iterable, query)` |
254
+ | every `Map` | `jqlSearch` (a value), `jqlFilter` (a `Map`), `jqlCount` |
255
+ | every `Set` | `jqlSearch`, `jqlFilter` (a `Set`), `jqlCount` |
256
+
257
+ It also exports `install()` and `uninstall()`. `install` refuses to replace a method of the
258
+ same name that something else defined, and is a no-op when the methods are already there —
259
+ including when another copy of JQL in the same process installed them, which npm produces
260
+ whenever two dependencies want different versions. The copy that got there first keeps the
261
+ field. `uninstall` removes JQL's methods and nothing else's.
262
+
263
+ ## Related
264
+
265
+ - [The specification](specification.md) — what the queries mean
266
+ - [TypeScript](../guides/typescript.md) — the types behind these signatures
@@ -0,0 +1,103 @@
1
+ # The command
2
+
3
+ `jql` — filtering JSON lines from a shell.
4
+
5
+ ← [Documentation](../index.md) · [Reference](index.md)
6
+
7
+ ---
8
+
9
+ The shape this library is for arrives as one JSON value per line — a log, an export, a
10
+ `kubectl` dump — more often than it arrives as an array in a program. The command is the
11
+ same engine over a stream, and everything it does is a public export used the way the
12
+ documentation says to use it.
13
+
14
+ ```sh
15
+ npx jql '{"status":"open"}' orders.jsonl
16
+ jql --text 'status:open total:>100' orders.jsonl
17
+ kubectl logs -f app | jql --text 'level:error' --fields ts,msg
18
+ ```
19
+
20
+ ## Usage
21
+
22
+ ```
23
+ jql [options] <query> [file...]
24
+ ```
25
+
26
+ `<query>` is a JQL query as JSON, or the [search-box syntax](text-syntax.md) with `--text`.
27
+ Files hold one JSON value per line; with none, or with `-`, it reads standard input.
28
+
29
+ | Option | Does |
30
+ | --- | --- |
31
+ | `--text` | read the query as search-box text rather than JSON |
32
+ | `--sort <field:order>` | order by a field, `asc` or `desc`; repeatable, the first key breaks ties last |
33
+ | `--skip <n>` / `--limit <n>` | pass over the first n matches / stop after n |
34
+ | `--fields <a,b>` | keep only these paths in each result |
35
+ | `--omit <a,b>` | drop these paths from each result |
36
+ | `--count` | print how many matched instead of the matches |
37
+ | `--group <field>` | print how many matched by that field, largest first; `--limit` then says how many groups, and `--sort`, `--skip`, `--fields`, `--omit` and `--count` are refused rather than ignored |
38
+ | `--explain` | say on stderr why the first line did or did not match |
39
+ | `--canonical` | print the query in its canonical form and stop |
40
+ | `--untrusted` | compile with the untrusted limits: `$regex` off, tighter caps |
41
+ | `--pretty` | print each result over several lines |
42
+ | `-h, --help` / `-v, --version` | |
43
+
44
+ Both `--name value` and `--name=value` work.
45
+
46
+ ## The two rules it is built on
47
+
48
+ **stdout is the artifact.** Matching lines and nothing else, so `jql … > kept.jsonl`
49
+ produces a file worth having. Counts, warnings and explanations go to stderr.
50
+
51
+ A matching line is written back **byte for byte as it arrived** — spacing, key order and all
52
+ — so filtering a file cannot change the data in it. That matters for numbers JavaScript
53
+ cannot hold exactly: a thirty-digit identifier read and written again comes back as
54
+ `1.2345678901234568e+29`, and `1e400` comes back as `null`. Asking for the result to be
55
+ reshaped (`--fields`, `--omit`, `--group`) or laid out again (`--pretty`) is asking for
56
+ something other than the line, and then it is written out as JSON. A leading byte-order mark
57
+ is dropped rather than treated as data.
58
+
59
+ **A value is validated, never coerced.** `--limit all` is an error, not a limit that quietly
60
+ stops limiting — `Number("all")` is `NaN`, every comparison against it is false, and the
61
+ failure would be silent in both directions. An option that cannot apply is refused for the
62
+ same reason: `--group` prints groups, so `--fields` beside it is an error rather than a flag
63
+ that goes nowhere.
64
+
65
+ **Everything streams except `--group`.** A filter, a sort with a limit and a count all read
66
+ one line at a time and hold only the page they are building, so a file larger than memory is
67
+ fine. Counting by a field is the exception: a group's total is not known until the last line,
68
+ so the matching items are held while the tally is built.
69
+
70
+ ## Exit codes
71
+
72
+ | Code | Means |
73
+ | --- | --- |
74
+ | 0 | it ran; no matches is an answer, not a failure |
75
+ | 1 | a file could not be read, or nothing in the input was JSON |
76
+ | 2 | bad arguments, or a query the engine refuses |
77
+
78
+ A line that is not JSON is reported on stderr and skipped; the lines around it are still
79
+ answered. If *every* line was unreadable, that is a failure rather than an empty result.
80
+
81
+ ## Examples
82
+
83
+ ```sh
84
+ # the ten slowest requests, as a table of two fields
85
+ jql --text 'path:/api' --sort durationMs:desc --limit 10 --fields path,durationMs access.jsonl
86
+
87
+ # how many of each verdict, for what the query matched
88
+ jql --group verdict '{"score":{"$gte":70}}' feed.jsonl
89
+
90
+ # why a line is not matching the filter you thought it would
91
+ jql --explain --text 'status:open total:>500' orders.jsonl > /dev/null
92
+
93
+ # a filter from somewhere you do not trust
94
+ jql --untrusted "$UNTRUSTED_QUERY" events.jsonl
95
+
96
+ # a redacted copy
97
+ jql --omit customer.email,customer.phone '{}' orders.jsonl > shareable.jsonl
98
+ ```
99
+
100
+ ## Related
101
+
102
+ - [Text syntax](text-syntax.md) — what `--text` takes
103
+ - [Library API](api.md) — the exports the command is built from
@@ -0,0 +1,12 @@
1
+ # Reference
2
+
3
+ The language, its text form, and the library.
4
+
5
+ ← [Documentation](../index.md)
6
+
7
+ ---
8
+
9
+ - [The specification](specification.md) — what a query means; the standard every implementation follows
10
+ - [Text syntax](text-syntax.md) — the search-box form, and the JSON each piece becomes
11
+ - [Library API](api.md) — every export of `@osqd/jql` and its entry points
12
+ - [The command](cli.md) — `jql`, for filtering JSON lines from a shell