@m6d/cortex-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -0
- package/bin/cortex.js +52 -0
- package/package.json +41 -0
- package/src/cc/client.ts +295 -0
- package/src/cc/flags.ts +135 -0
- package/src/cc/output.ts +77 -0
- package/src/cc/profiles.ts +105 -0
- package/src/cli.ts +56 -0
- package/src/commands/cc.ts +504 -0
- package/src/commands/graph.ts +229 -0
- package/src/commands/new.ts +333 -0
- package/src/commands/swagger.ts +261 -0
- package/src/config/load.ts +122 -0
- package/src/config/validate.ts +46 -0
- package/src/contracts/charts/chart-spec.ts +201 -0
- package/src/contracts/charts/index.ts +7 -0
- package/src/contracts/client-tools/index.ts +82 -0
- package/src/contracts/egress/index.ts +161 -0
- package/src/contracts/egress/scrub.ts +59 -0
- package/src/contracts/graph/clients/embed.ts +45 -0
- package/src/contracts/graph/clients/neo4j.ts +85 -0
- package/src/contracts/graph/helpers.ts +57 -0
- package/src/contracts/graph/index.ts +30 -0
- package/src/contracts/graph/schema.ts +65 -0
- package/src/contracts/graph/types.ts +125 -0
- package/src/contracts/markdown/index.ts +39 -0
- package/src/contracts/rich-text/index.ts +220 -0
- package/src/contracts/runtime/api.ts +328 -0
- package/src/contracts/runtime/auth.ts +41 -0
- package/src/contracts/runtime/dataset-sql-guide.ts +67 -0
- package/src/contracts/runtime/dataset-sql.ts +2522 -0
- package/src/contracts/runtime/datasets.fixtures.json +2849 -0
- package/src/contracts/runtime/datasets.ts +1391 -0
- package/src/contracts/runtime/grammar.ts +161 -0
- package/src/contracts/runtime/index.ts +14 -0
- package/src/contracts/runtime/signature.ts +237 -0
- package/src/contracts/runtime/sync.ts +172 -0
- package/src/contracts/runtime/triggers.ts +224 -0
- package/src/contracts/wire/index.ts +264 -0
- package/src/graph/expand-domains.ts +288 -0
- package/src/graph/generate-cypher.ts +201 -0
- package/src/graph/seed.ts +207 -0
- package/src/graph/validate.ts +78 -0
- package/src/scaffold/features.ts +190 -0
- package/src/scaffold/files.ts +411 -0
- package/src/scaffold/ports.ts +86 -0
- package/src/scaffold/templates/domains-README.md.txt +16 -0
- package/src/scaffold/templates/eslint.config.mjs.txt +20 -0
- package/src/scaffold/templates/fallowrc.jsonc.txt +42 -0
- package/src/scaffold/templates/gitignore.txt +19 -0
- package/src/scaffold/templates/index.ts.txt +18 -0
- package/src/scaffold/templates/prettier.config.mjs.txt +5 -0
- package/src/scaffold/templates/tsconfig.json.txt +25 -0
- package/src/swagger/extract-endpoints.ts +266 -0
- package/src/swagger/generated-block.ts +74 -0
- package/src/swagger/openapi-schema.ts +161 -0
- package/src/swagger/parse-response.ts +88 -0
- package/src/swagger/types.ts +25 -0
- package/src/ui/prompts.ts +125 -0
- package/src/ui/report.ts +107 -0
- package/tsconfig.json +46 -0
|
@@ -0,0 +1,1391 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
/*
|
|
4
|
+
* Datasets: governed tabular reads over a host's own data. A dataset host
|
|
5
|
+
* serves a manifest of the tables it exposes and runs one JSON query per
|
|
6
|
+
* read under the forwarded user identity, applying its own access rules
|
|
7
|
+
* before the query. Cortex validates a query against the manifest, caps it,
|
|
8
|
+
* forwards it, trims the answer, and logs the read; it never evaluates who
|
|
9
|
+
* may see which rows.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export const DATASET_IDENTIFIER_PATTERN = /^[a-zA-Z][a-zA-Z0-9_]*$/;
|
|
13
|
+
/** `column` on the dataset queried, or `relation.relation.column` along declared relations. */
|
|
14
|
+
export const DATASET_COLUMN_REF_PATTERN = /^[a-zA-Z][a-zA-Z0-9_]*(\.[a-zA-Z][a-zA-Z0-9_]*)*$/;
|
|
15
|
+
|
|
16
|
+
export const datasetIdentifierSchema = z.string().regex(DATASET_IDENTIFIER_PATTERN).max(64);
|
|
17
|
+
export const datasetColumnRefSchema = z.string().regex(DATASET_COLUMN_REF_PATTERN).max(256);
|
|
18
|
+
|
|
19
|
+
export const DATASET_COLUMN_TYPES = ["string", "number", "boolean", "date"] as const;
|
|
20
|
+
export type DatasetColumnType = (typeof DATASET_COLUMN_TYPES)[number];
|
|
21
|
+
|
|
22
|
+
export const datasetColumnSchema = z.object({
|
|
23
|
+
name: datasetIdentifierSchema,
|
|
24
|
+
type: z.enum(DATASET_COLUMN_TYPES),
|
|
25
|
+
description: z.string().max(200).default(""),
|
|
26
|
+
});
|
|
27
|
+
export type DatasetColumn = z.infer<typeof datasetColumnSchema>;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* A join the host can honor: `on` pairs a column of this dataset with a
|
|
31
|
+
* column of the target; `through` names the link dataset of a many-to-many,
|
|
32
|
+
* whose `on` pairs its own column with the target's.
|
|
33
|
+
*/
|
|
34
|
+
export const datasetRelationSchema = z.object({
|
|
35
|
+
dataset: datasetIdentifierSchema,
|
|
36
|
+
on: z.tuple([datasetIdentifierSchema, datasetIdentifierSchema]),
|
|
37
|
+
through: z
|
|
38
|
+
.object({
|
|
39
|
+
dataset: datasetIdentifierSchema,
|
|
40
|
+
on: z.tuple([datasetIdentifierSchema, datasetIdentifierSchema]),
|
|
41
|
+
})
|
|
42
|
+
.optional(),
|
|
43
|
+
});
|
|
44
|
+
export type DatasetRelation = z.infer<typeof datasetRelationSchema>;
|
|
45
|
+
|
|
46
|
+
export const DATASET_MAX_ROWS_DEFAULT = 500;
|
|
47
|
+
|
|
48
|
+
export const datasetDefinitionSchema = z.object({
|
|
49
|
+
name: datasetIdentifierSchema,
|
|
50
|
+
/** What the model reads in the catalog; the columns come only on request. */
|
|
51
|
+
description: z.string().max(500),
|
|
52
|
+
columns: z.array(datasetColumnSchema).min(1).max(200),
|
|
53
|
+
relations: z.record(datasetIdentifierSchema, datasetRelationSchema).default({}),
|
|
54
|
+
/** The most rows one query may answer with; the host clamps to it whatever the caller sent. */
|
|
55
|
+
maxRows: z.number().int().min(1).max(5000).default(DATASET_MAX_ROWS_DEFAULT),
|
|
56
|
+
});
|
|
57
|
+
export type DatasetDefinition = z.infer<typeof datasetDefinitionSchema>;
|
|
58
|
+
export type DatasetDefinitionInput = z.input<typeof datasetDefinitionSchema>;
|
|
59
|
+
|
|
60
|
+
/** What `GET {host}/` answers: every dataset the host exposes, under a version that changes when any of them does. */
|
|
61
|
+
export const datasetManifestSchema = z.object({
|
|
62
|
+
version: z.string().min(1).max(128),
|
|
63
|
+
datasets: z.array(datasetDefinitionSchema).max(500),
|
|
64
|
+
});
|
|
65
|
+
export type DatasetManifest = z.infer<typeof datasetManifestSchema>;
|
|
66
|
+
|
|
67
|
+
export const DATASET_FILTER_OPS = [
|
|
68
|
+
"=",
|
|
69
|
+
"!=",
|
|
70
|
+
"<",
|
|
71
|
+
"<=",
|
|
72
|
+
">",
|
|
73
|
+
">=",
|
|
74
|
+
"in",
|
|
75
|
+
"notIn",
|
|
76
|
+
"between",
|
|
77
|
+
"contains",
|
|
78
|
+
"startsWith",
|
|
79
|
+
"endsWith",
|
|
80
|
+
"isNull",
|
|
81
|
+
"isNotNull",
|
|
82
|
+
] as const;
|
|
83
|
+
export type DatasetFilterOp = (typeof DATASET_FILTER_OPS)[number];
|
|
84
|
+
|
|
85
|
+
/** The periods a date truncates to; a truncated value is the ISO date of the period's first day, weeks starting on Monday. */
|
|
86
|
+
export const DATASET_BUCKETS = ["day", "week", "month", "quarter", "year"] as const;
|
|
87
|
+
export type DatasetBucket = (typeof DATASET_BUCKETS)[number];
|
|
88
|
+
/** The parts a date answers as an integer: the period's number, or the weekday 1 (Monday) to 7 (Sunday). */
|
|
89
|
+
export const DATASET_DATE_PARTS = [...DATASET_BUCKETS, "dayOfWeek"] as const;
|
|
90
|
+
export type DatasetDatePart = (typeof DATASET_DATE_PARTS)[number];
|
|
91
|
+
/** Scalar functions over a value: the text ones over strings, `abs` and `round` (optional digits) over numbers, `coalesce` over two or more of one type. */
|
|
92
|
+
export const DATASET_FUNCTIONS = [
|
|
93
|
+
"lower",
|
|
94
|
+
"upper",
|
|
95
|
+
"trim",
|
|
96
|
+
"length",
|
|
97
|
+
"abs",
|
|
98
|
+
"round",
|
|
99
|
+
"coalesce",
|
|
100
|
+
] as const;
|
|
101
|
+
export type DatasetFunction = (typeof DATASET_FUNCTIONS)[number];
|
|
102
|
+
|
|
103
|
+
export const DATASET_AGGREGATES = ["count", "countDistinct", "sum", "avg", "min", "max"] as const;
|
|
104
|
+
export type DatasetAggregate = (typeof DATASET_AGGREGATES)[number];
|
|
105
|
+
|
|
106
|
+
const filterValueSchema = z.union([z.string(), z.number(), z.boolean()]);
|
|
107
|
+
|
|
108
|
+
export const DATASET_ARITHMETIC_OPS = ["+", "-", "*", "/"] as const;
|
|
109
|
+
export type DatasetArithmeticOp = (typeof DATASET_ARITHMETIC_OPS)[number];
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* A value read off a row: a column (or, on a grouped row, an aggregate or
|
|
113
|
+
* computed alias), a number, a text or boolean literal, arithmetic (division
|
|
114
|
+
* is exact, no integer division, and x / 0 is null), a date truncated to a period or one of its
|
|
115
|
+
* parts, a scalar function over values, or a CASE whose branches share a type.
|
|
116
|
+
*/
|
|
117
|
+
export type DatasetOperand =
|
|
118
|
+
| string
|
|
119
|
+
| number
|
|
120
|
+
| { value: string | boolean }
|
|
121
|
+
| { op: DatasetArithmeticOp; left: DatasetOperand; right: DatasetOperand }
|
|
122
|
+
| { fn: "dateTrunc"; unit: DatasetBucket; of: DatasetOperand }
|
|
123
|
+
| { fn: "datePart"; unit: DatasetDatePart; of: DatasetOperand }
|
|
124
|
+
| { fn: DatasetFunction; args: DatasetOperand[] }
|
|
125
|
+
| { case: { when: DatasetFilter; then: DatasetOperand }[]; else?: DatasetOperand };
|
|
126
|
+
/**
|
|
127
|
+
* Every object variant has its own key, and the `fn` variants are told apart
|
|
128
|
+
* by `fn` before their arguments are read, so a malformed value is checked by
|
|
129
|
+
* one variant per level: tried by each variant alike, `dateTrunc` and
|
|
130
|
+
* `datePart` would both read `of` and the error would double per level.
|
|
131
|
+
*/
|
|
132
|
+
export const datasetOperandSchema: z.ZodType<DatasetOperand, DatasetOperand> = z.lazy(() =>
|
|
133
|
+
z.union([
|
|
134
|
+
datasetColumnRefSchema,
|
|
135
|
+
z.number(),
|
|
136
|
+
z.object({ value: z.union([z.string().max(500), z.boolean()]) }),
|
|
137
|
+
z.object({
|
|
138
|
+
op: z.enum(DATASET_ARITHMETIC_OPS),
|
|
139
|
+
left: datasetOperandSchema,
|
|
140
|
+
right: datasetOperandSchema,
|
|
141
|
+
}),
|
|
142
|
+
z.discriminatedUnion("fn", [
|
|
143
|
+
z.object({
|
|
144
|
+
fn: z.literal("dateTrunc"),
|
|
145
|
+
unit: z.enum(DATASET_BUCKETS),
|
|
146
|
+
of: datasetOperandSchema,
|
|
147
|
+
}),
|
|
148
|
+
z.object({
|
|
149
|
+
fn: z.literal("datePart"),
|
|
150
|
+
unit: z.enum(DATASET_DATE_PARTS),
|
|
151
|
+
of: datasetOperandSchema,
|
|
152
|
+
}),
|
|
153
|
+
z.object({
|
|
154
|
+
fn: z.enum(DATASET_FUNCTIONS),
|
|
155
|
+
args: z.array(datasetOperandSchema).min(1).max(10),
|
|
156
|
+
}),
|
|
157
|
+
]),
|
|
158
|
+
z.object({
|
|
159
|
+
case: z
|
|
160
|
+
.array(z.object({ when: datasetFilterSchema, then: datasetOperandSchema }))
|
|
161
|
+
.min(1)
|
|
162
|
+
.max(20),
|
|
163
|
+
else: datasetOperandSchema.optional(),
|
|
164
|
+
}),
|
|
165
|
+
]),
|
|
166
|
+
);
|
|
167
|
+
/** A value under the key it comes back as; what select, groupBy and compute name. */
|
|
168
|
+
export const datasetNamedOperandSchema = z.object({
|
|
169
|
+
expr: datasetOperandSchema,
|
|
170
|
+
as: datasetIdentifierSchema,
|
|
171
|
+
});
|
|
172
|
+
export type DatasetNamedOperand = z.infer<typeof datasetNamedOperandSchema>;
|
|
173
|
+
/** The left side of a condition: a column or arithmetic, never a bare number. */
|
|
174
|
+
const datasetConditionLeftSchema = datasetOperandSchema.refine(
|
|
175
|
+
(operand) => typeof operand !== "number",
|
|
176
|
+
"a condition starts with a column",
|
|
177
|
+
);
|
|
178
|
+
|
|
179
|
+
export type DatasetCondition = {
|
|
180
|
+
col: Exclude<DatasetOperand, number>;
|
|
181
|
+
op: DatasetFilterOp;
|
|
182
|
+
/** A scalar, a list for `in` and `notIn`, a pair for `between`, nothing for the null checks, or `{ col }` to compare with another column or arithmetic. */
|
|
183
|
+
value?: string | number | boolean | (string | number | boolean)[] | { col: DatasetOperand };
|
|
184
|
+
};
|
|
185
|
+
export const datasetConditionSchema: z.ZodType<DatasetCondition, DatasetCondition> = z.object({
|
|
186
|
+
col: datasetConditionLeftSchema as z.ZodType<
|
|
187
|
+
Exclude<DatasetOperand, number>,
|
|
188
|
+
Exclude<DatasetOperand, number>
|
|
189
|
+
>,
|
|
190
|
+
op: z.enum(DATASET_FILTER_OPS),
|
|
191
|
+
value: z
|
|
192
|
+
.union([
|
|
193
|
+
filterValueSchema,
|
|
194
|
+
z.array(filterValueSchema).max(500),
|
|
195
|
+
z.object({ col: datasetOperandSchema }),
|
|
196
|
+
])
|
|
197
|
+
.optional(),
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* A condition, a group of filters that must all hold (`all`) or of which one
|
|
202
|
+
* must hold (`any`), or the negation of one (`not`, with SQL's null
|
|
203
|
+
* semantics: a row whose comparison is unknown is left out either way). A
|
|
204
|
+
* filter list is an `all`.
|
|
205
|
+
*/
|
|
206
|
+
export type DatasetFilter =
|
|
207
|
+
| DatasetCondition
|
|
208
|
+
| { any: DatasetFilter[] }
|
|
209
|
+
| { all: DatasetFilter[] }
|
|
210
|
+
| { not: DatasetFilter };
|
|
211
|
+
export const datasetFilterSchema: z.ZodType<DatasetFilter, DatasetFilter> = z.lazy(() =>
|
|
212
|
+
z.union([
|
|
213
|
+
datasetConditionSchema,
|
|
214
|
+
z.object({ any: z.array(datasetFilterSchema).min(1).max(20) }),
|
|
215
|
+
z.object({ all: z.array(datasetFilterSchema).min(1).max(20) }),
|
|
216
|
+
z.object({ not: datasetFilterSchema }),
|
|
217
|
+
]),
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
/** A column, answered under its own reference, or a value answered under its alias. */
|
|
221
|
+
export const datasetGroupBySchema = z.union([datasetColumnRefSchema, datasetNamedOperandSchema]);
|
|
222
|
+
const datasetSortSchema = z.object({
|
|
223
|
+
col: datasetColumnRefSchema,
|
|
224
|
+
dir: z.enum(["asc", "desc"]),
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
export const datasetAggregateSchema = z.object({
|
|
228
|
+
fn: z.enum(DATASET_AGGREGATES),
|
|
229
|
+
/** The column or arithmetic aggregated; `count` may omit it to count rows. */
|
|
230
|
+
col: (
|
|
231
|
+
datasetConditionLeftSchema as z.ZodType<
|
|
232
|
+
Exclude<DatasetOperand, number>,
|
|
233
|
+
Exclude<DatasetOperand, number>
|
|
234
|
+
>
|
|
235
|
+
)
|
|
236
|
+
.optional()
|
|
237
|
+
.describe("The column or arithmetic to aggregate; omit for count."),
|
|
238
|
+
as: datasetIdentifierSchema.describe(
|
|
239
|
+
"The key the value comes back under; also what sort may name.",
|
|
240
|
+
),
|
|
241
|
+
/** Only rows matching these (after the query filter) enter this aggregate: SQL's `FILTER (WHERE ...)`. Over no rows, count and sum answer 0, avg/min/max null. */
|
|
242
|
+
filter: z.array(datasetFilterSchema).max(20).default([]),
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* One read. Column references beyond the dataset's own walk the relations
|
|
247
|
+
* named in `join`, as `relation.column`; a grouped query answers one row per
|
|
248
|
+
* group with the group columns and the aggregate aliases as keys.
|
|
249
|
+
*/
|
|
250
|
+
const datasetQueryObjectSchema = z.object({
|
|
251
|
+
from: datasetIdentifierSchema,
|
|
252
|
+
/** Relation paths the query reaches, e.g. `["department", "plan.ownerDepartment"]`; a path a column reference names is joined without being listed. */
|
|
253
|
+
join: z
|
|
254
|
+
.array(datasetColumnRefSchema)
|
|
255
|
+
.max(8)
|
|
256
|
+
.default([])
|
|
257
|
+
.describe("Relations to join, by name; referencing relation.column joins it too."),
|
|
258
|
+
/** Columns and named values to return for an ungrouped query; when absent, every column the definition lists (filled in before the query reaches a host). As many as a definition may have columns. */
|
|
259
|
+
select: z
|
|
260
|
+
.array(z.union([datasetColumnRefSchema, datasetNamedOperandSchema]))
|
|
261
|
+
.max(200)
|
|
262
|
+
.optional()
|
|
263
|
+
.describe(
|
|
264
|
+
"Columns or named values to return when not grouping; aggregates go in aggregate, not here.",
|
|
265
|
+
),
|
|
266
|
+
filter: z.array(datasetFilterSchema).max(20).default([]),
|
|
267
|
+
groupBy: z
|
|
268
|
+
.array(datasetGroupBySchema)
|
|
269
|
+
.max(8)
|
|
270
|
+
.default([])
|
|
271
|
+
.describe(
|
|
272
|
+
"Group keys: a column name, or { expr, as } for a value such as a truncated date.",
|
|
273
|
+
),
|
|
274
|
+
aggregate: z
|
|
275
|
+
.array(datasetAggregateSchema)
|
|
276
|
+
.max(10)
|
|
277
|
+
.default([])
|
|
278
|
+
.describe(
|
|
279
|
+
"One entry per computed value; a grouped row holds the group keys and these aliases.",
|
|
280
|
+
),
|
|
281
|
+
/**
|
|
282
|
+
* An aggregate over the answered rows that share a partition (every row
|
|
283
|
+
* when `partitionBy` is empty), answered on each of those rows: SQL's
|
|
284
|
+
* `fn(col) OVER (PARTITION BY ...)`. Names are the row's own: columns and
|
|
285
|
+
* select aliases on a plain query, group keys and aggregate aliases on a
|
|
286
|
+
* grouped one. Worked out after `having`, before `compute` and `qualify`.
|
|
287
|
+
*/
|
|
288
|
+
window: z
|
|
289
|
+
.array(
|
|
290
|
+
datasetAggregateSchema.extend({
|
|
291
|
+
partitionBy: z.array(datasetColumnRefSchema).max(4).default([]),
|
|
292
|
+
}),
|
|
293
|
+
)
|
|
294
|
+
.max(10)
|
|
295
|
+
.default([]),
|
|
296
|
+
/** Values worked out per row from the aggregate aliases, group keys and window aliases: `{ as, expr }`, where `having`, `qualify`, `rank` and `sort` may name them. */
|
|
297
|
+
compute: z.array(datasetNamedOperandSchema).max(10).default([]),
|
|
298
|
+
/** Conditions on the grouped rows: they name aggregate aliases, computed aliases or group keys. SQL's HAVING. */
|
|
299
|
+
having: z.array(datasetFilterSchema).max(20).default([]),
|
|
300
|
+
/** Conditions on the finished rows, window and computed aliases included, before `rank`, `sort` and `limit`: SQL's QUALIFY. */
|
|
301
|
+
qualify: z.array(datasetFilterSchema).max(20).default([]),
|
|
302
|
+
/** The first `limit` rows of each partition, in `sort` order, taken before the query's own sort and limit: SQL's RANK() OVER (PARTITION BY ... ORDER BY ...) <= n, so rows tied on every sort key share a rank and all stay within the limit. */
|
|
303
|
+
rank: z
|
|
304
|
+
.object({
|
|
305
|
+
partitionBy: z.array(datasetColumnRefSchema).min(1).max(4),
|
|
306
|
+
sort: z.array(datasetSortSchema).min(1).max(5),
|
|
307
|
+
limit: z.number().int().min(1).max(100),
|
|
308
|
+
})
|
|
309
|
+
.optional(),
|
|
310
|
+
sort: z
|
|
311
|
+
.array(datasetSortSchema)
|
|
312
|
+
.max(5)
|
|
313
|
+
.default([])
|
|
314
|
+
.describe(
|
|
315
|
+
"By a column or select alias, a group key, an aggregate alias or a computed alias.",
|
|
316
|
+
),
|
|
317
|
+
limit: z.number().int().min(1).optional(),
|
|
318
|
+
/** Rows to skip before `limit`; only with a `sort`, since the row order is undefined without one. */
|
|
319
|
+
offset: z.number().int().min(0).optional(),
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
/** How deep a query's JSON may nest, objects and arrays alike; far past what a statement needs. */
|
|
323
|
+
export const DATASET_QUERY_MAX_DEPTH = 32;
|
|
324
|
+
|
|
325
|
+
/** Whether `value` nests at most `budget` objects and arrays deep; the walk stops at the budget, so a deeper value costs no more than an allowed one. */
|
|
326
|
+
function nestsWithin(value: unknown, budget: number): boolean {
|
|
327
|
+
if (typeof value !== "object" || value === null) return true;
|
|
328
|
+
return budget > 0 && Object.values(value).every((item) => nestsWithin(item, budget - 1));
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
export type DatasetQueryInput = z.input<typeof datasetQueryObjectSchema>;
|
|
332
|
+
|
|
333
|
+
/** A query, refused before it is read when it nests past the budget, so a body nested thousands deep costs what one at the budget does. */
|
|
334
|
+
export const datasetQuerySchema = z
|
|
335
|
+
.custom<DatasetQueryInput>(
|
|
336
|
+
(value) => nestsWithin(value, DATASET_QUERY_MAX_DEPTH),
|
|
337
|
+
`A query nests at most ${DATASET_QUERY_MAX_DEPTH} levels deep; write a flatter expression`,
|
|
338
|
+
)
|
|
339
|
+
.pipe(datasetQueryObjectSchema);
|
|
340
|
+
export type DatasetQuery = z.infer<typeof datasetQuerySchema>;
|
|
341
|
+
|
|
342
|
+
/** What `POST {host}/{name}` receives: the validated query, capped, plus what the host may want in its own logs. */
|
|
343
|
+
export const datasetRequestSchema = z.object({
|
|
344
|
+
query: datasetQuerySchema,
|
|
345
|
+
context: z.object({
|
|
346
|
+
threadId: z.string().max(128),
|
|
347
|
+
requestId: z.string().max(128).optional(),
|
|
348
|
+
}),
|
|
349
|
+
});
|
|
350
|
+
export type DatasetRequest = z.infer<typeof datasetRequestSchema>;
|
|
351
|
+
|
|
352
|
+
export const datasetResultSchema = z.object({
|
|
353
|
+
/** Keyed by the column reference as written in the query, or the aggregate alias. */
|
|
354
|
+
rows: z.array(z.record(z.string(), z.unknown())),
|
|
355
|
+
rowCount: z.number().int().nonnegative(),
|
|
356
|
+
/** More rows matched than were returned. */
|
|
357
|
+
truncated: z.boolean(),
|
|
358
|
+
});
|
|
359
|
+
export type DatasetResult = z.infer<typeof datasetResultSchema>;
|
|
360
|
+
|
|
361
|
+
export const DATASET_ERROR_KINDS = [
|
|
362
|
+
"invalid_query",
|
|
363
|
+
"unsupported",
|
|
364
|
+
"forbidden",
|
|
365
|
+
"timeout",
|
|
366
|
+
"internal",
|
|
367
|
+
] as const;
|
|
368
|
+
export type DatasetErrorKind = (typeof DATASET_ERROR_KINDS)[number];
|
|
369
|
+
|
|
370
|
+
/** The HTTP status each error kind travels under, both from a host and from the server's own refusals. */
|
|
371
|
+
export const DATASET_ERROR_STATUS = {
|
|
372
|
+
invalid_query: 400,
|
|
373
|
+
unsupported: 400,
|
|
374
|
+
forbidden: 403,
|
|
375
|
+
timeout: 504,
|
|
376
|
+
internal: 500,
|
|
377
|
+
} as const satisfies Record<DatasetErrorKind, number>;
|
|
378
|
+
|
|
379
|
+
export const datasetErrorSchema = z.object({
|
|
380
|
+
error: z.object({
|
|
381
|
+
kind: z.enum(DATASET_ERROR_KINDS),
|
|
382
|
+
detail: z.string().max(512),
|
|
383
|
+
}),
|
|
384
|
+
});
|
|
385
|
+
export type DatasetError = z.infer<typeof datasetErrorSchema>;
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* A dataset host as an agent's published configuration carries it: the
|
|
389
|
+
* datasets the agent exposes, already the published manifest's definitions,
|
|
390
|
+
* and which of them ride in every turn. The Control Center runs the queries
|
|
391
|
+
* for a host it registered, so the server needs no URL or credential here.
|
|
392
|
+
*/
|
|
393
|
+
export const datasetHostDeclarationSchema = z.object({
|
|
394
|
+
name: datasetIdentifierSchema,
|
|
395
|
+
manifestVersion: datasetManifestSchema.shape.version,
|
|
396
|
+
datasets: z.array(datasetDefinitionSchema).max(500),
|
|
397
|
+
pinned: z.array(datasetIdentifierSchema).max(50).default([]),
|
|
398
|
+
});
|
|
399
|
+
export type DatasetHostDeclaration = z.infer<typeof datasetHostDeclarationSchema>;
|
|
400
|
+
|
|
401
|
+
/** The definitions a query may reach, by dataset name. */
|
|
402
|
+
export type DatasetCatalog = ReadonlyMap<string, DatasetDefinition>;
|
|
403
|
+
|
|
404
|
+
export function datasetCatalogOf(definitions: Iterable<DatasetDefinition>) {
|
|
405
|
+
return new Map([...definitions].map((definition) => [definition.name, definition]));
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** The operators a filter may use on a column of the given type. */
|
|
409
|
+
const OPERATORS_BY_TYPE = {
|
|
410
|
+
string: ["=", "!=", "in", "notIn", "contains", "startsWith", "endsWith", "isNull", "isNotNull"],
|
|
411
|
+
number: ["=", "!=", "<", "<=", ">", ">=", "in", "notIn", "between", "isNull", "isNotNull"],
|
|
412
|
+
date: ["=", "!=", "<", "<=", ">", ">=", "in", "notIn", "between", "isNull", "isNotNull"],
|
|
413
|
+
boolean: ["=", "!=", "isNull", "isNotNull"],
|
|
414
|
+
} as const satisfies Record<DatasetColumnType, readonly DatasetFilterOp[]>;
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* The column a reference names, walking `relation.relation.column` through
|
|
418
|
+
* the catalog, or why it names none. `path` is the relation path the query
|
|
419
|
+
* has to join for it, empty for the dataset's own columns.
|
|
420
|
+
*/
|
|
421
|
+
export function resolveDatasetColumn(catalog: DatasetCatalog, from: string, ref: string) {
|
|
422
|
+
const segments = ref.split(".");
|
|
423
|
+
const columnName = segments.pop()!;
|
|
424
|
+
const missing = (error: string) => ({ ok: false as const, error });
|
|
425
|
+
let dataset = catalog.get(from);
|
|
426
|
+
if (!dataset) return missing(`Unknown dataset "${from}"`);
|
|
427
|
+
for (const segment of segments) {
|
|
428
|
+
const relation = dataset.relations[segment];
|
|
429
|
+
if (!relation) return missing(`"${dataset.name}" has no relation "${segment}"`);
|
|
430
|
+
const target = catalog.get(relation.dataset);
|
|
431
|
+
if (!target) {
|
|
432
|
+
return missing(`Relation "${segment}" points at unknown dataset "${relation.dataset}"`);
|
|
433
|
+
}
|
|
434
|
+
dataset = target;
|
|
435
|
+
}
|
|
436
|
+
const column = dataset.columns.find((candidate) => candidate.name === columnName);
|
|
437
|
+
if (!column) return missing(`"${dataset.name}" has no column "${columnName}"`);
|
|
438
|
+
return { ok: true as const, column, path: segments.join(".") };
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
function refError(kind: string, ref: string, reason: string) {
|
|
442
|
+
return `${kind} "${ref}": ${reason}`;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function filterValueIssue(filter: DatasetCondition) {
|
|
446
|
+
const { op, value } = filter;
|
|
447
|
+
if (op === "isNull" || op === "isNotNull") {
|
|
448
|
+
return value === undefined ? undefined : `${op} takes no value`;
|
|
449
|
+
}
|
|
450
|
+
if (op === "in" || op === "notIn") {
|
|
451
|
+
return Array.isArray(value) && value.length > 0
|
|
452
|
+
? undefined
|
|
453
|
+
: `${op} takes a non-empty list`;
|
|
454
|
+
}
|
|
455
|
+
if (op === "between") {
|
|
456
|
+
return Array.isArray(value) && value.length === 2 ? undefined : "between takes a pair";
|
|
457
|
+
}
|
|
458
|
+
return value !== undefined && !Array.isArray(value) ? undefined : `${op} takes one value`;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* The query as the host runs it: habits the model brings from SQL are
|
|
463
|
+
* absorbed here rather than refused. A reference prefixed with the dataset's
|
|
464
|
+
* own name (`orders.total` on `orders`) drops the prefix when no relation
|
|
465
|
+
* has that name; a relation any reference names is joined without being
|
|
466
|
+
* listed; a `select` on a grouped query that only repeats the group keys is
|
|
467
|
+
* dropped, and an ungrouped query without one selects the definition's
|
|
468
|
+
* columns. Every host receives this shape, so all of them agree on it.
|
|
469
|
+
*/
|
|
470
|
+
function normalizeDatasetQuery(catalog: DatasetCatalog, query: DatasetQuery) {
|
|
471
|
+
const dataset = catalog.get(query.from);
|
|
472
|
+
if (!dataset) return query;
|
|
473
|
+
const ownPrefix = `${query.from}.`;
|
|
474
|
+
const strip = (ref: string) =>
|
|
475
|
+
ref.startsWith(ownPrefix) && !Object.hasOwn(dataset.relations, query.from)
|
|
476
|
+
? ref.slice(ownPrefix.length)
|
|
477
|
+
: ref;
|
|
478
|
+
const stripFilter = (entry: DatasetFilter) => mapFilterRefs(entry, strip);
|
|
479
|
+
const stripNamed = (entry: DatasetNamedOperand) => ({
|
|
480
|
+
...entry,
|
|
481
|
+
expr: mapOperandRefs(entry.expr, strip),
|
|
482
|
+
});
|
|
483
|
+
const groupBy = query.groupBy.map((group) =>
|
|
484
|
+
typeof group === "string" ? strip(group) : stripNamed(group),
|
|
485
|
+
);
|
|
486
|
+
const aggregate = query.aggregate.map((entry) => ({
|
|
487
|
+
...entry,
|
|
488
|
+
...(entry.col === undefined
|
|
489
|
+
? {}
|
|
490
|
+
: { col: mapOperandRefs(entry.col, strip) as DatasetCondition["col"] }),
|
|
491
|
+
filter: entry.filter.map(stripFilter),
|
|
492
|
+
}));
|
|
493
|
+
const filter = query.filter.map(stripFilter);
|
|
494
|
+
const having = query.having.map(stripFilter);
|
|
495
|
+
const compute = query.compute.map(stripNamed);
|
|
496
|
+
const window = query.window.map((entry) => ({
|
|
497
|
+
...entry,
|
|
498
|
+
...(entry.col === undefined
|
|
499
|
+
? {}
|
|
500
|
+
: { col: mapOperandRefs(entry.col, strip) as DatasetCondition["col"] }),
|
|
501
|
+
filter: entry.filter.map(stripFilter),
|
|
502
|
+
partitionBy: entry.partitionBy.map(strip),
|
|
503
|
+
}));
|
|
504
|
+
const qualify = query.qualify.map(stripFilter);
|
|
505
|
+
const sort = query.sort.map((entry) => ({ ...entry, col: strip(entry.col) }));
|
|
506
|
+
const rank = query.rank && {
|
|
507
|
+
...query.rank,
|
|
508
|
+
partitionBy: query.rank.partitionBy.map(strip),
|
|
509
|
+
sort: query.rank.sort.map((entry) => ({ ...entry, col: strip(entry.col) })),
|
|
510
|
+
};
|
|
511
|
+
const grouped = groupBy.length > 0 || aggregate.length > 0;
|
|
512
|
+
const groupKeys = new Set(groupBy.map((group) => groupKeyName(group)));
|
|
513
|
+
const selected = query.select
|
|
514
|
+
?.map((entry) => (typeof entry === "string" ? strip(entry) : stripNamed(entry)))
|
|
515
|
+
.filter((entry) => !(grouped && typeof entry === "string" && groupKeys.has(entry)));
|
|
516
|
+
// An ungrouped query without a select reads the columns the catalog publishes, never whatever the host's table has grown since.
|
|
517
|
+
const select = selected?.length
|
|
518
|
+
? selected
|
|
519
|
+
: grouped
|
|
520
|
+
? undefined
|
|
521
|
+
: dataset.columns.map((column) => column.name);
|
|
522
|
+
const referenced = [
|
|
523
|
+
...(select ?? []).flatMap((entry) =>
|
|
524
|
+
typeof entry === "string" ? [entry] : operandRefs(entry.expr),
|
|
525
|
+
),
|
|
526
|
+
...filter.flatMap(filterRefs),
|
|
527
|
+
...groupBy.flatMap((group) =>
|
|
528
|
+
typeof group === "string" ? [group] : operandRefs(group.expr),
|
|
529
|
+
),
|
|
530
|
+
...aggregate.flatMap((entry) => [
|
|
531
|
+
...(entry.col === undefined ? [] : operandRefs(entry.col)),
|
|
532
|
+
...entry.filter.flatMap(filterRefs),
|
|
533
|
+
]),
|
|
534
|
+
...sort.map((entry) => entry.col),
|
|
535
|
+
...(rank ? [...rank.partitionBy, ...rank.sort.map((entry) => entry.col)] : []),
|
|
536
|
+
...window.flatMap((entry) => [
|
|
537
|
+
...(entry.col === undefined ? [] : operandRefs(entry.col)),
|
|
538
|
+
...entry.filter.flatMap(filterRefs),
|
|
539
|
+
...entry.partitionBy,
|
|
540
|
+
]),
|
|
541
|
+
...qualify.flatMap(filterRefs),
|
|
542
|
+
]
|
|
543
|
+
.filter((ref) => ref.includes("."))
|
|
544
|
+
.map((ref) => ref.slice(0, ref.lastIndexOf(".")));
|
|
545
|
+
const join = [...new Set([...query.join.map(strip), ...referenced])];
|
|
546
|
+
return {
|
|
547
|
+
...query,
|
|
548
|
+
join,
|
|
549
|
+
select,
|
|
550
|
+
filter,
|
|
551
|
+
groupBy,
|
|
552
|
+
aggregate,
|
|
553
|
+
having,
|
|
554
|
+
compute,
|
|
555
|
+
window,
|
|
556
|
+
qualify,
|
|
557
|
+
sort,
|
|
558
|
+
...(rank ? { rank } : {}),
|
|
559
|
+
} satisfies DatasetQuery;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/** The key a group answers under: the column reference, or the value's alias. */
|
|
563
|
+
export function groupKeyName(group: DatasetQuery["groupBy"][number]) {
|
|
564
|
+
return typeof group === "string" ? group : group.as;
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
function mapOperandRefs(operand: DatasetOperand, fn: (ref: string) => string): DatasetOperand {
|
|
568
|
+
if (typeof operand === "string") return fn(operand);
|
|
569
|
+
if (typeof operand === "number" || "value" in operand) return operand;
|
|
570
|
+
if ("op" in operand) {
|
|
571
|
+
return {
|
|
572
|
+
op: operand.op,
|
|
573
|
+
left: mapOperandRefs(operand.left, fn),
|
|
574
|
+
right: mapOperandRefs(operand.right, fn),
|
|
575
|
+
};
|
|
576
|
+
}
|
|
577
|
+
if ("case" in operand) {
|
|
578
|
+
return {
|
|
579
|
+
case: operand.case.map((branch) => ({
|
|
580
|
+
when: mapFilterRefs(branch.when, fn),
|
|
581
|
+
then: mapOperandRefs(branch.then, fn),
|
|
582
|
+
})),
|
|
583
|
+
...(operand.else === undefined ? {} : { else: mapOperandRefs(operand.else, fn) }),
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
if ("args" in operand) {
|
|
587
|
+
return { fn: operand.fn, args: operand.args.map((arg) => mapOperandRefs(arg, fn)) };
|
|
588
|
+
}
|
|
589
|
+
return { ...operand, of: mapOperandRefs(operand.of, fn) };
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
function mapFilterRefs(filter: DatasetFilter, fn: (ref: string) => string): DatasetFilter {
|
|
593
|
+
if ("any" in filter) return { any: filter.any.map((entry) => mapFilterRefs(entry, fn)) };
|
|
594
|
+
if ("all" in filter) return { all: filter.all.map((entry) => mapFilterRefs(entry, fn)) };
|
|
595
|
+
if ("not" in filter) return { not: mapFilterRefs(filter.not, fn) };
|
|
596
|
+
const value =
|
|
597
|
+
typeof filter.value === "object" && filter.value !== null && !Array.isArray(filter.value)
|
|
598
|
+
? { col: mapOperandRefs(filter.value.col, fn) }
|
|
599
|
+
: filter.value;
|
|
600
|
+
return {
|
|
601
|
+
col: mapOperandRefs(filter.col, fn) as DatasetCondition["col"],
|
|
602
|
+
op: filter.op,
|
|
603
|
+
...(value === undefined ? {} : { value }),
|
|
604
|
+
};
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/** Every column reference an operand names. */
|
|
608
|
+
export function operandRefs(operand: DatasetOperand): string[] {
|
|
609
|
+
if (typeof operand === "string") return [operand];
|
|
610
|
+
if (typeof operand === "number" || "value" in operand) return [];
|
|
611
|
+
if ("op" in operand) return [...operandRefs(operand.left), ...operandRefs(operand.right)];
|
|
612
|
+
if ("case" in operand) {
|
|
613
|
+
return [
|
|
614
|
+
...operand.case.flatMap((branch) => [
|
|
615
|
+
...filterRefs(branch.when),
|
|
616
|
+
...operandRefs(branch.then),
|
|
617
|
+
]),
|
|
618
|
+
...(operand.else === undefined ? [] : operandRefs(operand.else)),
|
|
619
|
+
];
|
|
620
|
+
}
|
|
621
|
+
if ("args" in operand) return operand.args.flatMap(operandRefs);
|
|
622
|
+
return operandRefs(operand.of);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/** Every column reference a filter names, on either side, at any depth. */
|
|
626
|
+
export function filterRefs(filter: DatasetFilter): string[] {
|
|
627
|
+
if ("any" in filter) return filter.any.flatMap(filterRefs);
|
|
628
|
+
if ("all" in filter) return filter.all.flatMap(filterRefs);
|
|
629
|
+
if ("not" in filter) return filterRefs(filter.not);
|
|
630
|
+
const right = isColumnValue(filter.value) ? operandRefs(filter.value.col) : [];
|
|
631
|
+
return [...operandRefs(filter.col), ...right];
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
/** A condition's right side that names a column or arithmetic rather than a literal. */
|
|
635
|
+
export function isColumnValue(value: DatasetCondition["value"]): value is { col: DatasetOperand } {
|
|
636
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/**
|
|
640
|
+
* What every reader of a query does before acting on it: normalizes,
|
|
641
|
+
* validates, and caps `limit` at the dataset's `maxRows` whatever the caller
|
|
642
|
+
* sent. The answer is bounded to that `limit` with `boundDatasetResult`.
|
|
643
|
+
*/
|
|
644
|
+
export function checkDatasetQuery(catalog: DatasetCatalog, raw: DatasetQuery) {
|
|
645
|
+
const normalized = normalizeDatasetQuery(catalog, raw);
|
|
646
|
+
const maxRows = catalog.get(raw.from)?.maxRows ?? DATASET_MAX_ROWS_DEFAULT;
|
|
647
|
+
return {
|
|
648
|
+
query: { ...normalized, limit: Math.min(normalized.limit ?? maxRows, maxRows) },
|
|
649
|
+
issues: validateDatasetQuery(catalog, normalized),
|
|
650
|
+
};
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
/** An answer cut to the query's `limit`: a host that sent more rows than asked is trimmed and marked truncated, never trusted to have capped itself. */
|
|
654
|
+
export function boundDatasetResult(
|
|
655
|
+
result: Omit<DatasetResult, "rows"> & { rows: ReadonlyArray<DatasetResult["rows"][number]> },
|
|
656
|
+
limit: number,
|
|
657
|
+
) {
|
|
658
|
+
return {
|
|
659
|
+
rows: result.rows.slice(0, limit),
|
|
660
|
+
rowCount: Math.min(result.rowCount, limit),
|
|
661
|
+
truncated: result.truncated || result.rows.length > limit,
|
|
662
|
+
} satisfies DatasetResult;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
/**
|
|
666
|
+
* Every way a query can be wrong given the catalog, as messages the model
|
|
667
|
+
* can act on; an empty list means the host will understand it. Checked
|
|
668
|
+
* before anything leaves the server, so a host can trust the shape and only
|
|
669
|
+
* has to translate.
|
|
670
|
+
*/
|
|
671
|
+
export function validateDatasetQuery(catalog: DatasetCatalog, query: DatasetQuery) {
|
|
672
|
+
if (!catalog.has(query.from)) return [`Unknown dataset "${query.from}"`];
|
|
673
|
+
const issues: string[] = [];
|
|
674
|
+
const joined = joinedPaths(catalog, query, issues);
|
|
675
|
+
const check = {
|
|
676
|
+
issue: (message: string) => issues.push(message),
|
|
677
|
+
columnOf(kind, ref) {
|
|
678
|
+
const resolved = resolveDatasetColumn(catalog, query.from, ref);
|
|
679
|
+
if (!resolved.ok) {
|
|
680
|
+
issues.push(refError(kind, ref, resolved.error));
|
|
681
|
+
return undefined;
|
|
682
|
+
}
|
|
683
|
+
if (resolved.path && !joined.has(resolved.path)) {
|
|
684
|
+
issues.push(
|
|
685
|
+
refError(
|
|
686
|
+
kind,
|
|
687
|
+
ref,
|
|
688
|
+
`relation "${resolved.path}" is not among this dataset's relations`,
|
|
689
|
+
),
|
|
690
|
+
);
|
|
691
|
+
return undefined;
|
|
692
|
+
}
|
|
693
|
+
return resolved.column;
|
|
694
|
+
},
|
|
695
|
+
} satisfies QueryCheck;
|
|
696
|
+
|
|
697
|
+
for (const filter of query.filter) checkFilter(check, filter);
|
|
698
|
+
const groupKeys = checkGroupBy(check, query.groupBy);
|
|
699
|
+
const aliases = checkAggregates(check, query.aggregate);
|
|
700
|
+
for (const alias of aliases.keys()) {
|
|
701
|
+
if (groupKeys.has(alias)) check.issue(`aggregate alias "${alias}" is also a group key`);
|
|
702
|
+
}
|
|
703
|
+
const grouped = query.groupBy.length > 0 || query.aggregate.length > 0;
|
|
704
|
+
if (query.having.length && !grouped) check.issue("having needs a grouped query");
|
|
705
|
+
if (query.compute.length && !grouped && query.window.length === 0) {
|
|
706
|
+
check.issue("compute needs a grouped query or a window");
|
|
707
|
+
}
|
|
708
|
+
// Models send every key; an empty select means the same as none.
|
|
709
|
+
if (grouped && query.select?.length) {
|
|
710
|
+
check.issue(
|
|
711
|
+
"a grouped query answers the group keys, the aggregate aliases and the computed aliases; put computed values in aggregate or compute and leave select out",
|
|
712
|
+
);
|
|
713
|
+
}
|
|
714
|
+
const selectAliases = checkSelect(check, query.select ?? []);
|
|
715
|
+
const row = grouped
|
|
716
|
+
? { names: new Map([...groupKeys, ...aliases]), kind: "grouped" as const }
|
|
717
|
+
: { names: selectAliases, kind: "plain" as const };
|
|
718
|
+
checkFinishedRow(check, query, row);
|
|
719
|
+
checkExpansion(check, query, catalog.get(query.from)?.columns.length ?? 0);
|
|
720
|
+
return issues;
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/** How large any one expression a host translates may grow once every alias it names is written out in place. */
|
|
724
|
+
export const DATASET_EXPANSION_BUDGET = 2_000;
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* Hosts write a computed alias out again wherever a later calculation or
|
|
728
|
+
* HAVING names it, as SQL requires, so ten aliases each naming the previous
|
|
729
|
+
* one ten times grow to 10^10 nodes from a few hundred bytes, with or without
|
|
730
|
+
* a column among them. Each alias costs what it expands to, worked out once in
|
|
731
|
+
* order, so the expansion is measured without being performed: a named select
|
|
732
|
+
* value or group key, an aggregate and a window cost their own trees wherever
|
|
733
|
+
* they are named, since hosts write them out. Every expression a host
|
|
734
|
+
* translates is held to the budget: select values, group keys, WHERE,
|
|
735
|
+
* aggregates and their filters, windows, calculations, HAVING and QUALIFY.
|
|
736
|
+
* Sort and rank name values and add none. A window is a correlated subquery
|
|
737
|
+
* that carries the row it counts over, so it also costs that row: WHERE and
|
|
738
|
+
* every value the row projects (`columnCount` of them for a plain query with
|
|
739
|
+
* no select).
|
|
740
|
+
*/
|
|
741
|
+
function checkExpansion(check: QueryCheck, query: DatasetQuery, columnCount: number) {
|
|
742
|
+
// The source row: what WHERE, select values, group keys and aggregates read, each column once.
|
|
743
|
+
const source = new Map<string, number>();
|
|
744
|
+
const named = [...(query.select ?? []), ...query.groupBy].flatMap((entry) =>
|
|
745
|
+
typeof entry === "string" ? [] : [[entry.as, operandCost(entry.expr, source)] as const],
|
|
746
|
+
);
|
|
747
|
+
const where = query.filter.map((filter) => filterCost(filter, source));
|
|
748
|
+
const costs = [...named.map(([, total]) => total), ...where];
|
|
749
|
+
const expanded = new Map(named);
|
|
750
|
+
const aggregates = query.aggregate.map((entry) => {
|
|
751
|
+
const parts = [
|
|
752
|
+
...(entry.col === undefined ? [] : [operandCost(entry.col, source)]),
|
|
753
|
+
...entry.filter.map((filter) => filterCost(filter, source)),
|
|
754
|
+
];
|
|
755
|
+
costs.push(...parts);
|
|
756
|
+
expanded.set(entry.as, nodeCost(parts));
|
|
757
|
+
return nodeCost(parts);
|
|
758
|
+
});
|
|
759
|
+
const grouped = query.groupBy.length > 0 || query.aggregate.length > 0;
|
|
760
|
+
const projected = grouped ? query.groupBy : (query.select ?? []);
|
|
761
|
+
const carried = Math.min(
|
|
762
|
+
DATASET_EXPANSION_BUDGET + 1,
|
|
763
|
+
[
|
|
764
|
+
...where,
|
|
765
|
+
...projected.map((entry) =>
|
|
766
|
+
typeof entry === "string" ? 1 : (expanded.get(entry.as) ?? 1),
|
|
767
|
+
),
|
|
768
|
+
...(grouped || projected.length ? [] : Array<number>(columnCount).fill(1)),
|
|
769
|
+
...aggregates,
|
|
770
|
+
].reduce((total, cost) => total + cost, 0),
|
|
771
|
+
);
|
|
772
|
+
// Every window reads the row as it was before any window, its partition and the row it carries included.
|
|
773
|
+
const windows = query.window.map((entry) => {
|
|
774
|
+
const total = nodeCost([
|
|
775
|
+
...(entry.col === undefined ? [] : [operandCost(entry.col, expanded)]),
|
|
776
|
+
...entry.filter.map((filter) => filterCost(filter, expanded)),
|
|
777
|
+
...entry.partitionBy.map((name) => operandCost(name, expanded)),
|
|
778
|
+
carried,
|
|
779
|
+
]);
|
|
780
|
+
costs.push(total);
|
|
781
|
+
return [entry.as, total] as const;
|
|
782
|
+
});
|
|
783
|
+
for (const [as, total] of windows) expanded.set(as, total);
|
|
784
|
+
for (const entry of query.compute) {
|
|
785
|
+
const total = operandCost(entry.expr, expanded);
|
|
786
|
+
expanded.set(entry.as, total);
|
|
787
|
+
costs.push(total);
|
|
788
|
+
}
|
|
789
|
+
for (const filter of query.having) costs.push(filterCost(filter, expanded));
|
|
790
|
+
// A grouped row is materialized before QUALIFY reads it; without group keys the
|
|
791
|
+
// .NET host writes the calculations QUALIFY names out in full.
|
|
792
|
+
const qualifying = query.groupBy.length ? source : expanded;
|
|
793
|
+
for (const filter of query.qualify) costs.push(filterCost(filter, qualifying));
|
|
794
|
+
if (costs.some((total) => total > DATASET_EXPANSION_BUDGET)) {
|
|
795
|
+
check.issue(
|
|
796
|
+
`Computed aliases expand to more than ${DATASET_EXPANSION_BUDGET} references; name intermediate results and reuse them less`,
|
|
797
|
+
);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/**
|
|
802
|
+
* How many times a host writes one part out when it spells the node holding
|
|
803
|
+
* it, the most either host does; every other part is written once. A divisor
|
|
804
|
+
* is tested for zero before the division, ROUND's digits scale the value
|
|
805
|
+
* before and after, and LENGTH's text is tested for NULL besides measured. A
|
|
806
|
+
* date is read once per part a truncation or part needs (Zeller's congruence
|
|
807
|
+
* for the week and weekday), and a truncation other than the week, packed as
|
|
808
|
+
* a number, is read 3 more times when it meets a real date in a COALESCE, a
|
|
809
|
+
* CASE or a comparison, so those factors are the product. A condition guards
|
|
810
|
+
* its sides against NULL, and a negated condition with no opposite operator
|
|
811
|
+
* (`between`, the text matches) tests its left side once more.
|
|
812
|
+
*/
|
|
813
|
+
const EXPANSION_WEIGHTS = {
|
|
814
|
+
divisor: 2,
|
|
815
|
+
roundDigits: 2,
|
|
816
|
+
lengthText: 2,
|
|
817
|
+
conditionLeft: 3,
|
|
818
|
+
conditionRight: 2,
|
|
819
|
+
dateTrunc: { day: 9, month: 6, quarter: 6, year: 3, week: 17 },
|
|
820
|
+
datePart: { day: 1, month: 1, quarter: 1, year: 1, dayOfWeek: 16, week: 17 },
|
|
821
|
+
} as const;
|
|
822
|
+
|
|
823
|
+
/** One node plus its children's costs, held at one past the budget so no sum can overflow. */
|
|
824
|
+
function nodeCost(children: number[]) {
|
|
825
|
+
return Math.min(
|
|
826
|
+
DATASET_EXPANSION_BUDGET + 1,
|
|
827
|
+
children.reduce((total, cost) => total + cost, 1),
|
|
828
|
+
);
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
/** How often a scalar function's argument is written out. */
|
|
832
|
+
function argumentWeight(fn: DatasetFunction, index: number) {
|
|
833
|
+
if (fn === "length") return EXPANSION_WEIGHTS.lengthText;
|
|
834
|
+
return fn === "round" && index === 1 ? EXPANSION_WEIGHTS.roundDigits : 1;
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
/**
|
|
838
|
+
* An operand's size written out: a literal or a column is 1, a reference to an
|
|
839
|
+
* earlier computed alias is that alias's cost (a self or forward one, refused
|
|
840
|
+
* elsewhere, is 1), and arithmetic, a date function, a scalar function, a CASE
|
|
841
|
+
* and each of its branches are 1 plus their parts, each part times its weight.
|
|
842
|
+
*/
|
|
843
|
+
function operandCost(operand: DatasetOperand, expanded: ReadonlyMap<string, number>): number {
|
|
844
|
+
if (typeof operand === "string") return expanded.get(operand) ?? 1;
|
|
845
|
+
if (typeof operand === "number" || "value" in operand) return 1;
|
|
846
|
+
if ("op" in operand) {
|
|
847
|
+
const divisor = operand.op === "/" ? EXPANSION_WEIGHTS.divisor : 1;
|
|
848
|
+
return nodeCost([
|
|
849
|
+
operandCost(operand.left, expanded),
|
|
850
|
+
operandCost(operand.right, expanded) * divisor,
|
|
851
|
+
]);
|
|
852
|
+
}
|
|
853
|
+
if ("case" in operand) {
|
|
854
|
+
return nodeCost([
|
|
855
|
+
...operand.case.map((branch) =>
|
|
856
|
+
nodeCost([filterCost(branch.when, expanded), operandCost(branch.then, expanded)]),
|
|
857
|
+
),
|
|
858
|
+
...(operand.else === undefined ? [] : [operandCost(operand.else, expanded)]),
|
|
859
|
+
]);
|
|
860
|
+
}
|
|
861
|
+
if ("args" in operand) {
|
|
862
|
+
return nodeCost(
|
|
863
|
+
operand.args.map(
|
|
864
|
+
(arg, index) => operandCost(arg, expanded) * argumentWeight(operand.fn, index),
|
|
865
|
+
),
|
|
866
|
+
);
|
|
867
|
+
}
|
|
868
|
+
const weight =
|
|
869
|
+
operand.fn === "dateTrunc"
|
|
870
|
+
? EXPANSION_WEIGHTS.dateTrunc[operand.unit]
|
|
871
|
+
: EXPANSION_WEIGHTS.datePart[operand.unit];
|
|
872
|
+
return nodeCost([operandCost(operand.of, expanded) * weight]);
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
/** A filter's size written out: 1 per condition or group plus its parts; a condition's left side is weighted, each literal on its right is 1, and a `{ col }` right side is its weighted operand cost. */
|
|
876
|
+
function filterCost(filter: DatasetFilter, expanded: ReadonlyMap<string, number>): number {
|
|
877
|
+
if ("not" in filter) return nodeCost([filterCost(filter.not, expanded)]);
|
|
878
|
+
if ("any" in filter || "all" in filter) {
|
|
879
|
+
const entries = "any" in filter ? filter.any : filter.all;
|
|
880
|
+
return nodeCost(entries.map((entry) => filterCost(entry, expanded)));
|
|
881
|
+
}
|
|
882
|
+
const right = isColumnValue(filter.value)
|
|
883
|
+
? operandCost(filter.value.col, expanded) * EXPANSION_WEIGHTS.conditionRight
|
|
884
|
+
: Array.isArray(filter.value)
|
|
885
|
+
? filter.value.length
|
|
886
|
+
: filter.value === undefined
|
|
887
|
+
? 0
|
|
888
|
+
: 1;
|
|
889
|
+
return nodeCost([operandCost(filter.col, expanded) * EXPANSION_WEIGHTS.conditionLeft, right]);
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
/**
|
|
893
|
+
* The row a window, a calculation, QUALIFY, sort and rank see: on a grouped
|
|
894
|
+
* query its keys and aggregates, on a plain one its columns and select
|
|
895
|
+
* aliases. Windows and calculations join it as they are worked out.
|
|
896
|
+
*/
|
|
897
|
+
function checkFinishedRow(
|
|
898
|
+
check: QueryCheck,
|
|
899
|
+
query: DatasetQuery,
|
|
900
|
+
row: { names: Map<string, DatasetColumnType>; kind: "grouped" | "plain" },
|
|
901
|
+
) {
|
|
902
|
+
const rowCheck = rowScope(check, row);
|
|
903
|
+
// On a plain row a selected column is a name too; a window or calculation may not take it.
|
|
904
|
+
const taken = new Set(
|
|
905
|
+
(query.select ?? []).flatMap((entry) => (typeof entry === "string" ? [entry] : [])),
|
|
906
|
+
);
|
|
907
|
+
checkWindows(rowCheck, query, row.names, taken);
|
|
908
|
+
checkComputeAndHaving(
|
|
909
|
+
rowCheck,
|
|
910
|
+
row.kind === "grouped" ? rowCheck : check,
|
|
911
|
+
query,
|
|
912
|
+
row.names,
|
|
913
|
+
taken,
|
|
914
|
+
);
|
|
915
|
+
for (const filter of query.qualify) checkFilter(rowCheck, filter, "qualify");
|
|
916
|
+
checkRowOrder(check, query, row);
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
/** A check whose references are the answered row's names; on a plain query a column is one too. */
|
|
920
|
+
function rowScope(check: QueryCheck, row: RowNames) {
|
|
921
|
+
return {
|
|
922
|
+
issue: (message: string) => check.issue(message),
|
|
923
|
+
columnOf(kind, ref) {
|
|
924
|
+
const type = row.names.get(ref);
|
|
925
|
+
if (type) return { name: ref, type, description: "" };
|
|
926
|
+
if (row.kind === "plain") return check.columnOf(kind, ref);
|
|
927
|
+
check.issue(
|
|
928
|
+
refError(
|
|
929
|
+
kind,
|
|
930
|
+
ref,
|
|
931
|
+
`${kind} names a group key, an aggregate alias, a window alias or a computed alias`,
|
|
932
|
+
),
|
|
933
|
+
);
|
|
934
|
+
return undefined;
|
|
935
|
+
},
|
|
936
|
+
} satisfies QueryCheck;
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
/**
|
|
940
|
+
* Each window over the row it partitions, typed like an aggregate. Every
|
|
941
|
+
* window reads the row as it was before any window (SQL nests none), so
|
|
942
|
+
* their aliases join the row only once all are checked.
|
|
943
|
+
*/
|
|
944
|
+
function checkWindows(
|
|
945
|
+
check: QueryCheck,
|
|
946
|
+
query: DatasetQuery,
|
|
947
|
+
rowValues: Map<string, DatasetColumnType>,
|
|
948
|
+
taken: ReadonlySet<string>,
|
|
949
|
+
) {
|
|
950
|
+
const answered = new Map<string, DatasetColumnType>();
|
|
951
|
+
// A window's own name, unless it is already the row's (that is the duplicate the loop reports).
|
|
952
|
+
const names = new Set(
|
|
953
|
+
query.window.flatMap((entry) =>
|
|
954
|
+
rowValues.has(entry.as) || taken.has(entry.as) ? [] : [entry.as],
|
|
955
|
+
),
|
|
956
|
+
);
|
|
957
|
+
const beforeWindows = {
|
|
958
|
+
issue: (message: string) => check.issue(message),
|
|
959
|
+
columnOf(kind, ref) {
|
|
960
|
+
if (!names.has(ref)) return check.columnOf(kind, ref);
|
|
961
|
+
check.issue(refError(kind, ref, "a window cannot read another window"));
|
|
962
|
+
return undefined;
|
|
963
|
+
},
|
|
964
|
+
} satisfies QueryCheck;
|
|
965
|
+
for (const entry of query.window) {
|
|
966
|
+
if (rowValues.has(entry.as) || answered.has(entry.as) || taken.has(entry.as)) {
|
|
967
|
+
check.issue(`window alias "${entry.as}" is used twice`);
|
|
968
|
+
}
|
|
969
|
+
if (entry.fn === "countDistinct") {
|
|
970
|
+
check.issue(
|
|
971
|
+
`window "${entry.as}": countDistinct cannot be a window; SQL allows no DISTINCT in OVER`,
|
|
972
|
+
);
|
|
973
|
+
}
|
|
974
|
+
for (const filter of entry.filter)
|
|
975
|
+
checkFilter(beforeWindows, filter, `window "${entry.as}" filter`);
|
|
976
|
+
for (const name of entry.partitionBy)
|
|
977
|
+
beforeWindows.columnOf(`window "${entry.as}" partitionBy`, name);
|
|
978
|
+
answered.set(entry.as, aggregateType(beforeWindows, entry, `window "${entry.as}"`));
|
|
979
|
+
}
|
|
980
|
+
for (const [as, type] of answered) rowValues.set(as, type);
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
/**
|
|
984
|
+
* Calculations are typed in order so a later one may name an earlier one, and
|
|
985
|
+
* join the row; HAVING then sees the grouped row (keys, aggregates and the
|
|
986
|
+
* calculations over them), never a window or a calculation over one, which
|
|
987
|
+
* are worked out after it and belong in QUALIFY.
|
|
988
|
+
*/
|
|
989
|
+
function checkComputeAndHaving(
|
|
990
|
+
rowCheck: QueryCheck,
|
|
991
|
+
havingCheck: QueryCheck,
|
|
992
|
+
query: DatasetQuery,
|
|
993
|
+
rowValues: Map<string, DatasetColumnType>,
|
|
994
|
+
taken: ReadonlySet<string>,
|
|
995
|
+
) {
|
|
996
|
+
const windows = new Set(query.window.map((entry) => entry.as));
|
|
997
|
+
const late = new Set(windows);
|
|
998
|
+
const later = new Set(query.compute.map((entry) => entry.as));
|
|
999
|
+
for (const entry of query.compute) {
|
|
1000
|
+
later.delete(entry.as);
|
|
1001
|
+
if (rowValues.has(entry.as) || taken.has(entry.as)) {
|
|
1002
|
+
rowCheck.issue(`compute alias "${entry.as}" is used twice`);
|
|
1003
|
+
}
|
|
1004
|
+
const refs = operandRefs(entry.expr);
|
|
1005
|
+
const ahead = refs.find(
|
|
1006
|
+
(ref) => (later.has(ref) || ref === entry.as) && !rowValues.has(ref),
|
|
1007
|
+
);
|
|
1008
|
+
if (ahead !== undefined) {
|
|
1009
|
+
rowCheck.issue(
|
|
1010
|
+
refError(
|
|
1011
|
+
"compute",
|
|
1012
|
+
ahead,
|
|
1013
|
+
"is computed after this entry; order compute so each names only earlier ones",
|
|
1014
|
+
),
|
|
1015
|
+
);
|
|
1016
|
+
}
|
|
1017
|
+
if (refs.some((ref) => late.has(ref))) late.add(entry.as);
|
|
1018
|
+
const type = operandType(rowCheck, "compute", entry.expr);
|
|
1019
|
+
rowValues.set(entry.as, type ?? "number");
|
|
1020
|
+
}
|
|
1021
|
+
const havingRow = {
|
|
1022
|
+
issue: (message: string) => havingCheck.issue(message),
|
|
1023
|
+
columnOf(kind, ref) {
|
|
1024
|
+
if (late.has(ref)) {
|
|
1025
|
+
havingCheck.issue(
|
|
1026
|
+
refError(kind, ref, "is worked out after HAVING; filter on it in QUALIFY"),
|
|
1027
|
+
);
|
|
1028
|
+
return undefined;
|
|
1029
|
+
}
|
|
1030
|
+
return havingCheck.columnOf(kind, ref);
|
|
1031
|
+
},
|
|
1032
|
+
} satisfies QueryCheck;
|
|
1033
|
+
for (const filter of query.having) checkFilter(havingRow, filter, "having");
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
/** Every selected column must be reachable and every named value typed; answers the names the row gains. */
|
|
1037
|
+
function checkSelect(check: QueryCheck, select: NonNullable<DatasetQuery["select"]>) {
|
|
1038
|
+
const aliases = new Map<string, DatasetColumnType>();
|
|
1039
|
+
const columns = new Set(select.filter((entry) => typeof entry === "string"));
|
|
1040
|
+
for (const entry of select) {
|
|
1041
|
+
if (typeof entry === "string") {
|
|
1042
|
+
check.columnOf("select", entry);
|
|
1043
|
+
continue;
|
|
1044
|
+
}
|
|
1045
|
+
if (aliases.has(entry.as) || columns.has(entry.as)) {
|
|
1046
|
+
check.issue(`select alias "${entry.as}" is used twice`);
|
|
1047
|
+
}
|
|
1048
|
+
aliases.set(entry.as, operandType(check, "select", entry.expr) ?? "string");
|
|
1049
|
+
}
|
|
1050
|
+
return aliases;
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
/** Sort, rank and offset all speak in the answered row's names. */
|
|
1054
|
+
function checkRowOrder(check: QueryCheck, query: DatasetQuery, row: RowNames) {
|
|
1055
|
+
checkSort(check, query.sort, "sort", row);
|
|
1056
|
+
if (query.rank) {
|
|
1057
|
+
for (const col of query.rank.partitionBy) checkRowName(check, "rank", col, row);
|
|
1058
|
+
checkSort(check, query.rank.sort, "rank", row);
|
|
1059
|
+
}
|
|
1060
|
+
if (query.offset !== undefined && query.sort.length === 0) {
|
|
1061
|
+
check.issue("offset needs a sort; without one the rows have no order to skip in");
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
/** What each clause check needs: a resolver that records why a reference fails, and a place for other issues. */
|
|
1066
|
+
type QueryCheck = {
|
|
1067
|
+
issue(message: string): void;
|
|
1068
|
+
columnOf(kind: string, ref: string): DatasetColumn | undefined;
|
|
1069
|
+
};
|
|
1070
|
+
|
|
1071
|
+
/** The relation paths the query joins, prefixes included: `plan.ownerDepartment` reaches `plan` too. */
|
|
1072
|
+
function joinedPaths(catalog: DatasetCatalog, query: DatasetQuery, issues: string[]) {
|
|
1073
|
+
const joined = new Set<string>();
|
|
1074
|
+
for (const path of query.join) {
|
|
1075
|
+
const probe = resolveDatasetColumn(
|
|
1076
|
+
catalog,
|
|
1077
|
+
query.from,
|
|
1078
|
+
`${path}.${firstColumnOf(catalog, query.from, path)}`,
|
|
1079
|
+
);
|
|
1080
|
+
if (!probe.ok) {
|
|
1081
|
+
issues.push(refError("join", path, probe.error));
|
|
1082
|
+
continue;
|
|
1083
|
+
}
|
|
1084
|
+
const segments = path.split(".");
|
|
1085
|
+
segments.forEach((_, index) => joined.add(segments.slice(0, index + 1).join(".")));
|
|
1086
|
+
}
|
|
1087
|
+
return joined;
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
/** An operand as the model wrote it, for messages: `a / target`. */
|
|
1091
|
+
export function describeOperand(operand: DatasetOperand): string {
|
|
1092
|
+
if (typeof operand !== "object") return String(operand);
|
|
1093
|
+
if ("value" in operand) return JSON.stringify(operand.value);
|
|
1094
|
+
if ("op" in operand) {
|
|
1095
|
+
return `${describeOperand(operand.left)} ${operand.op} ${describeOperand(operand.right)}`;
|
|
1096
|
+
}
|
|
1097
|
+
if ("case" in operand) return "CASE ... END";
|
|
1098
|
+
if ("args" in operand) return `${operand.fn}(${operand.args.map(describeOperand).join(", ")})`;
|
|
1099
|
+
return `${operand.fn}(${operand.unit}, ${describeOperand(operand.of)})`;
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
/** What each scalar function takes and answers; `coalesce` answers its arguments' shared type. */
|
|
1103
|
+
export const FUNCTION_SIGNATURES = {
|
|
1104
|
+
lower: { args: ["string"], answers: "string" },
|
|
1105
|
+
upper: { args: ["string"], answers: "string" },
|
|
1106
|
+
trim: { args: ["string"], answers: "string" },
|
|
1107
|
+
length: { args: ["string"], answers: "number" },
|
|
1108
|
+
abs: { args: ["number"], answers: "number" },
|
|
1109
|
+
round: { args: ["number", "number"], optional: 1, answers: "number" },
|
|
1110
|
+
coalesce: { args: [], answers: undefined },
|
|
1111
|
+
} as const satisfies Record<
|
|
1112
|
+
DatasetFunction,
|
|
1113
|
+
{
|
|
1114
|
+
args: readonly DatasetColumnType[];
|
|
1115
|
+
optional?: number;
|
|
1116
|
+
answers: DatasetColumnType | undefined;
|
|
1117
|
+
}
|
|
1118
|
+
>;
|
|
1119
|
+
|
|
1120
|
+
/**
|
|
1121
|
+
* The type an operand yields: the column's own, number for a literal number
|
|
1122
|
+
* or arithmetic, a date part, a truncated date, a function's answer, or the
|
|
1123
|
+
* type every branch of a CASE shares. Undefined once an issue was recorded.
|
|
1124
|
+
*/
|
|
1125
|
+
function operandType(
|
|
1126
|
+
check: QueryCheck,
|
|
1127
|
+
kind: string,
|
|
1128
|
+
operand: DatasetOperand,
|
|
1129
|
+
): DatasetColumnType | undefined {
|
|
1130
|
+
if (typeof operand === "number") return "number";
|
|
1131
|
+
if (typeof operand === "string") return check.columnOf(kind, operand)?.type;
|
|
1132
|
+
if ("value" in operand) return typeof operand.value === "string" ? "string" : "boolean";
|
|
1133
|
+
const shown = describeOperand(operand);
|
|
1134
|
+
const expect = (side: DatasetOperand, wanted: DatasetColumnType, what: string) => {
|
|
1135
|
+
const type = operandType(check, kind, side);
|
|
1136
|
+
if (type !== undefined && type !== wanted) {
|
|
1137
|
+
check.issue(refError(kind, shown, `${what} needs a ${wanted}, not a ${type}`));
|
|
1138
|
+
return false;
|
|
1139
|
+
}
|
|
1140
|
+
return type !== undefined;
|
|
1141
|
+
};
|
|
1142
|
+
if ("op" in operand) {
|
|
1143
|
+
const fine = [operand.left, operand.right].map((side) =>
|
|
1144
|
+
expect(side, "number", "arithmetic"),
|
|
1145
|
+
);
|
|
1146
|
+
return fine.every(Boolean) ? "number" : undefined;
|
|
1147
|
+
}
|
|
1148
|
+
if ("case" in operand) return caseType(check, kind, operand, shown);
|
|
1149
|
+
if ("args" in operand) return functionType(check, kind, operand, shown, expect);
|
|
1150
|
+
return expect(operand.of, "date", operand.fn)
|
|
1151
|
+
? operand.fn === "dateTrunc"
|
|
1152
|
+
? "date"
|
|
1153
|
+
: "number"
|
|
1154
|
+
: undefined;
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
/** Every branch of a CASE, and its ELSE, answer one type; a CASE without ELSE answers null where nothing matched. */
|
|
1158
|
+
function caseType(
|
|
1159
|
+
check: QueryCheck,
|
|
1160
|
+
kind: string,
|
|
1161
|
+
operand: Extract<DatasetOperand, { case: unknown }>,
|
|
1162
|
+
shown: string,
|
|
1163
|
+
) {
|
|
1164
|
+
const results = operand.case.map((branch) => {
|
|
1165
|
+
checkFilter(check, branch.when, kind);
|
|
1166
|
+
return operandType(check, kind, branch.then);
|
|
1167
|
+
});
|
|
1168
|
+
if (operand.else !== undefined) results.push(operandType(check, kind, operand.else));
|
|
1169
|
+
return sharedType(check, kind, shown, results, "branches answer");
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
/** One type across several values (CASE branches, coalesce arguments), or an issue naming the mix. */
|
|
1173
|
+
function sharedType(
|
|
1174
|
+
check: QueryCheck,
|
|
1175
|
+
kind: string,
|
|
1176
|
+
shown: string,
|
|
1177
|
+
types: (DatasetColumnType | undefined)[],
|
|
1178
|
+
what: string,
|
|
1179
|
+
) {
|
|
1180
|
+
const known = types.filter((type) => type !== undefined);
|
|
1181
|
+
if (new Set(known).size > 1) {
|
|
1182
|
+
check.issue(refError(kind, shown, `${what} a ${known.join(" and a ")}; use one type`));
|
|
1183
|
+
return undefined;
|
|
1184
|
+
}
|
|
1185
|
+
return known.length === types.length ? known[0] : undefined;
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
function functionType(
|
|
1189
|
+
check: QueryCheck,
|
|
1190
|
+
kind: string,
|
|
1191
|
+
operand: Extract<DatasetOperand, { args: unknown }>,
|
|
1192
|
+
shown: string,
|
|
1193
|
+
expect: (side: DatasetOperand, wanted: DatasetColumnType, what: string) => boolean,
|
|
1194
|
+
) {
|
|
1195
|
+
const signature: {
|
|
1196
|
+
args: readonly DatasetColumnType[];
|
|
1197
|
+
optional?: number;
|
|
1198
|
+
answers: DatasetColumnType | undefined;
|
|
1199
|
+
} = FUNCTION_SIGNATURES[operand.fn];
|
|
1200
|
+
if (operand.fn === "coalesce") {
|
|
1201
|
+
if (operand.args.length < 2) {
|
|
1202
|
+
check.issue(refError(kind, shown, "coalesce takes two or more values"));
|
|
1203
|
+
return undefined;
|
|
1204
|
+
}
|
|
1205
|
+
const types = operand.args.map((arg) => operandType(check, kind, arg));
|
|
1206
|
+
return sharedType(check, kind, shown, types, "mixes");
|
|
1207
|
+
}
|
|
1208
|
+
const least = signature.args.length - (signature.optional ?? 0);
|
|
1209
|
+
if (operand.args.length < least || operand.args.length > signature.args.length) {
|
|
1210
|
+
const count =
|
|
1211
|
+
least === signature.args.length ? `${least}` : `${least} or ${signature.args.length}`;
|
|
1212
|
+
check.issue(refError(kind, shown, `${operand.fn} takes ${count} value(s)`));
|
|
1213
|
+
return undefined;
|
|
1214
|
+
}
|
|
1215
|
+
const digits = operand.fn === "round" ? operand.args[1] : undefined;
|
|
1216
|
+
if (
|
|
1217
|
+
digits !== undefined &&
|
|
1218
|
+
(typeof digits !== "number" || !Number.isInteger(digits) || digits < 0 || digits > 10)
|
|
1219
|
+
) {
|
|
1220
|
+
check.issue(refError(kind, shown, "round takes a whole number of digits from 0 to 10"));
|
|
1221
|
+
return undefined;
|
|
1222
|
+
}
|
|
1223
|
+
const fine = operand.args.map((arg, index) => expect(arg, signature.args[index]!, operand.fn));
|
|
1224
|
+
return fine.every(Boolean) ? signature.answers : undefined;
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
const COLUMN_COMPARISONS: readonly DatasetFilterOp[] = ["=", "!=", "<", "<=", ">", ">="];
|
|
1228
|
+
|
|
1229
|
+
function checkFilter(check: QueryCheck, filter: DatasetFilter, kind = "filter") {
|
|
1230
|
+
if ("not" in filter) {
|
|
1231
|
+
checkFilter(check, filter.not, kind);
|
|
1232
|
+
return;
|
|
1233
|
+
}
|
|
1234
|
+
if ("any" in filter || "all" in filter) {
|
|
1235
|
+
for (const entry of "any" in filter ? filter.any : filter.all)
|
|
1236
|
+
checkFilter(check, entry, kind);
|
|
1237
|
+
return;
|
|
1238
|
+
}
|
|
1239
|
+
const shown = describeOperand(filter.col);
|
|
1240
|
+
const type = operandType(check, kind, filter.col);
|
|
1241
|
+
if (!type) return;
|
|
1242
|
+
const allowed: readonly DatasetFilterOp[] = OPERATORS_BY_TYPE[type];
|
|
1243
|
+
if (!allowed.includes(filter.op)) {
|
|
1244
|
+
check.issue(refError(kind, shown, `${filter.op} is not valid on a ${type} column`));
|
|
1245
|
+
return;
|
|
1246
|
+
}
|
|
1247
|
+
const issue = isColumnValue(filter.value)
|
|
1248
|
+
? columnValueIssue(check, kind, type, filter.op, filter.value.col)
|
|
1249
|
+
: (filterValueIssue(filter) ??
|
|
1250
|
+
filterValueTypeIssue(type, filter) ??
|
|
1251
|
+
truncatedDateIssue(filter));
|
|
1252
|
+
if (issue) check.issue(refError(kind, shown, issue));
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
/** A truncated date is a whole day, so a literal against it is one too ("2024-03-01"); a time or a bare month would never match. */
|
|
1256
|
+
function truncatedDateIssue(filter: DatasetCondition) {
|
|
1257
|
+
const truncated =
|
|
1258
|
+
typeof filter.col === "object" && "fn" in filter.col && filter.col.fn === "dateTrunc";
|
|
1259
|
+
if (!truncated || filter.value === undefined || isColumnValue(filter.value)) return undefined;
|
|
1260
|
+
const values = Array.isArray(filter.value) ? filter.value : [filter.value];
|
|
1261
|
+
const odd = values.find(
|
|
1262
|
+
(value) => typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(value),
|
|
1263
|
+
);
|
|
1264
|
+
return odd === undefined ? undefined : `${JSON.stringify(odd)} is not a whole day (YYYY-MM-DD)`;
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
/** A right side that is a column or arithmetic: only the six comparisons take it, and its type must match the left side's. */
|
|
1268
|
+
function columnValueIssue(
|
|
1269
|
+
check: QueryCheck,
|
|
1270
|
+
kind: string,
|
|
1271
|
+
type: DatasetColumnType,
|
|
1272
|
+
op: DatasetFilterOp,
|
|
1273
|
+
right: DatasetOperand,
|
|
1274
|
+
) {
|
|
1275
|
+
if (!COLUMN_COMPARISONS.includes(op)) return `${op} takes a literal, not a column`;
|
|
1276
|
+
const rightType = operandType(check, kind, right);
|
|
1277
|
+
return rightType && rightType !== type ? `compares a ${type} with a ${rightType}` : undefined;
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
/** A value the column's type cannot hold is refused here, not answered as an empty result by the host. */
|
|
1281
|
+
function filterValueTypeIssue(type: DatasetColumnType, filter: DatasetCondition) {
|
|
1282
|
+
if (filter.value === undefined || isColumnValue(filter.value)) return undefined;
|
|
1283
|
+
const values = Array.isArray(filter.value) ? filter.value : [filter.value];
|
|
1284
|
+
const fits = (value: string | number | boolean) => {
|
|
1285
|
+
switch (type) {
|
|
1286
|
+
case "number":
|
|
1287
|
+
return (
|
|
1288
|
+
typeof value === "number" ||
|
|
1289
|
+
(typeof value === "string" &&
|
|
1290
|
+
value.trim() !== "" &&
|
|
1291
|
+
Number.isFinite(Number(value)))
|
|
1292
|
+
);
|
|
1293
|
+
case "date":
|
|
1294
|
+
return typeof value === "string" && !Number.isNaN(Date.parse(value));
|
|
1295
|
+
case "boolean":
|
|
1296
|
+
return typeof value === "boolean" || value === "true" || value === "false";
|
|
1297
|
+
case "string":
|
|
1298
|
+
return true;
|
|
1299
|
+
}
|
|
1300
|
+
};
|
|
1301
|
+
const odd = values.find((value) => !fits(value));
|
|
1302
|
+
return odd === undefined ? undefined : `${JSON.stringify(odd)} is not a ${type}`;
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
/** Each group key with the type its value has: a column's own, or what the named value answers. */
|
|
1306
|
+
function checkGroupBy(check: QueryCheck, groupBy: DatasetQuery["groupBy"]) {
|
|
1307
|
+
const groupKeys = new Map<string, DatasetColumnType>();
|
|
1308
|
+
for (const group of groupBy) {
|
|
1309
|
+
const name = groupKeyName(group);
|
|
1310
|
+
if (typeof group !== "string" && operandRefs(group.expr).length === 0) {
|
|
1311
|
+
check.issue(
|
|
1312
|
+
refError("groupBy", name, "a group key reads a column; a constant groups nothing"),
|
|
1313
|
+
);
|
|
1314
|
+
}
|
|
1315
|
+
const type =
|
|
1316
|
+
typeof group === "string"
|
|
1317
|
+
? check.columnOf("groupBy", group)?.type
|
|
1318
|
+
: operandType(check, "groupBy", group.expr);
|
|
1319
|
+
// Row keys are the names as written, so a name may appear once.
|
|
1320
|
+
if (groupKeys.has(name)) check.issue(refError("groupBy", name, "is grouped twice"));
|
|
1321
|
+
groupKeys.set(name, type ?? "string");
|
|
1322
|
+
}
|
|
1323
|
+
return groupKeys;
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
/** Each alias with the type its value has, what HAVING and sort may name. */
|
|
1327
|
+
function checkAggregates(check: QueryCheck, aggregates: DatasetQuery["aggregate"]) {
|
|
1328
|
+
const aliases = new Map<string, DatasetColumnType>();
|
|
1329
|
+
for (const aggregate of aggregates) {
|
|
1330
|
+
if (aliases.has(aggregate.as))
|
|
1331
|
+
check.issue(`aggregate alias "${aggregate.as}" is used twice`);
|
|
1332
|
+
for (const filter of aggregate.filter)
|
|
1333
|
+
checkFilter(check, filter, `aggregate "${aggregate.as}" filter`);
|
|
1334
|
+
aliases.set(aggregate.as, aggregateType(check, aggregate));
|
|
1335
|
+
}
|
|
1336
|
+
return aliases;
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
/** The type an aggregate answers: number for counts, sums and averages, the operand's own for min and max. */
|
|
1340
|
+
function aggregateType(
|
|
1341
|
+
check: QueryCheck,
|
|
1342
|
+
aggregate: DatasetQuery["aggregate"][number],
|
|
1343
|
+
label = `aggregate "${aggregate.as}"`,
|
|
1344
|
+
) {
|
|
1345
|
+
if (aggregate.col === undefined) {
|
|
1346
|
+
if (aggregate.fn !== "count") check.issue(`${label}: ${aggregate.fn} needs a column`);
|
|
1347
|
+
return "number";
|
|
1348
|
+
}
|
|
1349
|
+
const kind = label.startsWith("window") ? label : "aggregate";
|
|
1350
|
+
const type = operandType(check, kind, aggregate.col);
|
|
1351
|
+
const numeric = aggregate.fn === "sum" || aggregate.fn === "avg";
|
|
1352
|
+
if (type && numeric && type !== "number") {
|
|
1353
|
+
check.issue(
|
|
1354
|
+
refError(kind, describeOperand(aggregate.col), `${aggregate.fn} needs a number column`),
|
|
1355
|
+
);
|
|
1356
|
+
}
|
|
1357
|
+
const counts = aggregate.fn === "count" || aggregate.fn === "countDistinct";
|
|
1358
|
+
return counts || numeric ? "number" : (type ?? "number");
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
/** What sort and rank may name: on a grouped query a group key or an alias; on a plain one a select alias or any reachable column. */
|
|
1362
|
+
type RowNames = { names: ReadonlyMap<string, DatasetColumnType>; kind: "grouped" | "plain" };
|
|
1363
|
+
|
|
1364
|
+
function checkSort(check: QueryCheck, sort: DatasetQuery["sort"], kind: string, row: RowNames) {
|
|
1365
|
+
for (const { col } of sort) checkRowName(check, kind, col, row);
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
function checkRowName(check: QueryCheck, kind: string, name: string, row: RowNames) {
|
|
1369
|
+
if (row.names.has(name)) return;
|
|
1370
|
+
if (row.kind === "plain") {
|
|
1371
|
+
check.columnOf(kind, name);
|
|
1372
|
+
return;
|
|
1373
|
+
}
|
|
1374
|
+
check.issue(
|
|
1375
|
+
refError(
|
|
1376
|
+
kind,
|
|
1377
|
+
name,
|
|
1378
|
+
"a grouped query names a group key, an aggregate alias, a window alias or a computed alias",
|
|
1379
|
+
),
|
|
1380
|
+
);
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1383
|
+
/** Any column of the dataset a relation path reaches, so a bare path can be probed through the column resolver. */
|
|
1384
|
+
function firstColumnOf(catalog: DatasetCatalog, from: string, path: string) {
|
|
1385
|
+
let dataset = catalog.get(from);
|
|
1386
|
+
for (const segment of path.split(".")) {
|
|
1387
|
+
const target = dataset?.relations[segment];
|
|
1388
|
+
dataset = target ? catalog.get(target.dataset) : undefined;
|
|
1389
|
+
}
|
|
1390
|
+
return dataset?.columns[0]?.name ?? "";
|
|
1391
|
+
}
|