measurr 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +176 -8
- package/dist/analytics.d.ts +61 -3
- package/dist/dataset.d.ts +47 -4
- package/dist/dialects/clickhouse.d.ts +26 -0
- package/dist/dialects/clickhouse.js +101 -0
- package/dist/handler.js +8 -46
- package/dist/{index-g255jb1s.js → index-115syy01.js} +1 -1
- package/dist/{index-zgb1e89d.js → index-g0ah4ehn.js} +122 -15
- package/dist/index-psw9s799.js +46 -0
- package/dist/{index-g2mjqx4f.js → index-rgqsrd72.js} +24 -2
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2 -2
- package/dist/metrics.d.ts +152 -0
- package/dist/metrics.js +167 -0
- package/dist/plan.d.ts +7 -1
- package/dist/query.d.ts +2 -2
- package/dist/shapes.d.ts +146 -1
- package/dist/shapes.js +84 -1
- package/dist/testing.js +5 -4
- package/package.json +15 -2
- package/src/analytics.ts +191 -17
- package/src/dataset.ts +82 -5
- package/src/dialects/clickhouse.ts +147 -0
- package/src/index.ts +14 -3
- package/src/metrics.ts +402 -0
- package/src/plan.ts +48 -7
- package/src/query.ts +17 -3
- package/src/shapes.ts +276 -1
- package/src/testing.ts +7 -2
package/README.md
CHANGED
|
@@ -7,8 +7,8 @@ returns shaped, typed rows.
|
|
|
7
7
|
|
|
8
8
|
- No SQL text in definitions: datasets are built from typed expression builders, so a
|
|
9
9
|
renamed column, `avg` over a string or `between` on an enum fails typecheck.
|
|
10
|
-
- Database-neutral definitions: dialects plug in through the `Dialect` interface
|
|
11
|
-
|
|
10
|
+
- Database-neutral definitions: dialects plug in through the `Dialect` interface. Postgres
|
|
11
|
+
(`measurr/postgres`) and ClickHouse (`measurr/clickhouse`) ship today.
|
|
12
12
|
- Tenant-scoped by construction: the engine adds the tenant filter to every statement and
|
|
13
13
|
refuses to run without a tenant. A caller can be scoped to one tenant or a list of them;
|
|
14
14
|
platform admins can query every tenant only through an explicit `allTenants` value that no
|
|
@@ -242,6 +242,95 @@ toLeaderboard(result, { by: 'region', measure: 'revenue' }) // ranked rows with
|
|
|
242
242
|
- `toDistribution`'s `total` is the measure over every row, so many-to-many shares can add up to
|
|
243
243
|
more than 1. `toPivot` sorts numeric keys (weekdays, hours).
|
|
244
244
|
|
|
245
|
+
### Presentation
|
|
246
|
+
|
|
247
|
+
`presentKpi`, `presentTimeseries`, `presentDistribution` and `presentLeaderboard` turn a result
|
|
248
|
+
into the wire outputs a dashboard draws (`KpiOutput`, `TimeSeriesOutput`, `DistributionOutput`,
|
|
249
|
+
`LeaderboardOutput`; `TableOutput` and `PivotOutput` for outputs you build yourself). A missing
|
|
250
|
+
value is 0, keys are strings, and the caller picks labels, units and chart types:
|
|
251
|
+
|
|
252
|
+
```ts
|
|
253
|
+
presentKpi(result, 'paidShare', { label: 'Paid', percent: true })
|
|
254
|
+
// { kind: 'kpi', value: 25, unit: 'PERCENTAGE', label: 'Paid' }
|
|
255
|
+
|
|
256
|
+
presentTimeseries(result, {
|
|
257
|
+
bucket: 'day', measure: 'orders', series: 'status',
|
|
258
|
+
keys: { PAID: 'paid' }, // wire keys, by String(key); 'null' names the empty group
|
|
259
|
+
labels: { paid: 'Settled' }, // labels by wire key
|
|
260
|
+
chartType: 'line'
|
|
261
|
+
})
|
|
262
|
+
// points are { ts: Date, value }: ts is the bucket's local start date as UTC midnight
|
|
263
|
+
|
|
264
|
+
presentDistribution(result, { by: 'tag', measure: 'orders', chartType: 'donut', order: ['t2'] })
|
|
265
|
+
// total is what the segments add up to, percent is 0-100; keys order omits go last
|
|
266
|
+
|
|
267
|
+
presentLeaderboard(result, { by: 'region', measure: 'orders', meta: (values) => ({ revenue: values.revenue }) })
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
`isKpi`, `isTimeSeries`, `isDistribution`, `isTable`, `isLeaderboard` and `isPivot` narrow a
|
|
271
|
+
`MetricOutput`. The module is pure, so a browser bundle can import the types and guards.
|
|
272
|
+
|
|
273
|
+
## Dashboard metrics
|
|
274
|
+
|
|
275
|
+
`measurr/metrics` names the numbers a dashboard shows: one query and the output it is drawn
|
|
276
|
+
in. The registry runs a batch for a caller and streams one chunk per metric.
|
|
277
|
+
|
|
278
|
+
```ts
|
|
279
|
+
import { createMetricRegistry, metricDefiner } from 'measurr/metrics'
|
|
280
|
+
import { presentKpi, presentTimeseries } from 'measurr/shapes'
|
|
281
|
+
|
|
282
|
+
// Types only, so metric modules need not import the analytics instance.
|
|
283
|
+
export const defineMetric = metricDefiner<(typeof datasets)[number], Role>()
|
|
284
|
+
|
|
285
|
+
const orderTrend = defineMetric({
|
|
286
|
+
key: 'orders.trend',
|
|
287
|
+
kind: 'timeseries',
|
|
288
|
+
description: 'Orders over time',
|
|
289
|
+
allowedRoles: ['owner', 'member'],
|
|
290
|
+
supportsTimeRange: true, // false for a snapshot: all time, no previous period
|
|
291
|
+
filterSchema: z.object({ status: z.enum(['OPEN', 'PAID']).optional() }),
|
|
292
|
+
query: ({ bucket, filters }) => ({
|
|
293
|
+
dataset: 'orders',
|
|
294
|
+
measures: ['orders'],
|
|
295
|
+
groupBy: [bucket],
|
|
296
|
+
filters: filters.status ? [{ dimension: 'status', op: 'in', values: [filters.status] }] : [],
|
|
297
|
+
limit: 100
|
|
298
|
+
}),
|
|
299
|
+
shape: (result, { bucket }) =>
|
|
300
|
+
presentTimeseries(result, { bucket, measure: 'orders', chartType: 'bar' })
|
|
301
|
+
})
|
|
302
|
+
|
|
303
|
+
export const metrics = createMetricRegistry({
|
|
304
|
+
analytics,
|
|
305
|
+
metrics: [orderTrend, orderCount] as const,
|
|
306
|
+
canRun: (metric, ctx) => ctx.isSuperAdmin || (metric.allowedRoles ?? []).includes(ctx.role),
|
|
307
|
+
onShapeError: (error, metric) => logger.error('metric.shape_failed', { key: metric.key, error })
|
|
308
|
+
})
|
|
309
|
+
|
|
310
|
+
const input = metrics.requestSchema.extend({ refresh: z.boolean().default(false) }).parse(body)
|
|
311
|
+
for await (const chunk of metrics.run(input, ctx)) send(chunk)
|
|
312
|
+
// { key, requestKey, result: { current, previous?, supportsTimeRange }, done }
|
|
313
|
+
// or { key, requestKey, error, done }
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
- `query` returns the query without `period` or `compareToPrevious`. The registry counts the
|
|
317
|
+
local days `from` and `to` (instants) fall on in `analytics.timezone(ctx)`, all time without
|
|
318
|
+
them, and always all time for a snapshot. `compareToPrevious` adds the previous period, whose
|
|
319
|
+
series points are put on the current buckets so a chart overlays them.
|
|
320
|
+
- `granularity` (hour to year, inferred from the range when omitted: day up to 31 days, week up
|
|
321
|
+
to 90, else month) picks `run.bucket`: `day`, `week` or `month`. Name time-bucket dimensions
|
|
322
|
+
after them so `groupBy: [bucket]` works.
|
|
323
|
+
- A metric that `canRun` refuses (`'Access denied'`), whose filters do not parse with its
|
|
324
|
+
`filterSchema` (`'Invalid filters: ...'`), whose query fails (the engine's message) or whose
|
|
325
|
+
`shape` throws (`'The metric could not be drawn'`) fails alone. `done` marks the last chunk.
|
|
326
|
+
- `metricDefiner<Datasets, Role, Extra>()` takes the app's own definition fields as `Extra`
|
|
327
|
+
(catalog metadata, UI hints for filters); the registry carries them to `canRun` and
|
|
328
|
+
`available` without reading them.
|
|
329
|
+
- Keys and outputs stay literal: a chunk narrows to its metric by `key`, and
|
|
330
|
+
`MetricOutputOf<typeof metrics, 'orders.trend'>`, `MetricFiltersOf`, `MetricKeysOfKind` and
|
|
331
|
+
`registry.get(key)` are typed per metric. `registry.available(ctx)` lists what `canRun`
|
|
332
|
+
allows, for a catalog.
|
|
333
|
+
|
|
245
334
|
## Serving queries
|
|
246
335
|
|
|
247
336
|
`measurr/handler` serves queries from any server framework:
|
|
@@ -266,6 +355,52 @@ try {
|
|
|
266
355
|
}
|
|
267
356
|
```
|
|
268
357
|
|
|
358
|
+
## Resolved measures
|
|
359
|
+
|
|
360
|
+
Some numbers one statement cannot give: sprays per employee when the employees live in another
|
|
361
|
+
database, an average per day of the period, a forecast. Declare such a measure with
|
|
362
|
+
`measure.resolved` and give the code to `createAnalytics`. It stays a measure like any other:
|
|
363
|
+
it is in the catalog and the schema, checked for access, scoped to the tenant and cached.
|
|
364
|
+
|
|
365
|
+
```ts
|
|
366
|
+
const sprays = defineDataset({
|
|
367
|
+
// ...
|
|
368
|
+
measures: {
|
|
369
|
+
sprays: measure.sum(status.col('sprays'), { label: 'Sprays' }),
|
|
370
|
+
spraysPerDay: measure.resolved({ label: 'Sprays per day', result: 'number', uses: ['sprays'] })
|
|
371
|
+
}
|
|
372
|
+
})
|
|
373
|
+
|
|
374
|
+
createAnalytics({
|
|
375
|
+
// ...
|
|
376
|
+
resolvers: {
|
|
377
|
+
sprays: {
|
|
378
|
+
spraysPerDay: ({ rows, totals, period }) => {
|
|
379
|
+
const days = 'all' in period ? null : daysBetween(period.from, period.to)
|
|
380
|
+
const perDay = (value: number | null) =>
|
|
381
|
+
value === null || days === null ? null : value / days
|
|
382
|
+
return { rows: rows.map(({ values }) => perDay(values.sprays)), totals: perDay(totals.sprays) }
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
})
|
|
387
|
+
```
|
|
388
|
+
|
|
389
|
+
- `uses` names the measures of the same dataset the resolver reads. The database computes them
|
|
390
|
+
for every row and for the totals, even when the query does not ask for them, and the result
|
|
391
|
+
holds only what it asked for. A measure with no `uses` still gets one value per group.
|
|
392
|
+
- The resolver gets the rows in order (`dimensions` for the grouped keys, `values` for the used
|
|
393
|
+
measures), the `totals`, `groupBy`, the `filters` (open-dimension labels resolved to keys),
|
|
394
|
+
the `period`, the `tenant` scope and `ctx`. It returns one value per row and one for the
|
|
395
|
+
totals. With `compareToPrevious` it is called once per period.
|
|
396
|
+
- `createAnalytics` throws when a resolved measure has no resolver, and when `resolvers` names
|
|
397
|
+
anything else. A wrong number of values fails the query; a value that is not a finite number
|
|
398
|
+
is null, and a `count` result is 0 instead.
|
|
399
|
+
- A resolved measure needs the access of the measures it uses, like a ratio. It cannot be sorted
|
|
400
|
+
by (the database limits the rows before the value exists), and `ratio` and `uses` cannot name
|
|
401
|
+
one.
|
|
402
|
+
- The contract kit leaves resolved measures out: they are app code, so test them with it.
|
|
403
|
+
|
|
269
404
|
## Tenants and platform admins
|
|
270
405
|
|
|
271
406
|
`tenant(ctx)` decides the scope of every statement, including the lookups that resolve filter
|
|
@@ -335,7 +470,7 @@ before any statement runs; a narrower scope is never ignored. `listDatasets(ctx)
|
|
|
335
470
|
| Concept | What it is |
|
|
336
471
|
| --- | --- |
|
|
337
472
|
| Dataset | One fact table: tenant column (or `null`), optional scope, time fields, default period, measures, dimensions, source. |
|
|
338
|
-
| Measure | `count`, `countWhere`, `countDistinct`, `sum`, `avg`, `median`, `share` (one condition over another)
|
|
473
|
+
| Measure | `count`, `countWhere`, `countDistinct`, `sum`, `avg`, `median`, `share` (one condition over another), `ratio` (one measure over another, after aggregation) or `resolved` (computed by the app after the query runs). Optionally restricted with `access`. |
|
|
339
474
|
| Dimension | `enum`, `computed` (closed value sets), `relation` (to-one), `manyToMany`, `number`, `timeBucket` (day, ISO week, month), `timePart` (ISO weekday, hour). Optionally restricted with `access`. |
|
|
340
475
|
| Query | `dataset`, 1-4 `measures`, 0-2 `groupBy`, `filters`, `period`, `time`, `compareToPrevious`, `sort` (by an asked measure or grouped dimension), `limit` (1-100, default 20). |
|
|
341
476
|
| Result | `{ period, rows, totals, previous?, notes, truncated }`; `period` is `{ from, to, timezone }` or `{ all: true, timezone }`. |
|
|
@@ -457,6 +592,34 @@ const main: AnalyticsSource = {
|
|
|
457
592
|
}
|
|
458
593
|
```
|
|
459
594
|
|
|
595
|
+
**`measurr/clickhouse`** `clickhouseDialect()`: typed `{pN:Type}` parameters, backquoted
|
|
596
|
+
identifiers, `-If` and `-OrNull` aggregates (an empty group is null, as in Postgres), an exact
|
|
597
|
+
interpolated median, `uniqExact` for distinct counts, and buckets with `toTimeZone`. Every
|
|
598
|
+
statement ends with `SETTINGS join_use_nulls = 1, enable_positional_arguments = 1`, so a left
|
|
599
|
+
join without a match gives null (an empty group) instead of an empty string. Pass the values
|
|
600
|
+
with `toQueryParams`:
|
|
601
|
+
|
|
602
|
+
```ts
|
|
603
|
+
import { clickhouseDialect, toQueryParams } from 'measurr/clickhouse'
|
|
604
|
+
|
|
605
|
+
const events: AnalyticsSource = {
|
|
606
|
+
dialect: clickhouseDialect(),
|
|
607
|
+
execute: async (statement) => {
|
|
608
|
+
const result = await clickhouse.query({
|
|
609
|
+
query: statement.text,
|
|
610
|
+
query_params: toQueryParams(statement),
|
|
611
|
+
format: 'JSONEachRow'
|
|
612
|
+
})
|
|
613
|
+
return result.json()
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
```
|
|
617
|
+
|
|
618
|
+
ClickHouse cannot run correlated subqueries, so a dataset on it cannot use `exists` or filter
|
|
619
|
+
by a many-to-many dimension (mark it `groupOnly`); `createAnalytics` fails at startup if it
|
|
620
|
+
does. Time columns are `DateTime` or `DateTime64`, which hold instants. Tested against
|
|
621
|
+
ClickHouse 24.
|
|
622
|
+
|
|
460
623
|
**`measurr/postgres`** `postgresDialect({ timestamps })`: `$n` parameters (untyped,
|
|
461
624
|
so Postgres infers them from the column, which keeps enum columns working), double-quoted
|
|
462
625
|
identifiers, `FILTER (WHERE ...)`, `count(DISTINCT ...)`, `percentile_cont`, correlated
|
|
@@ -511,13 +674,17 @@ Kept stable for publishing: `createAnalytics` and its options (with `authorize`,
|
|
|
511
674
|
`resultSchema`, `listDatasets`; `AnalyticsResultSchema`, `analyticsToolGuidance`,
|
|
512
675
|
`AnalyticsQueryEvent`; `defineDataset` (with `defaultPeriod` and `tenantColumn: null`),
|
|
513
676
|
`measure.*` and `dimension.*` (with `access`), `table` / `Table`, the expression builders; the
|
|
514
|
-
`Dialect`, `SqlRenderer` and `AnalyticsSource` contracts; `
|
|
677
|
+
`Dialect`, `SqlRenderer` and `AnalyticsSource` contracts; `measure.resolved` with the
|
|
678
|
+
`resolvers` option (`MeasureResolver`, `ResolverInput`, `ResolverOutput`); `postgresDialect`,
|
|
679
|
+
`clickhouseDialect` and `toQueryParams`,
|
|
515
680
|
`checkDatasetContract` / `assertDatasetContract`; `measurr/shapes` (`toKpi`, `toTimeseries`,
|
|
516
681
|
`toDistribution`, `toPivot`, `toLeaderboard`) and `measurr/handler` (`queryBatch`,
|
|
517
682
|
`streamBatch`, `toHttpError`); and the types `AnalyticsQuery`, `AnalyticsResult`,
|
|
518
683
|
`ResultPeriod`, `PeriodInput`, `Tenant`, `TenantId`, `AccessKey`, `AnalyticsCache`, `FieldKey`,
|
|
519
684
|
`Ratio`, `AnalyticsError` codes and their details (`isAnalyticsError`). `ExprNode` and
|
|
520
|
-
`SelectStatement` are exported for dialect authors and may grow new node kinds.
|
|
685
|
+
`SelectStatement` are exported for dialect authors and may grow new node kinds. The
|
|
686
|
+
presenters and wire outputs in `measurr/shapes`, `measurr/metrics` and `analytics.timezone`
|
|
687
|
+
are new and may still change before they join this list.
|
|
521
688
|
|
|
522
689
|
## Install
|
|
523
690
|
|
|
@@ -533,13 +700,14 @@ npm install measurr zod
|
|
|
533
700
|
bun install
|
|
534
701
|
bun run lint # types and Biome
|
|
535
702
|
bun test
|
|
536
|
-
# With a throwaway Postgres (the integration
|
|
537
|
-
ANALYTICS_TEST_DATABASE_URL=postgres://... bun test
|
|
703
|
+
# With a throwaway Postgres and ClickHouse (the integration tests create and drop their own tables):
|
|
704
|
+
ANALYTICS_TEST_DATABASE_URL=postgres://... ANALYTICS_TEST_CLICKHOUSE_URL=http://user:password@localhost:8123 bun test
|
|
538
705
|
bun run build && bun run verify:package
|
|
539
706
|
```
|
|
540
707
|
|
|
541
708
|
The integration test reads `ANALYTICS_TEST_DATABASE_URL`, not `DATABASE_URL`, because Bun
|
|
542
|
-
loads `.env` files and that name may point at a real database. CI runs
|
|
709
|
+
loads `.env` files and that name may point at a real database. CI runs them against Postgres 17
|
|
710
|
+
and ClickHouse 24.
|
|
543
711
|
|
|
544
712
|
Releases are automated with Release Please; see [docs/releasing.md](docs/releasing.md).
|
|
545
713
|
|
package/dist/analytics.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import type { CompiledStatement, Dialect } from './compile.js';
|
|
3
|
-
import type { AnyDataset, AnyDimension, Dataset } from './dataset.js';
|
|
4
|
-
import { type Tenant } from './plan.js';
|
|
5
|
-
import { type DatasetQuery, type QueryResult, type ResultOf } from './query.js';
|
|
3
|
+
import type { AnyDataset, AnyDimension, Dataset, DimensionValue, MeasureValue, ResolvedMeasure } from './dataset.js';
|
|
4
|
+
import { type NormalizedFilter, type Tenant } from './plan.js';
|
|
5
|
+
import { type DatasetQuery, type QueryResult, type ResultOf, type ResultPeriod } from './query.js';
|
|
6
6
|
export type Row = Record<string, unknown>;
|
|
7
7
|
/** A database: a dialect to compile with and a function that runs the compiled SQL. */
|
|
8
8
|
export type AnalyticsSource = {
|
|
@@ -59,6 +59,51 @@ export type AnalyticsQueryEvent = {
|
|
|
59
59
|
export type FieldKey<D> = D extends Dataset<string, infer M, infer Dm> ? keyof (M & Dm) & string : never;
|
|
60
60
|
/** The access keys a dataset's measures and dimensions are marked with. */
|
|
61
61
|
export type AccessKey<D> = D extends Dataset<string, infer M, infer Dm> ? NonNullable<(M & Dm)[keyof (M & Dm)]['access']> : never;
|
|
62
|
+
/** The resolved measures of a dataset. */
|
|
63
|
+
export type ResolvedKey<D> = D extends Dataset<string, infer M> ? {
|
|
64
|
+
[K in keyof M & string]: M[K] extends {
|
|
65
|
+
kind: 'resolved';
|
|
66
|
+
} ? K : never;
|
|
67
|
+
}[keyof M & string] : never;
|
|
68
|
+
/** The values of the measures resolved measure `K` uses. */
|
|
69
|
+
type UsedValues<D, K> = D extends Dataset<string, infer M> ? M[K & keyof M] extends ResolvedMeasure<MeasureResult, infer U> ? {
|
|
70
|
+
[Used in U & keyof M]: MeasureValue<M[Used]>;
|
|
71
|
+
} : never : never;
|
|
72
|
+
type MeasureResult = ResolvedMeasure['result'];
|
|
73
|
+
/** What a resolver gets for one period of one query. */
|
|
74
|
+
export type ResolverInput<D, K, Ctx> = D extends Dataset<string, infer _M, infer Dm> ? {
|
|
75
|
+
/**
|
|
76
|
+
* The result rows in order: their grouped dimensions and the measures `K` uses. One
|
|
77
|
+
* row when the query has no `groupBy`.
|
|
78
|
+
*/
|
|
79
|
+
rows: readonly {
|
|
80
|
+
dimensions: {
|
|
81
|
+
[G in keyof Dm]?: DimensionValue<Dm[G]>;
|
|
82
|
+
};
|
|
83
|
+
values: UsedValues<D, K>;
|
|
84
|
+
}[];
|
|
85
|
+
/** The used measures over every row of the period, ungrouped. */
|
|
86
|
+
totals: UsedValues<D, K>;
|
|
87
|
+
groupBy: readonly (keyof Dm & string)[];
|
|
88
|
+
/** The query's filters, open-dimension labels already resolved to keys. */
|
|
89
|
+
filters: readonly NormalizedFilter[];
|
|
90
|
+
/** The period being resolved: the query's, or its previous one. */
|
|
91
|
+
period: ResultPeriod;
|
|
92
|
+
tenant: Tenant;
|
|
93
|
+
ctx: Ctx;
|
|
94
|
+
} : never;
|
|
95
|
+
/** One value per row, in the same order, and one for the totals. */
|
|
96
|
+
export type ResolverOutput = {
|
|
97
|
+
rows: readonly (number | null)[];
|
|
98
|
+
totals: number | null;
|
|
99
|
+
};
|
|
100
|
+
export type MeasureResolver<D, K, Ctx> = (input: ResolverInput<D, K, Ctx>) => ResolverOutput | Promise<ResolverOutput>;
|
|
101
|
+
/** The code for every resolved measure, by dataset and measure. */
|
|
102
|
+
export type AnalyticsResolvers<DS extends readonly AnyDataset[], Ctx> = {
|
|
103
|
+
[D in DS[number] as [ResolvedKey<D>] extends [never] ? never : D['key']]: {
|
|
104
|
+
[K in ResolvedKey<D>]: MeasureResolver<D, K, Ctx>;
|
|
105
|
+
};
|
|
106
|
+
};
|
|
62
107
|
export type AnalyticsOptions<DS extends readonly AnyDataset[], Ctx> = {
|
|
63
108
|
datasets: DS;
|
|
64
109
|
/** Databases by name; a dataset's `source` picks one, or the only one is used. */
|
|
@@ -97,6 +142,13 @@ export type AnalyticsOptions<DS extends readonly AnyDataset[], Ctx> = {
|
|
|
97
142
|
* swallowed: a failing cache is a miss, never a failed query.
|
|
98
143
|
*/
|
|
99
144
|
cache?: AnalyticsCache<Ctx, DS[number]['key']> | undefined;
|
|
145
|
+
/**
|
|
146
|
+
* The code for `measure.resolved` measures, by dataset and measure key: called once per
|
|
147
|
+
* period with the result rows and the values of the measures each one uses. Required for
|
|
148
|
+
* every resolved measure (`createAnalytics` throws otherwise). What it throws fails the
|
|
149
|
+
* query; a value that is not a finite number counts as null.
|
|
150
|
+
*/
|
|
151
|
+
resolvers?: NoInfer<AnalyticsResolvers<DS, Ctx>> | undefined;
|
|
100
152
|
};
|
|
101
153
|
/**
|
|
102
154
|
* A store for finished results, such as Redis. Keys name the dataset and hash everything that
|
|
@@ -174,9 +226,15 @@ export type Analytics<DS extends readonly AnyDataset[], Ctx> = {
|
|
|
174
226
|
}): Promise<z.ZodType<DatasetQuery<DS[number]>>>;
|
|
175
227
|
/** The result schema for one dataset. */
|
|
176
228
|
resultSchema(dataset: DS[number]['key']): z.ZodType<QueryResult<Row, Row>>;
|
|
229
|
+
/**
|
|
230
|
+
* The timezone `ctx`'s periods and buckets resolve in: the `timezone` hook's answer, UTC
|
|
231
|
+
* when it has none. Rejects with `invalid_timezone` for a name the runtime does not know.
|
|
232
|
+
*/
|
|
233
|
+
timezone(ctx: Ctx): Promise<string>;
|
|
177
234
|
/** The datasets `ctx` may query, with the measures and dimensions it may use. */
|
|
178
235
|
listDatasets(ctx: Ctx): Promise<DatasetCatalogEntry<DS[number]['key']>[]>;
|
|
179
236
|
};
|
|
180
237
|
export type AnalyticsQuery<A> = A extends Analytics<infer DS, never> ? DatasetQuery<DS[number]> : never;
|
|
181
238
|
export type AnalyticsResult<A, Q> = A extends Analytics<infer DS, never> ? ResultOf<DS[number], Q> : never;
|
|
182
239
|
export declare function createAnalytics<const DS extends readonly AnyDataset[], Ctx>(options: AnalyticsOptions<DS, Ctx>): Analytics<DS, Ctx>;
|
|
240
|
+
export {};
|
package/dist/dataset.d.ts
CHANGED
|
@@ -53,7 +53,21 @@ export type RatioMeasure<A extends string = string, B extends string = string> =
|
|
|
53
53
|
readonly numerator: A;
|
|
54
54
|
readonly denominator: B;
|
|
55
55
|
};
|
|
56
|
-
|
|
56
|
+
/**
|
|
57
|
+
* A value the app computes after the query runs, from the measures it `uses` in each row and
|
|
58
|
+
* from anything else it can reach (another database, a forecast). The `resolvers` option of
|
|
59
|
+
* `createAnalytics` supplies the code.
|
|
60
|
+
*/
|
|
61
|
+
export type ResolvedMeasure<R extends MeasureResult = MeasureResult, U extends string = string> = {
|
|
62
|
+
readonly kind: 'resolved';
|
|
63
|
+
readonly result: R;
|
|
64
|
+
readonly label: string;
|
|
65
|
+
readonly description: string | undefined;
|
|
66
|
+
/** A resolved measure also needs the access of the measures it uses. */
|
|
67
|
+
readonly access: string | undefined;
|
|
68
|
+
readonly uses: readonly U[];
|
|
69
|
+
};
|
|
70
|
+
export type AnyMeasure = AggregateMeasure | ShareMeasure | RatioMeasure | ResolvedMeasure;
|
|
57
71
|
/** The TypeScript type of a measure's value in a result row. */
|
|
58
72
|
export type MeasureValue<M> = M extends {
|
|
59
73
|
result: 'count';
|
|
@@ -86,7 +100,23 @@ export declare const measure: {
|
|
|
86
100
|
* measures of the same dataset; `defineDataset` checks them.
|
|
87
101
|
*/
|
|
88
102
|
ratio<const A extends string, const B extends string, Access extends string = never>(numerator: A, denominator: B, options: Described<Access>): RatioMeasure<A, B> & Marked<NoInfer<Access>>;
|
|
103
|
+
resolved: typeof resolved;
|
|
89
104
|
};
|
|
105
|
+
/**
|
|
106
|
+
* A value the app computes after the query runs, for numbers one statement cannot give:
|
|
107
|
+
* another database, a forecast, a per-day average. `uses` names the measures of this dataset
|
|
108
|
+
* the resolver reads from each row; they are fetched even when the query does not ask for
|
|
109
|
+
* them. The code goes in `createAnalytics({ resolvers })`. Queries cannot sort by it, since
|
|
110
|
+
* the database limits the rows before the value exists.
|
|
111
|
+
*/
|
|
112
|
+
declare function resolved<R extends MeasureResult, Access extends string = never>(options: Described<Access> & {
|
|
113
|
+
result: R;
|
|
114
|
+
uses?: undefined;
|
|
115
|
+
}): ResolvedMeasure<R, never> & Marked<NoInfer<Access>>;
|
|
116
|
+
declare function resolved<R extends MeasureResult, const U extends string, Access extends string = never>(options: Described<Access> & {
|
|
117
|
+
result: R;
|
|
118
|
+
uses: readonly U[];
|
|
119
|
+
}): ResolvedMeasure<R, U> & Marked<NoInfer<Access>>;
|
|
90
120
|
/**
|
|
91
121
|
* How a dimension filters:
|
|
92
122
|
* - `closed`: a fixed set of keys (enum, computed) - `in`, `notIn`, `isEmpty`, `isNotEmpty`;
|
|
@@ -274,13 +304,26 @@ export type Dataset<K extends string = string, M extends Record<string, AnyMeasu
|
|
|
274
304
|
readonly dimensions: Dm;
|
|
275
305
|
};
|
|
276
306
|
export type AnyDataset = Dataset<string, any, any, string>;
|
|
277
|
-
/**
|
|
307
|
+
/** Measures the database computes: everything but resolved measures. */
|
|
308
|
+
export type SqlMeasureKey<M> = {
|
|
309
|
+
[Key in keyof M & string]: M[Key] extends {
|
|
310
|
+
kind: 'resolved';
|
|
311
|
+
} ? never : Key;
|
|
312
|
+
}[keyof M & string];
|
|
313
|
+
/**
|
|
314
|
+
* Ratio measures may only name other SQL measures of the same dataset, and resolved measures
|
|
315
|
+
* may only use SQL measures.
|
|
316
|
+
*/
|
|
278
317
|
type CheckMeasures<M> = {
|
|
279
318
|
[Key in keyof M]: M[Key] extends {
|
|
280
319
|
kind: 'ratio';
|
|
281
320
|
} ? {
|
|
282
|
-
numerator: Exclude<
|
|
283
|
-
denominator: Exclude<
|
|
321
|
+
numerator: Exclude<SqlMeasureKey<M>, Key>;
|
|
322
|
+
denominator: Exclude<SqlMeasureKey<M>, Key>;
|
|
323
|
+
} : M[Key] extends {
|
|
324
|
+
kind: 'resolved';
|
|
325
|
+
} ? {
|
|
326
|
+
uses: readonly SqlMeasureKey<M>[];
|
|
284
327
|
} : unknown;
|
|
285
328
|
};
|
|
286
329
|
/** Time dimensions may only name declared time fields. */
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type CompiledStatement, type Dialect } from '../compile.js';
|
|
2
|
+
/**
|
|
3
|
+
* ClickHouse: typed `{pN:Type}` parameters (pass them with `toQueryParams`), backquoted
|
|
4
|
+
* identifiers, `-If` and `-OrNull` aggregates, and buckets in the tenant's timezone. Time
|
|
5
|
+
* columns are `DateTime` or `DateTime64`, which hold an instant, so no timestamp option is
|
|
6
|
+
* needed.
|
|
7
|
+
*
|
|
8
|
+
* Correlated subqueries are not supported, so `exists` and filters on many-to-many dimensions
|
|
9
|
+
* (mark them `groupOnly`) fail when `createAnalytics` compiles the dataset.
|
|
10
|
+
*/
|
|
11
|
+
export declare function clickhouseDialect(): Dialect;
|
|
12
|
+
/**
|
|
13
|
+
* The compiled values as the `query_params` the ClickHouse client takes: `{ p1, p2, ... }`.
|
|
14
|
+
*
|
|
15
|
+
* ```ts
|
|
16
|
+
* execute: async (statement) => {
|
|
17
|
+
* const result = await clickhouse.query({
|
|
18
|
+
* query: statement.text,
|
|
19
|
+
* query_params: toQueryParams(statement),
|
|
20
|
+
* format: 'JSONEachRow'
|
|
21
|
+
* })
|
|
22
|
+
* return result.json()
|
|
23
|
+
* }
|
|
24
|
+
* ```
|
|
25
|
+
*/
|
|
26
|
+
export declare function toQueryParams(statement: CompiledStatement): Record<string, unknown>;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CompileError2,
|
|
3
|
+
decodeNumber2,
|
|
4
|
+
renderStatement2
|
|
5
|
+
} from "../index-nrjdssft.js";
|
|
6
|
+
import {
|
|
7
|
+
walk
|
|
8
|
+
} from "../index-e6k30sfp.js";
|
|
9
|
+
|
|
10
|
+
// src/dialects/clickhouse.ts
|
|
11
|
+
var SETTINGS = "SETTINGS join_use_nulls = 1, enable_positional_arguments = 1";
|
|
12
|
+
function clickhouseDialect() {
|
|
13
|
+
const zone = (expr, timezone) => `toTimeZone(${expr}, '${timezone.replaceAll("'", "\\'")}')`;
|
|
14
|
+
const renderer = {
|
|
15
|
+
name: "clickhouse",
|
|
16
|
+
quote: (identifier) => `\`${identifier.replaceAll("\\", "\\\\").replaceAll("`", "\\`")}\``,
|
|
17
|
+
param(value, index) {
|
|
18
|
+
const { type, bound } = typedParam(value);
|
|
19
|
+
return { placeholder: `{p${index}:${type}}`, value: bound };
|
|
20
|
+
},
|
|
21
|
+
aggregate(fn, arg, where) {
|
|
22
|
+
switch (fn) {
|
|
23
|
+
case "count":
|
|
24
|
+
return where ? `countIf(${where})` : "count()";
|
|
25
|
+
case "countDistinct":
|
|
26
|
+
return where ? `uniqExactIf(${arg}, ${where})` : `uniqExact(${arg})`;
|
|
27
|
+
case "sum":
|
|
28
|
+
case "avg": {
|
|
29
|
+
const args = where ? `${arg}, ${where}` : arg;
|
|
30
|
+
return `${fn}${where ? "If" : ""}OrNull(${args})`;
|
|
31
|
+
}
|
|
32
|
+
case "median":
|
|
33
|
+
return where ? `quantileExactInclusiveIfOrNull(0.5)(${arg}, ${where})` : `quantileExactInclusiveOrNull(0.5)(${arg})`;
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
divide: (left, right) => `(toFloat64(${left}) / nullIf(${right}, 0))`,
|
|
37
|
+
toText: (expr) => `toString(${expr})`,
|
|
38
|
+
timeBucket(unit, expr, timezone) {
|
|
39
|
+
const local = zone(expr, timezone);
|
|
40
|
+
switch (unit) {
|
|
41
|
+
case "day":
|
|
42
|
+
return `toString(toDate(${local}))`;
|
|
43
|
+
case "week":
|
|
44
|
+
return `toString(toMonday(${local}))`;
|
|
45
|
+
case "month":
|
|
46
|
+
return `toString(toStartOfMonth(${local}))`;
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
timePart: (part, expr, timezone) => part === "isodow" ? `toDayOfWeek(${zone(expr, timezone)})` : `toHour(${zone(expr, timezone)})`
|
|
50
|
+
};
|
|
51
|
+
return {
|
|
52
|
+
name: "clickhouse",
|
|
53
|
+
compile(statement) {
|
|
54
|
+
rejectCorrelatedSubqueries(statement);
|
|
55
|
+
const compiled = renderStatement2(statement, renderer);
|
|
56
|
+
return { ...compiled, text: `${compiled.text}
|
|
57
|
+
${SETTINGS}` };
|
|
58
|
+
},
|
|
59
|
+
decodeNumber: decodeNumber2
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
function toQueryParams(statement) {
|
|
63
|
+
return Object.fromEntries(statement.values.map((value, index) => [`p${index + 1}`, value]));
|
|
64
|
+
}
|
|
65
|
+
function typedParam(value) {
|
|
66
|
+
if (value === null)
|
|
67
|
+
return { type: "Nullable(String)", bound: null };
|
|
68
|
+
if (value instanceof Date) {
|
|
69
|
+
const utc = value.toISOString().replace("T", " ").replace("Z", "");
|
|
70
|
+
return { type: "DateTime64(3, 'UTC')", bound: utc };
|
|
71
|
+
}
|
|
72
|
+
switch (typeof value) {
|
|
73
|
+
case "string":
|
|
74
|
+
return { type: "String", bound: value };
|
|
75
|
+
case "boolean":
|
|
76
|
+
return { type: "Bool", bound: value };
|
|
77
|
+
case "number":
|
|
78
|
+
return {
|
|
79
|
+
type: Number.isSafeInteger(value) ? "Int64" : "Float64",
|
|
80
|
+
bound: value
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
function rejectCorrelatedSubqueries(statement) {
|
|
85
|
+
const nodes = [
|
|
86
|
+
...statement.select.map((item) => item.node),
|
|
87
|
+
...statement.where,
|
|
88
|
+
...statement.virtualJoins.map((join) => join.on)
|
|
89
|
+
];
|
|
90
|
+
for (const node of nodes) {
|
|
91
|
+
walk(node, (inner) => {
|
|
92
|
+
if (inner.kind === "exists") {
|
|
93
|
+
throw new CompileError2(`ClickHouse cannot run the correlated EXISTS that ${inner.table.name} needs: avoid exists() and mark many-to-many dimensions groupOnly`);
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
export {
|
|
99
|
+
clickhouseDialect,
|
|
100
|
+
toQueryParams
|
|
101
|
+
};
|
package/dist/handler.js
CHANGED
|
@@ -1,49 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
const results = {};
|
|
8
|
-
const errors = {};
|
|
9
|
-
for await (const chunk of streamBatch(analytics, queries, ctx)) {
|
|
10
|
-
if ("error" in chunk)
|
|
11
|
-
errors[chunk.key] = chunk.error;
|
|
12
|
-
else
|
|
13
|
-
results[chunk.key] = chunk.result;
|
|
14
|
-
}
|
|
15
|
-
return { results, errors };
|
|
16
|
-
}
|
|
17
|
-
async function* streamBatch(analytics, queries, ctx) {
|
|
18
|
-
const pending = new Map(Object.entries(queries).map(([key, query]) => {
|
|
19
|
-
const settled = analytics.query(query, ctx).then((result) => ({ key, result }), (error) => ({ key, error: batchError(error) }));
|
|
20
|
-
return [key, settled];
|
|
21
|
-
}));
|
|
22
|
-
while (pending.size > 0) {
|
|
23
|
-
const chunk = await Promise.race(pending.values());
|
|
24
|
-
pending.delete(chunk.key);
|
|
25
|
-
yield chunk;
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
function batchError(error) {
|
|
29
|
-
return error instanceof AnalyticsError2 ? { code: error.code, message: error.message } : { code: "internal", message: "The query failed" };
|
|
30
|
-
}
|
|
31
|
-
function toHttpError(error) {
|
|
32
|
-
if (error.code === "forbidden") {
|
|
33
|
-
return { status: 403, code: "FORBIDDEN", message: error.message };
|
|
34
|
-
}
|
|
35
|
-
if (error.code === "missing_tenant") {
|
|
36
|
-
return { status: 401, code: "UNAUTHORIZED", message: error.message };
|
|
37
|
-
}
|
|
38
|
-
return {
|
|
39
|
-
status: 400,
|
|
40
|
-
code: "BAD_REQUEST",
|
|
41
|
-
message: error.message,
|
|
42
|
-
data: { code: error.code, details: error.details }
|
|
43
|
-
};
|
|
44
|
-
}
|
|
2
|
+
queryBatch2,
|
|
3
|
+
streamBatch2,
|
|
4
|
+
toHttpError2
|
|
5
|
+
} from "./index-psw9s799.js";
|
|
6
|
+
import"./index-g0ah4ehn.js";
|
|
45
7
|
export {
|
|
46
|
-
queryBatch,
|
|
47
|
-
streamBatch,
|
|
48
|
-
toHttpError
|
|
8
|
+
queryBatch2 as queryBatch,
|
|
9
|
+
streamBatch2 as streamBatch,
|
|
10
|
+
toHttpError2 as toHttpError
|
|
49
11
|
};
|
|
@@ -146,4 +146,4 @@ function bucketStarts(from, to, unit) {
|
|
|
146
146
|
return starts;
|
|
147
147
|
}
|
|
148
148
|
|
|
149
|
-
export { MAX_PERIOD_DAYS, DEFAULT_PERIOD, isValidTimezone, PeriodError, resolvePeriod, previousPeriod, bucketStarts };
|
|
149
|
+
export { MAX_PERIOD_DAYS, DEFAULT_PERIOD, isValidTimezone, localDate, PeriodError, resolvePeriod, previousPeriod, bucketStarts };
|