@fjader/svelte 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/animation/EntranceGate.d.ts +52 -0
- package/dist/animation/EntranceGate.js +93 -0
- package/dist/animation/TransitionClock.d.ts +61 -30
- package/dist/animation/TransitionClock.js +67 -29
- package/dist/animation/transition-log.d.ts +41 -0
- package/dist/animation/transition-log.js +57 -0
- package/dist/components/axes/AxisBottom.svelte +4 -2
- package/dist/components/axes/AxisCenter.svelte +4 -2
- package/dist/components/axes/AxisLeft.svelte +4 -2
- package/dist/components/axes/BandAxisTicks.svelte +3 -2
- package/dist/components/charts/ChartCell.svelte +32 -5
- package/dist/components/charts/ChartCell.svelte.d.ts +9 -0
- package/dist/components/charts/ChartContainer.svelte +45 -3
- package/dist/components/charts/ChartContainer.svelte.d.ts +7 -1
- package/dist/components/charts/ChartHover.svelte.d.ts +14 -0
- package/dist/components/charts/ChartHover.svelte.js +22 -0
- package/dist/components/charts/ChartVisibility.svelte.d.ts +43 -0
- package/dist/components/charts/ChartVisibility.svelte.js +67 -0
- package/dist/components/guides/Grid.svelte +12 -1
- package/dist/components/marks/AreaMarkX.svelte +16 -4
- package/dist/components/marks/AreaMarkY.svelte +16 -4
- package/dist/components/marks/BarMarkX.svelte +7 -3
- package/dist/components/marks/BarMarkY.svelte +7 -3
- package/dist/components/marks/DotsMark.svelte +6 -3
- package/dist/components/marks/LineMark.svelte +11 -8
- package/dist/components/marks/ParallelCoordinatesMark.svelte +7 -2
- package/dist/components/marks/ParallelCoordinatesMark.svelte.d.ts +2 -0
- package/dist/components/marks/RectMark.svelte +6 -3
- package/dist/components/marks/RectXMark.svelte +7 -3
- package/dist/components/marks/RectYMark.svelte +7 -3
- package/dist/components/marks/RuleMarkX.svelte +30 -26
- package/dist/components/marks/RuleMarkY.svelte +30 -26
- package/dist/components/marks/TextMark.svelte +44 -5
- package/dist/components/marks/lifecycle/MarkLifecycleV2.svelte.d.ts +16 -0
- package/dist/components/marks/lifecycle/MarkLifecycleV2.svelte.js +398 -54
- package/dist/components/marks/lifecycle/domain-queries.d.ts +11 -2
- package/dist/components/marks/lifecycle/domain-queries.js +33 -6
- package/dist/components/marks/lifecycle/domain-registration.d.ts +6 -1
- package/dist/components/marks/lifecycle/domain-registration.js +15 -14
- package/dist/components/marks/lifecycle/other-rows.svelte.d.ts +9 -1
- package/dist/components/marks/lifecycle/other-rows.svelte.js +2 -2
- package/dist/components/marks/lifecycle/query-builder.d.ts +27 -2
- package/dist/components/marks/lifecycle/query-builder.js +23 -20
- package/dist/components/marks/utils/MarkColor.svelte.d.ts +3 -1
- package/dist/components/marks/utils/MarkColor.svelte.js +33 -3
- package/dist/components/marks/utils/MarkSeries.svelte.d.ts +14 -1
- package/dist/components/marks/utils/MarkSeries.svelte.js +25 -1
- package/dist/components/marks/utils/MarkSize.svelte.js +4 -0
- package/dist/components/marks/utils/reactive-config.d.ts +1 -1
- package/dist/components/marks/utils/reactive-config.js +3 -0
- package/dist/components/rendering/Areas.svelte +57 -19
- package/dist/components/rendering/Areas.svelte.d.ts +12 -0
- package/dist/components/rendering/BaseGPURenderer.svelte.d.ts +1 -1
- package/dist/components/rendering/Dots.svelte +23 -3
- package/dist/components/rendering/Dots.svelte.d.ts +6 -0
- package/dist/components/rendering/GridLine.svelte +13 -1
- package/dist/components/rendering/GridLine.svelte.d.ts +5 -0
- package/dist/components/rendering/Lines.svelte +48 -13
- package/dist/components/rendering/Lines.svelte.d.ts +12 -0
- package/dist/components/rendering/ParallelLines.svelte +5 -5
- package/dist/components/rendering/Rects.svelte +51 -11
- package/dist/components/rendering/Rects.svelte.d.ts +16 -2
- package/dist/components/rendering/RuleX.svelte +1 -1
- package/dist/components/rendering/RuleY.svelte +1 -1
- package/dist/components/rendering/WebGPUCanvas.svelte +4 -2
- package/dist/components/rendering/activity-reads.d.ts +41 -24
- package/dist/components/rendering/activity-reads.js +54 -31
- package/dist/components/rendering/uniform-sizes.d.ts +1 -1
- package/dist/components/rendering/uniform-sizes.js +1 -1
- package/dist/config/ChartConfig.svelte.d.ts +7 -0
- package/dist/config/ChartConfig.svelte.js +17 -0
- package/dist/gpu/GPUCoordinator.svelte.d.ts +31 -1
- package/dist/gpu/GPUCoordinator.svelte.js +164 -7
- package/dist/gpu/MorphedChannels.d.ts +11 -2
- package/dist/gpu/MorphedChannels.js +33 -11
- package/dist/gpu/ScaleTemplateSystem.d.ts +20 -2
- package/dist/gpu/ScaleTemplateSystem.js +58 -2
- package/dist/index.d.ts +17 -4
- package/dist/index.js +19 -4
- package/dist/interactors/TooltipManager.svelte.js +14 -5
- package/dist/query-engine/CrossfilterSelection.svelte.d.ts +4 -2
- package/dist/query-engine/CrossfilterSelection.svelte.js +2 -0
- package/dist/query-engine/FieldActivity.svelte.d.ts +2 -3
- package/dist/query-engine/FieldActivity.svelte.js +1 -2
- package/dist/query-engine/MarkHandle.svelte.d.ts +28 -1
- package/dist/query-engine/MarkHandle.svelte.js +33 -1
- package/dist/query-engine/QueryEngine.d.ts +225 -5
- package/dist/query-engine/QueryEngine.js +671 -41
- package/dist/query-engine/categorical-identity.d.ts +34 -8
- package/dist/query-engine/categorical-identity.js +138 -73
- package/dist/query-engine/debug/query-log.d.ts +39 -0
- package/dist/query-engine/debug/query-log.js +90 -0
- package/dist/query-engine/duckdb.d.ts +55 -8
- package/dist/query-engine/duckdb.js +82 -30
- package/dist/query-engine/index.d.ts +7 -4
- package/dist/query-engine/index.js +3 -3
- package/dist/query-engine/preagg/BuildQueue.d.ts +33 -13
- package/dist/query-engine/preagg/BuildQueue.js +51 -15
- package/dist/query-engine/preagg/Preagg.d.ts +46 -3
- package/dist/query-engine/preagg/Preagg.js +159 -31
- package/dist/query-engine/preagg/plan.d.ts +28 -1
- package/dist/query-engine/preagg/plan.js +128 -8
- package/dist/query-engine/sliced-lane.d.ts +108 -0
- package/dist/query-engine/sliced-lane.js +299 -0
- package/dist/query-engine/source-routing.d.ts +106 -0
- package/dist/query-engine/source-routing.js +189 -0
- package/dist/scales/BandScaleState.svelte.d.ts +9 -1
- package/dist/scales/BandScaleState.svelte.js +16 -5
- package/dist/scales/ChartScales.svelte.d.ts +3 -0
- package/dist/scales/ChartScales.svelte.js +26 -8
- package/dist/scales/ColorScaleState.svelte.d.ts +18 -2
- package/dist/scales/ColorScaleState.svelte.js +138 -26
- package/dist/scales/LinearScaleState.svelte.d.ts +7 -10
- package/dist/scales/LinearScaleState.svelte.js +21 -39
- package/dist/scales/ScaleGeneration.svelte.d.ts +48 -0
- package/dist/scales/ScaleGeneration.svelte.js +42 -0
- package/dist/scales/ScaleRegistry.svelte.js +3 -1
- package/dist/scales/SlotTransition.d.ts +15 -1
- package/dist/scales/SlotTransition.js +33 -4
- package/dist/scales/TimeScaleState.svelte.d.ts +7 -3
- package/dist/scales/TimeScaleState.svelte.js +7 -7
- package/dist/scales/color-utils.d.ts +0 -3
- package/dist/scales/color-utils.js +0 -3
- package/dist/scales/entered.d.ts +8 -0
- package/dist/scales/entered.js +10 -0
- package/dist/spec/LazyQueryEngine.js +2 -19
- package/dist/spec/MarkRegistry.d.ts +3 -1
- package/dist/spec/MarkRegistry.js +13 -1
- package/dist/spec/PlotFromSpec.svelte +1 -0
- package/dist/spec/export.js +11 -2
- package/dist/spec/types.d.ts +41 -13
- package/dist/sql/bin.d.ts +25 -0
- package/dist/sql/bin.js +108 -5
- package/dist/transforms/TransformCompiler.d.ts +10 -0
- package/dist/transforms/TransformCompiler.js +26 -0
- package/dist/transforms/index.js +144 -24
- package/dist/types.d.ts +136 -2
- package/package.json +2 -2
|
@@ -60,6 +60,40 @@ export interface IdentityExec {
|
|
|
60
60
|
exec(sql: string | string[]): Promise<void>;
|
|
61
61
|
/** Rows as plain objects. Callers adapt whatever their driver returns. */
|
|
62
62
|
queryRows(sql: string): Promise<Array<Record<string, unknown>>>;
|
|
63
|
+
/**
|
|
64
|
+
* Whether `table` is known to be a base table (not a view). Only then is a
|
|
65
|
+
* column's cardinality read from its statistics (#cardinalityFromStats):
|
|
66
|
+
* on a view, `stats(col) … LIMIT 1` runs the view, and a filtered view
|
|
67
|
+
* reports its base column's statistics. Absent: never.
|
|
68
|
+
*
|
|
69
|
+
* About the table's TYPE, not where it is stored: a table in an attached
|
|
70
|
+
* file or remote database answers yes (its statistics are metadata either
|
|
71
|
+
* way). Not a "fast source" test: routing asks that separately
|
|
72
|
+
* (SourceRouting), and must not reuse this.
|
|
73
|
+
*/
|
|
74
|
+
isBaseTable?(table: string): boolean;
|
|
75
|
+
/**
|
|
76
|
+
* Run statements as one message, returning the last one's rows (the
|
|
77
|
+
* engine's execBatch). Absent: they run one by one through exec and
|
|
78
|
+
* queryRows, with the same result.
|
|
79
|
+
*/
|
|
80
|
+
execBatch?(statements: string[]): Promise<Array<Record<string, unknown>>>;
|
|
81
|
+
/**
|
|
82
|
+
* Debug: told of every ensureColumns call (the table, the fields asked for,
|
|
83
|
+
* performance.now()), before anything is deduplicated. For measuring when
|
|
84
|
+
* marks ask, e.g. whether one table's fields arrive close enough together
|
|
85
|
+
* to share a drain.
|
|
86
|
+
*/
|
|
87
|
+
onRequest?(table: string, fields: string[], time: number): void;
|
|
88
|
+
/** Debug: one pass of a table's drain started, with the fields it took. */
|
|
89
|
+
onDrain?(table: string, fields: string[], time: number): void;
|
|
90
|
+
/**
|
|
91
|
+
* Whether anything listens to onRequest and onDrain. Checked first, so with
|
|
92
|
+
* no listener their arguments (a field list, a timestamp) are never built.
|
|
93
|
+
* The callbacks fire ONLY when this returns true: a host that implements
|
|
94
|
+
* them must implement this too, or it is never called.
|
|
95
|
+
*/
|
|
96
|
+
observing?(): boolean;
|
|
63
97
|
}
|
|
64
98
|
/**
|
|
65
99
|
* A thing that can have an identity: a source column, or a fixed SQL expression
|
|
@@ -290,14 +324,6 @@ export declare class CategoricalIdentity {
|
|
|
290
324
|
* before a reset names a row of a dropped dim, and is ignored.
|
|
291
325
|
*/
|
|
292
326
|
releaseSynthetic(synthetic: SyntheticId): void;
|
|
293
|
-
/**
|
|
294
|
-
* Re-read the dim into the field's value array, mutating in place.
|
|
295
|
-
*
|
|
296
|
-
* Re-reading the whole dim rather than only the new rows: categorical
|
|
297
|
-
* cardinality is small, this runs once per (field, table) pair at setup, and
|
|
298
|
-
* a full read cannot drift out of sync with the ids the way an incremental
|
|
299
|
-
* one can if an append is ever missed.
|
|
300
|
-
*/
|
|
301
327
|
/**
|
|
302
328
|
* Bumped whenever a dim's rows change: a table folding in, an Other bucket
|
|
303
329
|
* reserved, a reset. A query that reads a dim (dense padding, an id join)
|
|
@@ -304,19 +304,6 @@ export class CategoricalIdentity {
|
|
|
304
304
|
presentIds(field, table) {
|
|
305
305
|
return this.#dims.get(field)?.present.get(table)?.ids ?? null;
|
|
306
306
|
}
|
|
307
|
-
/**
|
|
308
|
-
* Record which of the dim's ids `table` holds: the ids of its distinct values,
|
|
309
|
-
* joined as the view joins (NULL to a NULL row, never to a synthetic one).
|
|
310
|
-
* Once per table, when it folds in.
|
|
311
|
-
*/
|
|
312
|
-
async #recordPresent(dim, table, expr, src) {
|
|
313
|
-
const name = `${dim.name}_t_${identityToken(table)}`;
|
|
314
|
-
await this.#exec.exec(`CREATE OR REPLACE TABLE ${quote(name)} AS ` +
|
|
315
|
-
`SELECT DISTINCT d.id FROM (SELECT DISTINCT ${expr} AS v FROM ${src}) s ` +
|
|
316
|
-
`JOIN ${quote(dim.name)} d ON s.v IS NOT DISTINCT FROM d.value AND d.label IS NULL`);
|
|
317
|
-
const rows = await this.#exec.queryRows(`SELECT id FROM ${quote(name)}`);
|
|
318
|
-
dim.present.set(table, { name, ids: new Set(rows.map((r) => Number(r.id))) });
|
|
319
|
-
}
|
|
320
307
|
/** Name of the view that carries id columns for `table`. */
|
|
321
308
|
viewTable(table) {
|
|
322
309
|
return `${identityToken(table)}_v_s${this.#session}`;
|
|
@@ -382,6 +369,9 @@ export class CategoricalIdentity {
|
|
|
382
369
|
const categorical = (fields.categorical ?? []).map(asField);
|
|
383
370
|
if (categorical.length === 0)
|
|
384
371
|
return;
|
|
372
|
+
if (this.#exec.observing?.()) {
|
|
373
|
+
this.#exec.onRequest?.(table, categorical.map((f) => f.key), performance.now());
|
|
374
|
+
}
|
|
385
375
|
let entry = this.#views.get(table);
|
|
386
376
|
if (!entry) {
|
|
387
377
|
entry = { current: new Set(), pending: new Set(), inflight: null, failures: new Map() };
|
|
@@ -468,6 +458,10 @@ export class CategoricalIdentity {
|
|
|
468
458
|
return;
|
|
469
459
|
const batch = [...entry.pending];
|
|
470
460
|
entry.pending.clear();
|
|
461
|
+
// Per pass: a request arriving during the first build is a second
|
|
462
|
+
// pass (and a second view build) inside this same call.
|
|
463
|
+
if (this.#exec.observing?.())
|
|
464
|
+
this.#exec.onDrain?.(table, batch, performance.now());
|
|
471
465
|
const succeeded = [];
|
|
472
466
|
for (const field of batch) {
|
|
473
467
|
try {
|
|
@@ -650,55 +644,83 @@ export class CategoricalIdentity {
|
|
|
650
644
|
// The value expression: a quoted column, or a fixed SQL expression. Both
|
|
651
645
|
// are evaluated with only the source table in scope.
|
|
652
646
|
const expr = dim.sql;
|
|
653
|
-
await this.#checkCardinality(field, expr, src);
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
`
|
|
682
|
-
|
|
683
|
-
//
|
|
684
|
-
//
|
|
685
|
-
//
|
|
686
|
-
|
|
687
|
-
`
|
|
688
|
-
|
|
689
|
-
|
|
647
|
+
await this.#checkCardinality(field, expr, src, table);
|
|
648
|
+
// The field's statements as ONE message (#batch): sent one by one, each
|
|
649
|
+
// step went to the back of the queue, behind every other mark's reads
|
|
650
|
+
// sent meanwhile; at 2M rows the categorical charts trailed the others
|
|
651
|
+
// by 420-500 ms. The last statement returns the values the JS cache
|
|
652
|
+
// needs, and this table's present ids with them.
|
|
653
|
+
const first = dim.tables.size === 0;
|
|
654
|
+
const presentName = `${dim.name}_t_${identityToken(table)}`;
|
|
655
|
+
const present = quote(presentName);
|
|
656
|
+
// `label` is NULL for every real value and carries the text for a
|
|
657
|
+
// synthetic one (see reserveSynthetic). It exists because `value` takes the
|
|
658
|
+
// SOURCE column's type: on a band axis over a year or a rating there is no
|
|
659
|
+
// INTEGER that means "Other", and writing the string into that column is a
|
|
660
|
+
// conversion error, not a fallback.
|
|
661
|
+
const statements = first
|
|
662
|
+
? [
|
|
663
|
+
`CREATE TABLE IF NOT EXISTS ${dimName} AS ` +
|
|
664
|
+
`SELECT value, NULL::VARCHAR AS label, ` +
|
|
665
|
+
`(ROW_NUMBER() OVER (ORDER BY value NULLS LAST) - 1)::INT AS id ` +
|
|
666
|
+
`FROM (SELECT DISTINCT ${expr} AS value FROM ${src})`,
|
|
667
|
+
// A second statement: CTAS cannot carry a constraint. `value` is
|
|
668
|
+
// unique by construction here (DISTINCT) and on append (WHERE NOT
|
|
669
|
+
// EXISTS), but reserveSynthetic is a writer that could break it, and
|
|
670
|
+
// a duplicate value does not tie-break: #createView joins ON
|
|
671
|
+
// t.field = d.value, so it FANS OUT every matching source row in
|
|
672
|
+
// every query against the view. Synthetic rows hold value NULL, and
|
|
673
|
+
// SQL treats NULLs as distinct under a unique index, so any number of
|
|
674
|
+
// them coexist (verified against DuckDB).
|
|
675
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS ${quote(`${dim.name}_value_uq`)} ON ${dimName}(value)`,
|
|
676
|
+
// The table that built the dim holds every real row of it (`chain`
|
|
677
|
+
// serializes a dim's work, so only one table takes this branch): its
|
|
678
|
+
// present ids are the real rows, no scan (the NULL row has no label
|
|
679
|
+
// and is present, as the join finds it; a synthetic row has one).
|
|
680
|
+
`CREATE OR REPLACE TABLE ${present} AS SELECT id FROM ${dimName} WHERE label IS NULL`,
|
|
681
|
+
`SELECT id, value, label FROM ${dimName} ORDER BY id`
|
|
682
|
+
]
|
|
683
|
+
: [
|
|
684
|
+
// Columns named rather than positional: a positional INSERT would
|
|
685
|
+
// silently put the id into `label`. IS NOT DISTINCT FROM, so a second
|
|
686
|
+
// table's NULLs find the NULL row; the label test keeps a synthetic
|
|
687
|
+
// row, also value NULL, from passing for it.
|
|
688
|
+
`INSERT INTO ${dimName} (value, label, id) ` +
|
|
689
|
+
`SELECT s.v, NULL, ((SELECT COALESCE(MAX(id), -1) FROM ${dimName}) + ` +
|
|
690
|
+
`ROW_NUMBER() OVER (ORDER BY s.v NULLS LAST))::INT ` +
|
|
691
|
+
`FROM (SELECT DISTINCT ${expr} AS v FROM ${src}) s ` +
|
|
692
|
+
`WHERE NOT EXISTS (SELECT 1 FROM ${dimName} d ` +
|
|
693
|
+
`WHERE d.value IS NOT DISTINCT FROM s.v AND d.label IS NULL)`,
|
|
694
|
+
// An appending table holds a subset: the ids of its distinct values,
|
|
695
|
+
// joined as the view joins.
|
|
696
|
+
`CREATE OR REPLACE TABLE ${present} AS ` +
|
|
697
|
+
`SELECT DISTINCT d.id FROM (SELECT DISTINCT ${expr} AS v FROM ${src}) s ` +
|
|
698
|
+
`JOIN ${dimName} d ON s.v IS NOT DISTINCT FROM d.value AND d.label IS NULL`,
|
|
699
|
+
`SELECT id, value, label, id IN (SELECT id FROM ${present}) AS present ` +
|
|
700
|
+
`FROM ${dimName} ORDER BY id`
|
|
701
|
+
];
|
|
702
|
+
const rows = await this.#batch(statements);
|
|
703
|
+
// A reset while the batch ran: its DROPs queued behind it, so these rows
|
|
704
|
+
// are of dims that no longer exist. Applying them would fill the cleared
|
|
705
|
+
// state with stale ids.
|
|
706
|
+
if (this.#generation !== generation)
|
|
707
|
+
return;
|
|
690
708
|
dim.tables.add(table);
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
// sizes the slot table and therefore what the shared buffer has to hold.
|
|
697
|
-
// Exact rather than approximate here: the rows exist, COUNT(*) on a
|
|
698
|
-
// categorical dim is trivial, and this is the number the error should
|
|
699
|
-
// quote.
|
|
709
|
+
this.#applyValues(field, rows);
|
|
710
|
+
const ids = first
|
|
711
|
+
? rows.filter((r) => r.label == null).map((r) => Number(r.id))
|
|
712
|
+
: rows.filter((r) => r.present === true).map((r) => Number(r.id));
|
|
713
|
+
dim.present.set(table, { name: presentName, ids: new Set(ids) });
|
|
700
714
|
this.#checkDimSize(dim, field, this.identity(field).values.length);
|
|
701
715
|
}
|
|
716
|
+
/** Statements as one message (IdentityExec.execBatch), or one by one; the last one's rows. */
|
|
717
|
+
async #batch(statements) {
|
|
718
|
+
if (this.#exec.execBatch)
|
|
719
|
+
return this.#exec.execBatch(statements);
|
|
720
|
+
for (const stmt of statements.slice(0, -1))
|
|
721
|
+
await this.#exec.exec(stmt);
|
|
722
|
+
return this.#exec.queryRows(statements[statements.length - 1]);
|
|
723
|
+
}
|
|
702
724
|
/**
|
|
703
725
|
* Refuse a band axis with more distinct values than can render.
|
|
704
726
|
*
|
|
@@ -708,23 +730,62 @@ export class CategoricalIdentity {
|
|
|
708
730
|
* the setup path that already blocks on one. Here it costs a query only for fields
|
|
709
731
|
* that actually get a dim, and it is where the refusal has to land anyway.
|
|
710
732
|
*
|
|
711
|
-
*
|
|
712
|
-
*
|
|
713
|
-
*
|
|
733
|
+
* For a plain column, its statistics first (#cardinalityFromStats): no scan.
|
|
734
|
+
* Otherwise `approx_count_distinct` rather than `COUNT(DISTINCT …)`: one
|
|
735
|
+
* HyperLogLog pass with no materialization, which matters precisely for the
|
|
736
|
+
* runaway expression this exists to catch.
|
|
714
737
|
*
|
|
715
738
|
* Neither syntax nor result type can answer this question — `year(ts)` is
|
|
716
739
|
* numeric with a tiny distinct set, a CASE can have ten thousand branches —
|
|
717
740
|
* so the count is the only honest gate.
|
|
718
741
|
*/
|
|
719
|
-
async #checkCardinality(field, expr, src) {
|
|
720
|
-
const
|
|
721
|
-
|
|
742
|
+
async #checkCardinality(field, expr, src, table) {
|
|
743
|
+
const scan = async () => Number((await this.#exec.queryRows(`SELECT approx_count_distinct(${expr}) AS n FROM ${src}`))[0]
|
|
744
|
+
?.n ?? 0);
|
|
745
|
+
const estimate = await this.#cardinalityFromStats(expr, src, table);
|
|
746
|
+
let n = estimate ?? (await scan());
|
|
747
|
+
// Statistics are never shrunk (deletes, updates), so an estimate over the
|
|
748
|
+
// limit can be stale: confirm it with the scan before refusing. A false
|
|
749
|
+
// refusal is user-visible; the scan runs only in this rare case.
|
|
750
|
+
if (estimate != null && n > MAX_IDENTITY_CARDINALITY)
|
|
751
|
+
n = await scan();
|
|
722
752
|
if (n > MAX_IDENTITY_CARDINALITY) {
|
|
723
753
|
throw new Error(`"${field}" has about ${n} distinct values in this table, over the ` +
|
|
724
754
|
`${MAX_IDENTITY_CARDINALITY} a categorical scale can render. Use a continuous ` +
|
|
725
755
|
`scale, bin the values, or reduce them with a transform.`);
|
|
726
756
|
}
|
|
727
757
|
}
|
|
758
|
+
/**
|
|
759
|
+
* A plain column's approximate distinct count from DuckDB's column
|
|
760
|
+
* statistics, without scanning: `stats(col)` reads what DuckDB keeps per
|
|
761
|
+
* column ("Approx Unique"). Measured on DuckDB-WASM: 1-3 ms, against
|
|
762
|
+
* 27-28 ms for approx_count_distinct over 2M rows; it agrees with the scan
|
|
763
|
+
* (167 and 167, 245 and 251). It can undercount a very large set (1.26M for
|
|
764
|
+
* 2M unique values): still far over the limit, and #checkDimSize counts
|
|
765
|
+
* the built dim exactly behind it. It can OVERcount, never shrinking after
|
|
766
|
+
* deletes, so an over-limit answer is confirmed by the scan
|
|
767
|
+
* (#checkCardinality).
|
|
768
|
+
*
|
|
769
|
+
* Base tables only (IdentityExec.isBaseTable): on a view, `stats … LIMIT 1`
|
|
770
|
+
* runs the view (a view that computes would then run twice, if the column
|
|
771
|
+
* has no estimate), and a filtered view reports its base column's
|
|
772
|
+
* statistics. Null for an expression field, a view, or statistics that say
|
|
773
|
+
* nothing: the caller scans as before.
|
|
774
|
+
*/
|
|
775
|
+
async #cardinalityFromStats(expr, src, table) {
|
|
776
|
+
if (!/^"(?:[^"]|"")+"$/.test(expr))
|
|
777
|
+
return null;
|
|
778
|
+
if (!this.#exec.isBaseTable?.(table))
|
|
779
|
+
return null;
|
|
780
|
+
try {
|
|
781
|
+
const rows = await this.#exec.queryRows(`SELECT stats(${expr}) AS s FROM ${src} LIMIT 1`);
|
|
782
|
+
const match = /Approx Unique: (\d+)/.exec(String(rows[0]?.s ?? ''));
|
|
783
|
+
return match ? Number(match[1]) : null;
|
|
784
|
+
}
|
|
785
|
+
catch {
|
|
786
|
+
return null;
|
|
787
|
+
}
|
|
788
|
+
}
|
|
728
789
|
/**
|
|
729
790
|
* Refuse or warn on the DIM's size, which is what actually has to render.
|
|
730
791
|
*
|
|
@@ -767,14 +828,6 @@ export class CategoricalIdentity {
|
|
|
767
828
|
`and a grid on that axis draws a line per value whether it is displayed or not.`);
|
|
768
829
|
}
|
|
769
830
|
}
|
|
770
|
-
/**
|
|
771
|
-
* Re-read the dim into the field's value array, mutating in place.
|
|
772
|
-
*
|
|
773
|
-
* Re-reading the whole dim rather than only the new rows: categorical
|
|
774
|
-
* cardinality is small, this runs once per (field, table) pair at setup, and
|
|
775
|
-
* a full read cannot drift out of sync with the ids the way an incremental
|
|
776
|
-
* one can if an append is ever missed.
|
|
777
|
-
*/
|
|
778
831
|
/**
|
|
779
832
|
* Bumped whenever a dim's rows change: a table folding in, an Other bucket
|
|
780
833
|
* reserved, a reset. A query that reads a dim (dense padding, an id join)
|
|
@@ -785,8 +838,20 @@ export class CategoricalIdentity {
|
|
|
785
838
|
return this.#version;
|
|
786
839
|
}
|
|
787
840
|
#version = 0;
|
|
841
|
+
/**
|
|
842
|
+
* Re-read the dim into the field's value array, mutating in place.
|
|
843
|
+
*
|
|
844
|
+
* Re-reading the whole dim rather than only the new rows: categorical
|
|
845
|
+
* cardinality is small, this runs once per (field, table) pair at setup, and
|
|
846
|
+
* a full read cannot drift out of sync with the ids the way an incremental
|
|
847
|
+
* one can if an append is ever missed.
|
|
848
|
+
*/
|
|
788
849
|
async #refreshValues(field) {
|
|
789
850
|
const rows = await this.#exec.queryRows(`SELECT id, value, label FROM ${quote(this.dimTable(field))} ORDER BY id`);
|
|
851
|
+
this.#applyValues(field, rows);
|
|
852
|
+
}
|
|
853
|
+
/** The dim's rows (`id, value, label`, in id order) into the JS cache. */
|
|
854
|
+
#applyValues(field, rows) {
|
|
790
855
|
// Ordered by id and dense by construction, so position is the id — and
|
|
791
856
|
// reset() derives value -> id from the same pass, so the two directions
|
|
792
857
|
// cannot disagree.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A statement log for measuring what reaches DuckDB: attach it to an engine,
|
|
3
|
+
* use the page, then read `report()` or `download()` the raw log.
|
|
4
|
+
*
|
|
5
|
+
* Built on `QueryEngine.observeStatements`, so it sees every statement the
|
|
6
|
+
* engine sends (queries, statements, DESCRIBE, Arrow inserts) with its class.
|
|
7
|
+
* Times are taken on the main thread when a statement is handed over and when
|
|
8
|
+
* it settles; while the main thread is busy (decoding, GPU uploads) they read
|
|
9
|
+
* high. The queue-wait estimate assumes each connection serves its statements
|
|
10
|
+
* first-in, first-out: a statement starts when it was sent or when the
|
|
11
|
+
* previous one ON ITS CONNECTION finished, whichever is later. Statements on a
|
|
12
|
+
* sliced lane interleave with the main connection's in the worker, so their
|
|
13
|
+
* "work" includes slices spent serving the other.
|
|
14
|
+
*/
|
|
15
|
+
import type { QueryEngine, StatementRecord } from '../QueryEngine';
|
|
16
|
+
/** What a statement is for, from its text: a coarse grouping for reports. */
|
|
17
|
+
export type StatementPurpose = 'describe' | 'ddl' | 'insert' | 'range' | 'data' | 'batch';
|
|
18
|
+
export interface LoggedStatement extends StatementRecord {
|
|
19
|
+
purpose: StatementPurpose;
|
|
20
|
+
/** The first table the statement reads, or '?'. */
|
|
21
|
+
table: string;
|
|
22
|
+
}
|
|
23
|
+
export interface QueryLog {
|
|
24
|
+
readonly statements: readonly LoggedStatement[];
|
|
25
|
+
/** Print a summary to the console and return it as rows. */
|
|
26
|
+
report(): ReportRow[];
|
|
27
|
+
/** Download the raw log as JSON (to the browser's downloads folder). */
|
|
28
|
+
download(name: string, extra?: Record<string, unknown>): void;
|
|
29
|
+
reset(): void;
|
|
30
|
+
detach(): void;
|
|
31
|
+
}
|
|
32
|
+
export interface ReportRow {
|
|
33
|
+
group: string;
|
|
34
|
+
n: number;
|
|
35
|
+
workMs: number;
|
|
36
|
+
waitMs: number;
|
|
37
|
+
}
|
|
38
|
+
/** Attach a statement log to an engine. Costs nothing once detached. */
|
|
39
|
+
export declare function attachQueryLog(engine: QueryEngine): QueryLog;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
function purposeOf(record) {
|
|
2
|
+
if (record.kind === 'describe')
|
|
3
|
+
return 'describe';
|
|
4
|
+
if (record.kind === 'insert')
|
|
5
|
+
return 'insert';
|
|
6
|
+
if (record.kind === 'batch')
|
|
7
|
+
return 'batch';
|
|
8
|
+
const s = record.sql.trimStart().toUpperCase();
|
|
9
|
+
if (/^(CREATE|DROP|INSERT|ALTER|DELETE|UPDATE)/.test(s))
|
|
10
|
+
return 'ddl';
|
|
11
|
+
// A scalar summary: min/max aggregates, no GROUP BY, no window.
|
|
12
|
+
if (/^SELECT/.test(s) && !/GROUP BY/.test(s) && /(MIN|MAX)\(/.test(s) && !/ OVER \(/.test(s)) {
|
|
13
|
+
return 'range';
|
|
14
|
+
}
|
|
15
|
+
return 'data';
|
|
16
|
+
}
|
|
17
|
+
function tableOf(sql) {
|
|
18
|
+
const m = /FROM\s+"?([A-Za-z_][\w]*)"?/i.exec(sql);
|
|
19
|
+
return m ? m[1] : '?';
|
|
20
|
+
}
|
|
21
|
+
/** Attach a statement log to an engine. Costs nothing once detached. */
|
|
22
|
+
export function attachQueryLog(engine) {
|
|
23
|
+
const statements = [];
|
|
24
|
+
/** Categorical identity requests, in order: when marks asked for which fields. */
|
|
25
|
+
const identityRequests = [];
|
|
26
|
+
const detachStatements = engine.observeStatements((record) => {
|
|
27
|
+
statements.push({ ...record, purpose: purposeOf(record), table: tableOf(record.sql) });
|
|
28
|
+
});
|
|
29
|
+
const detachIdentity = engine.observeIdentityRequests((request) => {
|
|
30
|
+
identityRequests.push(request);
|
|
31
|
+
});
|
|
32
|
+
/** Every preagg build step, in order (observeBuilds). Errors as text, for JSON. */
|
|
33
|
+
const builds = [];
|
|
34
|
+
const detachBuilds = engine.observeBuilds((event) => {
|
|
35
|
+
builds.push(event.error === undefined ? event : { ...event, error: String(event.error) });
|
|
36
|
+
});
|
|
37
|
+
const detach = () => {
|
|
38
|
+
detachStatements();
|
|
39
|
+
detachIdentity();
|
|
40
|
+
detachBuilds();
|
|
41
|
+
};
|
|
42
|
+
const report = () => {
|
|
43
|
+
const sorted = [...statements].sort((a, b) => a.start - b.start);
|
|
44
|
+
if (sorted.length === 0)
|
|
45
|
+
return [];
|
|
46
|
+
const lastEndOn = { main: 0, lane: 0 };
|
|
47
|
+
let lastEnd = 0;
|
|
48
|
+
const groups = new Map();
|
|
49
|
+
let work = 0;
|
|
50
|
+
for (const s of sorted) {
|
|
51
|
+
const conn = s.connection ?? 'main';
|
|
52
|
+
const start = Math.max(s.start, lastEndOn[conn]);
|
|
53
|
+
lastEndOn[conn] = Math.max(lastEndOn[conn], s.end);
|
|
54
|
+
lastEnd = Math.max(lastEnd, s.end);
|
|
55
|
+
const key = `${conn} ${s.class} ${s.purpose} ${s.table}`;
|
|
56
|
+
const g = groups.get(key) ?? { group: key, n: 0, workMs: 0, waitMs: 0 };
|
|
57
|
+
g.n++;
|
|
58
|
+
g.workMs += s.end - start;
|
|
59
|
+
g.waitMs += start - s.start;
|
|
60
|
+
work += s.end - start;
|
|
61
|
+
groups.set(key, g);
|
|
62
|
+
}
|
|
63
|
+
const span = lastEnd - sorted[0].start;
|
|
64
|
+
const rows = [...groups.values()]
|
|
65
|
+
.map((g) => ({ ...g, workMs: +g.workMs.toFixed(1), waitMs: +g.waitMs.toFixed(1) }))
|
|
66
|
+
.sort((a, b) => b.workMs - a.workMs);
|
|
67
|
+
console.log(`[query-log] ${sorted.length} statements, span ${span.toFixed(0)} ms, work ${work.toFixed(0)} ms`);
|
|
68
|
+
console.table(rows);
|
|
69
|
+
return rows;
|
|
70
|
+
};
|
|
71
|
+
const download = (name, extra = {}) => {
|
|
72
|
+
const json = JSON.stringify({ ...extra, statements, identityRequests, builds }, null, 1);
|
|
73
|
+
const a = document.createElement('a');
|
|
74
|
+
a.href = URL.createObjectURL(new Blob([json], { type: 'application/json' }));
|
|
75
|
+
a.download = name;
|
|
76
|
+
a.click();
|
|
77
|
+
URL.revokeObjectURL(a.href);
|
|
78
|
+
};
|
|
79
|
+
return {
|
|
80
|
+
statements,
|
|
81
|
+
report,
|
|
82
|
+
download,
|
|
83
|
+
reset: () => {
|
|
84
|
+
statements.length = 0;
|
|
85
|
+
identityRequests.length = 0;
|
|
86
|
+
builds.length = 0;
|
|
87
|
+
},
|
|
88
|
+
detach
|
|
89
|
+
};
|
|
90
|
+
}
|
|
@@ -1,21 +1,68 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* DuckDB-WASM initialization — replaces mosaic-core's wasmConnector.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* initDuckDB() returns a Promise<DuckDBConnection> suitable for passing to
|
|
5
|
+
* setQueryContext(). It loads the jsDelivr bundles unless the app passes its own.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* zero-copy typed arrays that go straight to GPU buffers.
|
|
7
|
+
* connectionFrom() adapts a duckdb-wasm connection the app created itself, for
|
|
8
|
+
* apps that need the AsyncDuckDB instance (registerFileBuffer, extensions, …).
|
|
10
9
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
10
|
+
* Both use the fast IPC query path: bypasses duckdb-wasm's Arrow JS
|
|
11
|
+
* deserialization and returns raw Arrow IPC bytes. QueryEngine decodes them with
|
|
12
|
+
* flechette into typed arrays for the GPU buffers. These are views on the IPC
|
|
13
|
+
* bytes only up to one record batch: DuckDB-WASM writes a batch per DuckDB vector
|
|
14
|
+
* (~2k rows), and flechette concatenates a column that spans several.
|
|
13
15
|
*/
|
|
16
|
+
import type { AsyncDuckDBConnection, DuckDBBundles } from '@duckdb/duckdb-wasm';
|
|
14
17
|
import type { DuckDBConnection } from './QueryEngine';
|
|
18
|
+
/**
|
|
19
|
+
* How long the worker runs a sliced statement before serving other messages:
|
|
20
|
+
* a quick query on another connection waits at most about this long behind a
|
|
21
|
+
* slow one. Measured in Phase 0: 1 ms at 5 ms, ~50 ms at the default (~100 ms).
|
|
22
|
+
*/
|
|
23
|
+
export declare const QUERY_POLLING_INTERVAL_MS = 5;
|
|
24
|
+
export interface InitDuckDBOptions {
|
|
25
|
+
/**
|
|
26
|
+
* DuckDB-WASM bundles to choose from, e.g. served by the app itself:
|
|
27
|
+
*
|
|
28
|
+
* import mvpWasm from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url';
|
|
29
|
+
* import mvpWorker from '@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url';
|
|
30
|
+
* …
|
|
31
|
+
* initFjader({ bundles: { mvp: { mainModule: mvpWasm, mainWorker: mvpWorker }, eh: … } });
|
|
32
|
+
*
|
|
33
|
+
* Only `mvp` is required; selectBundle picks `eh`/`coi` when given and supported.
|
|
34
|
+
* Relative URLs resolve against the page. Defaults to the jsDelivr bundles.
|
|
35
|
+
*/
|
|
36
|
+
bundles?: DuckDBBundles;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Adapt a duckdb-wasm connection to the DuckDBConnection QueryEngine expects,
|
|
40
|
+
* with the fast IPC query path, and sliced lanes: further connections on the
|
|
41
|
+
* same database (`openSlicedLane`).
|
|
42
|
+
*
|
|
43
|
+
* const db = new duckdb.AsyncDuckDB(logger, worker);
|
|
44
|
+
* await db.instantiate(…);
|
|
45
|
+
* await db.open({ query: { queryPollingInterval: 5 } }); // short slices, see below
|
|
46
|
+
* await db.registerFileBuffer('cached.parquet', bytes);
|
|
47
|
+
* initFjader(connectionFrom(await db.connect(), { pollingIntervalMs: 5 }));
|
|
48
|
+
*
|
|
49
|
+
* Sliced statements run in slices of the database's `queryPollingInterval`.
|
|
50
|
+
* At DuckDB's default (~100 ms) a quick query can wait ~50 ms behind a slow
|
|
51
|
+
* one; open the database with ~5 ms, as `initDuckDB` does. It is a setting of
|
|
52
|
+
* the database, which this function cannot change or read: say what you set
|
|
53
|
+
* as `pollingIntervalMs`. Without it, lanes measure their slices and warn if
|
|
54
|
+
* they run long.
|
|
55
|
+
*/
|
|
56
|
+
export declare function connectionFrom(conn: AsyncDuckDBConnection, options?: {
|
|
57
|
+
pollingIntervalMs?: number;
|
|
58
|
+
}): DuckDBConnection;
|
|
15
59
|
/**
|
|
16
60
|
* Initialize DuckDB-WASM and return a connection.
|
|
17
61
|
*
|
|
62
|
+
* Safe to call during SSR — returns a permanently pending promise on the
|
|
63
|
+
* server. The component re-runs on the client where Worker is available.
|
|
64
|
+
*
|
|
18
65
|
* Usage:
|
|
19
66
|
* const { queryEngine } = setQueryContext(initDuckDB());
|
|
20
67
|
*/
|
|
21
|
-
export declare function initDuckDB(): Promise<DuckDBConnection>;
|
|
68
|
+
export declare function initDuckDB(options?: InitDuckDBOptions): Promise<DuckDBConnection>;
|