@fjader/svelte 0.2.4 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/README.md +1 -1
  2. package/dist/animation/EntranceGate.d.ts +52 -0
  3. package/dist/animation/EntranceGate.js +93 -0
  4. package/dist/animation/TransitionClock.d.ts +61 -30
  5. package/dist/animation/TransitionClock.js +67 -29
  6. package/dist/animation/transition-log.d.ts +41 -0
  7. package/dist/animation/transition-log.js +57 -0
  8. package/dist/components/axes/AxisBottom.svelte +4 -2
  9. package/dist/components/axes/AxisCenter.svelte +4 -2
  10. package/dist/components/axes/AxisLeft.svelte +4 -2
  11. package/dist/components/axes/BandAxisTicks.svelte +3 -2
  12. package/dist/components/charts/ChartCell.svelte +32 -5
  13. package/dist/components/charts/ChartCell.svelte.d.ts +9 -0
  14. package/dist/components/charts/ChartContainer.svelte +45 -3
  15. package/dist/components/charts/ChartContainer.svelte.d.ts +7 -1
  16. package/dist/components/charts/ChartHover.svelte.d.ts +14 -0
  17. package/dist/components/charts/ChartHover.svelte.js +22 -0
  18. package/dist/components/charts/ChartVisibility.svelte.d.ts +43 -0
  19. package/dist/components/charts/ChartVisibility.svelte.js +67 -0
  20. package/dist/components/guides/Grid.svelte +12 -1
  21. package/dist/components/marks/AreaMarkX.svelte +16 -4
  22. package/dist/components/marks/AreaMarkY.svelte +16 -4
  23. package/dist/components/marks/BarMarkX.svelte +7 -3
  24. package/dist/components/marks/BarMarkY.svelte +7 -3
  25. package/dist/components/marks/DotsMark.svelte +6 -3
  26. package/dist/components/marks/LineMark.svelte +11 -8
  27. package/dist/components/marks/ParallelCoordinatesMark.svelte +7 -2
  28. package/dist/components/marks/ParallelCoordinatesMark.svelte.d.ts +2 -0
  29. package/dist/components/marks/RectMark.svelte +6 -3
  30. package/dist/components/marks/RectXMark.svelte +7 -3
  31. package/dist/components/marks/RectYMark.svelte +7 -3
  32. package/dist/components/marks/RuleMarkX.svelte +30 -26
  33. package/dist/components/marks/RuleMarkY.svelte +30 -26
  34. package/dist/components/marks/TextMark.svelte +44 -5
  35. package/dist/components/marks/lifecycle/MarkLifecycleV2.svelte.d.ts +16 -0
  36. package/dist/components/marks/lifecycle/MarkLifecycleV2.svelte.js +398 -54
  37. package/dist/components/marks/lifecycle/domain-queries.d.ts +11 -2
  38. package/dist/components/marks/lifecycle/domain-queries.js +33 -6
  39. package/dist/components/marks/lifecycle/domain-registration.d.ts +6 -1
  40. package/dist/components/marks/lifecycle/domain-registration.js +15 -14
  41. package/dist/components/marks/lifecycle/other-rows.svelte.d.ts +9 -1
  42. package/dist/components/marks/lifecycle/other-rows.svelte.js +2 -2
  43. package/dist/components/marks/lifecycle/query-builder.d.ts +27 -2
  44. package/dist/components/marks/lifecycle/query-builder.js +23 -20
  45. package/dist/components/marks/utils/MarkColor.svelte.d.ts +3 -1
  46. package/dist/components/marks/utils/MarkColor.svelte.js +33 -3
  47. package/dist/components/marks/utils/MarkSeries.svelte.d.ts +14 -1
  48. package/dist/components/marks/utils/MarkSeries.svelte.js +25 -1
  49. package/dist/components/marks/utils/MarkSize.svelte.js +4 -0
  50. package/dist/components/marks/utils/reactive-config.d.ts +1 -1
  51. package/dist/components/marks/utils/reactive-config.js +3 -0
  52. package/dist/components/rendering/Areas.svelte +57 -19
  53. package/dist/components/rendering/Areas.svelte.d.ts +12 -0
  54. package/dist/components/rendering/BaseGPURenderer.svelte.d.ts +1 -1
  55. package/dist/components/rendering/Dots.svelte +23 -3
  56. package/dist/components/rendering/Dots.svelte.d.ts +6 -0
  57. package/dist/components/rendering/GridLine.svelte +13 -1
  58. package/dist/components/rendering/GridLine.svelte.d.ts +5 -0
  59. package/dist/components/rendering/Lines.svelte +48 -13
  60. package/dist/components/rendering/Lines.svelte.d.ts +12 -0
  61. package/dist/components/rendering/ParallelLines.svelte +5 -5
  62. package/dist/components/rendering/Rects.svelte +51 -11
  63. package/dist/components/rendering/Rects.svelte.d.ts +16 -2
  64. package/dist/components/rendering/RuleX.svelte +1 -1
  65. package/dist/components/rendering/RuleY.svelte +1 -1
  66. package/dist/components/rendering/WebGPUCanvas.svelte +4 -2
  67. package/dist/components/rendering/activity-reads.d.ts +41 -24
  68. package/dist/components/rendering/activity-reads.js +54 -31
  69. package/dist/components/rendering/uniform-sizes.d.ts +1 -1
  70. package/dist/components/rendering/uniform-sizes.js +1 -1
  71. package/dist/config/ChartConfig.svelte.d.ts +7 -0
  72. package/dist/config/ChartConfig.svelte.js +17 -0
  73. package/dist/gpu/GPUCoordinator.svelte.d.ts +31 -1
  74. package/dist/gpu/GPUCoordinator.svelte.js +164 -7
  75. package/dist/gpu/MorphedChannels.d.ts +11 -2
  76. package/dist/gpu/MorphedChannels.js +33 -11
  77. package/dist/gpu/ScaleTemplateSystem.d.ts +20 -2
  78. package/dist/gpu/ScaleTemplateSystem.js +58 -2
  79. package/dist/index.d.ts +17 -4
  80. package/dist/index.js +19 -4
  81. package/dist/interactors/TooltipManager.svelte.js +14 -5
  82. package/dist/query-engine/CrossfilterSelection.svelte.d.ts +4 -2
  83. package/dist/query-engine/CrossfilterSelection.svelte.js +2 -0
  84. package/dist/query-engine/FieldActivity.svelte.d.ts +2 -3
  85. package/dist/query-engine/FieldActivity.svelte.js +1 -2
  86. package/dist/query-engine/MarkHandle.svelte.d.ts +28 -1
  87. package/dist/query-engine/MarkHandle.svelte.js +33 -1
  88. package/dist/query-engine/QueryEngine.d.ts +225 -5
  89. package/dist/query-engine/QueryEngine.js +671 -41
  90. package/dist/query-engine/categorical-identity.d.ts +34 -8
  91. package/dist/query-engine/categorical-identity.js +138 -73
  92. package/dist/query-engine/debug/query-log.d.ts +39 -0
  93. package/dist/query-engine/debug/query-log.js +90 -0
  94. package/dist/query-engine/duckdb.d.ts +55 -8
  95. package/dist/query-engine/duckdb.js +82 -30
  96. package/dist/query-engine/index.d.ts +7 -4
  97. package/dist/query-engine/index.js +3 -3
  98. package/dist/query-engine/preagg/BuildQueue.d.ts +33 -13
  99. package/dist/query-engine/preagg/BuildQueue.js +51 -15
  100. package/dist/query-engine/preagg/Preagg.d.ts +46 -3
  101. package/dist/query-engine/preagg/Preagg.js +159 -31
  102. package/dist/query-engine/preagg/plan.d.ts +28 -1
  103. package/dist/query-engine/preagg/plan.js +128 -8
  104. package/dist/query-engine/sliced-lane.d.ts +108 -0
  105. package/dist/query-engine/sliced-lane.js +299 -0
  106. package/dist/query-engine/source-routing.d.ts +106 -0
  107. package/dist/query-engine/source-routing.js +189 -0
  108. package/dist/scales/BandScaleState.svelte.d.ts +9 -1
  109. package/dist/scales/BandScaleState.svelte.js +16 -5
  110. package/dist/scales/ChartScales.svelte.d.ts +3 -0
  111. package/dist/scales/ChartScales.svelte.js +26 -8
  112. package/dist/scales/ColorScaleState.svelte.d.ts +18 -2
  113. package/dist/scales/ColorScaleState.svelte.js +138 -26
  114. package/dist/scales/LinearScaleState.svelte.d.ts +7 -10
  115. package/dist/scales/LinearScaleState.svelte.js +21 -39
  116. package/dist/scales/ScaleGeneration.svelte.d.ts +48 -0
  117. package/dist/scales/ScaleGeneration.svelte.js +42 -0
  118. package/dist/scales/ScaleRegistry.svelte.js +3 -1
  119. package/dist/scales/SlotTransition.d.ts +15 -1
  120. package/dist/scales/SlotTransition.js +33 -4
  121. package/dist/scales/TimeScaleState.svelte.d.ts +7 -3
  122. package/dist/scales/TimeScaleState.svelte.js +7 -7
  123. package/dist/scales/color-utils.d.ts +0 -3
  124. package/dist/scales/color-utils.js +0 -3
  125. package/dist/scales/entered.d.ts +8 -0
  126. package/dist/scales/entered.js +10 -0
  127. package/dist/spec/LazyQueryEngine.js +2 -19
  128. package/dist/spec/MarkRegistry.d.ts +3 -1
  129. package/dist/spec/MarkRegistry.js +13 -1
  130. package/dist/spec/PlotFromSpec.svelte +1 -0
  131. package/dist/spec/export.js +11 -2
  132. package/dist/spec/types.d.ts +41 -13
  133. package/dist/sql/bin.d.ts +25 -0
  134. package/dist/sql/bin.js +108 -5
  135. package/dist/transforms/TransformCompiler.d.ts +10 -0
  136. package/dist/transforms/TransformCompiler.js +26 -0
  137. package/dist/transforms/index.js +144 -24
  138. package/dist/types.d.ts +136 -2
  139. package/package.json +2 -2
@@ -60,6 +60,40 @@ export interface IdentityExec {
60
60
  exec(sql: string | string[]): Promise<void>;
61
61
  /** Rows as plain objects. Callers adapt whatever their driver returns. */
62
62
  queryRows(sql: string): Promise<Array<Record<string, unknown>>>;
63
+ /**
64
+ * Whether `table` is known to be a base table (not a view). Only then is a
65
+ * column's cardinality read from its statistics (#cardinalityFromStats):
66
+ * on a view, `stats(col) … LIMIT 1` runs the view, and a filtered view
67
+ * reports its base column's statistics. Absent: never.
68
+ *
69
+ * About the table's TYPE, not where it is stored: a table in an attached
70
+ * file or remote database answers yes (its statistics are metadata either
71
+ * way). Not a "fast source" test: routing asks that separately
72
+ * (SourceRouting), and must not reuse this.
73
+ */
74
+ isBaseTable?(table: string): boolean;
75
+ /**
76
+ * Run statements as one message, returning the last one's rows (the
77
+ * engine's execBatch). Absent: they run one by one through exec and
78
+ * queryRows, with the same result.
79
+ */
80
+ execBatch?(statements: string[]): Promise<Array<Record<string, unknown>>>;
81
+ /**
82
+ * Debug: told of every ensureColumns call (the table, the fields asked for,
83
+ * performance.now()), before anything is deduplicated. For measuring when
84
+ * marks ask, e.g. whether one table's fields arrive close enough together
85
+ * to share a drain.
86
+ */
87
+ onRequest?(table: string, fields: string[], time: number): void;
88
+ /** Debug: one pass of a table's drain started, with the fields it took. */
89
+ onDrain?(table: string, fields: string[], time: number): void;
90
+ /**
91
+ * Whether anything listens to onRequest and onDrain. Checked first, so with
92
+ * no listener their arguments (a field list, a timestamp) are never built.
93
+ * The callbacks fire ONLY when this returns true: a host that implements
94
+ * them must implement this too, or it is never called.
95
+ */
96
+ observing?(): boolean;
63
97
  }
64
98
  /**
65
99
  * A thing that can have an identity: a source column, or a fixed SQL expression
@@ -290,14 +324,6 @@ export declare class CategoricalIdentity {
290
324
  * before a reset names a row of a dropped dim, and is ignored.
291
325
  */
292
326
  releaseSynthetic(synthetic: SyntheticId): void;
293
- /**
294
- * Re-read the dim into the field's value array, mutating in place.
295
- *
296
- * Re-reading the whole dim rather than only the new rows: categorical
297
- * cardinality is small, this runs once per (field, table) pair at setup, and
298
- * a full read cannot drift out of sync with the ids the way an incremental
299
- * one can if an append is ever missed.
300
- */
301
327
  /**
302
328
  * Bumped whenever a dim's rows change: a table folding in, an Other bucket
303
329
  * reserved, a reset. A query that reads a dim (dense padding, an id join)
@@ -304,19 +304,6 @@ export class CategoricalIdentity {
304
304
  presentIds(field, table) {
305
305
  return this.#dims.get(field)?.present.get(table)?.ids ?? null;
306
306
  }
307
- /**
308
- * Record which of the dim's ids `table` holds: the ids of its distinct values,
309
- * joined as the view joins (NULL to a NULL row, never to a synthetic one).
310
- * Once per table, when it folds in.
311
- */
312
- async #recordPresent(dim, table, expr, src) {
313
- const name = `${dim.name}_t_${identityToken(table)}`;
314
- await this.#exec.exec(`CREATE OR REPLACE TABLE ${quote(name)} AS ` +
315
- `SELECT DISTINCT d.id FROM (SELECT DISTINCT ${expr} AS v FROM ${src}) s ` +
316
- `JOIN ${quote(dim.name)} d ON s.v IS NOT DISTINCT FROM d.value AND d.label IS NULL`);
317
- const rows = await this.#exec.queryRows(`SELECT id FROM ${quote(name)}`);
318
- dim.present.set(table, { name, ids: new Set(rows.map((r) => Number(r.id))) });
319
- }
320
307
  /** Name of the view that carries id columns for `table`. */
321
308
  viewTable(table) {
322
309
  return `${identityToken(table)}_v_s${this.#session}`;
@@ -382,6 +369,9 @@ export class CategoricalIdentity {
382
369
  const categorical = (fields.categorical ?? []).map(asField);
383
370
  if (categorical.length === 0)
384
371
  return;
372
+ if (this.#exec.observing?.()) {
373
+ this.#exec.onRequest?.(table, categorical.map((f) => f.key), performance.now());
374
+ }
385
375
  let entry = this.#views.get(table);
386
376
  if (!entry) {
387
377
  entry = { current: new Set(), pending: new Set(), inflight: null, failures: new Map() };
@@ -468,6 +458,10 @@ export class CategoricalIdentity {
468
458
  return;
469
459
  const batch = [...entry.pending];
470
460
  entry.pending.clear();
461
+ // Per pass: a request arriving during the first build is a second
462
+ // pass (and a second view build) inside this same call.
463
+ if (this.#exec.observing?.())
464
+ this.#exec.onDrain?.(table, batch, performance.now());
471
465
  const succeeded = [];
472
466
  for (const field of batch) {
473
467
  try {
@@ -650,55 +644,83 @@ export class CategoricalIdentity {
650
644
  // The value expression: a quoted column, or a fixed SQL expression. Both
651
645
  // are evaluated with only the source table in scope.
652
646
  const expr = dim.sql;
653
- await this.#checkCardinality(field, expr, src);
654
- if (dim.tables.size === 0) {
655
- // `label` is NULL for every real value and carries the text for a
656
- // synthetic one (see reserveSynthetic). It exists because `value` takes
657
- // the SOURCE column's type: on a band axis over a year or a rating
658
- // there is no INTEGER that means "Other", and writing the string into
659
- // that column is a conversion error, not a fallback.
660
- await this.#exec.exec(`CREATE TABLE IF NOT EXISTS ${dimName} AS ` +
661
- `SELECT value, NULL::VARCHAR AS label, ` +
662
- `(ROW_NUMBER() OVER (ORDER BY value NULLS LAST) - 1)::INT AS id ` +
663
- `FROM (SELECT DISTINCT ${expr} AS value FROM ${src})`);
664
- // A second statement: CTAS cannot carry a constraint. `value` is unique
665
- // by construction here (DISTINCT) and on append (WHERE NOT EXISTS), but
666
- // reserveSynthetic is a writer that could break it, and a duplicate
667
- // value does not tie-break — #createView joins ON t.field = d.value, so
668
- // it FANS OUT every matching source row in every query against the
669
- // view. Cheap insurance against a future writer forgetting.
670
- //
671
- // Synthetic rows hold value NULL, and SQL treats NULLs as distinct
672
- // under a unique index, so any number of them coexist here — verified
673
- // against DuckDB rather than assumed.
674
- await this.#exec.exec(`CREATE UNIQUE INDEX IF NOT EXISTS ${quote(`${dim.name}_value_uq`)} ON ${dimName}(value)`);
675
- }
676
- else {
677
- // Columns named rather than positional: the table has three of them and
678
- // a positional INSERT would silently put the id into `label`.
679
- await this.#exec.exec(`INSERT INTO ${dimName} (value, label, id) ` +
680
- `SELECT s.v, NULL, ((SELECT COALESCE(MAX(id), -1) FROM ${dimName}) + ` +
681
- `ROW_NUMBER() OVER (ORDER BY s.v NULLS LAST))::INT ` +
682
- `FROM (SELECT DISTINCT ${expr} AS v FROM ${src}) s ` +
683
- // IS NOT DISTINCT FROM, so a second table's NULLs find the NULL row
684
- // (NULL = NULL is never true, and `=` appended one per table). The
685
- // label test keeps a synthetic row, also value NULL, from passing
686
- // for it.
687
- `WHERE NOT EXISTS (SELECT 1 FROM ${dimName} d ` +
688
- `WHERE d.value IS NOT DISTINCT FROM s.v AND d.label IS NULL)`);
689
- }
647
+ await this.#checkCardinality(field, expr, src, table);
648
+ // The field's statements as ONE message (#batch): sent one by one, each
649
+ // step went to the back of the queue, behind every other mark's reads
650
+ // sent meanwhile; at 2M rows the categorical charts trailed the others
651
+ // by 420-500 ms. The last statement returns the values the JS cache
652
+ // needs, and this table's present ids with them.
653
+ const first = dim.tables.size === 0;
654
+ const presentName = `${dim.name}_t_${identityToken(table)}`;
655
+ const present = quote(presentName);
656
+ // `label` is NULL for every real value and carries the text for a
657
+ // synthetic one (see reserveSynthetic). It exists because `value` takes the
658
+ // SOURCE column's type: on a band axis over a year or a rating there is no
659
+ // INTEGER that means "Other", and writing the string into that column is a
660
+ // conversion error, not a fallback.
661
+ const statements = first
662
+ ? [
663
+ `CREATE TABLE IF NOT EXISTS ${dimName} AS ` +
664
+ `SELECT value, NULL::VARCHAR AS label, ` +
665
+ `(ROW_NUMBER() OVER (ORDER BY value NULLS LAST) - 1)::INT AS id ` +
666
+ `FROM (SELECT DISTINCT ${expr} AS value FROM ${src})`,
667
+ // A second statement: CTAS cannot carry a constraint. `value` is
668
+ // unique by construction here (DISTINCT) and on append (WHERE NOT
669
+ // EXISTS), but reserveSynthetic is a writer that could break it, and
670
+ // a duplicate value does not tie-break: #createView joins ON
671
+ // t.field = d.value, so it FANS OUT every matching source row in
672
+ // every query against the view. Synthetic rows hold value NULL, and
673
+ // SQL treats NULLs as distinct under a unique index, so any number of
674
+ // them coexist (verified against DuckDB).
675
+ `CREATE UNIQUE INDEX IF NOT EXISTS ${quote(`${dim.name}_value_uq`)} ON ${dimName}(value)`,
676
+ // The table that built the dim holds every real row of it (`chain`
677
+ // serializes a dim's work, so only one table takes this branch): its
678
+ // present ids are the real rows, no scan (the NULL row has no label
679
+ // and is present, as the join finds it; a synthetic row has one).
680
+ `CREATE OR REPLACE TABLE ${present} AS SELECT id FROM ${dimName} WHERE label IS NULL`,
681
+ `SELECT id, value, label FROM ${dimName} ORDER BY id`
682
+ ]
683
+ : [
684
+ // Columns named rather than positional: a positional INSERT would
685
+ // silently put the id into `label`. IS NOT DISTINCT FROM, so a second
686
+ // table's NULLs find the NULL row; the label test keeps a synthetic
687
+ // row, also value NULL, from passing for it.
688
+ `INSERT INTO ${dimName} (value, label, id) ` +
689
+ `SELECT s.v, NULL, ((SELECT COALESCE(MAX(id), -1) FROM ${dimName}) + ` +
690
+ `ROW_NUMBER() OVER (ORDER BY s.v NULLS LAST))::INT ` +
691
+ `FROM (SELECT DISTINCT ${expr} AS v FROM ${src}) s ` +
692
+ `WHERE NOT EXISTS (SELECT 1 FROM ${dimName} d ` +
693
+ `WHERE d.value IS NOT DISTINCT FROM s.v AND d.label IS NULL)`,
694
+ // An appending table holds a subset: the ids of its distinct values,
695
+ // joined as the view joins.
696
+ `CREATE OR REPLACE TABLE ${present} AS ` +
697
+ `SELECT DISTINCT d.id FROM (SELECT DISTINCT ${expr} AS v FROM ${src}) s ` +
698
+ `JOIN ${dimName} d ON s.v IS NOT DISTINCT FROM d.value AND d.label IS NULL`,
699
+ `SELECT id, value, label, id IN (SELECT id FROM ${present}) AS present ` +
700
+ `FROM ${dimName} ORDER BY id`
701
+ ];
702
+ const rows = await this.#batch(statements);
703
+ // A reset while the batch ran: its DROPs queued behind it, so these rows
704
+ // are of dims that no longer exist. Applying them would fill the cleared
705
+ // state with stale ids.
706
+ if (this.#generation !== generation)
707
+ return;
690
708
  dim.tables.add(table);
691
- await this.#refreshValues(field);
692
- await this.#recordPresent(dim, table, expr, src);
693
- // The gate above saw ONE table. The dim spans every table that registers
694
- // the field, so two tables of 6k each pass it individually and produce a
695
- // 12k dim — and the limit is a property of the dim, since that is what
696
- // sizes the slot table and therefore what the shared buffer has to hold.
697
- // Exact rather than approximate here: the rows exist, COUNT(*) on a
698
- // categorical dim is trivial, and this is the number the error should
699
- // quote.
709
+ this.#applyValues(field, rows);
710
+ const ids = first
711
+ ? rows.filter((r) => r.label == null).map((r) => Number(r.id))
712
+ : rows.filter((r) => r.present === true).map((r) => Number(r.id));
713
+ dim.present.set(table, { name: presentName, ids: new Set(ids) });
700
714
  this.#checkDimSize(dim, field, this.identity(field).values.length);
701
715
  }
716
+ /** Statements as one message (IdentityExec.execBatch), or one by one; the last one's rows. */
717
+ async #batch(statements) {
718
+ if (this.#exec.execBatch)
719
+ return this.#exec.execBatch(statements);
720
+ for (const stmt of statements.slice(0, -1))
721
+ await this.#exec.exec(stmt);
722
+ return this.#exec.queryRows(statements[statements.length - 1]);
723
+ }
702
724
  /**
703
725
  * Refuse a band axis with more distinct values than can render.
704
726
  *
@@ -708,23 +730,62 @@ export class CategoricalIdentity {
708
730
  * the setup path that already blocks on one. Here it costs a query only for fields
709
731
  * that actually get a dim, and it is where the refusal has to land anyway.
710
732
  *
711
- * `approx_count_distinct` rather than `COUNT(DISTINCT …)`: one HyperLogLog
712
- * pass with no materialization, which matters precisely for the runaway
713
- * expression this exists to catch.
733
+ * For a plain column, its statistics first (#cardinalityFromStats): no scan.
734
+ * Otherwise `approx_count_distinct` rather than `COUNT(DISTINCT …)`: one
735
+ * HyperLogLog pass with no materialization, which matters precisely for the
736
+ * runaway expression this exists to catch.
714
737
  *
715
738
  * Neither syntax nor result type can answer this question — `year(ts)` is
716
739
  * numeric with a tiny distinct set, a CASE can have ten thousand branches —
717
740
  * so the count is the only honest gate.
718
741
  */
719
- async #checkCardinality(field, expr, src) {
720
- const rows = await this.#exec.queryRows(`SELECT approx_count_distinct(${expr}) AS n FROM ${src}`);
721
- const n = Number(rows[0]?.n ?? 0);
742
+ async #checkCardinality(field, expr, src, table) {
743
+ const scan = async () => Number((await this.#exec.queryRows(`SELECT approx_count_distinct(${expr}) AS n FROM ${src}`))[0]
744
+ ?.n ?? 0);
745
+ const estimate = await this.#cardinalityFromStats(expr, src, table);
746
+ let n = estimate ?? (await scan());
747
+ // Statistics are never shrunk (deletes, updates), so an estimate over the
748
+ // limit can be stale: confirm it with the scan before refusing. A false
749
+ // refusal is user-visible; the scan runs only in this rare case.
750
+ if (estimate != null && n > MAX_IDENTITY_CARDINALITY)
751
+ n = await scan();
722
752
  if (n > MAX_IDENTITY_CARDINALITY) {
723
753
  throw new Error(`"${field}" has about ${n} distinct values in this table, over the ` +
724
754
  `${MAX_IDENTITY_CARDINALITY} a categorical scale can render. Use a continuous ` +
725
755
  `scale, bin the values, or reduce them with a transform.`);
726
756
  }
727
757
  }
758
+ /**
759
+ * A plain column's approximate distinct count from DuckDB's column
760
+ * statistics, without scanning: `stats(col)` reads what DuckDB keeps per
761
+ * column ("Approx Unique"). Measured on DuckDB-WASM: 1-3 ms, against
762
+ * 27-28 ms for approx_count_distinct over 2M rows; it agrees with the scan
763
+ * (167 and 167, 245 and 251). It can undercount a very large set (1.26M for
764
+ * 2M unique values): still far over the limit, and #checkDimSize counts
765
+ * the built dim exactly behind it. It can OVERcount, never shrinking after
766
+ * deletes, so an over-limit answer is confirmed by the scan
767
+ * (#checkCardinality).
768
+ *
769
+ * Base tables only (IdentityExec.isBaseTable): on a view, `stats … LIMIT 1`
770
+ * runs the view (a view that computes would then run twice, if the column
771
+ * has no estimate), and a filtered view reports its base column's
772
+ * statistics. Null for an expression field, a view, or statistics that say
773
+ * nothing: the caller scans as before.
774
+ */
775
+ async #cardinalityFromStats(expr, src, table) {
776
+ if (!/^"(?:[^"]|"")+"$/.test(expr))
777
+ return null;
778
+ if (!this.#exec.isBaseTable?.(table))
779
+ return null;
780
+ try {
781
+ const rows = await this.#exec.queryRows(`SELECT stats(${expr}) AS s FROM ${src} LIMIT 1`);
782
+ const match = /Approx Unique: (\d+)/.exec(String(rows[0]?.s ?? ''));
783
+ return match ? Number(match[1]) : null;
784
+ }
785
+ catch {
786
+ return null;
787
+ }
788
+ }
728
789
  /**
729
790
  * Refuse or warn on the DIM's size, which is what actually has to render.
730
791
  *
@@ -767,14 +828,6 @@ export class CategoricalIdentity {
767
828
  `and a grid on that axis draws a line per value whether it is displayed or not.`);
768
829
  }
769
830
  }
770
- /**
771
- * Re-read the dim into the field's value array, mutating in place.
772
- *
773
- * Re-reading the whole dim rather than only the new rows: categorical
774
- * cardinality is small, this runs once per (field, table) pair at setup, and
775
- * a full read cannot drift out of sync with the ids the way an incremental
776
- * one can if an append is ever missed.
777
- */
778
831
  /**
779
832
  * Bumped whenever a dim's rows change: a table folding in, an Other bucket
780
833
  * reserved, a reset. A query that reads a dim (dense padding, an id join)
@@ -785,8 +838,20 @@ export class CategoricalIdentity {
785
838
  return this.#version;
786
839
  }
787
840
  #version = 0;
841
+ /**
842
+ * Re-read the dim into the field's value array, mutating in place.
843
+ *
844
+ * Re-reading the whole dim rather than only the new rows: categorical
845
+ * cardinality is small, this runs once per (field, table) pair at setup, and
846
+ * a full read cannot drift out of sync with the ids the way an incremental
847
+ * one can if an append is ever missed.
848
+ */
788
849
  async #refreshValues(field) {
789
850
  const rows = await this.#exec.queryRows(`SELECT id, value, label FROM ${quote(this.dimTable(field))} ORDER BY id`);
851
+ this.#applyValues(field, rows);
852
+ }
853
+ /** The dim's rows (`id, value, label`, in id order) into the JS cache. */
854
+ #applyValues(field, rows) {
790
855
  // Ordered by id and dense by construction, so position is the id — and
791
856
  // reset() derives value -> id from the same pass, so the two directions
792
857
  // cannot disagree.
@@ -0,0 +1,39 @@
1
+ /**
2
+ * A statement log for measuring what reaches DuckDB: attach it to an engine,
3
+ * use the page, then read `report()` or `download()` the raw log.
4
+ *
5
+ * Built on `QueryEngine.observeStatements`, so it sees every statement the
6
+ * engine sends (queries, statements, DESCRIBE, Arrow inserts) with its class.
7
+ * Times are taken on the main thread when a statement is handed over and when
8
+ * it settles; while the main thread is busy (decoding, GPU uploads) they read
9
+ * high. The queue-wait estimate assumes each connection serves its statements
10
+ * first-in, first-out: a statement starts when it was sent or when the
11
+ * previous one ON ITS CONNECTION finished, whichever is later. Statements on a
12
+ * sliced lane interleave with the main connection's in the worker, so their
13
+ * "work" includes slices spent serving the other.
14
+ */
15
+ import type { QueryEngine, StatementRecord } from '../QueryEngine';
16
+ /** What a statement is for, from its text: a coarse grouping for reports. */
17
+ export type StatementPurpose = 'describe' | 'ddl' | 'insert' | 'range' | 'data' | 'batch';
18
+ export interface LoggedStatement extends StatementRecord {
19
+ purpose: StatementPurpose;
20
+ /** The first table the statement reads, or '?'. */
21
+ table: string;
22
+ }
23
+ export interface QueryLog {
24
+ readonly statements: readonly LoggedStatement[];
25
+ /** Print a summary to the console and return it as rows. */
26
+ report(): ReportRow[];
27
+ /** Download the raw log as JSON (to the browser's downloads folder). */
28
+ download(name: string, extra?: Record<string, unknown>): void;
29
+ reset(): void;
30
+ detach(): void;
31
+ }
32
+ export interface ReportRow {
33
+ group: string;
34
+ n: number;
35
+ workMs: number;
36
+ waitMs: number;
37
+ }
38
+ /** Attach a statement log to an engine. Costs nothing once detached. */
39
+ export declare function attachQueryLog(engine: QueryEngine): QueryLog;
@@ -0,0 +1,90 @@
1
+ function purposeOf(record) {
2
+ if (record.kind === 'describe')
3
+ return 'describe';
4
+ if (record.kind === 'insert')
5
+ return 'insert';
6
+ if (record.kind === 'batch')
7
+ return 'batch';
8
+ const s = record.sql.trimStart().toUpperCase();
9
+ if (/^(CREATE|DROP|INSERT|ALTER|DELETE|UPDATE)/.test(s))
10
+ return 'ddl';
11
+ // A scalar summary: min/max aggregates, no GROUP BY, no window.
12
+ if (/^SELECT/.test(s) && !/GROUP BY/.test(s) && /(MIN|MAX)\(/.test(s) && !/ OVER \(/.test(s)) {
13
+ return 'range';
14
+ }
15
+ return 'data';
16
+ }
17
+ function tableOf(sql) {
18
+ const m = /FROM\s+"?([A-Za-z_][\w]*)"?/i.exec(sql);
19
+ return m ? m[1] : '?';
20
+ }
21
+ /** Attach a statement log to an engine. Costs nothing once detached. */
22
+ export function attachQueryLog(engine) {
23
+ const statements = [];
24
+ /** Categorical identity requests, in order: when marks asked for which fields. */
25
+ const identityRequests = [];
26
+ const detachStatements = engine.observeStatements((record) => {
27
+ statements.push({ ...record, purpose: purposeOf(record), table: tableOf(record.sql) });
28
+ });
29
+ const detachIdentity = engine.observeIdentityRequests((request) => {
30
+ identityRequests.push(request);
31
+ });
32
+ /** Every preagg build step, in order (observeBuilds). Errors as text, for JSON. */
33
+ const builds = [];
34
+ const detachBuilds = engine.observeBuilds((event) => {
35
+ builds.push(event.error === undefined ? event : { ...event, error: String(event.error) });
36
+ });
37
+ const detach = () => {
38
+ detachStatements();
39
+ detachIdentity();
40
+ detachBuilds();
41
+ };
42
+ const report = () => {
43
+ const sorted = [...statements].sort((a, b) => a.start - b.start);
44
+ if (sorted.length === 0)
45
+ return [];
46
+ const lastEndOn = { main: 0, lane: 0 };
47
+ let lastEnd = 0;
48
+ const groups = new Map();
49
+ let work = 0;
50
+ for (const s of sorted) {
51
+ const conn = s.connection ?? 'main';
52
+ const start = Math.max(s.start, lastEndOn[conn]);
53
+ lastEndOn[conn] = Math.max(lastEndOn[conn], s.end);
54
+ lastEnd = Math.max(lastEnd, s.end);
55
+ const key = `${conn} ${s.class} ${s.purpose} ${s.table}`;
56
+ const g = groups.get(key) ?? { group: key, n: 0, workMs: 0, waitMs: 0 };
57
+ g.n++;
58
+ g.workMs += s.end - start;
59
+ g.waitMs += start - s.start;
60
+ work += s.end - start;
61
+ groups.set(key, g);
62
+ }
63
+ const span = lastEnd - sorted[0].start;
64
+ const rows = [...groups.values()]
65
+ .map((g) => ({ ...g, workMs: +g.workMs.toFixed(1), waitMs: +g.waitMs.toFixed(1) }))
66
+ .sort((a, b) => b.workMs - a.workMs);
67
+ console.log(`[query-log] ${sorted.length} statements, span ${span.toFixed(0)} ms, work ${work.toFixed(0)} ms`);
68
+ console.table(rows);
69
+ return rows;
70
+ };
71
+ const download = (name, extra = {}) => {
72
+ const json = JSON.stringify({ ...extra, statements, identityRequests, builds }, null, 1);
73
+ const a = document.createElement('a');
74
+ a.href = URL.createObjectURL(new Blob([json], { type: 'application/json' }));
75
+ a.download = name;
76
+ a.click();
77
+ URL.revokeObjectURL(a.href);
78
+ };
79
+ return {
80
+ statements,
81
+ report,
82
+ download,
83
+ reset: () => {
84
+ statements.length = 0;
85
+ identityRequests.length = 0;
86
+ builds.length = 0;
87
+ },
88
+ detach
89
+ };
90
+ }
@@ -1,21 +1,68 @@
1
1
  /**
2
2
  * DuckDB-WASM initialization — replaces mosaic-core's wasmConnector.
3
3
  *
4
- * Returns a Promise<DuckDBConnection> suitable for passing to setQueryContext().
5
- * Uses jsdelivr CDN bundles (same as mosaic-core's wasmConnector).
4
+ * initDuckDB() returns a Promise<DuckDBConnection> suitable for passing to
5
+ * setQueryContext(). It loads the jsDelivr bundles unless the app passes its own.
6
6
  *
7
- * Uses the fast IPC query path: bypasses duckdb-wasm's Arrow JS deserialization
8
- * and returns raw Arrow IPC bytes. QueryEngine decodes them with flechette for
9
- * zero-copy typed arrays that go straight to GPU buffers.
7
+ * connectionFrom() adapts a duckdb-wasm connection the app created itself, for
8
+ * apps that need the AsyncDuckDB instance (registerFileBuffer, extensions, …).
10
9
  *
11
- * Safe to call during SSR — returns a permanently pending promise on the
12
- * server. The component re-runs on the client where Worker is available.
10
+ * Both use the fast IPC query path: bypasses duckdb-wasm's Arrow JS
11
+ * deserialization and returns raw Arrow IPC bytes. QueryEngine decodes them with
12
+ * flechette into typed arrays for the GPU buffers. These are views on the IPC
13
+ * bytes only up to one record batch: DuckDB-WASM writes a batch per DuckDB vector
14
+ * (~2k rows), and flechette concatenates a column that spans several.
13
15
  */
16
+ import type { AsyncDuckDBConnection, DuckDBBundles } from '@duckdb/duckdb-wasm';
14
17
  import type { DuckDBConnection } from './QueryEngine';
18
+ /**
19
+ * How long the worker runs a sliced statement before serving other messages:
20
+ * a quick query on another connection waits at most about this long behind a
21
+ * slow one. Measured in Phase 0: 1 ms at 5 ms, ~50 ms at the default (~100 ms).
22
+ */
23
+ export declare const QUERY_POLLING_INTERVAL_MS = 5;
24
+ export interface InitDuckDBOptions {
25
+ /**
26
+ * DuckDB-WASM bundles to choose from, e.g. served by the app itself:
27
+ *
28
+ * import mvpWasm from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url';
29
+ * import mvpWorker from '@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url';
30
+ * …
31
+ * initFjader({ bundles: { mvp: { mainModule: mvpWasm, mainWorker: mvpWorker }, eh: … } });
32
+ *
33
+ * Only `mvp` is required; selectBundle picks `eh`/`coi` when given and supported.
34
+ * Relative URLs resolve against the page. Defaults to the jsDelivr bundles.
35
+ */
36
+ bundles?: DuckDBBundles;
37
+ }
38
+ /**
39
+ * Adapt a duckdb-wasm connection to the DuckDBConnection QueryEngine expects,
40
+ * with the fast IPC query path, and sliced lanes: further connections on the
41
+ * same database (`openSlicedLane`).
42
+ *
43
+ * const db = new duckdb.AsyncDuckDB(logger, worker);
44
+ * await db.instantiate(…);
45
+ * await db.open({ query: { queryPollingInterval: 5 } }); // short slices, see below
46
+ * await db.registerFileBuffer('cached.parquet', bytes);
47
+ * initFjader(connectionFrom(await db.connect(), { pollingIntervalMs: 5 }));
48
+ *
49
+ * Sliced statements run in slices of the database's `queryPollingInterval`.
50
+ * At DuckDB's default (~100 ms) a quick query can wait ~50 ms behind a slow
51
+ * one; open the database with ~5 ms, as `initDuckDB` does. It is a setting of
52
+ * the database, which this function cannot change or read: say what you set
53
+ * as `pollingIntervalMs`. Without it, lanes measure their slices and warn if
54
+ * they run long.
55
+ */
56
+ export declare function connectionFrom(conn: AsyncDuckDBConnection, options?: {
57
+ pollingIntervalMs?: number;
58
+ }): DuckDBConnection;
15
59
  /**
16
60
  * Initialize DuckDB-WASM and return a connection.
17
61
  *
62
+ * Safe to call during SSR — returns a permanently pending promise on the
63
+ * server. The component re-runs on the client where Worker is available.
64
+ *
18
65
  * Usage:
19
66
  * const { queryEngine } = setQueryContext(initDuckDB());
20
67
  */
21
- export declare function initDuckDB(): Promise<DuckDBConnection>;
68
+ export declare function initDuckDB(options?: InitDuckDBOptions): Promise<DuckDBConnection>;