@graphenedata/cli 0.0.14 → 0.0.16

package/dist/ui/internal/queryEngine.ts

@@ -1,21 +1,13 @@
 // The query engine gathers query requests and inputs from components, and issues requests to the server.
 // When inputs change, it takes care of notifying affected components and requesting new data.
 
-import {cacheRead, cacheWrite, getHashes} from './clientCache.ts'
-import {errorProvider} from './telemetry.ts'
-
-interface QueryResult {
-  rows?: any[]
-  errors?: Error[]
-  fields?: Field[]
-}
+import type {GrapheneError} from '../../lang/index.d.ts'
 
-interface Field {
-  name: string
-  type?: string
-}
+import {type QueryResult, type Field} from '../component-utilities/types.ts'
+import {cacheRead, cacheWrite, getHashes} from './clientCache.ts'
+import {getActivePageInputs, type ParamSnapshot} from './pageInputs.svelte.js'
 
-type ResultHandler = (res: QueryResult) => void
+type ResultHandler = (res: QueryResult | void) => void
 
 interface QueryNode {
   name?: string
@@ -24,32 +16,39 @@ interface QueryNode {
   callback?: ResultHandler
   loading: boolean
   fields: Map<string, string | string[]>
-  errors: Error[]
+  componentId?: string
+  error?: GrapheneError
 }
 
+export interface QueryRequest {
+  params: ParamSnapshot
+  gsql: string
+  hashes: string[]
+  repoId: string
+}
+
+export type QueryFetcher = (req: QueryRequest) => Promise<QueryResult>
+
 let runPending: Promise<void> | null = null
-let params = {} as Record<string, any>
 let queries = [] as QueryNode[]
-let queryResults = {} as Record<string, {rows: any[], fields?: Field[]}>
-
-function registerQuery (name: string, contents: string) {
-  queries = queries.filter(q => q.name !== name)
-  queries.push({name, contents, loading: false, fields: new Map(), errors: []})
-}
+let queryResults = {} as Record<string, {rows: any[]; fields?: Field[]}>
 
-const getRoutePath = () => typeof window === 'undefined' ? '/' : (window.location.pathname || '/')
+let queryFetcher: QueryFetcher = fetchWithCache
+export const setQueryFetcher = f => (queryFetcher = f)
 
-function updateParam (name: string, value: any) {
-  params[name] = value
-  runAll() // for now, do the easy thing and reload it all
+// Called by GrapheneQuery tags to register a named query on the page
+function registerQuery(name: string, contents: string) {
+  queries = queries.filter(q => q.name !== name)
+  queries.push({name, contents, loading: false, fields: new Map()})
 }
 
-function query (source: string, fields: Record<string, string | string[]>, callback: ResultHandler) {
+// Called by viz components to request a particular query of data
+function query(source: string, fields: Record<string, string | string[]>, callback: ResultHandler, componentId?: string) {
   // using Map here because it preserves the order in which we add fields to the select, which we use when we get the result.
   let map = new Map(Object.entries(fields))
   let exprs: string[] = []
   if (map.size > 0) {
-    map.forEach((value) => {
+    map.forEach(value => {
       if (Array.isArray(value)) exprs.push(...value)
       else exprs.push(value)
     })
@@ -57,146 +56,139 @@ function query (source: string, fields: Record<string, string | string[]>, callb
     exprs = ['*']
   }
   let contents = `from ${source} select ${exprs.join(', ')}`
-  queries.push({contents, callback, loading: false, fields: map, errors: [], source})
+  queries.push({contents, callback, loading: false, fields: map, source, componentId})
   runAll()
+  return componentId
 }
 
-function unsubscribe (callback: ResultHandler) {
+function unsubscribe(callback: ResultHandler) {
   queries = queries.filter(q => q.callback !== callback)
 }
 
-async function runNode (n: QueryNode) {
+function resetQueryEngine() {
+  queries = []
+  Object.keys(queryResults).forEach(key => delete queryResults[key])
+  getActivePageInputs().reset()
+}
+
+// Actually runs a given query that some frontend component is listening to.
+// This is pretty dumb at the moment, it simply concats all code fenced queries as table statements, then appends the actual query at the end.
+async function runNode(n: QueryNode) {
   if (!n.callback) throw new Error('running node nobody is listening to')
-  n.callback({}) // notify that the query is loading
+
+  n.callback() // notifies listeners we're back in the loading state
   n.loading = true
-  n.errors = []
+  n.error = undefined
 
+  // build up the request body. Hashes is the list of ETag hashes currently in our browser cache. We send all of them,
+  // letting the server determine the hash of this particular query, and whether data we already have is acceptable.
   let hashes = await getHashes()
   let tables = queries.filter(q => q.name)
-  let gsql = [
-    ...tables.map(q => `table ${q.name} as (${q.contents})`),
-    n.contents,
-  ].join('\n')
+  let gsql = [...tables.map(q => `table ${q.name} as (${q.contents})`), n.contents].join('\n')
+  let params = getActivePageInputs().getParams()
 
   try {
-    let response = await fetch('/_api/query', {
-      method: 'POST',
-      headers: {'Content-Type': 'application/json'},
-      body: JSON.stringify({params, gsql, hashes, routePath: getRoutePath(), repoId: window.$GRAPHENE?.repoId}),
-    })
-    let hash = response.headers.get('ETag') || ''
-
-    if (response.status == 304) { // cache hit. Read it out and use that
-      let body = await cacheRead(hash)
-      let result = translateData(body, n)
-      if (n.source) queryResults[n.source] = {rows: result.rows, fields: body.fields}
-      n.callback(result)
-    } else if (response.ok) { // cache miss. write it to the cache, and return the data
-      cacheWrite(hash, response.clone()) // clone allows us to write the raw response into the cache
-      let body = await response.json()
-      let fields = body.fields // grab before translateData mutates
-      let result = translateData(body, n) // nb that translateData modifies in place for performance
-      if (n.source) queryResults[n.source] = {rows: result.rows, fields}
-      n.callback(result)
-    } else { // request failed. Record it
-      let isJson = response.headers.get('Content-Type') === 'application/json'
-      let body = isJson ? await response.json() : await response.text()
-      n.errors = Array.isArray(body) ? body : [{message: body}]
-
-      let fieldIds = Array.from(n.fields.entries()).flatMap(([name, val]) => {
-        if (Array.isArray(val)) {
-          if (val.length === 0) return [] as string[]
-          if (val.length === 1) return [`${name}="${val[0]}"`]
-          return [`${name}="${val.join(', ')}"`]
-        }
-        if (typeof val === 'string' && val.trim().length === 0) return [] as string[]
-        if (val == null) return [] as string[]
-        return [`${name}="${val}"`]
-      })
-      let idStr = `Query (data="${n.source}" ` + fieldIds.join(' ') + ')'
-      n.errors.forEach(e => (e as any).id = idStr)
-      n.callback({errors: n.errors})
-    }
+    let res = await queryFetcher({params, gsql, hashes, repoId: window.$GRAPHENE?.repoId})
+    let result = translateData(res, n)
+    if (n.source) queryResults[n.source] = result // TODO do we still need queryResults? Seems like a hack
+    n.callback(result)
   } catch (e) {
-    n.errors = [e as Error]
+    let err = typeof e == 'string' ? new Error(e) : (e as Error)
+    let grapheneError = err as GrapheneError
+    n.error = {...grapheneError, componentId: n.componentId || grapheneError.componentId, message: err.message, stack: err.stack}
+    n.callback({rows: [], fields: [], error: n.error, sql: ''})
   } finally {
     n.loading = false
   }
 }
 
-function runAll () {
+async function fetchWithCache(req: QueryRequest): Promise<QueryResult> {
+  let response = await fetch('/_api/query', {
+    method: 'POST',
+    headers: {'Content-Type': 'application/json'},
+    body: JSON.stringify(req),
+  })
+  let hash = response.headers.get('ETag') || ''
+
+  // cache hit. Read data out of the browser cache and return it
+  if (response.status == 304) {
+    return await cacheRead(hash)
+  }
+
+  if (!response.ok) {
+    let body = (await response.json()) as GrapheneError
+    let err = new Error(body.message)
+    Object.assign(err, body)
+    throw err
+  }
+
+  cacheWrite(hash, response.clone())
+  return await response.json()
+}
+
+function runAll() {
   if (runPending) return runPending
-  runPending = Promise.resolve().then(_runAll).finally(() => runPending = null)
+  runPending = Promise.resolve()
+    .then(_runAll)
+    .finally(() => (runPending = null))
 }
 
-async function _runAll () {
-  await Promise.all(queries.map(async n => {
-    if (!n.callback) return
-    await runNode(n)
-  }))
+async function _runAll() {
+  await Promise.all(
+    queries.map(async n => {
+      if (!n.callback) return
+      await runNode(n)
+    }),
+  )
 }
 
-function translateData (data: any, node: QueryNode) {
+// This translates results we got back from the server into the format any frontend code expects.
+export function translateData(data: any, node: QueryNode): QueryResult {
   let rows = data.rows || []
-  rows.dataLoaded = true // evidence components need this to be set
-  rows._evidenceColumnTypes = []
-  let requestFields: string[] = []
-  node.fields.forEach((value) => {
-    if (Array.isArray(value)) requestFields.push(...value)
-    else requestFields.push(value)
-  })
+  let fields: Field[] = []
+
+  let requestFields = Array.from(node.fields.values()).flatMap(f => f)
 
   data.fields.forEach((field, index) => {
-    let name = field.name
+    let requested = requestFields[index]
+    let name = requested || field.name
+
+    // The key in row objects usually matches field.name, except in snowflake where it gets auto-capitalized
+    let rowKey = field.name
+    if (rows[0] && !Object.hasOwn(rows[0], field.name)) {
+      rowKey = Object.keys(rows[0]).find(k => k.toLowerCase() == field.name.toLowerCase())
+    }
 
-    // server gives names like `col_1` to unnamed expressions but we translate it back into the original expression like `avg(price)`
-    if (field.name.match(/col_\d+/)) {
-      name = requestFields[index]
+    // Result fields come back in select order, so we can map them back to the requested field names by index.
+    // Row objects are still keyed by the warehouse result name, which may differ by alias or by Snowflake uppercasing.
+    if (requested && rowKey && rowKey != requested) {
       rows.forEach(r => {
-        r[name] = r[field.name]
-        delete r[field.name]
+        r[requested] = r[rowKey]
+        delete r[rowKey]
       })
     }
 
-    // map graphene types down to the ones evidence expects
-    rows._evidenceColumnTypes.push({name, evidenceType: evidenceType(field.type)})
+    fields.push({...field, name})
   })
 
-  return {rows}
+  return {rows, fields}
 }
 
-export const isLoading = () => !!queries.find(q => q.loading)
-
-errorProvider('queryEngine', () => {
-  let unique = {}
-  queries.flatMap(q => q.errors).filter(q => !!q).forEach(e => {
-    unique[e.message + String((e as any).from?.lineText)] = e
+const isQueryLoading = () => !!queries.find(q => q.loading)
+
+if (typeof window !== 'undefined') {
+  Object.assign(window.$GRAPHENE, {
+    getParam: (name: string) => getActivePageInputs().getParam(name),
+    registerQuery,
+    subscribeParams: subscriber => getActivePageInputs().subscribeParams(subscriber),
+    syncParamsFromUrl: () => getActivePageInputs().syncFromUrl(),
+    updateParam: (name: string, value: any) => getActivePageInputs().updateParam(name, value),
+    updateParams: (nextParams: Record<string, any>) => getActivePageInputs().updateParams(nextParams),
+    query,
+    unsubscribe,
+    resetQueryEngine,
+    rerunQueries: runAll,
+    isQueryLoading,
+    queryResults,
   })
-  return Object.values(unique) as Error[]
-})
-
-async function waitForQueries (timeout = 20_000) {
-  let end = performance.now() + timeout
-  while (isLoading() && performance.now() < end) {
-    await new Promise(resolve => setTimeout(resolve, 25))
-  }
-  return !isLoading()
-}
-
-function evidenceType (type: string | undefined) {
-  if (type === 'string') return 'string'
-  if (type === 'number') return 'number'
-  if (type === 'boolean') return 'boolean'
-  if (type ===  'date' || type === 'timestamp') return 'date'
-  console.warn('Unsupported evidence type ' + type)
-  return 'string'
 }
-
-Object.assign(window.$GRAPHENE, {
-  registerQuery,
-  updateParam,
-  query,
-  unsubscribe,
-  waitForQueries,
-  queryResults,
-})

package/dist/ui/internal/runSocket.ts

@@ -0,0 +1,59 @@
+// WebSocket connection for the `graphene run` command.
+// Listens for run requests, waits for queries to finish, captures screenshots, and reports errors.
+
+import {getErrors} from './telemetry.ts'
+
+let socket: WebSocket | null = null
+connect()
+
+// html2canvas is dynamically loaded so we don't include it on pages that don't need it.
+let html2canvas: any
+async function loadHtml2Canvas() {
+  html2canvas ||= (await import('@graphenedata/html2canvas'))?.default
+}
+
+async function captureChart(chart: string) {
+  let escaped = window.CSS.escape(chart)
+  let chartEl = document.querySelector(`[data-chart-title="${escaped}"]`) as HTMLElement | null
+  chartEl ||= document.querySelector(`[data-component-id="${escaped}"]`) as HTMLElement | null
+  if (!chartEl) return undefined
+
+  await loadHtml2Canvas()
+  let canvas = await html2canvas(chartEl, {useCORS: true, allowTaint: true, scale: 1, liveDOM: true})
+  return canvas?.toDataURL('image/png')
+}
+
+function listComponentIds() {
+  return Array.from(document.querySelectorAll('[data-component-id]'))
+    .map(el => el.getAttribute('data-component-id') || '')
+    .filter(componentId => componentId.trim().length > 0)
+}
+
+async function takeScreenshot() {
+  await loadHtml2Canvas()
+  let canvas = await html2canvas(document.body, {useCORS: true, allowTaint: true, scale: 1, liveDOM: true})
+  return canvas?.toDataURL('image/png')
+}
+
+function connect() {
+  let wsUrl = `ws://${window.location.host}/_api/ws`
+  socket = new WebSocket(wsUrl)
+  socket.onclose = () => setTimeout(connect, 2000)
+  socket.onopen = () => socket!.send(JSON.stringify({type: 'register', url: window.location.href}))
+
+  socket.onmessage = async event => {
+    let {type, requestId, action, chart} = JSON.parse(event.data)
+    if (type !== 'check') return
+
+    if (action === 'list') {
+      await window.$GRAPHENE.pageReady
+      await new Promise(resolve => requestAnimationFrame(resolve))
+      socket!.send(JSON.stringify({type: 'checkResponse', requestId, componentIds: listComponentIds()}))
+      return
+    }
+
+    let finished = await window.$GRAPHENE.waitForLoad(20_000)
+    let screenshot = chart ? await captureChart(chart) : await takeScreenshot()
+    socket!.send(JSON.stringify({type: 'checkResponse', requestId, errors: getErrors(), stillLoading: !finished, screenshot}))
+  }
+}

package/dist/ui/internal/sidebar.svelte.js

@@ -0,0 +1,18 @@
+let state = $state({ open: false });
+let closeTimer;
+const sidebar = {
+  get open() {
+    return state.open;
+  },
+  enter() {
+    clearTimeout(closeTimer);
+    state.open = true;
+  },
+  leave() {
+    clearTimeout(closeTimer);
+    closeTimer = setTimeout(() => state.open = false, 120);
+  }
+};
+export {
+  sidebar
+};

package/dist/ui/internal/telemetry.ts

@@ -1,30 +1,65 @@
-type ErrorProvider = () => Error[]
+import {onDestroy} from 'svelte'
+
+import type {GrapheneError} from '../../lang/index.d.ts'
+
+// In local development, the same page could have components hot reload many times.
+// A flat list of errors would start to accumulate duplicates, and calling `graphene run` would report many (possibly stale) errors.
+// The current solution is to key errors by their componentId, and remove them when the component is destroyed.
+// This mostly works, but it has the downside that you can only show a single error per component, and does not help with `staticErrors`.
+// The other possible solution is that any error should prevent HMR and force a full refresh.
 
 window.$GRAPHENE ||= {}
 window.$GRAPHENE.getErrors = getErrors
 
-let staticErrors: Error[] = []
-let errorProviders: Record<string, ErrorProvider> = {}
+let staticErrors: GrapheneError[] = []
+let componentErrors = new Map<string, GrapheneError>()
 
-window.addEventListener('error', (event) => {
+window.addEventListener('error', event => {
   if ((event.error?.message || '').match(/Failed to fetch dynamically imported module.*\.md\?import/)) return
-  staticErrors.push(event.error)
-})
-window.addEventListener('unhandledrejection', (event) => {
-  staticErrors.push(event.reason)
+  logError(event.error)
 })
+window.addEventListener('unhandledrejection', event => logError(event.reason))
+
+// Logs errors that for whatever reason cannot be attached to a component
+export function logError(error: unknown) {
+  let err = error instanceof Error ? error : new Error(String(error))
+  staticErrors.push({message: err.message, stack: err.stack})
+}
+
+export function setErrorFor(key: string, error: GrapheneError | null) {
+  if (error) componentErrors.set(key, error)
+  else componentErrors.delete(key)
+}
+
+// Creates a logger for one component instance. Each component keeps its latest error
+// across rerenders, and we clear it when the component is destroyed during HMR/navigation.
+export function componentLogger(componentName: string, identifiers: Record<string, unknown> = {}) {
+  let id = computeComponentId(componentName, identifiers)
+  onDestroy(() => componentErrors.delete(id))
+
+  return {
+    id,
+    error(error: unknown, ctx: Partial<GrapheneError> = {}) {
+      if (!error) return componentErrors.delete(id)
+      let err = error instanceof Error ? error : new Error(String(error))
+      componentErrors.set(id, {message: err.message, stack: err.stack, componentId: id, ...ctx})
+    },
+  }
+}
 
-export function logError (e: Error | string, ctx?: any) {
-  if (typeof e === 'string') e = new Error(e)
-  if (ctx) Object.assign(e, ctx)
-  staticErrors.push(e)
+// Shared helper for logging when a svelte component is given props it did not expect.
+export function logExtraProps(logger: ReturnType<typeof componentLogger>, componentName: string, props: Record<string, unknown>) {
+  let unsupported = Object.keys(props).filter(prop => !['children', '$$slots', '$$events', '$$legacy'].includes(prop))
+  if (unsupported.length) logger.error(unsupported.map(prop => `Unsupported prop "${prop}" on ${componentName}.`).join(' '))
 }
 
-export function errorProvider (key:string, fn: ErrorProvider) {
-  errorProviders[key] = fn
+export function getErrors(): GrapheneError[] {
+  return staticErrors.concat(Array.from(componentErrors.values()))
 }
 
-export function getErrors (): Error[] {
-  let provided = Object.values(errorProviders).flatMap(p => p())
-  return staticErrors.concat(provided)
+export function computeComponentId(componentName: string, identifiers: Record<string, unknown> = {}) {
+  let attrs = Object.entries(identifiers).flatMap(([name, value]) =>
+    value === undefined || value === null || value === '' || (typeof value === 'object' && !Array.isArray(value)) ? [] : [`${name}="${Array.isArray(value) ? value.join(', ') : value}"`],
+  )
+  return `${componentName}${attrs.length ? ` (${attrs.join(' ')})` : ''}`
 }

package/dist/ui/internal/types.d.ts

@@ -0,0 +1,7 @@
+interface ImportMetaEnv {
+  readonly VITE_TEST: string
+}
+
+interface ImportMeta {
+  readonly env: ImportMetaEnv
+}

package/dist/ui/web.js

@@ -1,48 +1,84 @@
 import './internal/telemetry.ts'
-import './internal/checkSocket.ts'
+import './internal/queryEngine.ts'
+import './internal/runSocket.ts'
+import {getInstanceByDom} from 'echarts'
+
 import './app.css'
-import {mount} from 'svelte'
-import LocalApp from './internal/LocalApp.svelte'
+import {mount, unmount} from 'svelte'
 
-import Area from './components/Area.svelte'
 import AreaChart from './components/AreaChart.svelte'
-import Bar from './components/Bar.svelte'
 import BarChart from './components/BarChart.svelte'
 import BigValue from './components/BigValue.svelte'
-import Chart from './components/Chart.svelte'
 import Column from './components/Column.svelte'
 import DateRange from './components/DateRange.svelte'
 import Dropdown from './components/Dropdown.svelte'
 import DropdownOption from './components/DropdownOption.svelte'
 import ECharts from './components/ECharts.svelte'
-import ErrorChart from './components/ErrorChart.svelte'
 import GrapheneQuery from './components/GrapheneQuery.svelte'
 import InlineDelta from './components/InlineDelta.svelte'
-import Line from './components/Line.svelte'
 import LineChart from './components/LineChart.svelte'
 import PieChart from './components/PieChart.svelte'
 import QueryLoad from './components/QueryLoad.svelte'
 import Row from './components/Row.svelte'
+import ScatterPlot from './components/ScatterPlot.svelte'
 import SortIcon from './components/SortIcon.svelte'
 import Table from './components/Table.svelte'
 import TableCell from './components/TableCell.svelte'
 import TableGroupRow from './components/TableGroupRow.svelte'
 import TableGroupToggle from './components/TableGroupToggle.svelte'
+import TableHarness from './components/TableHarness.svelte'
 import TableHeader from './components/TableHeader.svelte'
 import TableRow from './components/TableRow.svelte'
 import TableSubtotalRow from './components/TableSubtotalRow.svelte'
 import TableTotalRow from './components/TableTotalRow.svelte'
 import TextInput from './components/TextInput.svelte'
+import Value from './components/Value.svelte'
+import ErrorChart from './internal/ErrorDisplay.svelte'
+import LocalApp from './internal/LocalApp.svelte'
 
+// Having a global $GRAPHENE allows us to provide an api that pages can use without having to import and bundle a bunch of components.
+// That means that as you navigate around, we only have to a very small amount of js for the page itself, and the bulk of the container and component
+// code only has to load once.
+// In theory we could do this with Vite splitting, but then we have a hard dependency on the exact format vite uses. Plus I find the easier to understand.
 window.$GRAPHENE = window.$GRAPHENE || {}
 
+let nextRenderId = 0
+let pendingRenders = new Set()
+
+window.$GRAPHENE.getChart = domNode => {
+  return getInstanceByDom(domNode)
+}
+
+window.$GRAPHENE.renderStart = id => {
+  let renderId = id == null ? `render:${++nextRenderId}` : String(id)
+  pendingRenders.add(renderId)
+  return renderId
+}
+
+window.$GRAPHENE.renderComplete = id => {
+  if (id == null) return
+  pendingRenders.delete(String(id))
+}
+
+window.$GRAPHENE.waitForLoad = async (timeout = 20_000) => {
+  let g = window.$GRAPHENE
+  let end = Date.now() + timeout
+  while (Date.now() < end) {
+    if (!g.isQueryLoading() && pendingRenders.size == 0) {
+      if (document.fonts?.ready) await document.fonts.ready
+      await new Promise(resolve => setTimeout(resolve, 300))
+      await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))
+      if (!g.isQueryLoading() && pendingRenders.size == 0) return true
+    }
+    await new Promise(resolve => setTimeout(resolve, 100))
+  }
+  return false
+}
+
 window.$GRAPHENE.components = {
-  Area,
   AreaChart,
-  Bar,
   BarChart,
   BigValue,
-  Chart,
   Column,
   DateRange,
   Dropdown,
@@ -51,21 +87,27 @@ window.$GRAPHENE.components = {
   ErrorChart,
   GrapheneQuery,
   InlineDelta,
-  Line,
   LineChart,
   PieChart,
   QueryLoad,
   Row,
+  ScatterPlot,
   SortIcon,
   Table,
   TableCell,
   TableGroupRow,
   TableGroupToggle,
   TableHeader,
+  TableHarness,
   TableRow,
   TableSubtotalRow,
   TableTotalRow,
   TextInput,
+  Value,
 }
 
-mount(LocalApp, {target: document.body})
+window.$GRAPHENE.svelte = {mount, unmount}
+
+if (window.location.pathname.replace(/\/+$/, '') !== '/__ct') {
+  mount(LocalApp, {target: document.body})
+}