@graphenedata/cli 0.0.1

package/dist/ui/internal/queryEngine.ts

@@ -0,0 +1,169 @@
+// The query engine gathers query requests and inputs from components, and issues requests to the server.
+// When inputs change, it takes care of notifying affected components and requesting new data.
+
+import {cacheRead, cacheWrite, getHashes} from './clientCache'
+import {errorProvider} from './telemetry.ts'
+
+interface QueryError {
+  file: string
+  message: string
+  from: {lineText: string}
+}
+
+interface QueryResult {
+  rows?: any[]
+  errors?: Error[]
+  fields?: Field[]
+}
+
+interface Field {
+  name: string
+  type?: string
+}
+
+type ResultHandler = (res: QueryResult) => void
+
+interface QueryNode {
+  name?: string
+  contents: string
+  callback?: ResultHandler
+  loading: boolean
+  fields: string[]
+  errors: Error[]
+}
+
+let runPending: Promise<void> | null = null
+let params = {} as Record<string, any>
+let queries = [] as QueryNode[]
+
+function registerQuery (name: string, contents: string) {
+  queries = queries.filter(q => q.name !== name)
+  queries.push({name, contents, loading: false, fields: [], errors: []})
+}
+
+function updateParam (name: string, value: any) {
+  params[name] = value
+  runAll() // for now, do the easy thing and reload it all
+}
+
+function query (source: string, fields: string[], callback: ResultHandler) {
+  let contents = `from ${source} select ${fields.join(', ')}`
+  queries.push({contents, callback, loading: false, fields, errors: []})
+  runAll()
+}
+
+function unsubscribe (callback: ResultHandler) {
+  queries = queries.filter(q => q.callback !== callback)
+}
+
+async function runNode (n: QueryNode) {
+  if (!n.callback) throw new Error('running node nobody is listening to')
+  n.callback({}) // notify that the query is loading
+  n.loading = true
+  n.errors = []
+
+  let hashes = await getHashes()
+  let tables = queries.filter(q => q.name)
+  let gsql = [
+    ...tables.map(q => `table ${q.name} as (${q.contents})`),
+    n.contents,
+  ].join('\n')
+
+  try {
+    let response = await fetch('/_api/query', {
+      method: 'POST',
+      headers: {'Content-Type': 'application/json'},
+      body: JSON.stringify({params, gsql, hashes}),
+    })
+    let hash = response.headers.get('ETag') || ''
+
+    if (response.status == 304) { // cache hit. Read it out and use that
+      let body = await cacheRead(hash)
+      n.callback(translateData(body, n))
+    } else if (response.ok) { // cache miss. write it to the cache, and return the data
+      cacheWrite(hash, response.clone()) // clone allows us to write the raw response into the cache
+      let body = await response.json()
+      n.callback(translateData(body, n)) // nb that translateData modifies in place for performance
+    } else { // request failed. Record it
+      let isJson = response.headers.get('Content-Type') === 'application/json'
+      let body = isJson ? await response.json() : await response.text()
+      n.errors = Array.isArray(body) ? body : [body]
+      n.callback({errors: n.errors})
+    }
+  } catch (e) {
+    n.errors = [e]
+  } finally {
+    n.loading = false
+  }
+}
+
+function runAll () {
+  if (runPending) return runPending
+  runPending = Promise.resolve().then(_runAll).finally(() => runPending = null)
+}
+
+async function _runAll () {
+  await Promise.all(queries.map(async n => {
+    if (!n.callback) return
+    await runNode(n)
+  }))
+}
+
+function translateData (data: any, node: QueryNode) {
+  let rows = data.rows || []
+  rows.dataLoaded = true // evidence components need this to be set
+  rows._evidenceColumnTypes = []
+
+  data.fields.forEach((field, index) => {
+    let name = field.name
+
+    // server gives names like `col_1` to unnamed expressions but we translate it back into the original expression like `avg(price)`
+    if (field.name.match(/col_\d+/)) {
+      name = node.fields[index]
+      rows.forEach(r => {
+        r[name] = r[field.name]
+        delete r[field.name]
+      })
+    }
+
+    // map graphene types down to the ones evidence expects
+    rows._evidenceColumnTypes.push({name, evidenceType: evidenceType(field.type)})
+  })
+
+  return {rows}
+}
+
+export const isLoading = () => !!queries.find(q => q.loading)
+
+errorProvider('queryEngine', () => {
+  let unique = {}
+  queries.flatMap(q => q.errors).filter(q => !!q).forEach(e => {
+    unique[e.message + String((e as any).from?.lineText)] = e
+  })
+  return Object.values(unique)
+})
+
+async function waitForQueries (timeout = 20_000) {
+  let end = performance.now() + timeout
+  while (isLoading() && performance.now() < end) {
+    await new Promise(resolve => setTimeout(resolve, 25))
+  }
+  return !isLoading()
+}
+
+function evidenceType (type: string | undefined) {
+  if (type === 'string') return 'string'
+  if (type === 'number') return 'number'
+  if (type === 'boolean') return 'boolean'
+  if (type ===  'date' || type === 'timestamp') return 'date'
+  console.warn('Unsupported evidence type ' + type)
+  return 'string'
+}
+
+Object.assign(window.$GRAPHENE, {
+  registerQuery,
+  updateParam,
+  query,
+  unsubscribe,
+  waitForQueries,
+})

package/dist/ui/internal/telemetry.ts

@@ -0,0 +1,28 @@
+type ErrorProvider = () => Error[]
+
+window.$GRAPHENE = {getErrors}
+
+let staticErrors: Error[] = []
+let errorProviders: Record<string, ErrorProvider> = {}
+
+window.addEventListener('error', (event) => {
+  console.log('recordedError')
+  staticErrors.push(event.error)
+})
+window.addEventListener('unhandledrejection', (event) => {
+  console.log('record unhandled')
+  staticErrors.push(event.reason)
+})
+
+export function error (e: Error, ctx?: any) {
+  staticErrors.push(e)
+}
+
+export function errorProvider (key:string, fn: ErrorProvider) {
+  errorProviders[key] = fn
+}
+
+export function getErrors (): Error[] {
+  return staticErrors.concat(Object.values(errorProviders)
+    .flatMap(p => p()))
+}

package/dist/ui/internal/theme.ts

@@ -0,0 +1,88 @@
+// Evidence has a complex theme loading system that we don't really want to keep.
+// This was generated by copying the runtime export of $evidence/themes from the evidence sample app.
+export const themes = {
+  'light': {
+    'colors': {
+      'primary': '#2563eb',
+      'accent': '#c2410c',
+      'base-100': '#ffffff',
+      'info': '#0284c7',
+      'positive': '#16a34a',
+      'warning': '#f8c900',
+      'negative': '#dc2626',
+      'primary-content': '#f6f8fb',
+      'accent-content': '#fbf7f6',
+      'base-200': '#f7f7f7',
+      'base-300': '#d6d6d6',
+      'base-heading': '#060606',
+      'base-content': '#2c2c2c',
+      'base-content-muted': '#717171',
+      'info-content': '#030709',
+      'positive-content': '#040906',
+      'negative-content': '#fbf6f6',
+      'warning-content': '#0a0803',
+    },
+    'colorPalettes': {
+      'default': [
+        '#236aa4',
+        '#45a1bf',
+        '#a5cdee',
+        '#8dacbf',
+        '#85c7c6',
+        '#d2c6ac',
+        '#f4b548',
+        '#8f3d56',
+        '#71b9f4',
+        '#46a485',
+      ],
+    },
+    'colorScales': {
+      'default': [
+        '#ADD8E6',
+        '#00008B',
+      ],
+    },
+  },
+  'dark': {
+    'colors': {
+      'primary': '#3b82f6',
+      'accent': '#fdba74',
+      'base-100': '#09090b',
+      'info': '#38bdf8',
+      'positive': '#4ade80',
+      'warning': '#fbbf24',
+      'negative': '#f87171',
+      'primary-content': '#030609',
+      'accent-content': '#090603',
+      'base-200': '#111113',
+      'base-300': '#29292b',
+      'base-heading': '#f8f8f9',
+      'base-content': '#cacacb',
+      'base-content-muted': '#7b7b7c',
+      'info-content': '#030809',
+      'positive-content': '#040906',
+      'negative-content': '#090303',
+      'warning-content': '#090803',
+    },
+    'colorPalettes': {
+      'default': [
+        '#236aa4',
+        '#45a1bf',
+        '#a5cdee',
+        '#8dacbf',
+        '#85c7c6',
+        '#d2c6ac',
+        '#f4b548',
+        '#8f3d56',
+        '#71b9f4',
+        '#46a485',
+      ],
+    },
+    'colorScales': {
+      'default': [
+        '#ADD8E6',
+        '#00008B',
+      ],
+    },
+  },
+}

package/dist/ui/layout.svelte

@@ -0,0 +1,3 @@
+<main>
+  <slot />
+</main>

package/dist/ui/playwright.config.ts

@@ -0,0 +1,30 @@
+import {defineConfig, devices} from '@playwright/test'
+
+export default defineConfig({
+  testDir: './tests',
+  outputDir: './tests/results',
+  timeout: 10_000,
+  expect: {
+    timeout: process.env.DEBUG ? 0 : 2_000,
+    toHaveScreenshot: {
+      pathTemplate: '{testDir}/snapshots/{testFilePath}/{arg}{ext}',
+    },
+  },
+  fullyParallel: false,
+  forbidOnly: !!process.env.CI,
+  retries: 0, // process.env.CI ? 1 : 0,
+  reporter: process.env.CI ? [['list'], ['github']] : 'list',
+  use: {
+    headless: true,
+    actionTimeout: 0,
+    trace: 'retain-on-failure',
+    video: 'off',
+    launchOptions: {devtools: !!process.env.DEBUG},
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: {...devices['Desktop Chrome'], browserName: 'chromium'},
+    },
+  ],
+})

package/dist/ui/web.js

@@ -0,0 +1,106 @@
+import {getErrors} from './internal/telemetry.ts'
+import './app.css'
+import {isLoading} from './internal/queryEngine.ts'
+
+import Area from './components/Area.svelte'
+import AreaChart from './components/AreaChart.svelte'
+import Bar from './components/Bar.svelte'
+import BarChart from './components/BarChart.svelte'
+import BigValue from './components/BigValue.svelte'
+import Chart from './components/Chart.svelte'
+import Column from './components/Column.svelte'
+import DateRange from './components/DateRange.svelte'
+import Dropdown from './components/Dropdown.svelte'
+import DropdownOption from './components/DropdownOption.svelte'
+import ECharts from './components/ECharts.svelte'
+import ErrorChart from './components/ErrorChart.svelte'
+import GrapheneQuery from './components/GrapheneQuery.svelte'
+import InlineDelta from './components/InlineDelta.svelte'
+import Line from './components/Line.svelte'
+import LineChart from './components/LineChart.svelte'
+import PieChart from './components/PieChart.svelte'
+import QueryLoad from './components/QueryLoad.svelte'
+import Row from './components/Row.svelte'
+import SortIcon from './components/SortIcon.svelte'
+import Table from './components/Table.svelte'
+import TableCell from './components/TableCell.svelte'
+import TableGroupRow from './components/TableGroupRow.svelte'
+import TableGroupToggle from './components/TableGroupToggle.svelte'
+import TableHeader from './components/TableHeader.svelte'
+import TableRow from './components/TableRow.svelte'
+import TableSubtotalRow from './components/TableSubtotalRow.svelte'
+import TableTotalRow from './components/TableTotalRow.svelte'
+import TextInput from './components/TextInput.svelte'
+
+window.$GRAPHENE.components = {
+  Area,
+  AreaChart,
+  Bar,
+  BarChart,
+  BigValue,
+  Chart,
+  Column,
+  DateRange,
+  Dropdown,
+  DropdownOption,
+  ECharts,
+  ErrorChart,
+  GrapheneQuery,
+  InlineDelta,
+  Line,
+  LineChart,
+  PieChart,
+  QueryLoad,
+  Row,
+  SortIcon,
+  Table,
+  TableCell,
+  TableGroupRow,
+  TableGroupToggle,
+  TableHeader,
+  TableRow,
+  TableSubtotalRow,
+  TableTotalRow,
+  TextInput,
+}
+
+
+let socket = null
+
+connectWebSocket()
+
+async function takeScreenshot (chartName = null) {
+  if (!window.html2canvas) {
+    let html2canvas = await import('html2canvas')
+    window.html2canvas = html2canvas.default
+  }
+
+  // wait some time for queries to finish loading
+  let startTime = Date.now()
+  while (isLoading() && Date.now() - startTime < 20_000) {
+    await new Promise(resolve => setTimeout(resolve, 100))
+  }
+
+  let targetElement = chartName ? document.querySelector(`[name="${chartName}"]`) : document.body
+  let canvas = targetElement && await window.html2canvas(targetElement, {useCORS: true, allowTaint: true, scale: 1})
+  return {stillLoading: isLoading(), screenshot: canvas?.toDataURL('image/png')}
+}
+
+function connectWebSocket () {
+  socket = new WebSocket(`ws://${window.location.host}/graphene-ws`)
+  socket.onclose = () => setTimeout(connectWebSocket, 2000)
+
+  socket.onopen = () => {
+    socket.send(JSON.stringify({type: 'register', url: window.location.href}))
+  }
+
+  socket.onmessage = async (event) => {
+    console.log('Got message', event.data)
+    let {type, requestId, chart} = JSON.parse(event.data)
+
+    if (type === 'view') {
+      let {screenshot, stillLoading} = await takeScreenshot(chart)
+      socket.send(JSON.stringify({type: 'viewResponse', requestId, screenshot, stillLoading, errors: getErrors()}))
+    }
+  }
+}

package/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@graphenedata/cli",
+  "main": "cli.ts",
+  "type": "module",
+  "author": "Graphene Systems Inc",
+  "version": "0.0.1",
+  "license": "Elastic-2.0",
+  "engines": {
+    "node": ">=16"
+  },
+  "files": [
+    "dist",
+    "package.json",
+    "LICENSE.md",
+    "THIRD_PARTY_NOTICES.md"
+  ],
+  "bin": {
+    "graphene": "./dist/cli/cli.js"
+  },
+  "exports": {
+    "./ui": {
+      "browser": "./dist/ui/web.js"
+    }
+  },
+  "dependencies": {
+    "@duckdb/node-api": "1.3.2-alpha.26",
+    "@google-cloud/bigquery": "^8.1.1",
+    "@lezer/common": "^1.2.3",
+    "@lezer/lr": "^1.4.2",
+    "@graphenedata/malloy": "0.0.304",
+    "@sveltejs/vite-plugin-svelte": "3.1.2",
+    "@tidyjs/tidy": "^2.5.2",
+    "chalk": "^5.3.0",
+    "chokidar": "3.6.0",
+    "chroma-js": "2.4.2",
+    "cli-table3": "^0.6.3",
+    "commander": "^11.0.0",
+    "debounce": "^1.2.1",
+    "dompurify": "^3.2.7",
+    "echarts": "^5.5.0",
+    "fs-extra": "11.2.0",
+    "glob": "^11.0.3",
+    "html2canvas": "^1.4.1",
+    "marked": "^16.3.0",
+    "mdsvex": "^0.12.6",
+    "nanoid": "3.3.8",
+    "rehype-stringify": "^10.0.1",
+    "remark": "^15.0.1",
+    "remark-mdx": "^3.1.1",
+    "remark-rehype": "^11.1.2",
+    "ssf": "^0.11.2",
+    "svelte": "4.2.19",
+    "unist-util-visit": "4.1.2",
+    "vite": "5.4.14",
+    "ws": "^8.18.0"
+  },
+  "devDependencies": {
+    "@types/fs-extra": "^11.0.4",
+    "@types/node": "^20.0.0",
+    "@types/ws": "^8.18.1",
+    "esbuild": "^0.21.5",
+    "vitest": "3.0.5",
+    "vscode-languageserver-types": "^3.17.0"
+  },
+  "scripts": {
+    "build": "rm -rf dist && node ./esbuild.mjs",
+    "test": "vitest run --reporter=default --reporter=json --outputFile=/tmp/graphene-test-results.json",
+    "test-one": "DEBUG=1 node --inspect ../scripts/turboTest.js",
+    "prepack": "pnpm run build"
+  }
+}