@sixb/core 0.1.21 → 0.1.22

package/src/artifacts/run-query.ts

@@ -0,0 +1,212 @@
+import { getAuthorizedOntologyView } from "../execution/authorized-object-reader"
+import type { AuthorizedOntologyView } from "../objects/authorized-ontology-view"
+import { ObjectQueryPlanningError, ObjectQueryValidationError } from "../objects/query/errors"
+import type { ObjectExpansion } from "../objects/query/ir"
+import { type ValidatedObjectQuery, validateObjectQuery } from "../objects/query/validate"
+import type { Schema, ValueType } from "../ontology/types"
+import type { SixbRuntimeContext } from "../runtime/types"
+import type { ObjectRow } from "../storage/objects/types"
+import { ArtifactRequestError } from "./errors"
+import { ARTIFACT_LIMITS } from "./limits"
+import type {
+  ArtifactQuery,
+  ArtifactQueryProperty,
+  ArtifactQueryResult,
+  ArtifactQuerySchema,
+  ArtifactQueryShape,
+} from "./queries"
+
+/** Rows read per call to the reader, so a large result never comes back in one response. */
+const PAGE_SIZE = 1_000
+
+/**
+ * Runs one of a version's queries as the runtime's caller, through their object reader: what they
+ * cannot view refuses the query, and what they are not cleared for is left out of each row.
+ *
+ * The query was valid when it was published, but the ontology may have changed since: a query
+ * that no longer validates, or no longer plans, is `query_invalid`. Rows are read a page at a
+ * time, up to `ARTIFACT_LIMITS.maxQueryRows`, within any `limit` the query declares.
+ */
+export async function runArtifactQuery(
+  runtime: Pick<SixbRuntimeContext, "ontology" | "objectReader">,
+  name: string,
+  query: ArtifactQuery
+): Promise<ArtifactQueryResult> {
+  try {
+    const shape = describeArtifactQuery(runtime, query)
+
+    if (query.kind === "facets") {
+      const { total, facets } = await runtime.objectReader.facet({
+        query: query.query,
+        facets: query.facets,
+      })
+      return { kind: "facets", total, facets, ...shape }
+    }
+
+    return { kind: "objects", ...(await readRows(runtime, query.query)), ...shape }
+  } catch (error) {
+    throw queryInvalid(name, error)
+  }
+}
+
+/**
+ * What a query returns and reads, and what its rows hold as the runtime's caller may view them. It
+ * throws what validating the query throws.
+ */
+export function describeArtifactQuery(
+  runtime: Pick<SixbRuntimeContext, "ontology" | "objectReader">,
+  query: ArtifactQuery
+): ArtifactQueryShape {
+  const validated = validateObjectQuery(query.query, { ontology: runtime.ontology })
+
+  return {
+    resultObjectTypeIds: [...validated.result.objectTypeIds],
+    readObjectTypeIds: [...validated.touchedObjectTypeIds],
+    schema: artifactQuerySchema(getAuthorizedOntologyView(runtime.objectReader), validated),
+  }
+}
+
+/**
+ * The properties of the types a validated query returns, and of those each link it expands on its
+ * rows reaches, read from `view`: a reader is never told of a property their view leaves out. A
+ * type the view leaves out adds none. Only the outermost expansions are read, where normalization
+ * puts every expansion of the rows; one nested in another is not.
+ */
+export function artifactQuerySchema(
+  view: AuthorizedOntologyView,
+  validated: ValidatedObjectQuery
+): ArtifactQuerySchema {
+  const valueTypes = view.getValueTypesById()
+  const results = validated.result.objectTypeIds
+  const root = validated.query
+
+  const links: Record<string, { properties: Record<string, ArtifactQueryProperty> }> = {}
+  for (const expansion of root.kind === "expand" ? root.expansions : []) {
+    links[expansion.linkId] = {
+      properties: propertiesOf(view, expansionTargets(view, results, expansion), valueTypes),
+    }
+  }
+
+  return { properties: propertiesOf(view, results, valueTypes), links }
+}
+
+function propertiesOf(
+  view: AuthorizedOntologyView,
+  objectTypeIds: readonly string[],
+  valueTypes: ReadonlyMap<string, ValueType>
+): Record<string, ArtifactQueryProperty> {
+  const properties: Record<string, ArtifactQueryProperty> = {}
+
+  for (const objectTypeId of objectTypeIds) {
+    for (const property of view.getObjectTypeById(objectTypeId)?.properties ?? []) {
+      properties[property.id] ??= queryProperty(property.schema, valueTypes)
+    }
+  }
+
+  return properties
+}
+
+/** What an expansion reaches from the rows, as the validator resolves it, within the view. */
+function expansionTargets(
+  view: AuthorizedOntologyView,
+  results: readonly string[],
+  expansion: ObjectExpansion
+): string[] {
+  const targets = new Set<string>()
+
+  if (expansion.direction === "outgoing") {
+    for (const objectTypeId of results) {
+      const link = view
+        .getObjectTypeById(objectTypeId)
+        ?.links.find((candidate) => candidate.id === expansion.linkId)
+      if (!link || link.targetObjectTypeId === "*") continue
+
+      for (const target of [link.targetObjectTypeId].flat()) targets.add(target)
+    }
+    return [...targets]
+  }
+
+  const sources =
+    expansion.sourceObjectTypeId === undefined
+      ? view.listObjectTypes()
+      : [view.getObjectTypeById(expansion.sourceObjectTypeId)].filter((type) => type !== null)
+
+  for (const source of sources) {
+    const link = source.links.find((candidate) => candidate.id === expansion.linkId)
+    if (!link) continue
+
+    const reaches =
+      link.targetObjectTypeId === "*" ||
+      results.some((result) => view.isValidLinkTarget(link.targetObjectTypeId, result))
+    if (reaches) targets.add(source.id)
+  }
+  return [...targets]
+}
+
+/** A property's schema as a value of a row holds it: a value type's resolved to its own. */
+function queryProperty(
+  schema: Schema,
+  valueTypes: ReadonlyMap<string, ValueType>,
+  depth = 0
+): ArtifactQueryProperty {
+  if (typeof schema === "string") return { type: schema }
+
+  switch (schema.type) {
+    case "valueTypeRef": {
+      const named = valueTypes.get(schema.valueTypeId)?.schema ?? schema._resolved
+      // A value type naming itself, through others or not, holds what nothing else says.
+      return named && depth < 8 ? queryProperty(named, valueTypes, depth + 1) : { type: "object" }
+    }
+    case "enum":
+      return { type: "enum", valueType: schema.valueType }
+    default:
+      return { type: schema.type }
+  }
+}
+
+async function readRows(
+  runtime: Pick<SixbRuntimeContext, "objectReader">,
+  query: ArtifactQuery["query"]
+): Promise<{ objects: ObjectRow[]; total: number; truncated: boolean }> {
+  const objects: ObjectRow[] = []
+  let total = 0
+  let pageToken: string | undefined
+
+  for (;;) {
+    // The last page asks only for what fits under the cap, and says whether more remain.
+    const pageSize = Math.min(PAGE_SIZE, ARTIFACT_LIMITS.maxQueryRows - objects.length)
+    const first = pageToken === undefined
+
+    const page = await runtime.objectReader.executeQuery({
+      query: { kind: "page", input: query, pageSize, ...(pageToken ? { pageToken } : {}) },
+      ...(first ? { includeTotal: true } : {}),
+    })
+
+    if (first) total = page.total ?? page.objects.length
+    objects.push(...page.objects)
+
+    if (!page.hasMore || page.nextPageToken === undefined) {
+      return { objects, total, truncated: false }
+    }
+    if (objects.length >= ARTIFACT_LIMITS.maxQueryRows) {
+      return { objects, total, truncated: true }
+    }
+
+    pageToken = page.nextPageToken
+  }
+}
+
+/** A query the ontology no longer admits, in the words validation gives; anything else as is. */
+function queryInvalid(name: string, error: unknown): unknown {
+  if (!(error instanceof ObjectQueryValidationError || error instanceof ObjectQueryPlanningError)) {
+    return error
+  }
+
+  const issues = error.issues.map((issue) => `${issue.path}: ${issue.message}`).join("; ")
+  const problem =
+    error instanceof ObjectQueryValidationError
+      ? "no longer matches the ontology"
+      : "can no longer run"
+
+  return new ArtifactRequestError("query_invalid", `[Sixb] Query '${name}' ${problem}: ${issues}`)
+}

package/src/artifacts/serialize-query.ts

@@ -0,0 +1,25 @@
+import { type SerializedExpandedObject, serializeExpandedObject } from "../objects/serialize"
+import type {
+  ArtifactFacetsQueryResult,
+  ArtifactObjectsQueryResult,
+  ArtifactQueryResult,
+} from "./queries"
+
+export type SerializedArtifactQueryResult =
+  | (Omit<ArtifactObjectsQueryResult, "objects"> & {
+      readonly objects: readonly SerializedExpandedObject[]
+    })
+  | ArtifactFacetsQueryResult
+
+/**
+ * An artifact query's result as `GET /api/artifacts/:id/versions/:v/queries/:name` answers it, and
+ * as a page's `sixb.query` resolves: its rows as the query route serializes them, or its facets.
+ * The route and the Agent's layout check both serve it, so a page reads the same JSON in either.
+ */
+export function serializeArtifactQueryResult(
+  result: ArtifactQueryResult
+): SerializedArtifactQueryResult {
+  return result.kind === "objects"
+    ? { ...result, objects: result.objects.map(serializeExpandedObject) }
+    : result
+}

package/src/artifacts/sheet-rules.ts

@@ -0,0 +1,180 @@
+import type { CsvTable } from "./csv"
+import {
+  checkFormula,
+  columnFormulaType,
+  evaluateComputedColumns,
+  FormulaCycleError,
+  FormulaSyntaxError,
+  formulaFitsColumn,
+  parseFormula,
+} from "./formula"
+import { ARTIFACT_LIMITS } from "./limits"
+import {
+  NUMERIC_COLUMN_TYPES,
+  sheetFilePath,
+  WORKBOOK_FILE,
+  type Workbook,
+  type WorkbookSheet,
+} from "./workbook"
+
+/**
+ * Whether a sheet's workbook and tables keep the rules publishing holds them to, whatever their
+ * cells hold: columns named once, every column the workbook names in its table, formulas that
+ * compute their columns, and charts that plot what they can draw. Returns the first rule broken,
+ * in the words publishing refuses the version with, or null. A sheet without a table here is not
+ * checked, nor a chart on it, so an editor can check only the sheet it changed.
+ */
+export function checkSheet(
+  workbook: Workbook,
+  tables: ReadonlyMap<string, CsvTable>
+): string | null {
+  for (const sheet of workbook.sheets) {
+    const table = tables.get(sheet.name)
+    if (!table) continue
+
+    const problem = sheetColumnsProblem(sheet, table) ?? sheetFormulasProblem(sheet, table)
+    if (problem) return problem
+  }
+
+  return sheetChartsProblem(workbook, tables)
+}
+
+/**
+ * A table's header and size against its sheet: named, unique, and holding what the sheet names. A
+ * live sheet's header is the columns its workbook lists.
+ */
+export function sheetColumnsProblem(sheet: WorkbookSheet, table: CsvTable): string | null {
+  const path = sheetFilePath(sheet.name)
+  const source = sheet.live ? `${WORKBOOK_FILE} sheet '${sheet.name}' "columns"` : `'${path}'`
+  const { header, rows } = table
+
+  if (header.length === 0) return `'${path}' is empty: its first line names the columns.`
+  if (header.length > ARTIFACT_LIMITS.maxSheetColumns) {
+    return `${source} has ${header.length} columns; a sheet holds at most ${ARTIFACT_LIMITS.maxSheetColumns}.`
+  }
+
+  const seen = new Set<string>()
+  for (const [index, name] of header.entries()) {
+    if (!name.trim())
+      return `${source} column ${index + 1} has no name${sheet.live ? "" : " in the header row"}.`
+    if (seen.has(name)) return `${source} has two columns named '${name}'.`
+    seen.add(name)
+  }
+
+  if (rows.length > ARTIFACT_LIMITS.maxSheetRows) {
+    return `'${path}' has ${rows.length} rows; a sheet holds at most ${ARTIFACT_LIMITS.maxSheetRows}.`
+  }
+  for (const [index, row] of rows.entries()) {
+    if (row.length > header.length) {
+      return `'${path}' row ${index + 2} has ${row.length} fields; the header names ${header.length} columns.`
+    }
+  }
+
+  for (const column of [
+    ...Object.keys(sheet.columns),
+    ...Object.keys(sheet.widths),
+    ...Object.keys(sheet.descriptions),
+    ...Object.keys(sheet.totals),
+  ]) {
+    if (!seen.has(column)) {
+      return `${WORKBOOK_FILE} sheet '${sheet.name}' names column '${column}', which ${sheet.live ? 'its "columns" does not list' : `'${path}' does not have`}.`
+    }
+  }
+
+  return null
+}
+
+/**
+ * Each computed column's formula against the columns it reads: it reads, its result fits the
+ * column, and no columns compute from each other.
+ */
+export function sheetFormulasProblem(sheet: WorkbookSheet, table: CsvTable): string | null {
+  const at = (column: string) => `${WORKBOOK_FILE} sheet '${sheet.name}' column '${column}'`
+  const types = new Map(
+    table.header.map((name) => [name, columnFormulaType(sheet.columns[name]?.type ?? "text")])
+  )
+
+  for (const [column, format] of Object.entries(sheet.columns)) {
+    if (!format.formula) continue
+
+    let node: ReturnType<typeof parseFormula>
+    try {
+      node = parseFormula(format.formula)
+    } catch (error) {
+      if (error instanceof FormulaSyntaxError) {
+        return `${at(column)} "formula" does not read: ${error.message}`
+      }
+      throw error
+    }
+
+    const checked = checkFormula(node, types)
+    if ("error" in checked) return `${at(column)} "formula": ${checked.error}.`
+    if (!formulaFitsColumn(checked.type, format.type)) {
+      return `${at(column)} "formula" computes ${checked.type === "date" ? "a date" : checked.type === "boolean" ? "true or false" : checked.type === "text" ? "text" : "a number"}, which a ${format.type} column can't show: change its type, or the formula.`
+    }
+  }
+
+  // Over no rows, the columns only find the order they compute in: a ring of them throws.
+  try {
+    evaluateComputedColumns(sheet, { header: table.header, rows: [] })
+  } catch (error) {
+    if (error instanceof FormulaCycleError) {
+      return `${WORKBOOK_FILE} sheet '${sheet.name}': ${error.message}, so none can be computed first.`
+    }
+    throw error
+  }
+
+  return null
+}
+
+/**
+ * Each chart against its sheet: the columns it plots exist and are numbers, its right axis reads
+ * one kind, and a pie has few enough slices to tell apart.
+ */
+export function sheetChartsProblem(
+  workbook: Workbook,
+  tables: ReadonlyMap<string, CsvTable>
+): string | null {
+  for (const chart of workbook.charts) {
+    const sheet = workbook.sheets.find((candidate) => candidate.name === chart.sheet)
+    const table = tables.get(chart.sheet)
+    if (!sheet || !table) continue
+
+    const plotted = [...chart.y, ...chart.line]
+    for (const column of [chart.x, ...plotted]) {
+      if (!table.header.includes(column)) {
+        return `${WORKBOOK_FILE} chart '${chart.name}' plots column '${column}', which sheet '${chart.sheet}' does not have.`
+      }
+    }
+
+    // A scatter places each row by its x, so x is a number too.
+    for (const column of chart.type === "scatter" ? [chart.x, ...plotted] : plotted) {
+      const type = sheet.columns[column]?.type ?? "text"
+      if (!NUMERIC_COLUMN_TYPES.includes(type)) {
+        return `${WORKBOOK_FILE} chart '${chart.name}' plots column '${column}', which is ${type}; give it a number, currency, or percent type in "columns".`
+      }
+    }
+
+    // Lines of another kind than the bars read on one right axis, which reads one kind.
+    const barType = sheet.columns[chart.y[0] ?? ""]?.type
+    const rightTypes = new Set(
+      chart.line.map((column) => sheet.columns[column]?.type).filter((type) => type !== barType)
+    )
+    if (rightTypes.size > 1) {
+      return `${WORKBOOK_FILE} chart '${chart.name}' draws lines of ${[...rightTypes].join(" and ")} over ${barType ?? "its"} bars; the right axis reads one kind. Chart the others separately.`
+    }
+
+    if (chart.type === "pie") {
+      // A pie's slices are its rows, or its x values when it summarizes them.
+      const index = table.header.indexOf(chart.x)
+      const slices = chart.aggregate
+        ? new Set(table.rows.map((row) => (row[index] ?? "").trim())).size
+        : table.rows.length
+      if (slices > ARTIFACT_LIMITS.maxPieSlices) {
+        return `${WORKBOOK_FILE} chart '${chart.name}' is a pie with ${slices} slices; a pie reads well with at most ${ARTIFACT_LIMITS.maxPieSlices}. Use a bar chart instead.`
+      }
+    }
+  }
+
+  return null
+}

package/src/artifacts/tones.ts

@@ -0,0 +1,20 @@
+/**
+ * The colors an artifact gives what it marks, a sheet's tag or a canvas's note: the theme's
+ * meanings, or one of its chart colors, so they follow the app's theme and dark mode.
+ */
+export const ARTIFACT_TONES = [
+  "neutral",
+  "positive",
+  "warning",
+  "negative",
+  "chart-1",
+  "chart-2",
+  "chart-3",
+  "chart-4",
+  "chart-5",
+] as const
+export type ArtifactTone = (typeof ARTIFACT_TONES)[number]
+
+export function isArtifactTone(value: unknown): value is ArtifactTone {
+  return (ARTIFACT_TONES as readonly unknown[]).includes(value)
+}