@sixb/core 0.1.20 → 0.1.22

package/src/artifacts/live-sheet.ts

@@ -0,0 +1,345 @@
+import type { CsvTable } from "./csv"
+import { formulaReferences, parseFormula } from "./formula/parser"
+import type { ArtifactQueryProperty, ArtifactQuerySchema } from "./queries"
+import {
+  NUMERIC_COLUMN_TYPES,
+  readSheetColumnFormat,
+  type SheetColumnFormat,
+  type SheetColumnType,
+  type SheetTotal,
+  sheetTotalProblem,
+  WORKBOOK_FILE,
+  type WorkbookSheet,
+} from "./workbook"
+
+/**
+ * A live sheet laid out against what its query's rows hold: publishing checks it as its author
+ * sees the rows, and the viewer draws it as each reader does.
+ */
+export interface LiveSheetLayout {
+  /**
+   * The sheet with a format for every column, as a CSV sheet's workbook gives them: what the
+   * publish rules, the grid and the charts read.
+   */
+  readonly sheet: WorkbookSheet
+  /** Each column, in order, with what it shows. */
+  readonly columns: readonly LiveSheetColumn[]
+  /** The first column that does not lay out, in the words publishing refuses it with; or null. */
+  readonly problem: string | null
+}
+
+export interface LiveSheetColumn {
+  readonly name: string
+  /** The property the column shows, of the rows or of what `linkId` reaches; none when computed. */
+  readonly from?: { readonly linkId?: string; readonly propertyId: string }
+  /** Whether the rows hold no such property for this reader: every cell of it is locked. */
+  readonly missing: boolean
+}
+
+/** What a row of a query's result holds, as core reads it and as the API serializes it. */
+export interface LiveSheetRow {
+  readonly properties: Readonly<Record<string, unknown>>
+  /** The properties left out because the reader is not cleared for them. */
+  readonly redactions?: Readonly<Record<string, unknown>>
+  readonly links?: Readonly<Record<string, LiveSheetRow | readonly LiveSheetRow[] | null>>
+}
+
+/** A live sheet's rows as a sheet's CSV would hold them, and the cells the reader may not see. */
+export interface LiveSheetTable {
+  /** Each cell as a CSV cell holds its value, computed columns empty. */
+  readonly table: CsvTable
+  /**
+   * By row, the columns of the cells the reader may not see: drawn locked, never as empty. A
+   * computed cell that reads a locked one is locked too.
+   */
+  readonly locked: ReadonlyMap<number, ReadonlySet<number>>
+}
+
+/**
+ * The column type a property's values take in a sheet: numbers, dates, true or false, an enum's
+ * values as tags, and anything else as text. The ontology names no money or ratio, so a number is
+ * a number; a column that wants a currency says so.
+ */
+export function sheetColumnType(property: ArtifactQueryProperty): SheetColumnType {
+  switch (property.type) {
+    case "integer":
+    case "double":
+    case "decimal":
+      return "number"
+    case "boolean":
+      return "boolean"
+    case "date":
+    case "timestamp":
+      return "date"
+    case "enum":
+      return property.valueType === "integer" ? "number" : "tag"
+    default:
+      return "text"
+  }
+}
+
+/**
+ * Lays a live sheet out against its query's schema: each column's property, and its type, its
+ * property's unless the workbook gives one. A column whose property the schema lacks shows locked
+ * cells, and `problem` names the first that does not lay out.
+ */
+export function liveSheetColumns(
+  sheet: WorkbookSheet,
+  schema: ArtifactQuerySchema
+): LiveSheetLayout {
+  const live = sheet.live
+  if (!live) throw new Error(`[Sixb] Sheet '${sheet.name}' is not a live sheet.`)
+
+  let problem: string | null = null
+  const formats: Record<string, SheetColumnFormat> = {}
+  const totals: Record<string, SheetTotal> = { ...sheet.totals }
+
+  const columns = live.columns.map((source): LiveSheetColumn => {
+    const columnAt = `sheet '${sheet.name}' column '${source.name}'`
+    const at = `${WORKBOOK_FILE} ${columnAt}`
+    const written = sheet.columns[source.name]
+
+    if (source.from === undefined) {
+      if (written) formats[source.name] = written
+      return { name: source.name, missing: false }
+    }
+
+    const [first = "", second] = source.from.split(".")
+    const from =
+      second === undefined ? { propertyId: first } : { linkId: first, propertyId: second }
+    const found = resolveProperty(schema, live.query, from)
+    if ("problem" in found) problem ??= `${at} "from": ${found.problem}`
+
+    const inferred = "property" in found ? sheetColumnType(found.property) : undefined
+    const format = layColumn(columnAt, source.from, written, source.untyped, inferred)
+    if (format.problem) problem ??= format.problem
+    formats[source.name] = format.format
+
+    const total = totals[source.name]
+    const totalProblem =
+      total && source.untyped
+        ? sheetTotalProblem(
+            `${WORKBOOK_FILE} sheet '${sheet.name}'`,
+            source.name,
+            total,
+            format.format.type
+          )
+        : null
+    if (totalProblem) {
+      problem ??= totalProblem
+      delete totals[source.name]
+    }
+
+    return { name: source.name, from, missing: !("property" in found) }
+  })
+
+  return { sheet: { ...sheet, columns: formats, totals }, columns, problem }
+}
+
+function resolveProperty(
+  schema: ArtifactQuerySchema,
+  query: string,
+  from: NonNullable<LiveSheetColumn["from"]>
+): { readonly property: ArtifactQueryProperty } | { readonly problem: string } {
+  const { linkId, propertyId } = from
+
+  if (linkId === undefined) {
+    const property = schema.properties[propertyId]
+    return property
+      ? { property }
+      : {
+          problem: `query '${query}' returns no property '${propertyId}'${known(schema.properties)}.`,
+        }
+  }
+
+  const link = schema.links[linkId]
+  if (!link) {
+    return {
+      problem: `query '${query}' does not expand link '${linkId}': add it to the query's "expand" node, or show a property of the rows.`,
+    }
+  }
+
+  const property = link.properties[propertyId]
+  return property
+    ? { property }
+    : { problem: `link '${linkId}' reaches no property '${propertyId}'${known(link.properties)}.` }
+}
+
+function known(properties: Readonly<Record<string, unknown>>): string {
+  const ids = Object.keys(properties)
+  return ids.length > 0 ? `; it has ${ids.join(", ")}` : ""
+}
+
+/** A column's format: the one the workbook gives, checked against its property, or its property's. */
+function layColumn(
+  columnAt: string,
+  from: string,
+  written: SheetColumnFormat | undefined,
+  untyped: Readonly<Record<string, unknown>> | undefined,
+  inferred: SheetColumnType | undefined
+): { readonly format: SheetColumnFormat; readonly problem?: string } {
+  if (written) {
+    if (inferred === undefined || fits(written.type, inferred)) return { format: written }
+    return {
+      format: written,
+      problem: `${WORKBOOK_FILE} ${columnAt} is ${written.type}, but '${from}' holds ${HOLDS[inferred]}: leave out "type" to show it as its property is, or show another property.`,
+    }
+  }
+
+  const type = inferred ?? "text"
+  const read = readSheetColumnFormat(untyped ?? {}, columnAt, type)
+  return "format" in read ? read : { format: { type }, problem: read.error }
+}
+
+const HOLDS: Readonly<Record<SheetColumnType, string>> = {
+  text: "text",
+  tag: "text",
+  number: "numbers",
+  currency: "numbers",
+  percent: "numbers",
+  date: "dates",
+  boolean: "true or false",
+  link: "text",
+  image: "text",
+}
+
+/** Whether a column of `type` reads every value of a property whose own type is `inferred`. */
+function fits(type: SheetColumnType, inferred: SheetColumnType): boolean {
+  if (NUMERIC_COLUMN_TYPES.includes(type)) return inferred === "number"
+  if (type === "date" || type === "boolean") return inferred === type
+  return true
+}
+
+/**
+ * A query's rows as a live sheet shows them: each column's value written as a CSV cell holds it,
+ * so the grid, the charts and the downloads read them as they read a CSV sheet's. A value the
+ * reader is not cleared for is a locked cell, as is every cell of a column the reader's rows do
+ * not hold.
+ */
+export function liveSheetTable(
+  layout: LiveSheetLayout,
+  rows: readonly LiveSheetRow[]
+): LiveSheetTable {
+  const { columns } = layout
+  const locked = new Map<number, Set<number>>()
+  const lock = (row: number, column: number) => {
+    const cells = locked.get(row) ?? new Set<number>()
+    cells.add(column)
+    locked.set(row, cells)
+  }
+
+  const tableRows = rows.map((row, rowIndex) =>
+    columns.map((column, columnIndex) => {
+      if (!column.from) return ""
+      if (column.missing) {
+        lock(rowIndex, columnIndex)
+        return ""
+      }
+
+      const cell = readCell(row, column.from)
+      if (cell.locked) lock(rowIndex, columnIndex)
+      return cell.locked ? "" : cell.text
+    })
+  )
+
+  lockComputed(layout, rows.length, locked)
+  return { table: { header: columns.map((column) => column.name), rows: tableRows }, locked }
+}
+
+function readCell(
+  row: LiveSheetRow,
+  from: NonNullable<LiveSheetColumn["from"]>
+): { readonly text: string; readonly locked: boolean } {
+  const { linkId, propertyId } = from
+  if (linkId === undefined) return propertyCell(row, propertyId)
+
+  const linked = row.links?.[linkId]
+  if (!linked) return { text: "", locked: false }
+  if (!isRowList(linked)) return propertyCell(linked, propertyId)
+
+  // A link that reaches several objects shows each one's value; one the reader may not see locks it.
+  const values = linked.map((each) => propertyCell(each, propertyId))
+  return {
+    text: values
+      .map((value) => value.text)
+      .filter(Boolean)
+      .join(", "),
+    locked: values.some((value) => value.locked),
+  }
+}
+
+function isRowList(
+  value: LiveSheetRow | readonly LiveSheetRow[]
+): value is readonly LiveSheetRow[] {
+  return Array.isArray(value)
+}
+
+function propertyCell(row: LiveSheetRow, propertyId: string) {
+  if (row.redactions && Object.hasOwn(row.redactions, propertyId)) {
+    return { text: "", locked: true }
+  }
+  return { text: cellText(row.properties[propertyId]), locked: false }
+}
+
+/** A value as a CSV cell holds it, which `parseSheetNumber` and `parseSheetDate` read back. */
+function cellText(value: unknown): string {
+  if (value === null || value === undefined) return ""
+  if (typeof value === "string") return value
+  if (typeof value === "number" || typeof value === "bigint") return String(value)
+  if (typeof value === "boolean") return value ? "true" : "false"
+  if (value instanceof Date) return value.toISOString()
+  if (Array.isArray(value)) return value.map(cellText).filter(Boolean).join(", ")
+  return JSON.stringify(value)
+}
+
+/**
+ * Locks each computed cell that reads a locked one, through as many computed columns as read each
+ * other: a formula over a value the reader may not see would show what it computes without it. A
+ * formula that reads a column whole, as `SUM([Amount])` does, reads every row's value: one locked
+ * cell in that column locks it on every row.
+ */
+function lockComputed(
+  layout: LiveSheetLayout,
+  rowCount: number,
+  locked: Map<number, Set<number>>
+): void {
+  const indexOf = new Map(layout.columns.map((column, index) => [column.name, index]))
+  const indexes = (names: ReadonlySet<string>) =>
+    [...names].map((name) => indexOf.get(name)).filter((read) => read !== undefined)
+
+  const computed = layout.columns.flatMap((column, index) => {
+    const formula = column.from ? undefined : layout.sheet.columns[column.name]?.formula
+    if (!formula) return []
+
+    const { row, whole } = formulaReferences(parseFormula(formula))
+    return [{ index, row: indexes(row), whole: indexes(whole) }]
+  })
+  if (computed.length === 0) return
+
+  for (let changed = true; changed; ) {
+    changed = false
+
+    const anywhere = new Set<number>()
+    for (const cells of locked.values()) for (const column of cells) anywhere.add(column)
+
+    for (const { index, whole } of computed) {
+      if (!whole.some((read) => anywhere.has(read))) continue
+
+      for (let row = 0; row < rowCount; row += 1) {
+        const cells = locked.get(row) ?? new Set<number>()
+        if (cells.has(index)) continue
+        cells.add(index)
+        locked.set(row, cells)
+        changed = true
+      }
+    }
+
+    for (const cells of locked.values()) {
+      for (const { index, row } of computed) {
+        if (cells.has(index) || !row.some((read) => cells.has(read))) continue
+        cells.add(index)
+        changed = true
+      }
+    }
+  }
+}

package/src/artifacts/outline.ts

@@ -0,0 +1,95 @@
+/**
+ * What a version contains, read from its files when it is published. Addresses resolve against
+ * it, and previews and hover cards show it without opening the files.
+ */
+export type ArtifactOutline =
+  | DocumentOutline
+  | SheetOutline
+  | SlidesOutline
+  | PageOutline
+  | CanvasOutline
+  | FileOutline
+
+export interface DocumentOutline {
+  readonly kind: "document"
+  readonly headings: readonly DocumentHeading[]
+}
+
+export interface DocumentHeading {
+  readonly depth: number
+  readonly text: string
+  /** What `#part` names in an address to this section. */
+  readonly slug: string
+}
+
+export interface SheetOutline {
+  readonly kind: "sheet"
+  readonly sheets: readonly SheetSummary[]
+  /** The names of the charts declared in the workbook. */
+  readonly charts: readonly string[]
+}
+
+export interface SheetSummary {
+  readonly name: string
+  /** The rows of its CSV; none for a live sheet, whose rows are each reader's. */
+  readonly rows: number
+  readonly columns: readonly string[]
+  /** A live sheet's query, whose rows it shows. */
+  readonly query?: string
+}
+
+export interface SlidesOutline {
+  readonly kind: "slides"
+  /** One entry per slide, in order; the title is the slide's first heading. */
+  readonly slides: readonly { readonly title?: string }[]
+}
+
+export interface PageOutline {
+  readonly kind: "page"
+  /** The page's `<title>`, else its first heading. */
+  readonly title?: string
+}
+
+export interface CanvasOutline {
+  readonly kind: "canvas"
+  /** The boards and sections an address can name, by id, in the canvas's order. */
+  readonly items: readonly CanvasOutlineItem[]
+}
+
+export type CanvasOutlineItem =
+  | {
+      readonly id: string
+      readonly type: "board"
+      readonly title?: string
+      /** The HTML file it shows, and its size: what its frame serves, and the shape it embeds at. */
+      readonly file: string
+      readonly w: number
+      readonly h: number
+    }
+  | { readonly id: string; readonly type: "section"; readonly title?: string }
+
+export interface FileOutline {
+  readonly kind: "file"
+  readonly mediaType: string
+}
+
+/**
+ * Turns a document's heading texts, in order, into their section slugs: lowercase, punctuation
+ * dropped, spaces as hyphens, and `-1`, `-2` after a repeat. Publishing and the renderer both name
+ * sections with it, so an address to a section finds the heading the reader sees.
+ */
+export function createHeadingSlugger(): (text: string) => string {
+  const used = new Set<string>()
+  return (text) => {
+    const base =
+      text
+        .trim()
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s_-]/gu, "")
+        .replace(/\s+/g, "-") || "section"
+    let slug = base
+    for (let repeat = 1; used.has(slug); repeat += 1) slug = `${base}-${repeat}`
+    used.add(slug)
+    return slug
+  }
+}

package/src/artifacts/publish/files.ts

@@ -0,0 +1,87 @@
+import { statFileRef } from "../../blob-storage/file-content"
+import type { BlobStorage } from "../../blob-storage/types"
+import { isValidFileRef } from "../../blob-storage/validation"
+import type { ArtifactFile } from "../../storage/artifacts/types"
+import { ArtifactRequestError } from "../errors"
+import { ARTIFACT_LIMITS } from "../limits"
+
+export type ArtifactBlobStorage = Pick<BlobStorage, "stat" | "open">
+
+/**
+ * Checks a version's files before anything reads them: safe unique paths, the per-version limits,
+ * and stored bytes that match each reference.
+ */
+export async function checkArtifactFiles(
+  blobStorage: ArtifactBlobStorage,
+  files: readonly ArtifactFile[]
+): Promise<void> {
+  if (files.length === 0) throw invalidArtifact("A version needs at least one file.")
+  if (files.length > ARTIFACT_LIMITS.maxFiles) {
+    throw invalidArtifact(
+      `A version holds at most ${ARTIFACT_LIMITS.maxFiles} files; this one has ${files.length}.`
+    )
+  }
+
+  const paths = new Set<string>()
+  let totalBytes = 0
+  for (const entry of files) {
+    if (!isArtifactPath(entry.path)) {
+      throw invalidArtifact(
+        `'${entry.path}' is not a usable file path: use a relative path with / separators, no . or .. segments, and at most ${ARTIFACT_LIMITS.maxPathLength} characters.`
+      )
+    }
+    if (paths.has(entry.path)) throw invalidArtifact(`'${entry.path}' appears twice.`)
+    paths.add(entry.path)
+    if (!isValidFileRef(entry.file)) {
+      throw invalidArtifact(`'${entry.path}' does not have a valid file reference.`)
+    }
+    if (entry.file.sizeBytes > ARTIFACT_LIMITS.maxFileBytes) {
+      throw invalidArtifact(
+        `'${entry.path}' is ${entry.file.sizeBytes} bytes; a file may be at most ${ARTIFACT_LIMITS.maxFileBytes}.`
+      )
+    }
+    totalBytes += entry.file.sizeBytes
+  }
+  if (totalBytes > ARTIFACT_LIMITS.maxVersionBytes) {
+    throw invalidArtifact(
+      `The files total ${totalBytes} bytes; a version may hold at most ${ARTIFACT_LIMITS.maxVersionBytes}.`
+    )
+  }
+
+  const stored = await Promise.all(files.map((entry) => statFileRef(blobStorage, entry.file)))
+  const missing = files.find((_, index) => !stored[index])
+  if (missing) throw invalidArtifact(`'${missing.path}' refers to a file that is not stored.`)
+}
+
+/** The file at `path`, read as UTF-8 text when it is no larger than `maxBytes`. */
+export async function readArtifactText(
+  blobStorage: ArtifactBlobStorage,
+  file: ArtifactFile,
+  maxBytes: number
+): Promise<string> {
+  if (file.file.sizeBytes > maxBytes) {
+    throw invalidArtifact(
+      `'${file.path}' is ${file.file.sizeBytes} bytes; it may be at most ${maxBytes}.`
+    )
+  }
+  const text = await new Response(await blobStorage.open(file.file.blobId)).text()
+  // A byte-order mark is not part of the text.
+  return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text
+}
+
+/** Relative, `/`-separated, no empty, `.`, or `..` segment, no backslash or control character. */
+export function isArtifactPath(path: string): boolean {
+  return (
+    typeof path === "string" &&
+    path.length > 0 &&
+    path.length <= ARTIFACT_LIMITS.maxPathLength &&
+    !path.startsWith("/") &&
+    !path.includes("\\") &&
+    !/[\u0000-\u001f\u007f]/.test(path) &&
+    path.split("/").every((segment) => segment.length > 0 && segment !== "." && segment !== "..")
+  )
+}
+
+export function invalidArtifact(message: string): ArtifactRequestError {
+  return new ArtifactRequestError("invalid_artifact", `[Sixb] ${message}`)
+}

package/src/artifacts/publish/index.ts

@@ -0,0 +1,14 @@
+/** Publishing for this repo's packages: the Agent publishes through the same checks as the API. */
+
+export { artifactAccessRule, artifactQueryChecker } from "../execution"
+export { isArtifactId } from "../ids"
+export { ARTIFACT_LIMITS } from "../limits"
+export { artifactQueryName } from "../queries"
+export {
+  artifactVersionWarnings,
+  type PreparedArtifactVersion,
+  type PublishArtifactInput,
+  prepareArtifactVersion,
+  publishArtifact,
+  storeArtifactVersion,
+} from "./publish"