@gmod/gbz-base 0.0.1 → 1.1.0

package/src/sqlite/btree.ts

@@ -0,0 +1,260 @@
+import { decodeRecord, readVarint } from './record.ts'
+
+import type { Pager } from './pager.ts'
+import type { SqlValue } from './record.ts'
+
+const INTERIOR_INDEX = 0x02
+const INTERIOR_TABLE = 0x05
+const LEAF_INDEX = 0x0a
+const LEAF_TABLE = 0x0d
+
+interface PageHeader {
+  type: number
+  cellCount: number
+  rightChild: number
+  cellPointers: number
+}
+
+function readHeader(page: Uint8Array, start: number): PageHeader {
+  const view = new DataView(page.buffer, page.byteOffset, page.byteLength)
+  const type = page[start]
+  if (
+    type !== INTERIOR_INDEX &&
+    type !== INTERIOR_TABLE &&
+    type !== LEAF_INDEX &&
+    type !== LEAF_TABLE
+  ) {
+    throw new Error(`SQLite page has unknown b-tree type ${type}`)
+  }
+  const interior = type === INTERIOR_INDEX || type === INTERIOR_TABLE
+  return {
+    type,
+    cellCount: view.getUint16(start + 3),
+    rightChild: interior ? view.getUint32(start + 8) : 0,
+    cellPointers: start + (interior ? 12 : 8),
+  }
+}
+
+function cellOffset(page: Uint8Array, header: PageHeader, index: number) {
+  const view = new DataView(page.buffer, page.byteOffset, page.byteLength)
+  return view.getUint16(header.cellPointers + 2 * index)
+}
+
+function readUint32(page: Uint8Array, offset: number) {
+  const view = new DataView(page.buffer, page.byteOffset, page.byteLength)
+  return view.getUint32(offset)
+}
+
+export class BTree {
+  private readonly usable: number
+  private pager: Pager
+
+  constructor(pager: Pager, reservedBytes: number) {
+    this.pager = pager
+    this.usable = pager.pageSize - reservedBytes
+  }
+
+  private localPayloadSize(total: number, isIndex: boolean) {
+    const usable = this.usable
+    const maxLocal = isIndex
+      ? Math.floor(((usable - 12) * 64) / 255) - 23
+      : usable - 35
+    if (total <= maxLocal) {
+      return total
+    }
+    const minLocal = Math.floor(((usable - 12) * 32) / 255) - 23
+    const spill = minLocal + ((total - minLocal) % (usable - 4))
+    return spill <= maxLocal ? spill : minLocal
+  }
+
+  private async payload(
+    page: Uint8Array,
+    offset: number,
+    total: number,
+    isIndex: boolean,
+  ) {
+    const local = this.localPayloadSize(total, isIndex)
+    if (local === total) {
+      return page.subarray(offset, offset + total)
+    }
+    const result = new Uint8Array(total)
+    result.set(page.subarray(offset, offset + local), 0)
+    let filled = local
+    let next = readUint32(page, offset + local)
+    while (next !== 0 && filled < total) {
+      const overflow = await this.pager.page(next)
+      const chunk = overflow.subarray(
+        4,
+        4 + Math.min(this.usable - 4, total - filled),
+      )
+      result.set(chunk, filled)
+      filled += chunk.length
+      next = readUint32(overflow, 0)
+    }
+    if (filled !== total) {
+      throw new Error(
+        'SQLite overflow chain ended before the payload was complete',
+      )
+    }
+    return result
+  }
+
+  private async pageAt(pageNumber: number) {
+    const page = await this.pager.page(pageNumber)
+    return { page, header: readHeader(page, pageNumber === 1 ? 100 : 0) }
+  }
+
+  async tableRowid(
+    root: number,
+    rowid: number,
+  ): Promise<SqlValue[] | undefined> {
+    let pageNumber = root
+    for (;;) {
+      const { page, header } = await this.pageAt(pageNumber)
+      if (header.type === LEAF_TABLE) {
+        let low = 0
+        let high = header.cellCount
+        while (low < high) {
+          const mid = (low + high) >> 1
+          const offset = cellOffset(page, header, mid)
+          const [size, afterSize] = readVarint(page, offset)
+          const [key, afterKey] = readVarint(page, afterSize)
+          if (key === rowid) {
+            return decodeRecord(await this.payload(page, afterKey, size, false))
+          }
+          if (key < rowid) {
+            low = mid + 1
+          } else {
+            high = mid
+          }
+        }
+        return undefined
+      }
+      if (header.type !== INTERIOR_TABLE) {
+        throw new Error('SQLite table b-tree contains an index page')
+      }
+      let low = 0
+      let high = header.cellCount
+      while (low < high) {
+        const mid = (low + high) >> 1
+        const offset = cellOffset(page, header, mid)
+        const [key] = readVarint(page, offset + 4)
+        if (key < rowid) {
+          low = mid + 1
+        } else {
+          high = mid
+        }
+      }
+      pageNumber =
+        low === header.cellCount
+          ? header.rightChild
+          : readUint32(page, cellOffset(page, header, low))
+    }
+  }
+
+  async *tableScan(
+    root: number,
+  ): AsyncGenerator<{ rowid: number; values: SqlValue[] }> {
+    const { page, header } = await this.pageAt(root)
+    if (header.type === LEAF_TABLE) {
+      for (let i = 0; i < header.cellCount; i++) {
+        const offset = cellOffset(page, header, i)
+        const [size, afterSize] = readVarint(page, offset)
+        const [rowid, afterKey] = readVarint(page, afterSize)
+        yield {
+          rowid,
+          values: decodeRecord(await this.payload(page, afterKey, size, false)),
+        }
+      }
+    } else {
+      const children: number[] = []
+      for (let i = 0; i < header.cellCount; i++) {
+        children.push(readUint32(page, cellOffset(page, header, i)))
+      }
+      children.push(header.rightChild)
+      this.pager.prefetch(children)
+      for (const child of children) {
+        yield* this.tableScan(child)
+      }
+    }
+  }
+
+  private async indexCell(page: Uint8Array, header: PageHeader, index: number) {
+    const offset = cellOffset(page, header, index)
+    const interior = header.type === INTERIOR_INDEX
+    const [size, afterSize] = readVarint(page, interior ? offset + 4 : offset)
+    const values = decodeRecord(await this.payload(page, afterSize, size, true))
+    return { values, leftChild: interior ? readUint32(page, offset) : 0 }
+  }
+
+  async *indexScanFrom(
+    root: number,
+    low: number[],
+  ): AsyncGenerator<SqlValue[]> {
+    const { page, header } = await this.pageAt(root)
+    let first = 0
+    let high = header.cellCount
+    while (first < high) {
+      const mid = (first + high) >> 1
+      const cell = await this.indexCell(page, header, mid)
+      if (compareKey(cell.values, low) < 0) {
+        first = mid + 1
+      } else {
+        high = mid
+      }
+    }
+    for (let i = first; i < header.cellCount; i++) {
+      const cell = await this.indexCell(page, header, i)
+      if (header.type === INTERIOR_INDEX) {
+        yield* this.indexScanFrom(cell.leftChild, low)
+      }
+      yield cell.values
+    }
+    if (header.type === INTERIOR_INDEX) {
+      yield* this.indexScanFrom(header.rightChild, low)
+    }
+  }
+
+  async indexSeekLE(
+    root: number,
+    key: number[],
+  ): Promise<SqlValue[] | undefined> {
+    let best: SqlValue[] | undefined
+    let pageNumber = root
+    for (;;) {
+      const { page, header } = await this.pageAt(pageNumber)
+      let low = 0
+      let high = header.cellCount
+      let child = 0
+      while (low < high) {
+        const mid = (low + high) >> 1
+        const cell = await this.indexCell(page, header, mid)
+        if (compareKey(cell.values, key) <= 0) {
+          best = cell.values
+          low = mid + 1
+        } else {
+          child = cell.leftChild
+          high = mid
+        }
+      }
+      if (header.type === LEAF_INDEX) {
+        return best
+      }
+      pageNumber = low === header.cellCount ? header.rightChild : child
+    }
+  }
+}
+
+function compareKey(values: SqlValue[], key: number[]) {
+  for (let i = 0; i < key.length; i++) {
+    const a = values[i]
+    const b = key[i]
+    if (typeof a !== 'number' || b === undefined) {
+      throw new Error('SQLite index key is not numeric')
+    }
+    if (a !== b) {
+      return a < b ? -1 : 1
+    }
+  }
+  return 0
+}

package/src/sqlite/database.ts

@@ -0,0 +1,117 @@
+import { BTree } from './btree.ts'
+import { Pager } from './pager.ts'
+
+import type { ByteSource } from '../filehandle.ts'
+import type { PagerOptions } from './pager.ts'
+import type { SqlValue } from './record.ts'
+
+export interface SqliteObject {
+  type: string
+  name: string
+  tableName: string
+  rootPage: number
+  sql: string
+}
+
+export class SqliteDatabase {
+  readonly pager: Pager
+  readonly btree: BTree
+  readonly objects: Map<string, SqliteObject>
+
+  private constructor(
+    pager: Pager,
+    btree: BTree,
+    objects: Map<string, SqliteObject>,
+  ) {
+    this.pager = pager
+    this.btree = btree
+    this.objects = objects
+  }
+
+  static async open(source: ByteSource, opts: PagerOptions = {}) {
+    const { size } = await source.stat()
+    const firstBlock = await source.read(
+      Math.min(opts.blockSize ?? 65536, size),
+      0,
+    )
+    const header = firstBlock.subarray(0, 100)
+    const magic = new TextDecoder().decode(header.subarray(0, 15))
+    if (magic !== 'SQLite format 3') {
+      throw new Error('Not a SQLite database')
+    }
+    const view = new DataView(
+      header.buffer,
+      header.byteOffset,
+      header.byteLength,
+    )
+    const rawPageSize = view.getUint16(16)
+    const pageSize = rawPageSize === 1 ? 65536 : rawPageSize
+    const reserved = header[20] ?? 0
+    const encoding = view.getUint32(56)
+    if (encoding !== 1) {
+      throw new Error(`SQLite text encoding ${encoding} is not UTF-8`)
+    }
+    const pager = new Pager(source, pageSize, size, opts)
+    pager.seed(0, firstBlock)
+    pager.fetches += 1
+    pager.bytesFetched += firstBlock.length
+    const btree = new BTree(pager, reserved)
+    const objects = new Map<string, SqliteObject>()
+    for await (const { values } of btree.tableScan(1)) {
+      const [type, name, tableName, rootPage, sql] = values
+      if (
+        typeof type === 'string' &&
+        typeof name === 'string' &&
+        typeof tableName === 'string' &&
+        typeof rootPage === 'number'
+      ) {
+        objects.set(name, {
+          type,
+          name,
+          tableName,
+          rootPage,
+          sql: typeof sql === 'string' ? sql : '',
+        })
+      }
+    }
+    return new SqliteDatabase(pager, btree, objects)
+  }
+
+  rootPage(name: string) {
+    const object = this.objects.get(name)
+    if (!object) {
+      throw new Error(`SQLite database has no object named ${name}`)
+    }
+    return object.rootPage
+  }
+
+  indexOn(tableName: string) {
+    const index = [...this.objects.values()].find(
+      o => o.type === 'index' && o.tableName === tableName,
+    )
+    if (!index) {
+      throw new Error(`SQLite table ${tableName} has no index`)
+    }
+    return index.rootPage
+  }
+
+  byRowid(table: string, rowid: number) {
+    return this.btree.tableRowid(this.rootPage(table), rowid)
+  }
+
+  scan(table: string) {
+    return this.btree.tableScan(this.rootPage(table))
+  }
+
+  indexSeekLE(table: string, key: number[]): Promise<SqlValue[] | undefined> {
+    return this.btree.indexSeekLE(this.indexOn(table), key)
+  }
+
+  indexScanFrom(table: string, low: number[]) {
+    return this.btree.indexScanFrom(this.indexOn(table), low)
+  }
+
+  has(name: string) {
+    return this.objects.has(name)
+  }
+}

package/src/sqlite/pager.ts

@@ -0,0 +1,148 @@
+import type { ByteSource } from '../filehandle.ts'
+
+export interface PagerOptions {
+  blockSize?: number
+  maxBlocks?: number
+}
+
+export class Pager {
+  private blocks = new Map<number, Promise<Uint8Array>>()
+  private readonly blockSize: number
+  private readonly maxBlocks: number
+  bytesFetched = 0
+  fetches = 0
+
+  private source: ByteSource
+  readonly pageSize: number
+  private fileSize: number
+
+  constructor(
+    source: ByteSource,
+    pageSize: number,
+    fileSize: number,
+    opts: PagerOptions = {},
+  ) {
+    this.source = source
+    this.pageSize = pageSize
+    this.fileSize = fileSize
+    const requested = opts.blockSize ?? 65536
+    this.blockSize = Math.max(
+      pageSize,
+      Math.ceil(requested / pageSize) * pageSize,
+    )
+    this.maxBlocks = opts.maxBlocks ?? 256
+  }
+
+  private async read(length: number, start: number) {
+    let attempt = 0
+    for (;;) {
+      try {
+        const bytes = await this.source.read(length, start)
+        this.bytesFetched += bytes.length
+        return bytes
+      } catch (error) {
+        attempt += 1
+        if (attempt >= 3) {
+          throw error
+        }
+        await new Promise(resolve => setTimeout(resolve, 200 * attempt))
+      }
+    }
+  }
+
+  seed(index: number, bytes: Uint8Array) {
+    if (
+      bytes.length ===
+      Math.min(this.blockSize, this.fileSize - index * this.blockSize)
+    ) {
+      this.blocks.set(index, Promise.resolve(bytes))
+    }
+  }
+
+  private block(index: number) {
+    const cached = this.blocks.get(index)
+    if (cached) {
+      this.blocks.delete(index)
+      this.blocks.set(index, cached)
+      return cached
+    }
+    const start = index * this.blockSize
+    const length = Math.min(this.blockSize, this.fileSize - start)
+    const pending = this.read(length, start)
+    this.fetches += 1
+    this.blocks.set(index, pending)
+    this.forgetOnFailure(pending, [index])
+    this.evict()
+    return pending
+  }
+
+  prefetch(pageNumbers: number[]) {
+    const wanted = [
+      ...new Set(
+        pageNumbers.map(pageNumber =>
+          Math.floor(((pageNumber - 1) * this.pageSize) / this.blockSize),
+        ),
+      ),
+    ]
+      .filter(index => !this.blocks.has(index))
+      .sort((a, b) => a - b)
+    let runStart = 0
+    while (runStart < wanted.length) {
+      let runEnd = runStart + 1
+      while (
+        runEnd < wanted.length &&
+        wanted[runEnd] === wanted[runEnd - 1]! + 1
+      ) {
+        runEnd += 1
+      }
+      this.fetchRun(wanted[runStart]!, runEnd - runStart)
+      runStart = runEnd
+    }
+  }
+
+  private fetchRun(firstIndex: number, count: number) {
+    const start = firstIndex * this.blockSize
+    const length = Math.min(count * this.blockSize, this.fileSize - start)
+    const pending = this.read(length, start)
+    this.fetches += 1
+    const indexes: number[] = []
+    for (let i = 0; i < count; i++) {
+      const within = i * this.blockSize
+      indexes.push(firstIndex + i)
+      this.blocks.set(
+        firstIndex + i,
+        pending.then(bytes => bytes.subarray(within, within + this.blockSize)),
+      )
+    }
+    this.forgetOnFailure(pending, indexes)
+    this.evict()
+  }
+
+  private forgetOnFailure(pending: Promise<Uint8Array>, indexes: number[]) {
+    pending.catch(() => {
+      for (const index of indexes) {
+        this.blocks.delete(index)
+      }
+    })
+  }
+
+  private evict() {
+    while (this.blocks.size > this.maxBlocks) {
+      const oldest = this.blocks.keys().next().value
+      if (oldest === undefined) {
+        break
+      }
+      this.blocks.delete(oldest)
+    }
+  }
+
+  async page(pageNumber: number) {
+    const offset = (pageNumber - 1) * this.pageSize
+    if (pageNumber < 1 || offset + this.pageSize > this.fileSize) {
+      throw new Error(`SQLite page ${pageNumber} is outside the file`)
+    }
+    const bytes = await this.block(Math.floor(offset / this.blockSize))
+    const within = offset % this.blockSize
+    return bytes.subarray(within, within + this.pageSize)
+  }
+}

package/src/sqlite/record.ts

@@ -0,0 +1,100 @@
+export type SqlValue = null | number | string | Uint8Array
+
+export function readVarint(
+  bytes: Uint8Array,
+  offset: number,
+): [value: number, next: number] {
+  let value = 0
+  for (let i = 0; i < 8; i++) {
+    const byte = bytes[offset + i]
+    if (byte === undefined) {
+      throw new Error('SQLite varint runs past the end of the buffer')
+    }
+    value = value * 128 + (byte & 0x7f)
+    if ((byte & 0x80) === 0) {
+      return [value, offset + i + 1]
+    }
+  }
+  const last = bytes[offset + 8]
+  if (last === undefined) {
+    throw new Error('SQLite varint runs past the end of the buffer')
+  }
+  value = value * 256 + last
+  if (!Number.isSafeInteger(value)) {
+    throw new Error('SQLite varint exceeds the safe integer range')
+  }
+  return [value, offset + 9]
+}
+
+function readSignedInt(view: DataView, offset: number, width: number) {
+  switch (width) {
+    case 1:
+      return view.getInt8(offset)
+    case 2:
+      return view.getInt16(offset)
+    case 3:
+      return (view.getInt8(offset) << 16) | view.getUint16(offset + 1)
+    case 4:
+      return view.getInt32(offset)
+    case 6:
+      return view.getInt16(offset) * 4294967296 + view.getUint32(offset + 2)
+    default: {
+      const big = view.getBigInt64(offset)
+      if (
+        big > BigInt(Number.MAX_SAFE_INTEGER) ||
+        big < BigInt(Number.MIN_SAFE_INTEGER)
+      ) {
+        throw new Error(`SQLite integer ${big} exceeds the safe integer range`)
+      }
+      return Number(big)
+    }
+  }
+}
+
+const decoder = new TextDecoder()
+
+export function decodeRecord(payload: Uint8Array): SqlValue[] {
+  const [headerSize, firstType] = readVarint(payload, 0)
+  const types: number[] = []
+  let offset = firstType
+  while (offset < headerSize) {
+    const [type, next] = readVarint(payload, offset)
+    types.push(type)
+    offset = next
+  }
+  const view = new DataView(
+    payload.buffer,
+    payload.byteOffset,
+    payload.byteLength,
+  )
+  let body = headerSize
+  return types.map(type => {
+    if (type === 0) {
+      return null
+    }
+    if (type >= 1 && type <= 6) {
+      const width = type <= 4 ? type : type === 5 ? 6 : 8
+      const value = readSignedInt(view, body, width)
+      body += width
+      return value
+    }
+    if (type === 7) {
+      const value = view.getFloat64(body)
+      body += 8
+      return value
+    }
+    if (type === 8) {
+      return 0
+    }
+    if (type === 9) {
+      return 1
+    }
+    if (type >= 12) {
+      const length = (type - 12) >> 1
+      const slice = payload.subarray(body, body + length)
+      body += length
+      return type % 2 === 0 ? slice : decoder.decode(slice)
+    }
+    throw new Error(`SQLite serial type ${type} is reserved`)
+  })
+}