@gmod/gbz-base 1.0.0 → 2.0.0

package/src/sqlite/btree.ts

@@ -15,8 +15,20 @@ interface PageHeader {
   cellPointers: number
 }
 
+function readUint16(page: Uint8Array, offset: number) {
+  return page[offset]! * 256 + page[offset + 1]!
+}
+
+function readUint32(page: Uint8Array, offset: number) {
+  return (
+    page[offset]! * 16777216 +
+    page[offset + 1]! * 65536 +
+    page[offset + 2]! * 256 +
+    page[offset + 3]!
+  )
+}
+
 function readHeader(page: Uint8Array, start: number): PageHeader {
-  const view = new DataView(page.buffer, page.byteOffset, page.byteLength)
   const type = page[start]
   if (
     type !== INTERIOR_INDEX &&
@@ -29,25 +41,27 @@ function readHeader(page: Uint8Array, start: number): PageHeader {
   const interior = type === INTERIOR_INDEX || type === INTERIOR_TABLE
   return {
     type,
-    cellCount: view.getUint16(start + 3),
-    rightChild: interior ? view.getUint32(start + 8) : 0,
+    cellCount: readUint16(page, start + 3),
+    rightChild: interior ? readUint32(page, start + 8) : 0,
     cellPointers: start + (interior ? 12 : 8),
   }
 }
 
 function cellOffset(page: Uint8Array, header: PageHeader, index: number) {
-  const view = new DataView(page.buffer, page.byteOffset, page.byteLength)
-  return view.getUint16(header.cellPointers + 2 * index)
+  return readUint16(page, header.cellPointers + 2 * index)
 }
 
-function readUint32(page: Uint8Array, offset: number) {
-  const view = new DataView(page.buffer, page.byteOffset, page.byteLength)
-  return view.getUint32(offset)
+interface IndexCell {
+  values: SqlValue[]
+  leftChild: number
 }
 
+const MAX_DECODED_INDEX_PAGES = 4096
+
 export class BTree {
   private readonly usable: number
   private pager: Pager
+  private decodedIndexPages = new Map<number, (IndexCell | undefined)[]>()
 
   constructor(pager: Pager, reservedBytes: number) {
     this.pager = pager
@@ -167,19 +181,54 @@ export class BTree {
         }
       }
     } else {
+      const children: number[] = []
       for (let i = 0; i < header.cellCount; i++) {
-        yield* this.tableScan(readUint32(page, cellOffset(page, header, i)))
+        children.push(readUint32(page, cellOffset(page, header, i)))
+      }
+      children.push(header.rightChild)
+      this.pager.prefetch(children)
+      for (const child of children) {
+        yield* this.tableScan(child)
       }
-      yield* this.tableScan(header.rightChild)
     }
   }
 
-  private async indexCell(page: Uint8Array, header: PageHeader, index: number) {
-    const offset = cellOffset(page, header, index)
-    const interior = header.type === INTERIOR_INDEX
-    const [size, afterSize] = readVarint(page, interior ? offset + 4 : offset)
-    const values = decodeRecord(await this.payload(page, afterSize, size, true))
-    return { values, leftChild: interior ? readUint32(page, offset) : 0 }
+  private decodedCells(pageNumber: number, header: PageHeader) {
+    let cells = this.decodedIndexPages.get(pageNumber)
+    if (cells) {
+      this.decodedIndexPages.delete(pageNumber)
+    } else {
+      cells = new Array<IndexCell | undefined>(header.cellCount)
+      if (this.decodedIndexPages.size >= MAX_DECODED_INDEX_PAGES) {
+        const oldest = this.decodedIndexPages.keys().next().value
+        if (oldest !== undefined) {
+          this.decodedIndexPages.delete(oldest)
+        }
+      }
+    }
+    this.decodedIndexPages.set(pageNumber, cells)
+    return cells
+  }
+
+  private async indexCell(
+    pageNumber: number,
+    page: Uint8Array,
+    header: PageHeader,
+    index: number,
+  ): Promise<IndexCell> {
+    const cells = this.decodedCells(pageNumber, header)
+    let cell = cells[index]
+    if (!cell) {
+      const offset = cellOffset(page, header, index)
+      const interior = header.type === INTERIOR_INDEX
+      const [size, afterSize] = readVarint(page, interior ? offset + 4 : offset)
+      cell = {
+        values: decodeRecord(await this.payload(page, afterSize, size, true)),
+        leftChild: interior ? readUint32(page, offset) : 0,
+      }
+      cells[index] = cell
+    }
+    return cell
   }
 
   async *indexScanFrom(
@@ -191,7 +240,7 @@ export class BTree {
     let high = header.cellCount
     while (first < high) {
       const mid = (first + high) >> 1
-      const cell = await this.indexCell(page, header, mid)
+      const cell = await this.indexCell(root, page, header, mid)
       if (compareKey(cell.values, low) < 0) {
         first = mid + 1
       } else {
@@ -199,7 +248,7 @@ export class BTree {
       }
     }
     for (let i = first; i < header.cellCount; i++) {
-      const cell = await this.indexCell(page, header, i)
+      const cell = await this.indexCell(root, page, header, i)
       if (header.type === INTERIOR_INDEX) {
         yield* this.indexScanFrom(cell.leftChild, low)
       }
@@ -223,7 +272,7 @@ export class BTree {
       let child = 0
       while (low < high) {
         const mid = (low + high) >> 1
-        const cell = await this.indexCell(page, header, mid)
+        const cell = await this.indexCell(pageNumber, page, header, mid)
         if (compareKey(cell.values, key) <= 0) {
           best = cell.values
           low = mid + 1

package/src/sqlite/pager.ts

@@ -33,6 +33,23 @@ export class Pager {
     this.maxBlocks = opts.maxBlocks ?? 256
   }
 
+  private async read(length: number, start: number) {
+    let attempt = 0
+    for (;;) {
+      try {
+        const bytes = await this.source.read(length, start)
+        this.bytesFetched += bytes.length
+        return bytes
+      } catch (error) {
+        attempt += 1
+        if (attempt >= 3) {
+          throw error
+        }
+        await new Promise(resolve => setTimeout(resolve, 200 * attempt))
+      }
+    }
+  }
+
   seed(index: number, bytes: Uint8Array) {
     if (
       bytes.length ===
@@ -51,19 +68,72 @@ export class Pager {
     }
     const start = index * this.blockSize
     const length = Math.min(this.blockSize, this.fileSize - start)
-    const pending = this.source.read(length, start).then(bytes => {
-      this.bytesFetched += bytes.length
-      return bytes
-    })
+    const pending = this.read(length, start)
     this.fetches += 1
     this.blocks.set(index, pending)
-    if (this.blocks.size > this.maxBlocks) {
+    this.forgetOnFailure(pending, [index])
+    this.evict()
+    return pending
+  }
+
+  prefetch(pageNumbers: number[]) {
+    const wanted = [
+      ...new Set(
+        pageNumbers.map(pageNumber =>
+          Math.floor(((pageNumber - 1) * this.pageSize) / this.blockSize),
+        ),
+      ),
+    ]
+      .filter(index => !this.blocks.has(index))
+      .sort((a, b) => a - b)
+    let runStart = 0
+    while (runStart < wanted.length) {
+      let runEnd = runStart + 1
+      while (
+        runEnd < wanted.length &&
+        wanted[runEnd] === wanted[runEnd - 1]! + 1
+      ) {
+        runEnd += 1
+      }
+      this.fetchRun(wanted[runStart]!, runEnd - runStart)
+      runStart = runEnd
+    }
+  }
+
+  private fetchRun(firstIndex: number, count: number) {
+    const start = firstIndex * this.blockSize
+    const length = Math.min(count * this.blockSize, this.fileSize - start)
+    const pending = this.read(length, start)
+    this.fetches += 1
+    const indexes: number[] = []
+    for (let i = 0; i < count; i++) {
+      const within = i * this.blockSize
+      indexes.push(firstIndex + i)
+      this.blocks.set(
+        firstIndex + i,
+        pending.then(bytes => bytes.subarray(within, within + this.blockSize)),
+      )
+    }
+    this.forgetOnFailure(pending, indexes)
+    this.evict()
+  }
+
+  private forgetOnFailure(pending: Promise<Uint8Array>, indexes: number[]) {
+    pending.catch(() => {
+      for (const index of indexes) {
+        this.blocks.delete(index)
+      }
+    })
+  }
+
+  private evict() {
+    while (this.blocks.size > this.maxBlocks) {
       const oldest = this.blocks.keys().next().value
-      if (oldest !== undefined) {
-        this.blocks.delete(oldest)
+      if (oldest === undefined) {
+        break
       }
+      this.blocks.delete(oldest)
     }
-    return pending
   }
 
   async page(pageNumber: number) {