hypgrep 0.1.1

package/LICENSE

@@ -0,0 +1,7 @@
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,90 @@
+# HypGrep
+
+[![mit license](https://img.shields.io/badge/License-MIT-orange.svg)](https://opensource.org/licenses/MIT)
+![coverage](https://img.shields.io/badge/Coverage-95-darkred)
+
+Build a compact full-text search index for a Parquet file using [`hyparquet`](https://github.com/hyparam/hyparquet) and [`hyparquet-writer`](https://github.com/hyparam/hyparquet-writer).
+
+## Why?
+
+Enable efficient full-text search on large Parquet datasets from any client without a server. Store your Parquet dataset on S3, generate a compact index file, and query it directly from a browser or other clients using HTTP range requests. The index tells you exactly which row blocks to fetch, so you only download the data you need.
+
+Perfect for serverless architectures where you want to offer search capabilities without managing infrastructure.
+
+## CLI usage
+
+```bash
+npx hypgrep dataset.parquet [dataset.index.parquet]
+```
+
+To install as a system-wide CLI tool:
+
+```bash
+npm install -g hypgrep
+hypgrep dataset.parquet [dataset.index.parquet]
+```
+
+## Find rows in a parquet file in JavaScript
+
+Use `parquetFind` to find rows matching a query while preserving natural row order (like Ctrl+F):
+
+```javascript
+import { parquetFind } from 'hypgrep'
+
+for await (const row of parquetFind({
+  query: 'serverless',
+  url: 'https://s3.hyperparam.app/hypgrep/wiki_en.parquet',
+})) {
+  console.log(row) // { title: '...', text: '...' }
+}
+```
+
+## Ranked search
+
+Use `parquetSearch` to rank results by BM25 relevance score (like a search engine):
+
+```javascript
+import { parquetSearch } from 'hypgrep'
+
+for await (const row of parquetSearch({
+  query: 'serverless',
+  url: 'https://s3.hyperparam.app/hypgrep/wiki_en.parquet',
+})) {
+  console.log(row) // highest relevance first
+}
+```
+
+## Create an index in JavaScript
+
+```javascript
+import { asyncBufferFromFile } from 'hyparquet'
+import { fileWriter } from 'hyparquet-writer'
+import { createIndex } from 'hypgrep'
+
+// Generate dataset.index.parquet from dataset.parquet
+const sourceFile = await asyncBufferFromFile('dataset.parquet')
+const indexFile = fileWriter('dataset.index.parquet')
+await createIndex({ sourceFile, indexFile })
+```
+
+## Local parquet files
+
+To search against local parquet files, provide an `asyncBufferFactory` that loads the file from the local filesystem:
+
+```js
+import { asyncBufferFromFile } from 'hyparquet'
+import { parquetFind } from 'hypgrep'
+
+// Loads parquet file from local filesystem
+function asyncBufferFactory({ url }) {
+  return asyncBufferFromFile(url)
+}
+
+for await (const row of parquetFind({
+  query: 'serverless',
+  url: 'dataset.parquet',
+  asyncBufferFactory,
+})) {
+  console.log(row)
+}
+```

package/bin/cli.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+
+import { fileWriter } from 'hyparquet-writer'
+import { pathToFileURL } from 'node:url'
+import { createIndex } from '../src/createIndex.js'
+import { inspect } from './inspect.js'
+import { loadParquet } from './load.js'
+
+/**
+ * Command line entry point.
+ */
+async function main() {
+  const sourcePath = process.argv[2]
+  if (!sourcePath) {
+    console.error('Usage: npx hypgrep dataset.parquet [dataset.index.parquet]')
+    process.exit(1)
+  }
+
+  // Load source
+  const { file: sourceFile, metadata: sourceMetadata } = await loadParquet(sourcePath)
+
+  // Check for HypGrep file
+  if (sourceMetadata.key_value_metadata?.some(kv => kv.key === 'hypgrep.version')) {
+    console.error('Input is already a HypGrep file')
+    console.error(`HypGrep: ${sourcePath}`)
+    await inspect({ indexPath: sourcePath })
+    process.exit(1)
+  }
+
+  // Infer index path if needed
+  const indexPath = process.argv[3] ?? `${sourcePath.replace(/\.parquet$/i, '')}.index.parquet`
+  if (indexPath.startsWith('http://') || indexPath.startsWith('https://')) {
+    console.error('Must specify local index path to generate index for a remote source')
+    process.exit(1)
+  }
+
+  console.log(`Source: ${sourcePath}`)
+  console.log(`Source: ${sourceFile.byteLength.toLocaleString()} bytes`)
+  console.log(`Source: ${sourceMetadata.num_rows.toLocaleString()} rows`)
+  console.log(`Source: ${sourceMetadata.row_groups.length.toLocaleString()} rowgroups`)
+  console.log(`HypGrep: ${indexPath}`)
+  const startTime = performance.now()
+
+  // Create index
+  const indexFile = fileWriter(indexPath)
+  await createIndex({ sourceFile, sourceMetadata, indexFile })
+
+  // Print statistics
+  await inspect({ sourcePath, indexPath })
+
+  console.log()
+  console.log(`Created ${indexPath} in ${((performance.now() - startTime) / 1000).toFixed(1)} s`)
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main().catch(error => {
+    console.error(error.message)
+    process.exit(1)
+  })
+}

package/bin/inspect.js

@@ -0,0 +1,67 @@
+import { parseKvMetadata } from '../src/queryIndex.js'
+import { loadParquet } from './load.js'
+
+/**
+ * @import { FileMetaData } from 'hyparquet'
+ */
+
+/**
+ * Print statistics about an index file.
+ *
+ * @param {object} options
+ * @param {string} options.indexPath
+ * @param {string} [options.sourcePath]
+ */
+export async function inspect({ indexPath, sourcePath }) {
+  // Load the index file
+  const { file: indexFile, metadata: indexMetadata } = await loadParquet(indexPath)
+
+  console.log(`HypGrep: ${indexFile.byteLength.toLocaleString()} bytes`)
+  console.log(`HypGrep: ${indexMetadata.num_rows.toLocaleString()} rows`)
+  console.log(`HypGrep: ${indexMetadata.row_groups.length.toLocaleString()} index rowgroups`)
+
+  // Calculate index stats
+  let avgIndexRowGroupSize = 0
+  for (const rowGroup of indexMetadata.row_groups) {
+    avgIndexRowGroupSize += Number(rowGroup.total_byte_size)
+  }
+  avgIndexRowGroupSize /= indexMetadata.row_groups.length || 1
+
+  // Guess source path if not provided
+  sourcePath ??= indexPath.replace(/\.index\.parquet$/i, '.parquet')
+  const sourceMetadata = await loadParquet(sourcePath)
+    .then(({ metadata }) => metadata)
+    .catch(() => undefined)
+  if (!sourceMetadata) {
+    throw new Error(`Failed to load source parquet: ${sourcePath}`)
+  }
+
+  // Calculate source stats
+  const { blockSize } = parseKvMetadata(indexMetadata.key_value_metadata ?? [])
+  const numBlocks = Math.ceil(Number(sourceMetadata.num_rows) / blockSize)
+
+  console.log(`HypGrep: ${sourceMetadata.row_groups.length.toLocaleString()} source rowgroups`)
+  console.log(`HypGrep: ${numBlocks.toLocaleString()} blocks`)
+
+  let avgSourceRowGroupSize = 0
+  for (const rowGroup of sourceMetadata.row_groups) {
+    avgSourceRowGroupSize += Number(rowGroup.total_byte_size)
+  }
+  avgSourceRowGroupSize /= sourceMetadata.row_groups.length || 1
+
+  // Average single-term query size
+  const indexMetadataLength = Math.round(indexMetadata.metadata_length).toLocaleString()
+  const indexAvgRowGroup = Math.round(avgIndexRowGroupSize).toLocaleString()
+  const sourceMetadataLength = Math.round(sourceMetadata.metadata_length).toLocaleString()
+  const sourceAvgRowGroup = Math.round(avgSourceRowGroupSize).toLocaleString()
+  const queryAvg = Math.round(
+    indexMetadata.metadata_length + avgIndexRowGroupSize + avgSourceRowGroupSize
+  ).toLocaleString()
+
+  // Print statistics
+  console.log(`Query: ~${queryAvg} bytes`)
+  console.log(` + ${indexMetadataLength} bytes (index metadata)`)
+  console.log(` + ${indexAvgRowGroup} bytes (index rowgroup)`)
+  console.log(` + ${sourceMetadataLength} bytes (source metadata)`)
+  console.log(` + ${sourceAvgRowGroup} bytes (source rowgroup)`)
+}

package/bin/load.js

@@ -0,0 +1,43 @@
+import fs from 'node:fs/promises'
+import { asyncBufferFromFile, asyncBufferFromUrl, parquetMetadataAsync } from 'hyparquet'
+
+/**
+ * @import {AsyncBuffer, FileMetaData} from 'hyparquet'
+ */
+
+/**
+ * @param {string} path
+ * @returns {Promise<{ file: AsyncBuffer, metadata: FileMetaData }>}
+ */
+export async function loadParquet(path) {
+  /** @type {AsyncBuffer | undefined} */
+  let file
+  if (path.startsWith('http://') || path.startsWith('https://')) {
+    try {
+      file = await asyncBufferFromUrl({ url: path })
+    } catch (error) {
+      console.error(`Failed to load Parquet file from URL: ${path}`)
+      throw error
+    }
+  } else {
+    try {
+      await fs.access(path)
+    } catch {
+      throw new Error(`Parquet file not found: ${path}`)
+    }
+    try {
+      file = await asyncBufferFromFile(path)
+    } catch (error) {
+      console.error(`Failed to load Parquet file from path: ${path}`)
+      throw error
+    }
+  }
+  /** @type {FileMetaData | undefined} */
+  let metadata
+  try {
+    metadata = await parquetMetadataAsync(file)
+  } catch (error) {
+    throw new Error(`Failed to read Parquet metadata from file: ${path}`, { cause: error })
+  }
+  return { file, metadata }
+}

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "hypgrep",
+  "version": "0.1.1",
+  "author": "Hyperparam",
+  "homepage": "https://hyperparam.app",
+  "license": "MIT",
+  "keywords": [
+    "parquet",
+    "index",
+    "search",
+    "full-text-search",
+    "hyparquet",
+    "serverless"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/hyparam/hypgrep.git"
+  },
+  "type": "module",
+  "sideEffects": false,
+  "bin": {
+    "hypgrep": "./bin/cli.js"
+  },
+  "types": "src/index.d.ts",
+  "main": "src/index.js",
+  "export": {
+    ".": {
+      "import": "./src/index.js",
+      "types": "./src/index.d.ts"
+    }
+  },
+  "files": [
+    "bin",
+    "src"
+  ],
+  "scripts": {
+    "benchmark": "node benchmark.js",
+    "coverage": "vitest run --coverage --coverage.include=src",
+    "lint": "eslint",
+    "lint:fix": "eslint --fix",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "hyparquet": "1.25.8",
+    "hyparquet-compressors": "1.1.1",
+    "hyparquet-writer": "0.15.1"
+  },
+  "devDependencies": {
+    "@types/node": "25.9.1",
+    "@vitest/coverage-v8": "4.1.6",
+    "eslint": "9.39.4",
+    "eslint-plugin-jsdoc": "62.9.0",
+    "typescript": "6.0.3",
+    "vitest": "4.1.6"
+  }
+}

package/src/constants.js

@@ -0,0 +1,8 @@
+// Version of the parquet index format
+export const hypGrepVersion = 0
+
+// Number of rows per virtual block
+export const defaultBlockSize = 500
+
+// Row group size in the index file
+export const defaultIndexRowGroupSize = 40000

package/src/createIndex.js

@@ -0,0 +1,162 @@
+import { parquetMetadataAsync, parquetReadObjects } from 'hyparquet'
+import { parquetWrite } from 'hyparquet-writer'
+import { defaultBlockSize, defaultIndexRowGroupSize, hypGrepVersion } from './constants.js'
+import { tokenize } from './tokenize.js'
+import { getTextColumnsFromSchema } from './utils.js'
+
+/**
+ * @import { BlockStats, CreateIndexOptions, IndexRow } from './types.js'
+ * @import { ColumnSource } from 'hyparquet-writer'
+ */
+
+/**
+ * Create a full-text search index parquet next to the given parquet file.
+ *
+ * @param {CreateIndexOptions} options
+ * @returns {Promise<void>}
+ */
+export async function createIndex({
+  sourceFile,
+  sourceMetadata,
+  indexFile,
+  blockSize = defaultBlockSize,
+  indexRowGroupSize = defaultIndexRowGroupSize,
+}) {
+  const metadata = sourceMetadata ?? await parquetMetadataAsync(sourceFile)
+  const numRows = Number(metadata.num_rows)
+  const textColumns = getTextColumnsFromSchema(metadata)
+  if (textColumns.length === 0) {
+    throw new Error('No string columns found to index')
+  }
+
+  // Map from term -> array of entries (entries are in blockId order)
+  /** @type {Map<string, IndexRow[]>} */
+  const termIndex = new Map()
+
+  let blockId = 0
+  for (let rowStart = 0; rowStart < numRows; rowStart += blockSize) {
+    const rowEnd = Math.min(rowStart + blockSize, numRows)
+
+    const rows = await parquetReadObjects({
+      file: sourceFile,
+      metadata,
+      rowStart,
+      rowEnd,
+      columns: textColumns,
+    })
+
+    const { termDocCount, termFreqMap } = collectBlockStats(rows, textColumns)
+
+    // Build index entries for this block
+    for (const [term, docCount] of termDocCount.entries()) {
+      const termFreq = termFreqMap.get(term) || docCount
+      const entry = { term, blockId, docCount, termFreq }
+      const existing = termIndex.get(term)
+      if (existing) {
+        existing.push(entry)
+      } else {
+        termIndex.set(term, [entry])
+      }
+    }
+
+    blockId += 1
+  }
+
+  // Sort by term
+  const sortedTerms = Array.from(termIndex.keys()).sort()
+
+  // Flatten into sorted indexRows
+  /** @type {IndexRow[]} */
+  const indexRows = []
+  for (const term of sortedTerms) {
+    const entries = termIndex.get(term)
+    if (!entries) continue
+    indexRows.push(...entries)
+  }
+
+  const kvMetadata = [
+    { key: 'hypgrep.version', value: String(hypGrepVersion) },
+    { key: 'hypgrep.block_size', value: String(blockSize) },
+    { key: 'hypgrep.text_columns', value: textColumns.join(',') },
+    { key: 'hypgrep.source_rows', value: String(numRows) },
+    // Can save network requests on the source file
+    { key: 'hypgrep.source_bytelength', value: String(sourceFile.byteLength) },
+  ]
+
+  const columnData = buildColumnData(indexRows)
+  await parquetWrite({
+    writer: indexFile,
+    columnData,
+    rowGroupSize: indexRowGroupSize,
+    kvMetadata,
+  })
+}
+
+/**
+ * Collect term statistics for a single logical block of rows.
+ *
+ * @param {Record<string, any>[]} rows
+ * @param {string[]} textColumns
+ * @returns {BlockStats}
+ */
+function collectBlockStats(rows, textColumns) {
+  const termDocCount = new Map()
+  const termFreqMap = new Map()
+
+  for (const row of rows) {
+    if (!row) continue
+
+    const seenInRow = new Set()
+
+    for (const columnName of textColumns) {
+      const value = row[columnName]
+      if (typeof value !== 'string' || value.length === 0) continue
+
+      const tokens = tokenize(value)
+
+      for (const token of tokens) {
+        seenInRow.add(token)
+        const prevFreq = termFreqMap.get(token) || 0
+        termFreqMap.set(token, prevFreq + 1)
+      }
+    }
+
+    for (const token of seenInRow) {
+      const prevDocCount = termDocCount.get(token) || 0
+      termDocCount.set(token, prevDocCount + 1)
+    }
+  }
+
+  return { termDocCount, termFreqMap }
+}
+
+/**
+ * Convert row-oriented index entries to column-oriented data for hyparquet-writer.
+ *
+ * @param {IndexRow[]} indexRows
+ * @returns {ColumnSource[]}
+ */
+function buildColumnData(indexRows) {
+  const { length } = indexRows
+  const terms = new Array(length)
+  const blockIds = new Array(length)
+  const docCounts = new Array(length)
+  const termFreqs = new Array(length)
+
+  for (let i = 0; i < length; i += 1) {
+    const row = indexRows[i]
+    terms[i] = row.term
+    blockIds[i] = row.blockId
+    docCounts[i] = row.docCount
+    termFreqs[i] = row.termFreq
+  }
+
+  return [
+    // Delta byte array encoding works well for sorted string columns
+    { name: 'term', data: terms, type: 'STRING', encoding: 'DELTA_BYTE_ARRAY' },
+    // Delta binary packed works well for incrementing integers
+    { name: 'blockId', data: blockIds, type: 'INT32', encoding: 'DELTA_BINARY_PACKED' },
+    { name: 'docCount', data: docCounts, type: 'INT32' },
+    { name: 'termFreq', data: termFreqs, type: 'INT32' },
+  ]
+}

package/src/index.d.ts

@@ -0,0 +1,29 @@
+import type { CreateIndexOptions, ParquetSearchOptions, QueryIndexOptions, QueryResult } from './types.js'
+
+export const hypGrepVersion: number
+
+/**
+ * Create a full-text search index parquet next to the given parquet file.
+ */
+export function createIndex(options: CreateIndexOptions): Promise<void>
+
+/**
+ * Uses hypgrep to find rows matching a query from a source parquet file.
+ */
+export function parquetFind(options: ParquetSearchOptions): AsyncGenerator<Record<string, any>, void, unknown>
+
+/**
+ * Uses hypgrep to query a source parquet file and return matching rows ranked by relevance.
+ */
+export function parquetSearch(options: ParquetSearchOptions): AsyncGenerator<Record<string, any>, void, unknown>
+
+/**
+ * Query a search index to find matching row groups from the source parquet.
+ */
+export function queryIndex(options: QueryIndexOptions): Promise<QueryResult>
+
+/**
+ * Tokenize text into normalized terms.
+ * Lowercases and splits on non-alphanumeric boundaries.
+ */
+export function tokenize(text: string): string[]

package/src/index.js

@@ -0,0 +1,6 @@
+export { hypGrepVersion } from './constants.js'
+export { createIndex } from './createIndex.js'
+export { parquetFind } from './parquetFind.js'
+export { parquetSearch } from './parquetSearch.js'
+export { queryIndex } from './queryIndex.js'
+export { tokenize } from './tokenize.js'

package/src/parquetFind.js

@@ -0,0 +1,113 @@
+import { asyncBufferFromUrl, parquetMetadataAsync, parquetReadObjects } from 'hyparquet'
+import { queryIndex } from './queryIndex.js'
+import { tokenize } from './tokenize.js'
+
+/**
+ * Find rows matching a query, maintaining natural row order.
+ *
+ * @import {ParquetSearchOptions, TermResults} from '../src/types.js'
+ * @param {ParquetSearchOptions} options
+ * @returns {AsyncGenerator<Record<string, any>, void, unknown>}
+ */
+export async function* parquetFind({
+  query,
+  url,
+  limit = Infinity,
+  prefix = true,
+  signal,
+  asyncBufferFactory = asyncBufferFromUrl,
+  sourceFile,
+  sourceMetadata,
+  indexFile,
+  indexMetadata,
+  ...hyparquetOptions
+}) {
+  if (!query || limit <= 0) return
+  signal?.throwIfAborted()
+  // Query the index to get matching blocks
+  indexFile ??= await asyncBufferFactory({ url: `${url.replace(/\.parquet$/i, '')}.index.parquet` })
+  const queryResult = await queryIndex({ query, indexFile, indexMetadata, prefix })
+  if (!queryResult) return
+  const { blocks, textColumns, sourceByteLength } = queryResult
+
+  // If no matching blocks, return empty result
+  if (blocks.length === 0) return
+
+  // Sort blocks by blockId for natural row order
+  blocks.sort((a, b) => a.blockId - b.blockId)
+  signal?.throwIfAborted()
+
+  // Construct source file if not provided, use byteLength from index metadata if available
+  const file = sourceFile ?? await asyncBufferFactory({ url, byteLength: sourceByteLength })
+  // Get source metadata once before loop only if needed
+  const metadata = sourceMetadata ?? await parquetMetadataAsync(file)
+
+  // Tokenize query terms for matching
+  const queryTerms = new Set(tokenize(query))
+
+  // For each matching block (in natural order), read rows from the source parquet
+  let count = 0
+  for (const block of blocks) {
+    signal?.throwIfAborted()
+    const blockRows = await parquetReadObjects({
+      ...hyparquetOptions,
+      file,
+      metadata,
+      rowStart: block.rowStart,
+      rowEnd: block.rowEnd,
+      useOffsetIndex: true,
+    })
+
+    // Yield matching rows in natural order (no sorting)
+    for (let i = 0; i < blockRows.length; i++) {
+      const row = blockRows[i]
+      if (matchesRow(row, textColumns, queryTerms, block.terms, prefix)) {
+        yield { __index__: block.rowStart + i, ...row }
+        if (++count >= limit) return
+      }
+    }
+  }
+}
+
+/**
+ * Check if a row matches any of the query terms.
+ *
+ * @param {Record<string, any>} row
+ * @param {string[]} textColumns
+ * @param {Set<string>} queryTerms
+ * @param {TermResults} termStats
+ * @param {boolean} prefix
+ * @returns {boolean}
+ */
+function matchesRow(row, textColumns, queryTerms, termStats, prefix) {
+  const rowTokens = new Set()
+
+  // Collect all tokens from text columns
+  for (const col of textColumns) {
+    const value = row[col]
+    if (typeof value === 'string') {
+      for (const token of tokenize(value)) {
+        rowTokens.add(token)
+      }
+    }
+  }
+
+  // Check if any query term matches
+  for (const queryTerm of queryTerms) {
+    if (prefix) {
+      // Prefix matching: find row tokens that start with query term
+      for (const token of rowTokens) {
+        if (token.startsWith(queryTerm) && termStats[token]) {
+          return true
+        }
+      }
+    } else {
+      // Exact matching
+      if (rowTokens.has(queryTerm) && termStats[queryTerm]) {
+        return true
+      }
+    }
+  }
+
+  return false
+}

package/src/parquetSearch.js

@@ -0,0 +1,128 @@
+import { asyncBufferFromUrl, parquetMetadataAsync, parquetReadObjects } from 'hyparquet'
+import { queryIndex } from './queryIndex.js'
+import { tokenize } from './tokenize.js'
+
+/**
+ * Uses the hypgrep to query a source parquet file and return matching rows.
+ *
+ * @import {ParquetSearchOptions, TermResults} from '../src/types.js'
+ * @param {ParquetSearchOptions} options
+ * @returns {AsyncGenerator<Record<string, any>, void, unknown>}
+ */
+export async function* parquetSearch({
+  query,
+  url,
+  limit = Infinity,
+  prefix = true,
+  signal,
+  asyncBufferFactory = asyncBufferFromUrl,
+  sourceFile,
+  sourceMetadata,
+  indexFile,
+  indexMetadata,
+  ...hyparquetOptions
+}) {
+  if (!query || limit <= 0) return
+  signal?.throwIfAborted()
+  // Query the index to get matching blocks
+  indexFile ??= await asyncBufferFactory({ url: `${url.replace(/\.parquet$/i, '')}.index.parquet` })
+  const queryResult = await queryIndex({ query, indexFile, indexMetadata, prefix })
+  if (!queryResult) return
+  const { blocks, textColumns, sourceByteLength } = queryResult
+
+  // Sort blocks by score descending (most relevant first)
+  blocks.sort((a, b) => b.score - a.score)
+
+  // If no matching blocks, return empty result
+  if (blocks.length === 0) return
+  signal?.throwIfAborted()
+
+  // Construct source file if not provided, use byteLength from index metadata if available
+  const file = sourceFile ?? await asyncBufferFactory({ url, byteLength: sourceByteLength })
+  // Get source metadata once before loop only if needed
+  const metadata = sourceMetadata ?? await parquetMetadataAsync(file)
+
+  // Tokenize query terms for matching
+  const queryTerms = new Set(tokenize(query))
+
+  // For each matching block, read rows from the source parquet
+  let count = 0
+  for (const block of blocks) {
+    signal?.throwIfAborted()
+    const blockRows = await parquetReadObjects({
+      ...hyparquetOptions,
+      file,
+      metadata,
+      rowStart: block.rowStart,
+      rowEnd: block.rowEnd,
+      useOffsetIndex: true,
+    })
+
+    // Score and collect matching rows within the block
+    /** @type {{index: number, row: Record<string, any>, score: number}[]} */
+    const scoredRows = []
+    for (let i = 0; i < blockRows.length; i++) {
+      const row = blockRows[i]
+      const score = scoreRow(row, textColumns, queryTerms, block.terms, prefix)
+      if (score > 0) {
+        scoredRows.push({ index: block.rowStart + i, row, score })
+      }
+    }
+
+    // Sort by score descending within block
+    scoredRows.sort((a, b) => b.score - a.score)
+
+    // Yield rows in score order
+    for (const { index, row } of scoredRows) {
+      yield { __index__: index, ...row }
+      if (++count >= limit) return
+    }
+  }
+}
+
+/**
+ * Score a row based on which query terms it matches, weighted by IDF.
+ *
+ * @param {Record<string, any>} row
+ * @param {string[]} textColumns
+ * @param {Set<string>} queryTerms
+ * @param {TermResults} termStats
+ * @param {boolean} prefix
+ * @returns {number} score (0 if no match)
+ */
+function scoreRow(row, textColumns, queryTerms, termStats, prefix) {
+  let score = 0
+  const rowTokens = new Set()
+
+  // Collect all tokens from text columns
+  for (const col of textColumns) {
+    const value = row[col]
+    if (typeof value === 'string') {
+      for (const token of tokenize(value)) {
+        rowTokens.add(token)
+      }
+    }
+  }
+
+  // Score based on matching query terms weighted by IDF
+  for (const queryTerm of queryTerms) {
+    if (prefix) {
+      // Prefix matching: find row tokens that start with query term
+      for (const token of rowTokens) {
+        if (token.startsWith(queryTerm)) {
+          // Use the matched token's stats from the index
+          const stats = termStats[token]
+          score += stats?.idf ?? 1
+        }
+      }
+    } else {
+      // Exact matching
+      if (rowTokens.has(queryTerm)) {
+        const stats = termStats[queryTerm]
+        score += stats?.idf ?? 1
+      }
+    }
+  }
+
+  return score
+}

package/src/queryIndex.js

@@ -0,0 +1,171 @@
+import { parquetMetadataAsync, parquetQuery } from 'hyparquet'
+import { defaultBlockSize, hypGrepVersion } from './constants.js'
+import { tokenize } from './tokenize.js'
+
+/**
+ * @import { FileMetaData, KeyValue, ParquetQueryFilter } from 'hyparquet'
+ * @import { BlockResult, HypGrepMetadata, QueryIndexOptions, QueryResult, TermResults } from './types.js'
+ */
+
+/**
+ * Build a pushdown filter to efficiently query for terms in the index.
+ * Optionally uses prefix matching.
+ *
+ * @param {string[]} terms
+ * @param {boolean} prefix - whether to use prefix matching
+ * @returns {ParquetQueryFilter}
+ */
+function termsFilter(terms, prefix) {
+  if (prefix) {
+    const $or = terms.map(t => {
+      const lastChar = t.charCodeAt(t.length - 1)
+      const upperBound = t.slice(0, -1) + String.fromCharCode(lastChar + 1)
+      return { term: { $gte: t, $lt: upperBound } }
+    })
+    return { $or }
+  } else {
+    return { term: { $in: terms } }
+  }
+}
+
+/**
+ * Query a search index to find matching row groups from the source parquet.
+ * Returns undefined if query is empty so the search index is not used.
+ *
+ * @param {QueryIndexOptions} options
+ * @returns {Promise<QueryResult | undefined>}
+ */
+export async function queryIndex({ query, indexFile, indexMetadata, prefix = true }) {
+  // Tokenize the query using the same logic as indexing
+  const queryTerms = tokenize(query)
+  if (queryTerms.length === 0) return undefined
+
+  // Read index kv metadata
+  indexMetadata ??= await parquetMetadataAsync(indexFile)
+  const kvMetadata = indexMetadata.key_value_metadata || []
+  const { blockSize, textColumns, sourceByteLength, sourceRows } = parseKvMetadata(kvMetadata)
+
+  // Read index rows matching any of the query terms
+  const indexRows = await parquetQuery({
+    file: indexFile,
+    metadata: indexMetadata,
+    // use hyparquet pushdown filtering
+    filter: termsFilter(queryTerms, prefix),
+  })
+
+  // Pre-compute corpusDocFreq by summing docCount per term
+  /** @type {Map<string, number>} */
+  const corpusDocFreq = new Map()
+  for (const row of indexRows) {
+    const prev = corpusDocFreq.get(row.term) || 0
+    corpusDocFreq.set(row.term, prev + row.docCount)
+  }
+
+  // Map to accumulate scores per blockId
+  /** @type {Map<number, number>} */
+  const blockScores = new Map()
+  // Map to accumulate term statistics per blockId
+  /** @type {Map<number, TermResults>} */
+  const blockTerms = new Map()
+
+  // For each query term, find matching blocks and accumulate scores
+  for (const queryTerm of queryTerms) {
+    for (const indexRow of indexRows) {
+      // Check if this index term matches (exact or prefix)
+      const matches = prefix
+        ? indexRow.term.startsWith(queryTerm)
+        : indexRow.term === queryTerm
+      if (matches) {
+        const currentScore = blockScores.get(indexRow.blockId) || 0
+
+        // Use actual index term's corpus doc freq for scoring
+        const termCorpusDocFreq = corpusDocFreq.get(indexRow.term) || 0
+
+        // BM25 scoring
+        // IDF component: log((N - df + 0.5) / (df + 0.5) + 1)
+        const idf = Math.log((sourceRows - termCorpusDocFreq + 0.5) / (termCorpusDocFreq + 0.5) + 1)
+
+        // BM25 parameters
+        const k1 = 1.2 // controls term frequency saturation
+        const b = 0.75 // controls length normalization
+
+        // TF component with saturation and length normalization
+        const tf = indexRow.termFreq
+        const tfComponent = tf * (k1 + 1) / (tf + k1 * (1 - b + b * indexRow.docCount / blockSize))
+
+        const termScore = idf * tfComponent
+
+        blockScores.set(indexRow.blockId, currentScore + termScore)
+
+        // Collect term statistics
+        if (!blockTerms.has(indexRow.blockId)) {
+          blockTerms.set(indexRow.blockId, {})
+        }
+        const terms = blockTerms.get(indexRow.blockId)
+        if (!terms) continue
+        terms[indexRow.term] = {
+          docs: indexRow.docCount,
+          frequency: indexRow.termFreq,
+          idf,
+        }
+      }
+    }
+  }
+
+  // Convert block scores to BlockResults
+  /** @type {BlockResult[]} */
+  const blocks = []
+  const numRows = Number(indexMetadata.num_rows)
+  for (const [blockId, score] of blockScores.entries()) {
+    const rowStart = blockId * blockSize
+    const rowEnd = Math.min((blockId + 1) * blockSize, numRows)
+    const terms = blockTerms.get(blockId) || {}
+    blocks.push({ blockId, rowStart, rowEnd, score, terms })
+  }
+
+  return { blocks, textColumns, sourceByteLength }
+}
+
+/**
+ * Parse key-value metadata from the index file
+ *
+ * @param {KeyValue[]} kvMetadata
+ * @returns {HypGrepMetadata}
+ */
+export function parseKvMetadata(kvMetadata) {
+  let blockSize = defaultBlockSize
+  /** @type {string[]} */
+  let textColumns = []
+  /** @type {number | undefined} */
+  let sourceByteLength
+  /** @type {number | undefined} */
+  let sourceRows
+
+  for (const { key, value } of kvMetadata) {
+    if (key === 'hypgrep.block_size') {
+      blockSize = Number(value)
+    }
+    if (key === 'hypgrep.version') {
+      if (Number(value) !== hypGrepVersion) {
+        throw new Error(`Unsupported hypgrep version ${value}`)
+      }
+    }
+    if (key === 'hypgrep.text_columns' && value) {
+      textColumns = value.split(',')
+    }
+    if (key === 'hypgrep.source_rows') {
+      sourceRows = Number(value)
+    }
+    if (key === 'hypgrep.source_bytelength') {
+      sourceByteLength = Number(value)
+    }
+  }
+  if (sourceRows === undefined) {
+    throw new Error('Missing hypgrep.source_rows in index metadata')
+  }
+  if (sourceByteLength === undefined) {
+    throw new Error('Missing hypgrep.source_bytelength in index metadata')
+  }
+
+  return { blockSize, textColumns, sourceByteLength, sourceRows }
+}

package/src/stemmer.js

@@ -0,0 +1,78 @@
+
+const vowels = 'aeiouy'
+
+// ordered longest-first
+const suffixRules = [
+  { suffix: 'ing', minStem: 4, needsVowel: true },
+  { suffix: 'ed', minStem: 3, needsVowel: true },
+  { suffix: 'ly', minStem: 3 },
+  // handle plural-ish endings carefully
+  { suffix: 'es', minStem: 3, needsVowel: true, plural: true },
+  { suffix: 's', minStem: 3, needsVowel: true, plural: true },
+  // bad with i/y: er, ers, est, ies
+  // other options: ment, ingly, edly, ness
+]
+
+/**
+ * Simple prefix stemmer, removes common English suffixes.
+ * Based on a simplified version of the Porter stemming algorithm.
+ * Importantly, only removes suffixes.
+ *
+ * @param {string} term - lowercase word to stem
+ * @returns {string} stemmed word
+ */
+export function stemmer(term) {
+  // too short to bother
+  if (term.length < 4) return term
+
+  // skip anything that isn't a simple lowercase word
+  if (!isLowerAlpha(term)) return term
+
+  for (let i = 0; i < suffixRules.length; i += 1) {
+    const rule = suffixRules[i]
+    const { suffix } = rule
+
+    if (!term.endsWith(suffix)) continue
+
+    const stem = term.slice(0, term.length - suffix.length)
+    if (stem.length < rule.minStem) continue
+
+    if (rule.needsVowel && !hasVowel(stem)) continue
+
+    if (rule.plural) {
+      // fix: class, boss
+      if (term.endsWith('ss')) continue
+      // fix: virus, status
+      if (term.endsWith('us')) continue
+      // fix: this, analysis
+      if (term.endsWith('is')) continue
+    }
+
+    return stem
+  }
+
+  return term
+}
+
+/**
+ * @param {string} s
+ * @returns {boolean}
+ */
+function hasVowel(s) {
+  for (let i = 0; i < s.length; i += 1) {
+    if (vowels.includes(s[i])) return true
+  }
+  return false
+}
+
+/**
+ * @param {string} s
+ * @returns {boolean}
+ */
+function isLowerAlpha(s) {
+  for (let i = 0; i < s.length; i += 1) {
+    const code = s.charCodeAt(i)
+    if (code < 97 || code > 122) return false
+  }
+  return true
+}

package/src/tokenize.js

@@ -0,0 +1,72 @@
+import { stemmer } from './stemmer.js'
+
+/**
+ * Common English stop words to filter from index.
+ * These high-frequency, low-value words are excluded to reduce index size.
+ */
+const STOP_WORDS = new Set([
+  'the', 'be', 'to', 'of', 'and', 'a', 'in', 'that', 'have', 'it',
+  'for', 'not', 'on', 'with', 'he', 'as', 'you', 'do', 'at', 'this',
+  'but', 'his', 'by', 'from', 'they', 'we', 'say', 'her', 'she', 'or',
+  'an', 'will', 'my', 'one', 'all', 'would', 'there', 'their', 'what',
+  'so', 'up', 'out', 'if', 'about', 'who', 'get', 'which', 'go', 'me',
+  'when', 'make', 'can', 'like', 'no', 'just', 'him', 'know', 'take',
+  'into', 'your', 'some', 'could', 'them', 'than', 'then', 'now', 'only',
+  'its', 'also', 'other', 'how', 'our', 'may', 'these', 'was', 'been',
+  'has', 'had', 'are', 'is', 'am', 'were', 'does', 'did', 'being',
+])
+
+/**
+ * Split camelCase and PascalCase words by inserting spaces before uppercase letters.
+ * Converts "parseUserInput" to "parse User Input", "XMLParser" to "XMLParser", etc.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function splitCamelCase(text) {
+  // Insert space before uppercase letters that follow lowercase letters or digits
+  return text.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+}
+
+/**
+ * Normalize Unicode text by removing diacritics/accents.
+ * Converts "café" to "cafe", "résumé" to "resume", etc.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function normalizeUnicode(text) {
+  // NFD decomposes combined characters into base + combining marks
+  // Then remove combining marks (Unicode category: Mark, Nonspacing)
+  return text.normalize('NFD').replace(/[\u0300-\u036f]/g, '')
+}
+
+/**
+ * Tokenize text into normalized terms.
+ * Splits camelCase, lowercases, normalizes Unicode, splits on non-alphanumeric boundaries,
+ * filters stop words, and applies Porter stemming.
+ *
+ * @param {string} text
+ * @returns {string[]}
+ */
+export function tokenize(text) {
+  // Split camelCase/PascalCase before normalization
+  const split = splitCamelCase(text)
+  // Normalize Unicode (remove accents) before lowercasing
+  const normalized = normalizeUnicode(split)
+  const lower = normalized.toLowerCase()
+
+  // Split on non-alphanumeric boundaries
+  const rawTokens = lower.split(/[^a-z0-9]+/g)
+  const tokens = []
+
+  for (const token of rawTokens) {
+    if (!token) continue
+    if (token.length < 2) continue
+    if (STOP_WORDS.has(token)) continue
+    // Apply Porter stemming to reduce words to their root form
+    tokens.push(stemmer(token))
+  }
+
+  return tokens
+}

package/src/types.d.ts

@@ -0,0 +1,92 @@
+import type { AsyncBuffer, asyncBufferFromUrl, Compressors, FileMetaData, ParquetParsers, ParquetQueryFilter } from 'hyparquet'
+import type { Writer } from 'hyparquet-writer'
+
+export interface CreateIndexOptions {
+  sourceFile: AsyncBuffer // file reader for the source parquet file
+  sourceMetadata?: FileMetaData // optional source parquet metadata
+  indexFile: Writer // file writer for the output index parquet file
+  blockSize?: number // number of rows per logical block
+  indexRowGroupSize?: number // row group size in the index file
+}
+
+export interface QueryIndexOptions {
+  query: string // the search query string
+  indexFile: AsyncBuffer // file reader for the index parquet file
+  indexMetadata?: FileMetaData // optional index parquet metadata
+  prefix?: boolean // enable prefix matching (default: true)
+}
+
+export interface ParquetSearchOptions {
+  query: string // the search query string
+  url: string // URL or file path to the source parquet file
+  limit?: number // maximum number of matching rows to return
+  prefix?: boolean // enable prefix matching (default: true)
+
+  // fetch options
+  signal?: AbortSignal // optional AbortSignal to cancel the search operation
+  asyncBufferFactory?: typeof asyncBufferFromUrl // optional factory to create AsyncBuffers for source and index files
+  sourceFile?: AsyncBuffer // file reader for the source parquet file
+  sourceMetadata?: FileMetaData // optional source parquet metadata
+  indexFile?: AsyncBuffer // file reader for the index parquet file
+  indexMetadata?: FileMetaData // optional index parquet metadata
+
+  // misc options passed through to hyparquet
+  columns?: string[]
+  filter?: ParquetQueryFilter
+  compressors?: Compressors
+  utf8?: boolean
+  parsers?: ParquetParsers
+}
+
+/**
+ * Represents a single entry in the search index.
+ */
+export interface IndexRow {
+  term: string // normalized search term
+  blockId: number // logical block ID this term appears in
+  docCount: number // number of documents in the block containing this term
+  termFreq: number // total frequency of the term in the block
+}
+
+export interface QueryResult {
+  blocks: BlockResult[] // list of matching blocks
+  textColumns: string[] // list of indexed text columns in the source parquet
+  sourceByteLength: number // byte length of the source parquet file
+}
+
+/**
+ * Represents a matching block of rows from the source parquet.
+ */
+export interface BlockResult {
+  blockId: number
+  rowStart: number // starting row index (inclusive) in the source parquet
+  rowEnd: number // ending row index (exclusive) in the source parquet
+  score: number // relevance score based on term frequency
+  terms: TermResults // per-term statistics
+}
+
+export type TermResults = Record<string, TermResult>
+interface TermResult {
+  docs: number // number of documents in the block containing this term
+  frequency: number // total occurrences of the term in the block
+  idf: number // inverse document frequency for this term
+}
+
+/**
+ * Metadata about the source parquet file and index settings.
+ * Parsed from the index file KV metadata
+ */
+export interface HypGrepMetadata {
+  blockSize: number // number of rows per logical block
+  textColumns: string[] // list of indexed text columns
+  sourceRows: number // number of rows in the source parquet file
+  sourceByteLength: number // byte length of the source parquet file
+}
+
+/**
+ * Statistics collected for a single logical block during index creation.
+ */
+export interface BlockStats {
+  termDocCount: Map<string, number> // number of documents containing each term
+  termFreqMap: Map<string, number> // total frequency of each term in the block
+}

package/src/utils.js

@@ -0,0 +1,44 @@
+import { parquetSchema } from 'hyparquet'
+
+/**
+ * @import {FileMetaData, SchemaTree} from 'hyparquet'
+ */
+
+/**
+ * Get string column names from the parquet schema.
+ *
+ * @param {FileMetaData} metadata
+ * @returns {string[]}
+ */
+export function getTextColumnsFromSchema(metadata) {
+  const schemaTree = parquetSchema(metadata)
+  /** @type {string[]} */
+  const textColumns = []
+
+  /**
+   * @param {SchemaTree} node
+   */
+  function traverse(node) {
+    const { element, children } = node
+
+    // Check if this is a string column
+    const isString =
+      element.converted_type === 'UTF8' ||
+      element.logical_type?.type === 'STRING'
+
+    // If it's a leaf node (no children) and it's a string, add it
+    if (isString && (!children || children.length === 0)) {
+      textColumns.push(element.name)
+    }
+
+    // Traverse children
+    if (children) {
+      for (const child of children) {
+        traverse(child)
+      }
+    }
+  }
+
+  traverse(schemaTree)
+  return textColumns
+}