lakeql 0.1.7 → 0.1.8

package/dist/{chunk-RZL45ZSN.js → chunk-MXGEAVHL.js}

@@ -1,6 +1,7 @@
 import { LakeqlError, TimestampValue, isTimestampValue, timestampEpochForUnit, compareTimestampValues, timestampValueFromIso, matches, ensureGeoBackendForExprs, encodeJsonLine, jsonSafeValue, evaluate, timestampFromEpoch, requireGeoBackend, regexpReplaceValue, regexpMatchesValue, toGeometry, parseGeometry, envelopeOf, envelopeFromGeometry, bboxIntersects } from './chunk-TFD5RFKB.js';
 import { parquetReadObjects, parquetMetadataAsync, parquetSchema, parquetRead } from 'hyparquet';
 import { getSchemaPath, isFlatColumn } from 'hyparquet/src/schema.js';
+import { decompress } from 'fzstd';
 import { DEFAULT_PARSERS, convert, convertWithDictionary } from 'hyparquet/src/convert.js';
 import { Encodings, PageTypes } from 'hyparquet/src/constants.js';
 import { decompressPage, readDataPage, readDataPageV2 } from 'hyparquet/src/datapage.js';
@@ -4601,6 +4602,8 @@ var QueryResult = class {
         now: config.now,
         startedAt
       };
+      if (config.limit !== void 0)
+        scanOptions.canStopEarly = true;
       const partitionValues = config.hive ? parseHivePartitions(object.path) : {};
       const physicalColumns = readColumns?.filter((column) => !(column in partitionValues));
       if (physicalColumns !== void 0 && physicalColumns.length > 0) {
@@ -4655,6 +4658,8 @@ var QueryResult = class {
         now: config.now,
         startedAt
       };
+      if (config.limit !== void 0)
+        scanOptions.canStopEarly = true;
       const partitionValues = config.hive ? parseHivePartitions(object.path) : {};
       const physicalColumns = columns?.filter((column) => !(column in partitionValues));
       if (physicalColumns !== void 0 && physicalColumns.length > 0) {
@@ -4711,6 +4716,11 @@ var QueryResult = class {
     const scanVectorBatches = config.scanner.scanVectorBatches;
     if (scanVectorBatches === void 0)
       return;
+    const lateMaterialized = await this.lateMaterializedLimitRows(startedAt);
+    if (lateMaterialized !== void 0) {
+      yield* rowsAsBatches(lateMaterialized, config.batchSize ?? 4096);
+      return;
+    }
     let offsetSkipped = 0;
     let returned = 0;
     const { planned: paths, skipped: skippedFiles } = await this.planObjects();
@@ -4728,6 +4738,8 @@ var QueryResult = class {
         now: config.now,
         startedAt
       };
+      if (config.limit !== void 0)
+        scanOptions.canStopEarly = true;
       if (columns !== void 0 && columns.length > 0)
         scanOptions.columns = columns;
       if (config.where !== void 0)
@@ -4763,6 +4775,70 @@ var QueryResult = class {
     stats.elapsedMs = config.now() - startedAt;
     config.metrics?.timing("lakeql.query.elapsed", stats.elapsedMs, { queryId: stats.queryId });
   }
+  async lateMaterializedLimitRows(startedAt) {
+    const config = this.config;
+    if (config.limit === void 0 || config.where === void 0 || config.select === void 0 || config.scanner.scanVectorBatches === void 0 || !vectorExprSupported(config.where) || !Object.values(config.projections ?? {}).every(vectorExprSupported)) {
+      return void 0;
+    }
+    const predicateColumns = predicateReadColumns(config.where);
+    const outputColumns = outputReadColumns(config.select, config.projections);
+    if (predicateColumns.length === 0 || outputColumns.length === 0)
+      return void 0;
+    const lateColumns = outputColumns.filter((column) => !predicateColumns.includes(column));
+    if (lateColumns.length === 0)
+      return void 0;
+    const refs = [];
+    const maxRefs = (config.offset ?? 0) + config.limit;
+    const scanVectorBatches = config.scanner.scanVectorBatches;
+    const { planned: paths, skipped: skippedFiles } = await this.planObjects();
+    this.stats.filesSkipped = skippedFiles;
+    for (const object of paths) {
+      this.stats.filesPlanned += 1;
+      this.stats.filesRead += 1;
+      this.stats.bytesRequested += object.size;
+      enforceBudget(config.budget, this.stats, config.now, startedAt);
+      const scanOptions = {
+        columns: predicateColumns,
+        where: config.where,
+        canStopEarly: true,
+        batchSize: columnarBatchSize(config.batchSize),
+        stats: this.stats,
+        budget: config.budget,
+        now: config.now,
+        startedAt
+      };
+      for await (const { rowOffset, batch } of scanVectorBatches.call(config.scanner, object.path, scanOptions)) {
+        const selection = predicateSelection(batch, config.where);
+        const selected = [...selectedRowIndices(batch.rowCount, selection)];
+        this.stats.rowsMatched += selected.length;
+        for (const index of selected) {
+          refs.push({
+            path: object.path,
+            rowIndex: rowOffset + index,
+            keys: rankKeyRow(batch, index, predicateColumns)
+          });
+          enforceBufferedRowsBudget(config.budget, refs.length);
+          enforceOperatorMemoryBudget(config.budget, estimateOperatorMemoryBytes(refs));
+          if (refs.length >= maxRefs)
+            break;
+        }
+        enforceBudget(config.budget, this.stats, config.now, startedAt);
+        if (refs.length >= maxRefs)
+          break;
+      }
+      if (refs.length >= maxRefs)
+        break;
+    }
+    const selectedRefs = refs.slice(config.offset ?? 0, maxRefs);
+    const rowsByRef = await this.materializeRowRefs(selectedRefs, outputColumns, predicateColumns, startedAt, config.batchSize ?? 4096);
+    const rows = selectedRefs.map((ref) => rowsByRef.get(rowRefKey(ref))).filter((row) => row !== void 0).map((row) => project(row, config.select, config.projections));
+    for (const _row of rows) {
+      this.stats.rowsReturned += 1;
+      enforceBudget(config.budget, this.stats, config.now, startedAt);
+    }
+    this.stats.elapsedMs = config.now() - startedAt;
+    return rows;
+  }
   async toArray() {
     const rows = [];
     for await (const row of this.rows())
@@ -5243,7 +5319,7 @@ var QueryResult = class {
     this.stats.elapsedMs = config.now() - startedAt;
     return rows.map((row) => project(row, config.select, void 0));
   }
-  async materializeRowRefs(refs, columns, rankColumns, startedAt) {
+  async materializeRowRefs(refs, columns, rankColumns, startedAt, maxWindowRows = columnarBatchSize(this.config.batchSize)) {
     const scanVectorBatches = vectorBatchScanner(this.config.scanner);
     if (scanVectorBatches === void 0)
       return /* @__PURE__ */ new Map();
@@ -5253,7 +5329,7 @@ var QueryResult = class {
     const lateColumns = columns.filter((column) => !rankColumns.includes(column));
     if (lateColumns.length === 0)
       return rows;
-    for (const window of materializationWindows(refs, columnarBatchSize(this.config.batchSize))) {
+    for (const window of materializationWindows(refs, maxWindowRows)) {
       const scanOptions = {
         columns: lateColumns,
         rowStart: window.rowStart,
@@ -6701,6 +6777,24 @@ function projectedReadColumns(select, where, orderBy = void 0, projections = voi
   collectExprColumns(where, columns);
   return columns.size === 0 ? void 0 : [...columns].sort();
 }
+function predicateReadColumns(where) {
+  const columns = /* @__PURE__ */ new Set();
+  collectExprColumns(where, columns);
+  return [...columns].sort();
+}
+function outputReadColumns(select, projections) {
+  const columns = /* @__PURE__ */ new Set();
+  for (const column of select)
+    columns.add(column);
+  for (const expr of Object.values(projections ?? {}))
+    collectExprColumns(expr, columns);
+  return [...columns].sort();
+}
+async function* rowsAsBatches(rows, batchSize) {
+  for (let index = 0; index < rows.length; index += batchSize) {
+    yield rows.slice(index, index + batchSize);
+  }
+}
 function aggregateReadColumns(groupColumns, spec, where) {
   const columns = /* @__PURE__ */ new Set();
   for (const column of groupColumns)
@@ -7807,6 +7901,39 @@ function copyBytes2(data) {
 }
 
 // ../core/dist/store.js
+function uriObjectStore(store, authorities) {
+  const mappings = (Array.isArray(authorities) ? authorities : [authorities]).map(normalizeUriAuthority);
+  const wrapped = {
+    async get(path) {
+      return await store.get(normalizeObjectStorePath(path, mappings));
+    },
+    async getRange(path, range) {
+      return await store.getRange(normalizeObjectStorePath(path, mappings), range);
+    },
+    async put(path, body, options) {
+      return await store.put(normalizeObjectStorePath(path, mappings), body, options);
+    },
+    async delete(path) {
+      return await store.delete(normalizeObjectStorePath(path, mappings));
+    },
+    async *list(prefix, options) {
+      yield* store.list(normalizeObjectStorePath(prefix, mappings), options);
+    },
+    async head(path) {
+      return await store.head(normalizeObjectStorePath(path, mappings));
+    }
+  };
+  if (isConditionalObjectStore(store)) {
+    const conditional = {
+      ...wrapped,
+      async conditionalPut(path, body, options) {
+        return await store.conditionalPut(normalizeObjectStorePath(path, mappings), body, options);
+      }
+    };
+    return conditional;
+  }
+  return wrapped;
+}
 function withObjectStoreReadControls(store, controls) {
   if (controls.maxConcurrentReads === void 0 && controls.signal === void 0 && controls.maxElapsedMs === void 0) {
     return store;
@@ -7876,6 +8003,97 @@ async function controlledRead(controls, semaphore, read) {
     release?.();
   }
 }
+function normalizeUriAuthority(mapping) {
+  const scheme = requiredUriPart(mapping.scheme, "scheme").toLowerCase();
+  const authority = requiredUriPart(mapping.authority, "authority");
+  const prefix = trimSlashes(mapping.prefix ?? "");
+  return { scheme, authority, prefix };
+}
+function normalizeObjectStorePath(path, mappings) {
+  const maybeUri = absoluteObjectStoreUri(path);
+  if (maybeUri === void 0)
+    return path;
+  const match = mappings.find((mapping) => mapping.scheme === maybeUri.scheme && mapping.authority === maybeUri.authority);
+  if (match === void 0) {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Object URI does not match configured store", {
+      path,
+      scheme: maybeUri.scheme,
+      authority: maybeUri.authority,
+      configuredAuthorities: mappings.map((mapping) => `${mapping.scheme}://${mapping.authority}`)
+    });
+  }
+  const key = maybeUri.path;
+  if (match.prefix === "")
+    return key;
+  if (key === match.prefix)
+    return "";
+  const prefix = `${match.prefix}/`;
+  if (key.startsWith(prefix))
+    return key.slice(prefix.length);
+  throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Object URI is outside configured prefix", {
+    path,
+    prefix: match.prefix
+  });
+}
+function absoluteObjectStoreUri(path) {
+  if (!/^[A-Za-z][A-Za-z0-9+.-]*:\/\//u.test(path))
+    return void 0;
+  rejectPathEscape(rawUriPath(path), path);
+  let url;
+  try {
+    url = new URL(path);
+  } catch (cause) {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Invalid object URI", { path, cause });
+  }
+  const objectPath = trimLeadingSlash(url.pathname);
+  rejectPathEscape(objectPath, path);
+  return {
+    scheme: url.protocol.slice(0, -1).toLowerCase(),
+    authority: url.host,
+    path: objectPath
+  };
+}
+function rejectPathEscape(objectPath, original) {
+  const decoded = objectPath.split("/").map((segment) => {
+    try {
+      return decodeURIComponent(segment);
+    } catch {
+      return segment;
+    }
+  }).join("/");
+  if (decoded === ".." || decoded.startsWith("../") || decoded.includes("/../") || decoded.endsWith("/..")) {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "Object URI path may not escape its store", {
+      path: original
+    });
+  }
+}
+function rawUriPath(uri) {
+  const authorityStart = uri.indexOf("://") + 3;
+  const pathStart = uri.indexOf("/", authorityStart);
+  if (pathStart === -1)
+    return "";
+  const queryStart = uri.indexOf("?", pathStart);
+  const fragmentStart = uri.indexOf("#", pathStart);
+  const endCandidates = [queryStart, fragmentStart].filter((index) => index !== -1);
+  const pathEnd = endCandidates.length === 0 ? uri.length : Math.min(...endCandidates);
+  return trimLeadingSlash(uri.slice(pathStart, pathEnd));
+}
+function requiredUriPart(value, name) {
+  const trimmed = value.trim();
+  if (trimmed === "") {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Object URI ${name} is required`);
+  }
+  return trimmed;
+}
+function trimSlashes(value) {
+  return value.replace(/^\/+/u, "").replace(/\/+$/u, "");
+}
+function trimLeadingSlash(value) {
+  return value.replace(/^\/+/u, "");
+}
+function isConditionalObjectStore(store) {
+  return typeof store.conditionalPut === "function";
+}
 async function* controlledList(store, prefix, options, controls) {
   throwIfAborted(controls.signal);
   for await (const object of store.list(prefix, options)) {
@@ -8365,23 +8583,27 @@ function timestampStatsValue(value) {
 }
 
 // ../parquet/dist/schema.js
-function rejectUnsupportedParquetSchema(metadata) {
+function rejectUnsupportedParquetSchema(metadata, options = {}) {
   const schema = metadata.schema;
   if (!Array.isArray(schema) || schema.length === 0)
     return;
+  const selected = options.columns === void 0 ? void 0 : new Set(options.columns);
   const root = schema[0];
   const childCount = schemaChildCount(root);
   let index = 1;
   for (let child = 0; child < childCount && index < schema.length; child += 1) {
-    index = rejectUnsupportedParquetSchemaNode(schema, index, []);
+    index = rejectUnsupportedParquetSchemaNode(schema, index, [], selected);
   }
 }
-function rejectUnsupportedParquetSchemaNode(schema, index, path) {
+function rejectUnsupportedParquetSchemaNode(schema, index, path, selected) {
   const element = schema[index];
   if (element === void 0)
     return index + 1;
   const name = String(element.name ?? `field_${index}`);
   const nodePath = [...path, name];
+  if (selected !== void 0 && path.length === 0 && !selected.has(name)) {
+    return skipParquetSchemaSubtree(schema, index);
+  }
   const childCount = schemaChildCount(element);
   rejectUnsupportedParquetLeaf(element, nodePath);
   if (childCount === 0)
@@ -8551,6 +8773,11 @@ function bytesToArrayBuffer2(bytes) {
   new Uint8Array(out).set(bytes);
   return out;
 }
+var lakeqlParquetCompressors = {
+  ZSTD(input, outputLength) {
+    return decompress(input, new Uint8Array(outputLength));
+  }
+};
 
 // ../parquet/dist/decoded-column-cache.js
 var DecodedColumnCache = class {
@@ -8802,6 +9029,7 @@ async function readParquetColumnBatch(file, metadata, columns, rowStart, rowEnd)
     columns,
     rowStart,
     rowEnd,
+    compressors: lakeqlParquetCompressors,
     parsers: lakeqlParquetParsers,
     onChunk(chunk) {
       appendColumnChunk(columnValues, chunk, rowStart, rowEnd);
@@ -9017,7 +9245,7 @@ async function* readParquetVectorBatchesFromFile(file, metadata, options) {
       rowGroupStart = rowGroupEnd;
       continue;
     }
-    const vectorSources = columnVectorSources(file, metadata, rowGroup, columns, rowGroupStart, Math.max(rowGroupStart, requestedStart), Math.min(rowGroupEnd, requestedEnd), options);
+    const vectorSources = columnVectorSources(file, metadata, rowGroup, columns, rowGroupStart, rowGroupEnd, Math.max(rowGroupStart, requestedStart), Math.min(rowGroupEnd, requestedEnd), options);
     if (vectorSources === void 0)
       return;
     recordRowGroupRead(options.stats);
@@ -9110,14 +9338,14 @@ function canRepresentDirectVectorLeaf(leaf) {
 function usesDictionaryEncoding(column) {
   return column.encodings?.some(isDictionaryEncoding) === true || column.encoding_stats?.some((stats) => isDictionaryEncoding(stats.encoding)) === true;
 }
-function columnVectorSources(file, metadata, rowGroup, columns, rowGroupStart, requestedStart, requestedEnd, options) {
+function columnVectorSources(file, metadata, rowGroup, columns, rowGroupStart, rowGroupEnd, requestedStart, requestedEnd, options) {
   const sources = [];
   for (const column of columns) {
     const metadataForColumn = directLeafColumnMetadata(rowGroup, column);
     if (metadataForColumn !== void 0 && canDirectVector(metadata, metadataForColumn)) {
       sources.push({
         columns: [column],
-        iterator: readColumnVectorBatches(file, metadata, metadataForColumn, column, rowGroupStart, requestedStart, requestedEnd, options)[Symbol.asyncIterator]()
+        iterator: readColumnVectorBatches(file, metadata, metadataForColumn, column, rowGroupStart, rowGroupEnd, requestedStart, requestedEnd, options)[Symbol.asyncIterator]()
       });
       continue;
     }
@@ -9204,11 +9432,15 @@ async function* readNestedColumnVectorBatches(file, metadata, column, requestedS
     yield { rowOffset: rowStart, batch };
   }
 }
-async function* readColumnVectorBatches(file, metadata, columnMetadata, column, rowGroupStart, requestedStart, requestedEnd, options) {
+async function* readColumnVectorBatches(file, metadata, columnMetadata, column, rowGroupStart, rowGroupEnd, requestedStart, requestedEnd, options) {
   const chunkStart = safeNumber2(columnMetadata.dictionary_page_offset ?? columnMetadata.data_page_offset);
   const compressedSize = safeNumber2(columnMetadata.total_compressed_size);
   if (chunkStart === void 0 || compressedSize === void 0)
     return;
+  if (options.canStopEarly === true || requestedStart > rowGroupStart || requestedEnd < rowGroupEnd) {
+    yield* readColumnWindowVectorBatches(file, metadata, columnMetadata, column, rowGroupStart, requestedStart, requestedEnd, chunkStart, chunkStart + compressedSize, options);
+    return;
+  }
   const buffer = await file.slice(chunkStart, chunkStart + compressedSize);
   const reader = { view: new DataView(buffer), offset: 0 };
   const schemaPath = getSchemaPath(metadata.schema, columnMetadata.path_in_schema);
@@ -9220,6 +9452,7 @@ async function* readColumnVectorBatches(file, metadata, columnMetadata, column,
     element: leaf.element,
     schemaPath,
     parsers: { ...DEFAULT_PARSERS, ...lakeqlParquetParsers },
+    compressors: lakeqlParquetCompressors,
     ...columnMetadata
   };
   let dictionary;
@@ -9284,6 +9517,117 @@ async function* readColumnVectorBatches(file, metadata, columnMetadata, column,
     pageRowStart = pageRowEnd;
   }
 }
+async function* readColumnWindowVectorBatches(file, metadata, columnMetadata, column, rowGroupStart, requestedStart, requestedEnd, chunkStart, chunkEnd, options) {
+  const schemaPath = getSchemaPath(metadata.schema, columnMetadata.path_in_schema);
+  const leaf = schemaPath[schemaPath.length - 1];
+  if (leaf === void 0)
+    return;
+  const columnDecoder = {
+    pathInSchema: columnMetadata.path_in_schema,
+    element: leaf.element,
+    schemaPath,
+    parsers: { ...DEFAULT_PARSERS, ...lakeqlParquetParsers },
+    compressors: lakeqlParquetCompressors,
+    ...columnMetadata
+  };
+  let dictionary;
+  let pageRowStart = rowGroupStart;
+  let offset = 0;
+  while (chunkStart + offset < chunkEnd && pageRowStart < requestedEnd) {
+    const page = await readPageWindow(file, chunkStart, chunkEnd, offset);
+    if (page === void 0)
+      return;
+    offset += page.headerBytes + page.header.compressed_page_size;
+    if (page.header.type === "DICTIONARY_PAGE") {
+      dictionary = dictionaryPageValues(page.compressedBytes, page.header, columnDecoder, column, rowGroupStart, page.bodyOffset, file, options);
+      continue;
+    }
+    const rowCount = dataPageRowCount(page.header);
+    if (rowCount === void 0)
+      continue;
+    const pageRowEnd = pageRowStart + rowCount;
+    if (pageRowEnd <= requestedStart || pageRowStart >= requestedEnd) {
+      pageRowStart = pageRowEnd;
+      continue;
+    }
+    const start = Math.max(pageRowStart, requestedStart);
+    const end = Math.min(pageRowEnd, requestedEnd);
+    if (start < end) {
+      const cache = options.decodedColumnCache;
+      const key = cache === void 0 || options.decodedColumnCacheKey === void 0 ? void 0 : decodedColumnPageCacheKey({
+        path: options.decodedColumnCacheKey,
+        byteLength: file.byteLength,
+        ...file.etag === void 0 ? {} : { etag: file.etag },
+        column,
+        rowGroupStart,
+        pageRowStart,
+        pageRowEnd,
+        pageOffset: page.bodyOffset,
+        compressedPageSize: page.header.compressed_page_size
+      });
+      const cached = key === void 0 || cache === void 0 ? void 0 : cache.getVector(key);
+      let vector;
+      if (cached !== void 0) {
+        vector = cached;
+      } else {
+        const values = dataPageValues(page.compressedBytes, page.header, columnDecoder, dictionary);
+        if (values === void 0)
+          continue;
+        vector = flatPageVector(values.values, values.definitionLevels, 0, rowCount, values.dictionary);
+        if (key !== void 0 && cache !== void 0)
+          cache.setVector(key, vector);
+      }
+      if (key !== void 0 && options.stats !== void 0) {
+        if (cached === void 0)
+          options.stats.cacheMisses += 1;
+        else
+          options.stats.cacheHits += 1;
+      }
+      yield {
+        rowOffset: start,
+        batch: batchFromVectors({
+          [column]: sliceVector(vector, start - pageRowStart, end - pageRowStart)
+        })
+      };
+    }
+    pageRowStart = pageRowEnd;
+  }
+}
+async function readPageWindow(file, chunkStart, chunkEnd, offset) {
+  const absoluteOffset = chunkStart + offset;
+  if (absoluteOffset >= chunkEnd)
+    return void 0;
+  const header = await readPageHeader(file, absoluteOffset, chunkEnd);
+  const bodyOffset = absoluteOffset + header.headerBytes;
+  const bodyEnd = bodyOffset + header.header.compressed_page_size;
+  if (bodyEnd > chunkEnd)
+    return void 0;
+  const body = await file.slice(bodyOffset, bodyEnd);
+  return {
+    ...header,
+    bodyOffset,
+    compressedBytes: new Uint8Array(body)
+  };
+}
+async function readPageHeader(file, absoluteOffset, chunkEnd) {
+  let size = 256;
+  let lastError;
+  while (absoluteOffset + size <= chunkEnd || size === 256) {
+    const end = Math.min(chunkEnd, absoluteOffset + size);
+    const bytes = await file.slice(absoluteOffset, end);
+    const reader = { view: new DataView(bytes), offset: 0 };
+    try {
+      const header = parquetHeader(reader);
+      return { header, headerBytes: reader.offset };
+    } catch (cause) {
+      lastError = cause;
+      if (end === chunkEnd)
+        break;
+      size *= 4;
+    }
+  }
+  throw lastError instanceof Error ? lastError : new Error("Unable to read Parquet page header");
+}
 function dataPageRowCount(header) {
   if (header.type === "DATA_PAGE")
     return header.data_page_header?.num_values;
@@ -9312,7 +9656,7 @@ function dictionaryPageValues(compressedBytes, header, columnDecoder, column, ro
       options.stats.cacheHits += 1;
     return cached;
   }
-  const page = decompressPage(compressedBytes, Number(header.uncompressed_page_size), columnDecoder.codec, void 0);
+  const page = decompressPage(compressedBytes, Number(header.uncompressed_page_size), columnDecoder.codec, columnDecoder.compressors);
   const pageReader = {
     view: new DataView(page.buffer, page.byteOffset, page.byteLength),
     offset: 0
@@ -9330,7 +9674,7 @@ function dataPageValues(compressedBytes, header, columnDecoder, dictionary) {
     const dataHeader = header.data_page_header;
     if (dataHeader === void 0)
       return void 0;
-    const page = decompressPage(compressedBytes, Number(header.uncompressed_page_size), columnDecoder.codec, void 0);
+    const page = decompressPage(compressedBytes, Number(header.uncompressed_page_size), columnDecoder.codec, columnDecoder.compressors);
     const { definitionLevels, dataPage } = readDataPage(page, dataHeader, columnDecoder);
     const compactDataPage = compactPresentValues(dataPage);
     const pageDictionary = dictionary !== void 0 && isDictionaryEncoding(dataHeader.encoding) ? dictionary : void 0;
@@ -9678,7 +10022,7 @@ var ParquetScanAdapter = class {
     const batchSize = options.batchSize || this.defaultBatchSize;
     const file = this.scanBuffer(path, await asyncBufferFromStore(this.store, path, options));
     const metadata = await this.metadata(path, file, options);
-    rejectUnsupportedParquetSchema(metadata);
+    rejectUnsupportedParquetSchema(metadata, { columns: options.columns });
     const readColumns = options.columns;
     if (readColumns) {
       recordReadColumns(options.stats, readColumns);
@@ -9702,6 +10046,7 @@ var ParquetScanAdapter = class {
           rowFormat: "object",
           rowStart,
           rowEnd,
+          compressors: lakeqlParquetCompressors,
           parsers: lakeqlParquetParsers
         };
         if (readColumns)
@@ -9729,7 +10074,7 @@ var ParquetScanAdapter = class {
   async *scanVectorBatches(path, options) {
     const file = this.scanBuffer(path, await asyncBufferFromStore(this.store, path, options));
     const metadata = await this.metadata(path, file, options);
-    rejectUnsupportedParquetSchema(metadata);
+    rejectUnsupportedParquetSchema(metadata, { columns: options.columns });
     try {
       const vectorOptions = {
         batchSize: options.batchSize || this.defaultBatchSize,
@@ -9737,6 +10082,7 @@ var ParquetScanAdapter = class {
         ...options.rowEnd === void 0 ? {} : { rowEnd: options.rowEnd },
         ...options.columns === void 0 ? {} : { columns: options.columns },
         ...options.where === void 0 ? {} : { where: options.where },
+        ...options.canStopEarly === void 0 ? {} : { canStopEarly: options.canStopEarly },
         ...this.decodedColumnCache === void 0 ? {} : {
           decodedColumnCache: this.decodedColumnCache,
           decodedColumnCacheKey: path
@@ -9756,6 +10102,7 @@ var ParquetScanAdapter = class {
         ...options.rowEnd === void 0 ? {} : { rowEnd: options.rowEnd },
         ...options.columns === void 0 ? {} : { columns: options.columns },
         ...options.where === void 0 ? {} : { where: options.where },
+        ...options.canStopEarly === void 0 ? {} : { canStopEarly: options.canStopEarly },
         ...this.decodedColumnCache === void 0 ? {} : {
           decodedColumnCache: this.decodedColumnCache,
           decodedColumnCacheKey: path
@@ -10441,7 +10788,7 @@ async function* readParquetObjectBatches(store, path, options = {}) {
   const file = await asyncBufferFromStore(store, path);
   try {
     const metadata = await readParquetMetadataFromFile(file);
-    rejectUnsupportedParquetSchema(metadata);
+    rejectUnsupportedParquetSchema(metadata, { columns: options.columns });
     yield* readParquetObjectBatchesFromFile(file, metadata, options);
   } catch (cause) {
     if (cause instanceof LakeqlError)
@@ -10453,7 +10800,7 @@ async function* readParquetColumnBatches(store, path, options = {}) {
   const file = await asyncBufferFromStore(store, path);
   try {
     const metadata = await readParquetMetadataFromFile(file);
-    rejectUnsupportedParquetSchema(metadata);
+    rejectUnsupportedParquetSchema(metadata, { columns: options.columns });
     yield* readParquetColumnBatchesFromFile(file, metadata, options);
   } catch (cause) {
     if (cause instanceof LakeqlError)
@@ -11031,4 +11378,4 @@ function createParquetLake(config) {
   });
 }
 
-export { AggregationBuilder, CacheApiCache, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, SharedMemoryCache, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, concatBatches, createBookmark, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, enforceVectorGroupByBudget, eq, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, ilike, isIn, isNotNull, isNull, jsonWorkUnitBoundary, like, lit, lookupJoin, lt, lte, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanParquetTaskBatches, scanParquetTaskColumnBatches, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };
+export { AggregationBuilder, CacheApiCache, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, SharedMemoryCache, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, concatBatches, createBookmark, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, enforceVectorGroupByBudget, eq, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, ilike, isIn, isNotNull, isNull, jsonWorkUnitBoundary, like, lit, lookupJoin, lt, lte, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanParquetTaskBatches, scanParquetTaskColumnBatches, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, uriObjectStore, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask };

package/dist/cloudflare.d.ts

@@ -1,8 +1,24 @@
-import { ObjectStore } from './index.js';
-export { AggregateExpr, AggregateGroupSnapshot, AggregateOp, AggregateOperatorState, AggregateOptions, AggregateParquetGroupTaskOptions, AggregateParquetGroupTasksOptions, AggregateParquetTaskOptions, AggregateParquetTasksOptions, AggregateResult, AggregateSnapshotValue, AggregateSpec, AggregateStateSnapshot, AggregationBuilder, ApplyIcebergDeletesOptions, ArithmeticExpr, BBox, BBoxIndex, Batch, BatchExprValues, BetweenExpr, Bookmark, BookmarkInit, BookmarkPosition, BookmarkQuery, BroadcastJoinOptions, CacheAdapter, CacheApiCache, CacheApiCacheOptions, CacheEntry, CachePolicy, CallExpr, CaseExpr, CaseWhenExpr, CheckpointAdapter, CheckpointStore, Clock, ColumnExpr, ColumnInput, CompareExpr, CompareOp, ConditionalObjectStore, ConditionalPutOptions, CreateParquetTableAsOptions, CreateParquetTableAsQuery, CreateParquetTableAsResult, CsvStreamOptions, DecodedIcebergDeletes, DecodedIcebergParquetDeletes, ERROR_CODES, EngineFilePlan, EngineTable, ErrorDetails, ExplainJson, ExplainResult, Expr, GeoBackend, GeoJsonGeometry, IcebergAppendFile, IcebergAppendOptions, IcebergAppendOutputManifestOptions, IcebergAppendResult, IcebergCatalog, IcebergCommitCatalog, IcebergCommitInput, IcebergCommitResult, IcebergDeleteFile, IcebergDeletionVector, IcebergEnginePlan, IcebergEngineTable, IcebergEqualityDelete, IcebergField, IcebergGlueCatalogOptions, IcebergNessieCatalogOptions, IcebergParquetDeleteFile, IcebergParquetDeleteFileContent, IcebergPartitionField, IcebergPartitionSpec, IcebergPlan, IcebergPositionDelete, IcebergReadMode, IcebergRestCatalog, IcebergRestCatalogOptions, IcebergRowBatch, IcebergSortOrder, IcebergTable, IcebergTableIdentifier, IcebergUnsupportedCatalog, IdGenerator, InExpr, InMemoryLakeOptions, InMemoryRowsScanner, InMemoryRowsStore, InMemoryTableOptions, IndexPruneResult, IndexValue, InsertValidationRules, JoinKey, JoinType, JsonExpr, JsonOrderByTerm, JsonQueryV1, Lake, LakeConfig, LakeqlError, LakeqlErrorCode, LikeExpr, ListOptions, LiteralExpr, LoadIcebergEngineTableOptions, LoadIcebergTableFromObjectStoreOptions, LoadIcebergTableFromRestOptions, LoadIcebergTableOptions, LoadParquetEngineTableOptions, LoadTableOptions, LockAdapter, LogHook, LogicalExpr, LookupJoinFunction, LookupJoinOptions, Manifest, ManifestDeleteFile, ManifestFile, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, MemorySpillAdapterOptions, MetadataFile, MetricsHook, MinMaxColumnIndex, NotExpr, NullCheckExpr, ObjectHead, ObjectInfo, ObjectStoreCacheOptions, ObjectStoreIcebergCommitCatalog, ObjectStoreReadControls, OperatorSnapshotValue, OrderByTerm, OutputManifest, OutputManifestEntry, PACKAGE, ParquetColumnBatch, ParquetEnginePlan, ParquetEngineTable, ParquetLakeConfig, ParquetMetadata, ParquetRowBatch, ParquetRowGroupPlan, ParquetScanAdapter, PartitionedParquetOutputEntryOptions, PathQueryInit, PlanIcebergFilesOptions, PlanParquetRowGroupsOptions, PlanParquetTaskWorkUnitsOptions, PlannedIcebergFile, PlannedParquetRowGroup, PredicatePlan, PredicatePlanOptions, ProjectIcebergRowOptions, PutOptions, QueryBudget, QueryBuilder, QueryPolicy, QueryPolicyContext, QueryResult, QueryRunOptions, QueryStats, QueueAdapter, ReadParquetBatchOptions, ReadParquetOptions, ResumableBatchOptions, ResumedQuery, Row, RuntimeSubstrate, Scalar, ScanAdapter, ScanBatch, ScanColumnBatch, ScanEngineOptions, ScanOptions, ScanParquetTaskOptions, ScanPlannedIcebergRowsOptions, ScanTaskPlan, ScanTaskPlanOptions, ScanVectorBatch, Selection, SharedCacheEntry, SharedCacheSetOptions, SharedMemoryCache, SidecarFileIndex, SliceOptions, SliceResult, Snapshot, SortOperatorState, SortOptions, SortResult, SortRunState, SpillAdapter, SpillRef, SpillUsage, SqlBoolean, TaskCheckpoint, TaskInput, TaskManifest, TaskManifestTask, TaskState, TaskTransitionInput, TimestampUnit, TimestampValue, TopKOperatorState, TopKOptions, TopKResult, ValueInput, Vector, VectorAggregateOptions, VectorAggregateSnapshotValue, VectorAggregateState, VectorAggregateStateSnapshot, VectorAggregateStateSnapshots, VectorAggregateStates, VectorAggregateValue, VectorGroup, VectorGroupByGroupSnapshot, VectorGroupByOptions, VectorGroupBySnapshotValue, VectorGroupByState, VectorGroupByStateSnapshot, VectorHashJoinOptions, VectorProjectionSpec, VectorTopKOptions, WorkUnitFanInOptions, WriteParquetOptions, WriteParquetRowsOptions, WritePartitionedParquetFile, WritePartitionedParquetResult, WritePartitionedParquetTaskOptions, WritePartitionedParquetTaskResult, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, applyIcebergDeletes, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxIntersects, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, compareTimestampValues, concatBatches, createBookmark, createInMemoryLake, createLake, createOutputManifest, createOutputManifestFromCheckpoints, createLake as createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, encodeJsonLine, enforceVectorGroupByBudget, ensureGeoBackendForExprs, envelopeFromGeometry, envelopeOf, eq, evaluate, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, inMemoryRowsScanner, isIn, isLakeqlError, isNotNull, isNull, isTimestampValue, jsonSafeValue, jsonWorkUnitBoundary, like, lit, loadGeoBackend, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, or, parquetScanner, parseGeometry, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planFiles, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, requireGeoBackend, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanBatches, scanParquetTaskBatches, scanParquetTaskColumnBatches, scanPlannedIcebergRows, scanRows, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, setGeoBackend, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, timestampEpochForUnit, timestampFromEpoch, timestampToIsoString, timestampValueFromIso, toGeometry, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorHashJoin, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask } from './index.js';
+import { CacheAdapter, ObjectStore } from './index.js';
+export { AggregateExpr, AggregateGroupSnapshot, AggregateOp, AggregateOperatorState, AggregateOptions, AggregateParquetGroupTaskOptions, AggregateParquetGroupTasksOptions, AggregateParquetTaskOptions, AggregateParquetTasksOptions, AggregateResult, AggregateSnapshotValue, AggregateSpec, AggregateStateSnapshot, AggregationBuilder, ApplyIcebergDeletesOptions, ArithmeticExpr, BBox, BBoxIndex, Batch, BatchExprValues, BetweenExpr, Bookmark, BookmarkInit, BookmarkPosition, BookmarkQuery, BroadcastJoinOptions, CacheApiCache, CacheApiCacheOptions, CacheEntry, CachePolicy, CallExpr, CaseExpr, CaseWhenExpr, CheckpointAdapter, CheckpointStore, Clock, ColumnExpr, ColumnInput, CompareExpr, CompareOp, ConditionalObjectStore, ConditionalPutOptions, CreateParquetTableAsOptions, CreateParquetTableAsQuery, CreateParquetTableAsResult, CsvStreamOptions, DecodedIcebergDeletes, DecodedIcebergParquetDeletes, ERROR_CODES, EngineFilePlan, EngineTable, ErrorDetails, ExplainJson, ExplainResult, Expr, GeoBackend, GeoJsonGeometry, IcebergAppendFile, IcebergAppendOptions, IcebergAppendOutputManifestOptions, IcebergAppendResult, IcebergCatalog, IcebergCommitCatalog, IcebergCommitInput, IcebergCommitResult, IcebergDeleteFile, IcebergDeletionVector, IcebergEnginePlan, IcebergEngineTable, IcebergEqualityDelete, IcebergField, IcebergGlueCatalogOptions, IcebergLoadTableOptions, IcebergNessieCatalogOptions, IcebergParquetDeleteFile, IcebergParquetDeleteFileContent, IcebergPartitionField, IcebergPartitionSpec, IcebergPlan, IcebergPositionDelete, IcebergReadMode, IcebergRestAccessDelegation, IcebergRestCatalog, IcebergRestCatalogConfig, IcebergRestCatalogOptions, IcebergRestLoadContext, IcebergRestLoadTableOptions, IcebergRestLoadTableResult, IcebergRestStorageCredential, IcebergRowBatch, IcebergSortOrder, IcebergTable, IcebergTableIdentifier, IcebergUnsupportedCatalog, IdGenerator, InExpr, InMemoryLakeOptions, InMemoryRowsScanner, InMemoryRowsStore, InMemoryTableOptions, IndexPruneResult, IndexValue, InsertValidationRules, JoinKey, JoinType, JsonExpr, JsonOrderByTerm, JsonQueryV1, Lake, LakeConfig, LakeqlError, LakeqlErrorCode, LikeExpr, ListOptions, LiteralExpr, LoadIcebergEngineTableOptions, LoadIcebergTableFromObjectStoreOptions, LoadIcebergTableFromRestOptions, LoadIcebergTableOptions, LoadParquetEngineTableOptions, LoadTableOptions, LockAdapter, LogHook, LogicalExpr, LookupJoinFunction, LookupJoinOptions, Manifest, ManifestDeleteFile, ManifestFile, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, MemorySpillAdapterOptions, MetadataFile, MetricsHook, MinMaxColumnIndex, NotExpr, NullCheckExpr, ObjectHead, ObjectInfo, ObjectStoreCacheOptions, ObjectStoreIcebergCommitCatalog, ObjectStoreJsonCache, ObjectStoreJsonCacheOptions, ObjectStoreReadControls, ObjectStoreUriAuthority, OperatorSnapshotValue, OrderByTerm, OutputManifest, OutputManifestEntry, PACKAGE, ParquetColumnBatch, ParquetEnginePlan, ParquetEngineTable, ParquetLakeConfig, ParquetMetadata, ParquetRowBatch, ParquetRowGroupPlan, ParquetScanAdapter, PartitionedParquetOutputEntryOptions, PathQueryInit, PlanIcebergFilesOptions, PlanParquetRowGroupsOptions, PlanParquetTaskWorkUnitsOptions, PlannedIcebergFile, PlannedParquetRowGroup, PredicatePlan, PredicatePlanOptions, ProjectIcebergRowOptions, PutOptions, QueryBudget, QueryBuilder, QueryPolicy, QueryPolicyContext, QueryResult, QueryRunOptions, QueryStats, QueueAdapter, ReadParquetBatchOptions, ReadParquetOptions, ResumableBatchOptions, ResumedQuery, Row, RuntimeSubstrate, Scalar, ScanAdapter, ScanBatch, ScanColumnBatch, ScanEngineOptions, ScanOptions, ScanParquetTaskOptions, ScanPlannedIcebergRowsOptions, ScanTaskPlan, ScanTaskPlanOptions, ScanVectorBatch, Selection, SharedCacheEntry, SharedCacheSetOptions, SharedMemoryCache, SidecarFileIndex, SliceOptions, SliceResult, Snapshot, SortOperatorState, SortOptions, SortResult, SortRunState, SpillAdapter, SpillRef, SpillUsage, SqlBoolean, TaskCheckpoint, TaskInput, TaskManifest, TaskManifestTask, TaskState, TaskTransitionInput, TimestampUnit, TimestampValue, TopKOperatorState, TopKOptions, TopKResult, ValueInput, Vector, VectorAggregateOptions, VectorAggregateSnapshotValue, VectorAggregateState, VectorAggregateStateSnapshot, VectorAggregateStateSnapshots, VectorAggregateStates, VectorAggregateValue, VectorGroup, VectorGroupByGroupSnapshot, VectorGroupByOptions, VectorGroupBySnapshotValue, VectorGroupByState, VectorGroupByStateSnapshot, VectorHashJoinOptions, VectorProjectionSpec, VectorTopKOptions, WorkUnitFanInOptions, WriteParquetOptions, WriteParquetRowsOptions, WritePartitionedParquetFile, WritePartitionedParquetResult, WritePartitionedParquetTaskOptions, WritePartitionedParquetTaskResult, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, applyIcebergDeletes, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxIntersects, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, compareTimestampValues, concatBatches, createBookmark, createInMemoryLake, createLake, createOutputManifest, createOutputManifestFromCheckpoints, createLake as createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, encodeJsonLine, enforceVectorGroupByBudget, ensureGeoBackendForExprs, envelopeFromGeometry, envelopeOf, eq, evaluate, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, inMemoryRowsScanner, isIn, isLakeqlError, isNotNull, isNull, isTimestampValue, jsonSafeValue, jsonWorkUnitBoundary, like, lit, loadGeoBackend, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, objectStoreJsonCache, or, parquetScanner, parseGeometry, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planFiles, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, requireGeoBackend, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanBatches, scanParquetTaskBatches, scanParquetTaskColumnBatches, scanPlannedIcebergRows, scanRows, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, setGeoBackend, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, timestampEpochForUnit, timestampFromEpoch, timestampToIsoString, timestampValueFromIso, toGeometry, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, uriObjectStore, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorHashJoin, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask } from './index.js';
 import 'hyparquet-writer';
 import 'hyparquet';
 
+interface D1DatabaseLike {
+    prepare(query: string): D1PreparedStatementLike;
+}
+interface D1PreparedStatementLike {
+    bind(...values: unknown[]): D1PreparedStatementLike;
+    first<T = unknown>(column?: string): Promise<T | null>;
+    run(): Promise<unknown>;
+}
+interface CloudflareD1JsonCacheOptions {
+    db: D1DatabaseLike;
+    table?: string;
+    prefix?: string;
+    ttlMs?: number;
+    now?: () => number;
+    createTable?: boolean;
+}
 interface R2ObjectBody {
     key: string;
     size: number;
@@ -39,5 +55,6 @@ interface R2BucketLike {
     }>;
 }
 declare function r2Store(bucket: R2BucketLike): ObjectStore;
+declare function cloudflareD1JsonCache<T = unknown>(options: CloudflareD1JsonCacheOptions): CacheAdapter<T>;
 
-export { ObjectStore, r2Store };
+export { CacheAdapter, ObjectStore, cloudflareD1JsonCache, r2Store };