lakeql 0.1.2 → 0.1.8

package/dist/index.js

@@ -1,2 +1,3 @@
-export { IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, ObjectStoreIcebergCommitCatalog, PACKAGE, applyIcebergDeletes, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, planFiles, scanBatches, scanPlannedIcebergRows, scanRows } from './chunk-MUTXXFOW.js';
-export { AggregationBuilder, CacheApiCache, ERROR_CODES, LaQLError, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, add, advanceTaskCheckpoint, and, assertBookmarkMatches, asyncBufferFromStore, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, classifyPredicate, col, createBookmark, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, encodeJsonLine, eq, evaluate, fingerprint, fn, gt, gte, ilike, isIn, isLaQLError, isNotNull, isNull, jsonSafeValue, like, lit, lookupJoin, lt, lte, matches, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planRowGroups, planRowGroupsFromMetadata, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, rowGroupMayMatch, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask } from './chunk-D3A4VN3U.js';
+export { IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, InMemoryRowsScanner, InMemoryRowsStore, ObjectStoreIcebergCommitCatalog, ObjectStoreJsonCache, PACKAGE, applyIcebergDeletes, createInMemoryLake, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, inMemoryRowsScanner, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, objectStoreJsonCache, planFiles, scanBatches, scanPlannedIcebergRows, scanRows, vectorHashJoin } from './chunk-5K5JMJ2M.js';
+export { AggregationBuilder, CacheApiCache, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, SharedMemoryCache, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, concatBatches, createBookmark, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, enforceVectorGroupByBudget, eq, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, ilike, isIn, isNotNull, isNull, jsonWorkUnitBoundary, like, lit, lookupJoin, lt, lte, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanParquetTaskBatches, scanParquetTaskColumnBatches, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, uriObjectStore, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask } from './chunk-MXGEAVHL.js';
+export { ERROR_CODES, LakeqlError, TimestampValue, bboxIntersects, compareTimestampValues, encodeJsonLine, ensureGeoBackendForExprs, envelopeFromGeometry, envelopeOf, evaluate, isLakeqlError, isTimestampValue, jsonSafeValue, loadGeoBackend, matches, parseGeometry, requireGeoBackend, setGeoBackend, timestampEpochForUnit, timestampFromEpoch, timestampToIsoString, timestampValueFromIso, toGeometry } from './chunk-TFD5RFKB.js';

package/dist/node.d.ts

@@ -1,8 +1,16 @@
-import { ObjectInfo, ObjectStore, ConditionalObjectStore } from './index.js';
-export { AggregateExpr, AggregateGroupSnapshot, AggregateOp, AggregateOperatorState, AggregateOptions, AggregateResult, AggregateSnapshotValue, AggregateSpec, AggregateStateSnapshot, AggregationBuilder, ApplyIcebergDeletesOptions, ArithmeticExpr, BBoxIndex, BetweenExpr, Bookmark, BookmarkInit, BookmarkPosition, BookmarkQuery, BroadcastJoinOptions, CacheAdapter, CacheApiCache, CacheApiCacheOptions, CacheEntry, CallExpr, CaseExpr, CaseWhenExpr, CheckpointAdapter, CheckpointStore, Clock, ColumnExpr, ColumnInput, CompareExpr, CompareOp, ConditionalPutOptions, CreateParquetTableAsOptions, CreateParquetTableAsQuery, CreateParquetTableAsResult, CsvStreamOptions, DecodedIcebergDeletes, DecodedIcebergParquetDeletes, ERROR_CODES, EngineFilePlan, EngineTable, ErrorDetails, ExplainJson, ExplainResult, Expr, IcebergAppendFile, IcebergAppendOptions, IcebergAppendOutputManifestOptions, IcebergAppendResult, IcebergCatalog, IcebergCommitCatalog, IcebergCommitInput, IcebergCommitResult, IcebergDeleteFile, IcebergDeletionVector, IcebergEnginePlan, IcebergEngineTable, IcebergEqualityDelete, IcebergField, IcebergGlueCatalogOptions, IcebergNessieCatalogOptions, IcebergParquetDeleteFile, IcebergParquetDeleteFileContent, IcebergPartitionField, IcebergPartitionSpec, IcebergPlan, IcebergPositionDelete, IcebergReadMode, IcebergRestCatalog, IcebergRestCatalogOptions, IcebergRowBatch, IcebergSortOrder, IcebergTable, IcebergTableIdentifier, IcebergUnsupportedCatalog, IdGenerator, InExpr, IndexPruneResult, IndexValue, InsertValidationRules, JoinKey, JoinType, JsonExpr, JsonOrderByTerm, JsonQueryV1, LaQLError, LaQLErrorCode, Lake, LakeConfig, LikeExpr, ListOptions, LiteralExpr, LoadIcebergEngineTableOptions, LoadIcebergTableFromObjectStoreOptions, LoadIcebergTableFromRestOptions, LoadIcebergTableOptions, LoadParquetEngineTableOptions, LoadTableOptions, LockAdapter, LogHook, LogicalExpr, LookupJoinFunction, LookupJoinOptions, Manifest, ManifestDeleteFile, ManifestFile, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, MemorySpillAdapterOptions, MetadataFile, MetricsHook, MinMaxColumnIndex, NotExpr, NullCheckExpr, ObjectHead, ObjectStoreIcebergCommitCatalog, ObjectStoreReadControls, OperatorSnapshotValue, OrderByTerm, OutputManifest, OutputManifestEntry, PACKAGE, ParquetEnginePlan, ParquetEngineTable, ParquetLakeConfig, ParquetMetadata, ParquetRowBatch, ParquetRowGroupPlan, ParquetScanAdapter, PartitionedParquetOutputEntryOptions, PathQueryInit, PlanIcebergFilesOptions, PlanParquetRowGroupsOptions, PlannedIcebergFile, PlannedParquetRowGroup, PredicatePlan, PredicatePlanOptions, ProjectIcebergRowOptions, PutOptions, QueryBudget, QueryBuilder, QueryPolicy, QueryPolicyContext, QueryResult, QueryRunOptions, QueryStats, QueueAdapter, ReadParquetBatchOptions, ReadParquetOptions, ResumableBatchOptions, ResumedQuery, Row, RuntimeSubstrate, Scalar, ScanAdapter, ScanBatch, ScanEngineOptions, ScanOptions, ScanPlannedIcebergRowsOptions, ScanTaskPlan, ScanTaskPlanOptions, SidecarFileIndex, SliceOptions, SliceResult, Snapshot, SortOperatorState, SortOptions, SortResult, SortRunState, SpillAdapter, SpillRef, SpillUsage, SqlBoolean, TaskCheckpoint, TaskInput, TaskManifest, TaskManifestTask, TaskState, TaskTransitionInput, TopKOperatorState, TopKOptions, TopKResult, ValueInput, WriteParquetOptions, WriteParquetRowsOptions, WritePartitionedParquetFile, WritePartitionedParquetResult, WritePartitionedParquetTaskOptions, WritePartitionedParquetTaskResult, add, advanceTaskCheckpoint, and, applyIcebergDeletes, assertBookmarkMatches, asyncBufferFromStore, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, classifyPredicate, col, createBookmark, createLake, createOutputManifest, createOutputManifestFromCheckpoints, createLake as createParquetLake, createParquetTableAs, createTaskManifest, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, encodeJsonLine, eq, evaluate, fingerprint, fn, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, isIn, isLaQLError, isNotNull, isNull, jsonSafeValue, like, lit, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planFiles, planRowGroups, planRowGroupsFromMetadata, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, rowGroupMayMatch, scanBatches, scanPlannedIcebergRows, scanRows, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask } from './index.js';
-import 'hyparquet/src/types.js';
-import 'hyparquet';
+import { CacheAdapter, ObjectInfo, ObjectStore, ConditionalObjectStore } from './index.js';
+export { AggregateExpr, AggregateGroupSnapshot, AggregateOp, AggregateOperatorState, AggregateOptions, AggregateParquetGroupTaskOptions, AggregateParquetGroupTasksOptions, AggregateParquetTaskOptions, AggregateParquetTasksOptions, AggregateResult, AggregateSnapshotValue, AggregateSpec, AggregateStateSnapshot, AggregationBuilder, ApplyIcebergDeletesOptions, ArithmeticExpr, BBox, BBoxIndex, Batch, BatchExprValues, BetweenExpr, Bookmark, BookmarkInit, BookmarkPosition, BookmarkQuery, BroadcastJoinOptions, CacheApiCache, CacheApiCacheOptions, CacheEntry, CachePolicy, CallExpr, CaseExpr, CaseWhenExpr, CheckpointAdapter, CheckpointStore, Clock, ColumnExpr, ColumnInput, CompareExpr, CompareOp, ConditionalPutOptions, CreateParquetTableAsOptions, CreateParquetTableAsQuery, CreateParquetTableAsResult, CsvStreamOptions, DecodedIcebergDeletes, DecodedIcebergParquetDeletes, ERROR_CODES, EngineFilePlan, EngineTable, ErrorDetails, ExplainJson, ExplainResult, Expr, GeoBackend, GeoJsonGeometry, IcebergAppendFile, IcebergAppendOptions, IcebergAppendOutputManifestOptions, IcebergAppendResult, IcebergCatalog, IcebergCommitCatalog, IcebergCommitInput, IcebergCommitResult, IcebergDeleteFile, IcebergDeletionVector, IcebergEnginePlan, IcebergEngineTable, IcebergEqualityDelete, IcebergField, IcebergGlueCatalogOptions, IcebergLoadTableOptions, IcebergNessieCatalogOptions, IcebergParquetDeleteFile, IcebergParquetDeleteFileContent, IcebergPartitionField, IcebergPartitionSpec, IcebergPlan, IcebergPositionDelete, IcebergReadMode, IcebergRestAccessDelegation, IcebergRestCatalog, IcebergRestCatalogConfig, IcebergRestCatalogOptions, IcebergRestLoadContext, IcebergRestLoadTableOptions, IcebergRestLoadTableResult, IcebergRestStorageCredential, IcebergRowBatch, IcebergSortOrder, IcebergTable, IcebergTableIdentifier, IcebergUnsupportedCatalog, IdGenerator, InExpr, InMemoryLakeOptions, InMemoryRowsScanner, InMemoryRowsStore, InMemoryTableOptions, IndexPruneResult, IndexValue, InsertValidationRules, JoinKey, JoinType, JsonExpr, JsonOrderByTerm, JsonQueryV1, Lake, LakeConfig, LakeqlError, LakeqlErrorCode, LikeExpr, ListOptions, LiteralExpr, LoadIcebergEngineTableOptions, LoadIcebergTableFromObjectStoreOptions, LoadIcebergTableFromRestOptions, LoadIcebergTableOptions, LoadParquetEngineTableOptions, LoadTableOptions, LockAdapter, LogHook, LogicalExpr, LookupJoinFunction, LookupJoinOptions, Manifest, ManifestDeleteFile, ManifestFile, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, MemorySpillAdapterOptions, MetadataFile, MetricsHook, MinMaxColumnIndex, NotExpr, NullCheckExpr, ObjectHead, ObjectStoreCacheOptions, ObjectStoreIcebergCommitCatalog, ObjectStoreJsonCache, ObjectStoreJsonCacheOptions, ObjectStoreReadControls, ObjectStoreUriAuthority, OperatorSnapshotValue, OrderByTerm, OutputManifest, OutputManifestEntry, PACKAGE, ParquetColumnBatch, ParquetEnginePlan, ParquetEngineTable, ParquetLakeConfig, ParquetMetadata, ParquetRowBatch, ParquetRowGroupPlan, ParquetScanAdapter, PartitionedParquetOutputEntryOptions, PathQueryInit, PlanIcebergFilesOptions, PlanParquetRowGroupsOptions, PlanParquetTaskWorkUnitsOptions, PlannedIcebergFile, PlannedParquetRowGroup, PredicatePlan, PredicatePlanOptions, ProjectIcebergRowOptions, PutOptions, QueryBudget, QueryBuilder, QueryPolicy, QueryPolicyContext, QueryResult, QueryRunOptions, QueryStats, QueueAdapter, ReadParquetBatchOptions, ReadParquetOptions, ResumableBatchOptions, ResumedQuery, Row, RuntimeSubstrate, Scalar, ScanAdapter, ScanBatch, ScanColumnBatch, ScanEngineOptions, ScanOptions, ScanParquetTaskOptions, ScanPlannedIcebergRowsOptions, ScanTaskPlan, ScanTaskPlanOptions, ScanVectorBatch, Selection, SharedCacheEntry, SharedCacheSetOptions, SharedMemoryCache, SidecarFileIndex, SliceOptions, SliceResult, Snapshot, SortOperatorState, SortOptions, SortResult, SortRunState, SpillAdapter, SpillRef, SpillUsage, SqlBoolean, TaskCheckpoint, TaskInput, TaskManifest, TaskManifestTask, TaskState, TaskTransitionInput, TimestampUnit, TimestampValue, TopKOperatorState, TopKOptions, TopKResult, ValueInput, Vector, VectorAggregateOptions, VectorAggregateSnapshotValue, VectorAggregateState, VectorAggregateStateSnapshot, VectorAggregateStateSnapshots, VectorAggregateStates, VectorAggregateValue, VectorGroup, VectorGroupByGroupSnapshot, VectorGroupByOptions, VectorGroupBySnapshotValue, VectorGroupByState, VectorGroupByStateSnapshot, VectorHashJoinOptions, VectorProjectionSpec, VectorTopKOptions, WorkUnitFanInOptions, WriteParquetOptions, WriteParquetRowsOptions, WritePartitionedParquetFile, WritePartitionedParquetResult, WritePartitionedParquetTaskOptions, WritePartitionedParquetTaskResult, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, applyIcebergDeletes, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxIntersects, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, compareTimestampValues, concatBatches, createBookmark, createInMemoryLake, createLake, createOutputManifest, createOutputManifestFromCheckpoints, createLake as createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, encodeJsonLine, enforceVectorGroupByBudget, ensureGeoBackendForExprs, envelopeFromGeometry, envelopeOf, eq, evaluate, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, ilike, inMemoryRowsScanner, isIn, isLakeqlError, isNotNull, isNull, isTimestampValue, jsonSafeValue, jsonWorkUnitBoundary, like, lit, loadGeoBackend, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, lookupJoin, lt, lte, matches, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, objectStoreJsonCache, or, parquetScanner, parseGeometry, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planFiles, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, requireGeoBackend, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanBatches, scanParquetTaskBatches, scanParquetTaskColumnBatches, scanPlannedIcebergRows, scanRows, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, setGeoBackend, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, timestampEpochForUnit, timestampFromEpoch, timestampToIsoString, timestampValueFromIso, toGeometry, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, uriObjectStore, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorHashJoin, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask } from './index.js';
 import 'hyparquet-writer';
+import 'hyparquet';
+
+interface FsJsonCacheOptions {
+    root: string;
+    prefix?: string;
+    ttlMs?: number;
+    now?: () => number;
+    space?: number;
+}
+declare function fsJsonCache<T = unknown>(options: FsJsonCacheOptions): CacheAdapter<T>;
 
 interface HttpStoreOptions {
     baseUrl: string;
@@ -12,16 +20,30 @@ interface HttpStoreOptions {
 }
 declare function httpStore(options: HttpStoreOptions): ObjectStore;
 
+interface S3Credentials {
+    accessKeyId: string;
+    secretAccessKey: string;
+    sessionToken?: string;
+    expiresAt?: Date;
+}
+type S3CredentialsProvider = () => S3Credentials | Promise<S3Credentials>;
 interface S3StoreOptions {
     endpoint: string;
     bucket: string;
     region: string;
-    accessKeyId: string;
-    secretAccessKey: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
     sessionToken?: string;
+    credentials?: S3CredentialsProvider;
+    credentialRefreshWindowMs?: number;
     fetch?: typeof fetch;
     now?: () => Date;
 }
+interface S3JsonCacheOptions extends S3StoreOptions {
+    prefix: string;
+    ttlMs?: number;
+}
 declare function s3Store(options: S3StoreOptions): ConditionalObjectStore;
+declare function s3JsonCache<T = unknown>(options: S3JsonCacheOptions): CacheAdapter<T>;
 
-export { ConditionalObjectStore, ObjectInfo, ObjectStore, httpStore, s3Store };
+export { CacheAdapter, ConditionalObjectStore, ObjectInfo, ObjectStore, fsJsonCache, httpStore, s3JsonCache, s3Store };

package/dist/node.js

@@ -1,9 +1,110 @@
-export { IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, ObjectStoreIcebergCommitCatalog, PACKAGE, applyIcebergDeletes, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, planFiles, scanBatches, scanPlannedIcebergRows, scanRows } from './chunk-MUTXXFOW.js';
-import { LaQLError } from './chunk-D3A4VN3U.js';
-export { AggregationBuilder, CacheApiCache, ERROR_CODES, LaQLError, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, add, advanceTaskCheckpoint, and, assertBookmarkMatches, asyncBufferFromStore, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, classifyPredicate, col, createBookmark, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, encodeJsonLine, eq, evaluate, fingerprint, fn, gt, gte, ilike, isIn, isLaQLError, isNotNull, isNull, jsonSafeValue, like, lit, lookupJoin, lt, lte, matches, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planRowGroups, planRowGroupsFromMetadata, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, rowGroupMayMatch, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask } from './chunk-D3A4VN3U.js';
+import { objectStoreJsonCache } from './chunk-5K5JMJ2M.js';
+export { IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, InMemoryRowsScanner, InMemoryRowsStore, ObjectStoreIcebergCommitCatalog, ObjectStoreJsonCache, PACKAGE, applyIcebergDeletes, createInMemoryLake, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, inMemoryRowsScanner, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, objectStoreJsonCache, planFiles, scanBatches, scanPlannedIcebergRows, scanRows, vectorHashJoin } from './chunk-5K5JMJ2M.js';
+export { AggregationBuilder, CacheApiCache, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, SharedMemoryCache, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, concatBatches, createBookmark, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, enforceVectorGroupByBudget, eq, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, ilike, isIn, isNotNull, isNull, jsonWorkUnitBoundary, like, lit, lookupJoin, lt, lte, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanParquetTaskBatches, scanParquetTaskColumnBatches, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, uriObjectStore, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask } from './chunk-MXGEAVHL.js';
+import { LakeqlError } from './chunk-TFD5RFKB.js';
+export { ERROR_CODES, LakeqlError, TimestampValue, bboxIntersects, compareTimestampValues, encodeJsonLine, ensureGeoBackendForExprs, envelopeFromGeometry, envelopeOf, evaluate, isLakeqlError, isTimestampValue, jsonSafeValue, loadGeoBackend, matches, parseGeometry, requireGeoBackend, setGeoBackend, timestampEpochForUnit, timestampFromEpoch, timestampToIsoString, timestampValueFromIso, toGeometry } from './chunk-TFD5RFKB.js';
+import { readFile, mkdir, writeFile, rm } from 'fs/promises';
+import { join } from 'path';
 import { AwsClient } from 'aws4fetch';
 import { XMLParser } from 'fast-xml-parser';
 
+function fsJsonCache(options) {
+  return new FsJsonCache(options);
+}
+var FsJsonCache = class {
+  root;
+  prefix;
+  ttlMs;
+  now;
+  space;
+  constructor(options) {
+    this.root = options.root;
+    this.prefix = normalizedPrefix(options.prefix ?? "lakeql");
+    this.ttlMs = options.ttlMs;
+    this.now = options.now ?? Date.now;
+    this.space = options.space;
+  }
+  async get(key) {
+    const path = this.pathFor(key);
+    let bytes;
+    try {
+      bytes = await readFile(path);
+    } catch (cause) {
+      if (isNodeError(cause) && cause.code === "ENOENT")
+        return void 0;
+      throw cause;
+    }
+    const parsed = JSON.parse(bytes.toString("utf8"));
+    if (!isStoredFsJsonCacheEntry(parsed)) {
+      throw new LakeqlError("LAKEQL_CATALOG_ERROR", `Invalid filesystem cache entry at ${path}`, {
+        path
+      });
+    }
+    if (parsed.expiresAt !== void 0 && parsed.expiresAt <= this.now()) {
+      await this.delete(key);
+      return void 0;
+    }
+    const entry = { value: parsed.value };
+    if (parsed.expiresAt !== void 0)
+      entry.expiresAt = parsed.expiresAt;
+    return entry;
+  }
+  async set(key, entry) {
+    const path = this.pathFor(key);
+    await mkdir(join(this.root, this.prefix), { recursive: true });
+    const stored = { value: entry.value };
+    if (entry.expiresAt !== void 0) {
+      stored.expiresAt = entry.expiresAt;
+    } else if (this.ttlMs !== void 0) {
+      stored.expiresAt = this.now() + this.ttlMs;
+    }
+    await writeFile(path, jsonCacheValue(stored, key, this.space));
+  }
+  async delete(key) {
+    await rm(this.pathFor(key), { force: true });
+  }
+  pathFor(key) {
+    if (key.length === 0) {
+      throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "filesystem cache key must not be empty");
+    }
+    return join(this.root, this.prefix, `${encodeURIComponent(key)}.json`);
+  }
+};
+function normalizedPrefix(prefix) {
+  const parts = prefix.split("/").filter((part) => part.length > 0);
+  if (parts.length === 0 || parts.some((part) => part === "." || part === "..")) {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "filesystem cache prefix must be relative", {
+      prefix
+    });
+  }
+  return parts.join("/");
+}
+function isStoredFsJsonCacheEntry(value) {
+  return typeof value === "object" && value !== null && "value" in value;
+}
+function jsonCacheValue(value, key, space) {
+  if (value.value === void 0) {
+    throw new LakeqlError("LAKEQL_TYPE_ERROR", "filesystem JSON cache value is not JSON serializable", { key });
+  }
+  try {
+    const json = JSON.stringify(value, void 0, space);
+    if (json === void 0) {
+      throw new LakeqlError("LAKEQL_TYPE_ERROR", "filesystem JSON cache value is not JSON serializable", { key });
+    }
+    return json;
+  } catch (cause) {
+    if (cause instanceof LakeqlError)
+      throw cause;
+    throw new LakeqlError("LAKEQL_TYPE_ERROR", "filesystem JSON cache value is not JSON serializable", {
+      key,
+      cause
+    });
+  }
+}
+function isNodeError(value) {
+  return value instanceof Error && "code" in value;
+}
+
 // ../http/dist/index.js
 function httpStore(options) {
   return new HttpObjectStore(options);
@@ -28,7 +129,10 @@ var HttpObjectStore = class {
   }
   async getRange(path, range) {
     if (range.offset < 0 || range.length < 0) {
-      throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `Invalid range for ${path}`, { path, range });
+      throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `Invalid range for ${path}`, {
+        path,
+        range
+      });
     }
     const response = await this.fetchPath(path, {
       method: "GET",
@@ -54,7 +158,7 @@ var HttpObjectStore = class {
   }
   async *list(prefix, options) {
     if (!this.objects) {
-      throw new LaQLError("LAQL_UNSUPPORTED_PUSHDOWN", "HTTP store list requires an object index", {
+      throw new LakeqlError("LAKEQL_UNSUPPORTED_PUSHDOWN", "HTTP store list requires an object index", {
         prefix
       });
     }
@@ -79,7 +183,7 @@ var HttpObjectStore = class {
     const contentLength = response.headers.get("content-length");
     const size = total ?? (contentLength ? Number(contentLength) : Number.NaN);
     if (!Number.isFinite(size)) {
-      throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `Missing size for ${path}`, { path });
+      throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `Missing size for ${path}`, { path });
     }
     const head = { size };
     const etag = response.headers.get("etag");
@@ -105,7 +209,7 @@ var HttpObjectStore = class {
     const base = new URL(this.baseUrl);
     const url = new URL(encodeObjectPath(path), base);
     if (url.origin !== base.origin || !url.pathname.startsWith(base.pathname)) {
-      throw new LaQLError("LAQL_VALIDATION_ERROR", `Object path escapes HTTP base URL: ${path}`, {
+      throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Object path escapes HTTP base URL: ${path}`, {
         path,
         baseUrl: this.baseUrl
       });
@@ -115,7 +219,7 @@ var HttpObjectStore = class {
 };
 function encodeObjectPath(path) {
   if (/^(?:[a-z][a-z0-9+.-]*:)?\/\//iu.test(path) || path.startsWith("/")) {
-    throw new LaQLError("LAQL_VALIDATION_ERROR", `Object path must be relative: ${path}`, {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Object path must be relative: ${path}`, {
       path
     });
   }
@@ -126,12 +230,12 @@ function encodeObjectPath(path) {
     try {
       decoded = decodeURIComponent(segment);
     } catch {
-      throw new LaQLError("LAQL_VALIDATION_ERROR", `Object path has invalid encoding: ${path}`, {
+      throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Object path has invalid encoding: ${path}`, {
         path
       });
     }
     if (decoded === "." || decoded === "..") {
-      throw new LaQLError("LAQL_VALIDATION_ERROR", `Object path contains traversal: ${path}`, {
+      throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Object path contains traversal: ${path}`, {
         path
       });
     }
@@ -151,9 +255,9 @@ function assertOk(response, path) {
   if (response.ok || response.status === 206)
     return;
   if (response.status === 404) {
-    throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No object at ${path}`, { path });
+    throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `No object at ${path}`, { path });
   }
-  throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `HTTP object request failed for ${path}`, {
+  throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `HTTP object request failed for ${path}`, {
     path,
     status: response.status
   });
@@ -169,25 +273,26 @@ function bodyInit(body) {
 function s3Store(options) {
   return new S3ObjectStore(options);
 }
+function s3JsonCache(options) {
+  return objectStoreJsonCache({
+    store: s3Store(options),
+    prefix: options.prefix,
+    ...options.ttlMs !== void 0 ? { ttlMs: options.ttlMs } : {}
+  });
+}
 var S3ObjectStore = class {
   options;
   fetchImpl;
   now;
-  aws;
+  credentialsSnapshot;
+  credentialsPromise;
   constructor(options) {
     this.options = options;
     this.fetchImpl = options.fetch ?? fetch;
     this.now = options.now ?? (() => /* @__PURE__ */ new Date());
-    const awsOptions = {
-      accessKeyId: options.accessKeyId,
-      secretAccessKey: options.secretAccessKey,
-      region: options.region,
-      service: "s3",
-      retries: 0
-    };
-    if (options.sessionToken !== void 0)
-      awsOptions.sessionToken = options.sessionToken;
-    this.aws = new AwsClient(awsOptions);
+    if (options.credentials === void 0) {
+      this.credentialsSnapshot = staticCredentials(options);
+    }
   }
   async get(path) {
     const response = await this.request("GET", path);
@@ -258,7 +363,7 @@ var S3ObjectStore = class {
     assertOk2(response, path);
     const contentLength = response.headers.get("content-length");
     if (contentLength === null) {
-      throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `Missing S3 content-length for ${path}`, {
+      throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `Missing S3 content-length for ${path}`, {
         path
       });
     }
@@ -282,19 +387,41 @@ var S3ObjectStore = class {
       url,
       headers,
       region: this.options.region,
-      accessKeyId: this.options.accessKeyId,
-      secretAccessKey: this.options.secretAccessKey,
+      ...await this.currentCredentials(),
       now: this.now()
     };
-    if (this.options.sessionToken !== void 0)
-      signRequest.sessionToken = this.options.sessionToken;
     const requestBody = body === void 0 ? void 0 : bodyInit2(body);
-    const signed = await signS3Request(signRequest, this.aws, requestBody);
+    const signed = await signS3Request(signRequest, void 0, requestBody);
     const init = { method, headers: signed };
     if (requestBody !== void 0)
       init.body = requestBody;
     return this.fetchImpl(url, init);
   }
+  async currentCredentials() {
+    if (this.options.credentials === void 0) {
+      if (this.credentialsSnapshot === void 0)
+        this.credentialsSnapshot = staticCredentials(this.options);
+      return this.credentialsSnapshot;
+    }
+    if (this.credentialsSnapshot !== void 0 && !credentialsNeedRefresh(this.credentialsSnapshot, this.now(), this.credentialRefreshWindowMs())) {
+      return this.credentialsSnapshot;
+    }
+    if (this.credentialsPromise === void 0) {
+      this.credentialsPromise = Promise.resolve(this.options.credentials()).then((credentials) => {
+        const normalized = validateCredentials(credentials);
+        this.credentialsSnapshot = normalized;
+        this.credentialsPromise = void 0;
+        return normalized;
+      }).catch((cause) => {
+        this.credentialsPromise = void 0;
+        throw cause;
+      });
+    }
+    return await this.credentialsPromise;
+  }
+  credentialRefreshWindowMs() {
+    return this.options.credentialRefreshWindowMs ?? 3e5;
+  }
 };
 async function signS3Request(request, client = s3SignerClient(request), body) {
   const headers = new Headers(request.headers);
@@ -325,6 +452,39 @@ function s3SignerClient(request) {
     options.sessionToken = request.sessionToken;
   return new AwsClient(options);
 }
+function staticCredentials(options) {
+  if (options.accessKeyId === void 0 || options.secretAccessKey === void 0) {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "S3 credentials are required", {
+      bucket: options.bucket
+    });
+  }
+  return validateCredentials({
+    accessKeyId: options.accessKeyId,
+    secretAccessKey: options.secretAccessKey,
+    ...options.sessionToken !== void 0 ? { sessionToken: options.sessionToken } : {}
+  });
+}
+function validateCredentials(credentials) {
+  const accessKeyId = credentials.accessKeyId.trim();
+  const secretAccessKey = credentials.secretAccessKey.trim();
+  if (accessKeyId === "" || secretAccessKey === "") {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "S3 credentials are required");
+  }
+  const normalized = {
+    accessKeyId,
+    secretAccessKey
+  };
+  if (credentials.sessionToken !== void 0)
+    normalized.sessionToken = credentials.sessionToken;
+  if (credentials.expiresAt !== void 0)
+    normalized.expiresAt = credentials.expiresAt;
+  return normalized;
+}
+function credentialsNeedRefresh(credentials, now, refreshWindowMs) {
+  if (credentials.expiresAt === void 0)
+    return false;
+  return credentials.expiresAt.getTime() - now.getTime() <= refreshWindowMs;
+}
 var listObjectsParser = new XMLParser({
   ignoreAttributes: true,
   parseTagValue: false,
@@ -372,7 +532,7 @@ function ensureSlash(value) {
 }
 function encodeObjectPath2(path) {
   if (/^(?:[a-z][a-z0-9+.-]*:)?\/\//iu.test(path) || path.startsWith("/")) {
-    throw new LaQLError("LAQL_VALIDATION_ERROR", `Object path must be relative: ${path}`, {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Object path must be relative: ${path}`, {
       path
     });
   }
@@ -383,12 +543,12 @@ function encodeObjectPath2(path) {
     try {
       decoded = decodeURIComponent(segment);
     } catch {
-      throw new LaQLError("LAQL_VALIDATION_ERROR", `Object path has invalid encoding: ${path}`, {
+      throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Object path has invalid encoding: ${path}`, {
         path
       });
     }
     if (decoded === "." || decoded === "..") {
-      throw new LaQLError("LAQL_VALIDATION_ERROR", `Object path contains traversal: ${path}`, {
+      throw new LakeqlError("LAKEQL_VALIDATION_ERROR", `Object path contains traversal: ${path}`, {
         path
       });
     }
@@ -399,9 +559,9 @@ function assertOk2(response, path) {
   if (response.ok || response.status === 206)
     return;
   if (response.status === 404) {
-    throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `No object at ${path}`, { path });
+    throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `No object at ${path}`, { path });
   }
-  throw new LaQLError("LAQL_OBJECT_NOT_FOUND", `S3 request failed for ${path}`, {
+  throw new LakeqlError("LAKEQL_OBJECT_NOT_FOUND", `S3 request failed for ${path}`, {
     path,
     status: response.status
   });
@@ -420,4 +580,4 @@ function arrayBuffer(bytes) {
   return copy.buffer;
 }
 
-export { httpStore, s3Store };
+export { fsJsonCache, httpStore, s3JsonCache, s3Store };

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "lakeql",
-  "version": "0.1.2",
+  "sideEffects": false,
+  "version": "0.1.8",
   "description": "lakeql: lightweight TypeScript query engine for Iceberg + Parquet on object storage",
   "keywords": [
     "lakehouse",
@@ -39,7 +40,7 @@
   "repository": {
     "type": "git",
     "url": "https://github.com/earonesty/lakeql.git",
-    "directory": "packages/laql"
+    "directory": "packages/lakeql"
   },
   "homepage": "https://lakeql.com",
   "engines": {
@@ -54,20 +55,22 @@
     "avsc": "^5.7.9",
     "aws4fetch": "^1.0.20",
     "fast-xml-parser": "^5.9.0",
+    "fzstd": "^0.1.1",
     "h3-js": "^4.4.0",
     "hyparquet": "^1.26.0",
     "hyparquet-writer": "^0.15.6"
   },
   "devDependencies": {
     "tsup": "^8.5.0",
-    "@laql/core": "0.1.0",
-    "@laql/http": "0.0.2",
-    "@laql/iceberg": "0.0.2",
-    "@laql/parquet": "0.0.2",
-    "@laql/fixtures": "0.0.1",
-    "@laql/r2": "0.0.2",
-    "@laql/s3": "0.0.2",
-    "@laql/sql": "0.0.2"
+    "lakeql-fixtures": "0.0.1",
+    "lakeql-core": "0.1.3",
+    "lakeql-fs": "0.0.2",
+    "lakeql-iceberg": "0.0.5",
+    "lakeql-parquet": "0.0.5",
+    "lakeql-http": "0.0.5",
+    "lakeql-r2": "0.0.5",
+    "lakeql-s3": "0.0.5",
+    "lakeql-sql": "0.0.5"
   },
   "scripts": {
     "build": "tsup"