lakeql 0.1.7 → 0.1.8

package/dist/node.js

@@ -1,10 +1,110 @@
-export { IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, InMemoryRowsScanner, InMemoryRowsStore, ObjectStoreIcebergCommitCatalog, PACKAGE, applyIcebergDeletes, createInMemoryLake, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, inMemoryRowsScanner, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, planFiles, scanBatches, scanPlannedIcebergRows, scanRows, vectorHashJoin } from './chunk-BFLGC6Y5.js';
-export { AggregationBuilder, CacheApiCache, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, SharedMemoryCache, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, concatBatches, createBookmark, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, enforceVectorGroupByBudget, eq, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, ilike, isIn, isNotNull, isNull, jsonWorkUnitBoundary, like, lit, lookupJoin, lt, lte, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanParquetTaskBatches, scanParquetTaskColumnBatches, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask } from './chunk-RZL45ZSN.js';
+import { objectStoreJsonCache } from './chunk-5K5JMJ2M.js';
+export { IcebergRestCatalog, IcebergTable, IcebergUnsupportedCatalog, InMemoryRowsScanner, InMemoryRowsStore, ObjectStoreIcebergCommitCatalog, ObjectStoreJsonCache, PACKAGE, applyIcebergDeletes, createInMemoryLake, icebergGlueCatalog, icebergNessieCatalog, icebergRestCatalog, inMemoryRowsScanner, loadIcebergTable, loadIcebergTableFromObjectStore, loadIcebergTableFromRest, loadTable, objectStoreJsonCache, planFiles, scanBatches, scanPlannedIcebergRows, scanRows, vectorHashJoin } from './chunk-5K5JMJ2M.js';
+export { AggregationBuilder, CacheApiCache, Lake, MemoryCache, MemoryCheckpointAdapter, MemoryObjectStore, MemorySpillAdapter, ParquetScanAdapter, QueryBuilder, QueryResult, ResumedQuery, SharedMemoryCache, add, advanceTaskCheckpoint, aggregateParquetGroupTask, aggregateParquetGroupTasks, aggregateParquetGroupTasksBatch, aggregateParquetTask, aggregateParquetTasks, and, assertBookmarkMatches, asyncBufferFromStore, batchExprValues, batchFromColumns, batchFromVectors, bboxMayIntersect, between, broadcastJoin, buildBBoxIndex, buildMinMaxIndex, cacheApiCache, cachedObjectStore, classifyPredicate, col, concatBatches, createBookmark, createParquetLake as createLake, createOutputManifest, createOutputManifestFromCheckpoints, createParquetLake, createParquetTableAs, createTaskManifest, createVectorAggregateStates, createVectorGroupByState, deserializeAggregateOperatorState, deserializeSortOperatorState, deserializeTopKOperatorState, div, enforceVectorGroupByBudget, eq, fanInWorkUnits, finalizeVectorAggregateStates, finalizeVectorGroupByBatch, finalizeVectorGroupByRows, fingerprint, fn, gatherBatch, getOrCreateVectorGroup, gt, gte, ilike, isIn, isNotNull, isNull, jsonWorkUnitBoundary, like, lit, lookupJoin, lt, lte, materializeBatchRows, materializeSelectedBatchRows, memoryCache, memoryCheckpointAdapter, memorySpillAdapter, memoryStore, mergeVectorAggregateStateSnapshots, mergeVectorAggregateStates, mergeVectorGroupByStates, mod, mul, ne, not, notIn, or, parquetScanner, parseHivePartitions, parseJsonQuery, partitionedParquetOutputEntries, planParquetTaskWorkUnits, planRowGroups, planRowGroupsFromMetadata, predicateSelection, pruneFilesWithIndex, readControlSignal, readIcebergParquetDeletes, readOutputManifest, readParquetColumnBatches, readParquetMetadata, readParquetObjectBatches, readParquetObjects, rejectUnsupportedParquetSchema, restoreVectorAggregateStates, restoreVectorGroupByState, rowGroupMayMatch, rowGroupMustMatch, scalarVectorValue, scanParquetTaskBatches, scanParquetTaskColumnBatches, selectedRowCount, selectedRowIndices, serializeAggregateOperatorState, serializeSortOperatorState, serializeTopKOperatorState, signPaginationToken, snapshotVectorAggregateStates, snapshotVectorGroupByState, stableStringify, sub, throwIfAborted, transitionTaskCheckpoint, tryPredicateSelection, updateVectorAggregateStateValue, updateVectorAggregateStates, updateVectorGroupAggregateValue, updateVectorGroupByState, uriObjectStore, vectorAggregateBatch, vectorFromValues, vectorGroupByBatch, vectorLength, vectorOrderByBatch, vectorProjectBatch, vectorSortIndices, vectorTopKBatch, vectorTopKIndices, vectorValue, verifyPaginationToken, withObjectStoreReadControls, writeOutputManifest, writeParquet, writePartitionedParquet, writePartitionedParquetTask } from './chunk-MXGEAVHL.js';
 import { LakeqlError } from './chunk-TFD5RFKB.js';
 export { ERROR_CODES, LakeqlError, TimestampValue, bboxIntersects, compareTimestampValues, encodeJsonLine, ensureGeoBackendForExprs, envelopeFromGeometry, envelopeOf, evaluate, isLakeqlError, isTimestampValue, jsonSafeValue, loadGeoBackend, matches, parseGeometry, requireGeoBackend, setGeoBackend, timestampEpochForUnit, timestampFromEpoch, timestampToIsoString, timestampValueFromIso, toGeometry } from './chunk-TFD5RFKB.js';
+import { readFile, mkdir, writeFile, rm } from 'fs/promises';
+import { join } from 'path';
 import { AwsClient } from 'aws4fetch';
 import { XMLParser } from 'fast-xml-parser';
 
+function fsJsonCache(options) {
+  return new FsJsonCache(options);
+}
+var FsJsonCache = class {
+  root;
+  prefix;
+  ttlMs;
+  now;
+  space;
+  constructor(options) {
+    this.root = options.root;
+    this.prefix = normalizedPrefix(options.prefix ?? "lakeql");
+    this.ttlMs = options.ttlMs;
+    this.now = options.now ?? Date.now;
+    this.space = options.space;
+  }
+  async get(key) {
+    const path = this.pathFor(key);
+    let bytes;
+    try {
+      bytes = await readFile(path);
+    } catch (cause) {
+      if (isNodeError(cause) && cause.code === "ENOENT")
+        return void 0;
+      throw cause;
+    }
+    const parsed = JSON.parse(bytes.toString("utf8"));
+    if (!isStoredFsJsonCacheEntry(parsed)) {
+      throw new LakeqlError("LAKEQL_CATALOG_ERROR", `Invalid filesystem cache entry at ${path}`, {
+        path
+      });
+    }
+    if (parsed.expiresAt !== void 0 && parsed.expiresAt <= this.now()) {
+      await this.delete(key);
+      return void 0;
+    }
+    const entry = { value: parsed.value };
+    if (parsed.expiresAt !== void 0)
+      entry.expiresAt = parsed.expiresAt;
+    return entry;
+  }
+  async set(key, entry) {
+    const path = this.pathFor(key);
+    await mkdir(join(this.root, this.prefix), { recursive: true });
+    const stored = { value: entry.value };
+    if (entry.expiresAt !== void 0) {
+      stored.expiresAt = entry.expiresAt;
+    } else if (this.ttlMs !== void 0) {
+      stored.expiresAt = this.now() + this.ttlMs;
+    }
+    await writeFile(path, jsonCacheValue(stored, key, this.space));
+  }
+  async delete(key) {
+    await rm(this.pathFor(key), { force: true });
+  }
+  pathFor(key) {
+    if (key.length === 0) {
+      throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "filesystem cache key must not be empty");
+    }
+    return join(this.root, this.prefix, `${encodeURIComponent(key)}.json`);
+  }
+};
+function normalizedPrefix(prefix) {
+  const parts = prefix.split("/").filter((part) => part.length > 0);
+  if (parts.length === 0 || parts.some((part) => part === "." || part === "..")) {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "filesystem cache prefix must be relative", {
+      prefix
+    });
+  }
+  return parts.join("/");
+}
+function isStoredFsJsonCacheEntry(value) {
+  return typeof value === "object" && value !== null && "value" in value;
+}
+function jsonCacheValue(value, key, space) {
+  if (value.value === void 0) {
+    throw new LakeqlError("LAKEQL_TYPE_ERROR", "filesystem JSON cache value is not JSON serializable", { key });
+  }
+  try {
+    const json = JSON.stringify(value, void 0, space);
+    if (json === void 0) {
+      throw new LakeqlError("LAKEQL_TYPE_ERROR", "filesystem JSON cache value is not JSON serializable", { key });
+    }
+    return json;
+  } catch (cause) {
+    if (cause instanceof LakeqlError)
+      throw cause;
+    throw new LakeqlError("LAKEQL_TYPE_ERROR", "filesystem JSON cache value is not JSON serializable", {
+      key,
+      cause
+    });
+  }
+}
+function isNodeError(value) {
+  return value instanceof Error && "code" in value;
+}
+
 // ../http/dist/index.js
 function httpStore(options) {
   return new HttpObjectStore(options);
@@ -173,25 +273,26 @@ function bodyInit(body) {
 function s3Store(options) {
   return new S3ObjectStore(options);
 }
+function s3JsonCache(options) {
+  return objectStoreJsonCache({
+    store: s3Store(options),
+    prefix: options.prefix,
+    ...options.ttlMs !== void 0 ? { ttlMs: options.ttlMs } : {}
+  });
+}
 var S3ObjectStore = class {
   options;
   fetchImpl;
   now;
-  aws;
+  credentialsSnapshot;
+  credentialsPromise;
   constructor(options) {
     this.options = options;
     this.fetchImpl = options.fetch ?? fetch;
     this.now = options.now ?? (() => /* @__PURE__ */ new Date());
-    const awsOptions = {
-      accessKeyId: options.accessKeyId,
-      secretAccessKey: options.secretAccessKey,
-      region: options.region,
-      service: "s3",
-      retries: 0
-    };
-    if (options.sessionToken !== void 0)
-      awsOptions.sessionToken = options.sessionToken;
-    this.aws = new AwsClient(awsOptions);
+    if (options.credentials === void 0) {
+      this.credentialsSnapshot = staticCredentials(options);
+    }
   }
   async get(path) {
     const response = await this.request("GET", path);
@@ -286,19 +387,41 @@ var S3ObjectStore = class {
       url,
       headers,
       region: this.options.region,
-      accessKeyId: this.options.accessKeyId,
-      secretAccessKey: this.options.secretAccessKey,
+      ...await this.currentCredentials(),
       now: this.now()
     };
-    if (this.options.sessionToken !== void 0)
-      signRequest.sessionToken = this.options.sessionToken;
     const requestBody = body === void 0 ? void 0 : bodyInit2(body);
-    const signed = await signS3Request(signRequest, this.aws, requestBody);
+    const signed = await signS3Request(signRequest, void 0, requestBody);
     const init = { method, headers: signed };
     if (requestBody !== void 0)
       init.body = requestBody;
     return this.fetchImpl(url, init);
   }
+  async currentCredentials() {
+    if (this.options.credentials === void 0) {
+      if (this.credentialsSnapshot === void 0)
+        this.credentialsSnapshot = staticCredentials(this.options);
+      return this.credentialsSnapshot;
+    }
+    if (this.credentialsSnapshot !== void 0 && !credentialsNeedRefresh(this.credentialsSnapshot, this.now(), this.credentialRefreshWindowMs())) {
+      return this.credentialsSnapshot;
+    }
+    if (this.credentialsPromise === void 0) {
+      this.credentialsPromise = Promise.resolve(this.options.credentials()).then((credentials) => {
+        const normalized = validateCredentials(credentials);
+        this.credentialsSnapshot = normalized;
+        this.credentialsPromise = void 0;
+        return normalized;
+      }).catch((cause) => {
+        this.credentialsPromise = void 0;
+        throw cause;
+      });
+    }
+    return await this.credentialsPromise;
+  }
+  credentialRefreshWindowMs() {
+    return this.options.credentialRefreshWindowMs ?? 3e5;
+  }
 };
 async function signS3Request(request, client = s3SignerClient(request), body) {
   const headers = new Headers(request.headers);
@@ -329,6 +452,39 @@ function s3SignerClient(request) {
     options.sessionToken = request.sessionToken;
   return new AwsClient(options);
 }
+function staticCredentials(options) {
+  if (options.accessKeyId === void 0 || options.secretAccessKey === void 0) {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "S3 credentials are required", {
+      bucket: options.bucket
+    });
+  }
+  return validateCredentials({
+    accessKeyId: options.accessKeyId,
+    secretAccessKey: options.secretAccessKey,
+    ...options.sessionToken !== void 0 ? { sessionToken: options.sessionToken } : {}
+  });
+}
+function validateCredentials(credentials) {
+  const accessKeyId = credentials.accessKeyId.trim();
+  const secretAccessKey = credentials.secretAccessKey.trim();
+  if (accessKeyId === "" || secretAccessKey === "") {
+    throw new LakeqlError("LAKEQL_VALIDATION_ERROR", "S3 credentials are required");
+  }
+  const normalized = {
+    accessKeyId,
+    secretAccessKey
+  };
+  if (credentials.sessionToken !== void 0)
+    normalized.sessionToken = credentials.sessionToken;
+  if (credentials.expiresAt !== void 0)
+    normalized.expiresAt = credentials.expiresAt;
+  return normalized;
+}
+function credentialsNeedRefresh(credentials, now, refreshWindowMs) {
+  if (credentials.expiresAt === void 0)
+    return false;
+  return credentials.expiresAt.getTime() - now.getTime() <= refreshWindowMs;
+}
 var listObjectsParser = new XMLParser({
   ignoreAttributes: true,
   parseTagValue: false,
@@ -424,4 +580,4 @@ function arrayBuffer(bytes) {
   return copy.buffer;
 }
 
-export { httpStore, s3Store };
+export { fsJsonCache, httpStore, s3JsonCache, s3Store };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "lakeql",
   "sideEffects": false,
-  "version": "0.1.7",
+  "version": "0.1.8",
   "description": "lakeql: lightweight TypeScript query engine for Iceberg + Parquet on object storage",
   "keywords": [
     "lakehouse",
@@ -55,6 +55,7 @@
     "avsc": "^5.7.9",
     "aws4fetch": "^1.0.20",
     "fast-xml-parser": "^5.9.0",
+    "fzstd": "^0.1.1",
     "h3-js": "^4.4.0",
     "hyparquet": "^1.26.0",
     "hyparquet-writer": "^0.15.6"
@@ -62,13 +63,14 @@
   "devDependencies": {
     "tsup": "^8.5.0",
     "lakeql-fixtures": "0.0.1",
-    "lakeql-core": "0.1.2",
-    "lakeql-http": "0.0.4",
-    "lakeql-iceberg": "0.0.4",
-    "lakeql-parquet": "0.0.4",
-    "lakeql-r2": "0.0.4",
-    "lakeql-s3": "0.0.4",
-    "lakeql-sql": "0.0.4"
+    "lakeql-core": "0.1.3",
+    "lakeql-fs": "0.0.2",
+    "lakeql-iceberg": "0.0.5",
+    "lakeql-parquet": "0.0.5",
+    "lakeql-http": "0.0.5",
+    "lakeql-r2": "0.0.5",
+    "lakeql-s3": "0.0.5",
+    "lakeql-sql": "0.0.5"
   },
   "scripts": {
     "build": "tsup"