@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/lib/loader-utils/normalize-loader.ts

@@ -2,8 +2,8 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {Loader} from '@loaders.gl/loader-utils';
-import {assert} from '@loaders.gl/loader-utils';
+import type {Loader} from '@loaders.gl/core/loader-utils';
+import {assert} from '@loaders.gl/core/loader-utils';
 
 export function isLoaderObject(loader?: any): boolean {
   if (!loader) {

package/src/lib/loader-utils/option-defaults.ts

@@ -2,8 +2,8 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {LoaderOptions} from '@loaders.gl/loader-utils';
-import {isBrowser} from '@loaders.gl/loader-utils';
+import type {LoaderOptions} from '@loaders.gl/core/loader-utils';
+import {isBrowser} from '@loaders.gl/core/loader-utils';
 import {ConsoleLog} from './loggers';
 
 export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {

package/src/lib/loader-utils/option-utils.ts

@@ -10,7 +10,7 @@ import {
   isObject,
   StrictLoaderOptions,
   path
-} from '@loaders.gl/loader-utils';
+} from '@loaders.gl/core/loader-utils';
 import {probeLog, NullLog} from './loggers';
 import {DEFAULT_LOADER_OPTIONS, REMOVED_LOADER_OPTIONS} from './option-defaults';
 import {stripQueryString} from '../utils/url-utils';

package/src/lib/utils/resource-utils.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {isResponse, isBlob} from '@loaders.gl/loader-utils';
+import {isResponse, isBlob} from '@loaders.gl/core/loader-utils';
 import {parseMIMEType, parseMIMETypeFromURL} from './mime-type-utils';
 import {stripQueryString} from './url-utils';
 

package/src/lib/utils/response-utils.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {encodeArrayBufferToBase64, isResponse} from '@loaders.gl/loader-utils';
+import {encodeArrayBufferToBase64, isResponse} from '@loaders.gl/core/loader-utils';
 import {FetchError} from '../fetch/fetch-error';
 import {getResourceContentLength, getResourceUrl, getResourceMIMEType} from './resource-utils';
 import {shortenUrlForDisplay} from './url-utils';

package/src/loader-utils/format-types.ts

@@ -0,0 +1,49 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * Physical serialization used by a format.
+ */
+export type FormatEncoding =
+  | 'text'
+  | 'json'
+  | 'xml'
+  | 'csv'
+  | 'binary'
+  | 'protobuf'
+  | 'flatbuffers'
+  | 'arrow'
+  | 'parquet'
+  | 'sqlite'
+  | 'zip'
+  | 'image'
+  | 'video';
+
+/**
+ * An object that describes a format
+ */
+export type Format = {
+  /** Human readable name */
+  name: string;
+  /** Unique lower-case id string for this format. Used for e.g. LoaderOptions */
+  id: string;
+  /** loaders.gl module that contains the implementation of this format */
+  module: string;
+  /** Which category does this loader belong to */
+  category?: string;
+  /** Physical serialization used by this format. */
+  encoding?: FormatEncoding;
+  /** Logical format represented by this loader or writer. */
+  format?: string;
+  /** File extensions that are potential matches with this loader. */
+  extensions: string[];
+  /** MIMETypes that indicate a match with this loader. @note Some MIMETypes are generic and supported by many loaders */
+  mimeTypes: string[];
+  /** Is this a binary format */
+  binary?: boolean;
+  /** Is this a text format */
+  text?: boolean;
+  /** Test some initial bytes of content to see if this loader might be a match */
+  tests?: (((ArrayBuffer: ArrayBuffer) => boolean) | ArrayBuffer | string)[];
+};

package/src/loader-utils/index.ts

@@ -0,0 +1,522 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
+
+// TYPES
+
+export type {
+  // misc
+  DataType,
+  SyncDataType,
+  BatchableDataType,
+  TransformBatches,
+  // numeric array types
+  TypedArray,
+  BigTypedArray,
+  TypedArrayConstructor,
+  BigTypedArrayConstructor,
+  NumberArray,
+  NumericArray,
+  // fetch
+  FetchLike
+} from './types';
+
+// formats
+
+export type {Format, FormatEncoding} from './format-types';
+
+// loaders
+
+export type {
+  Loader,
+  LoaderWithParser,
+  LoaderContext,
+  StrictLoaderOptions,
+  LoaderOptions,
+  LoaderShapeType,
+  LoaderOptionsWithShape,
+  LoaderOptionsType,
+  LoaderReturnType,
+  LoaderBatchType,
+  LoaderArrayOptionsType,
+  LoaderArrayReturnType,
+  LoaderArrayBatchType
+} from './loader-types';
+export type {SearchParams} from './loader-types';
+
+export type {ExperimentalScanOptions} from './lib/scan-utils/experimental-scan-options';
+
+export {parseFromContext, parseSyncFromContext, parseInBatchesFromContext} from './loader-types';
+
+// writers
+
+export type {
+  Writer,
+  WriterWithEncoder,
+  WriterOptions,
+  WriterOptionsType,
+  WriterDataType,
+  WriterBatchType
+} from './writer-types';
+
+// GENERAL UTILS
+export {assert} from './lib/env-utils/assert';
+export {
+  isBrowser,
+  isWorker,
+  nodeVersion,
+  self,
+  window,
+  global,
+  document
+} from './lib/env-utils/globals';
+
+export {log} from './lib/log-utils/log';
+
+export type {ReadableStreamType} from './lib/javascript-utils/is-type';
+export {
+  isObject,
+  isPureObject,
+  isArrayBuffer,
+  isArrayBufferLike,
+  isPromise,
+  isIterable,
+  isAsyncIterable,
+  isIterator,
+  isResponse,
+  isFile,
+  isBlob,
+  isWritableDOMStream,
+  isReadableDOMStream,
+  isWritableNodeStream,
+  isReadableNodeStream,
+  isReadableStream,
+  isWritableStream
+} from './lib/javascript-utils/is-type';
+
+// Options and modules
+export type {RequiredOptions} from './lib/option-utils/merge-options';
+export {mergeOptions, getRequiredOptions} from './lib/option-utils/merge-options';
+
+// Modules (external libraries)
+export {initializeWasmModule} from './lib/module-utils/initialize-wasm-module';
+export {registerJSModules} from './lib/module-utils/js-module-utils';
+export {checkJSModule, getJSModule, getJSModuleOrNull} from './lib/module-utils/js-module-utils';
+
+// LOADERS.GL-SPECIFIC WORKER UTILS
+export {
+  createLoaderWorker,
+  processLoaderWorkerBatches
+} from './lib/worker-loader-utils/create-loader-worker';
+export {
+  parseWithWorker,
+  parseWithWorkerInBatches,
+  canParseWithWorker,
+  shouldParseWithWorker
+} from './lib/worker-loader-utils/parse-with-worker';
+export {canEncodeWithWorker} from './lib/worker-loader-utils/encode-with-worker';
+
+// PARSER UTILS
+export {parseJSON} from './lib/parser-utils/parse-json';
+
+// MEMORY COPY UTILS
+export {
+  sliceArrayBuffer,
+  concatenateArrayBuffers,
+  concatenateArrayBuffersFromArray,
+  concatenateTypedArrays,
+  compareArrayBuffers
+} from './lib/binary-utils/array-buffer-utils';
+export {BinaryChunkReader} from './lib/binary-utils/binary-chunk-reader';
+export type {
+  BinaryChunkReaderCheckpoint,
+  BinaryChunkReaderOptions
+} from './lib/binary-utils/binary-chunk-reader';
+export {padToNBytes, copyToArray, copyArrayBuffer} from './lib/binary-utils/memory-copy-utils';
+export {
+  padStringToByteAlignment,
+  copyStringToDataView,
+  copyBinaryToDataView,
+  copyPaddedArrayBufferToDataView,
+  copyPaddedStringToDataView
+} from './lib/binary-utils/dataview-copy-utils';
+export {getFirstCharacters, getMagicString} from './lib/binary-utils/get-first-characters';
+export {encodeArrayBufferToBase64} from './lib/binary-utils/encode-array-buffer-to-base64';
+
+// ITERATOR UTILS
+export {
+  makeTextEncoderIterator,
+  makeTextDecoderIterator,
+  makeLineIterator,
+  makeNumberedLineIterator
+} from './lib/iterators/text-iterators';
+export {
+  forEach,
+  concatenateArrayBuffersAsync,
+  toArrayBufferIterator
+} from './lib/iterators/async-iteration';
+
+// REQUEST UTILS
+export {default as RequestScheduler} from './lib/request-utils/request-scheduler';
+export {
+  resolveCredentials,
+  resolveAuthenticationOptions,
+  resolveLoaderAuthenticationOptions
+} from './lib/request-utils/authentication';
+export type {
+  Authentication,
+  AuthenticationConstructor,
+  AuthenticationRequest,
+  AuthenticatedRequest,
+  AuthenticateRequest,
+  RequestAuthentication,
+  Credential,
+  CredentialOptions
+} from './lib/request-utils/authentication';
+export {RequestCache} from './lib/request-utils/request-cache';
+export type {
+  RequestCacheProps,
+  RequestCacheRemovalReason
+} from './lib/request-utils/request-cache';
+export {parseContentType} from './lib/request-utils/parse-content-type';
+export {
+  createAuthenticatedFetch,
+  TokenAuthentication,
+  QueryParameterAuthentication,
+  BearerTokenAuthentication,
+  createBearerTokenCredential,
+  createQueryParameterCredential,
+  getAuthenticatedFetch,
+  redactCredentialURL
+} from './lib/request-utils/request-credentials';
+export type {
+  AuthenticatedFetchOptions,
+  BearerTokenCredentialOptions,
+  QueryParameterCredentialOptions,
+  RequestCredential,
+  TokenProvider,
+  TokenProviderContext,
+  TokenProviderReason,
+  TokenProviderResponse,
+  TokenValue
+} from './lib/request-utils/request-credentials';
+export {
+  RangeRequestScheduler,
+  createRangeStats,
+  fetchHttpRange,
+  getRangeStats
+} from './lib/request-utils/range-request-scheduler';
+export type {
+  RangeFetchRequest,
+  RangeRequest,
+  RangeRequestEvent,
+  RangeRequestSchedulerProps,
+  RangeRequestTransportResult,
+  RangeStats
+} from './lib/request-utils/range-request-scheduler';
+export {RangeRequestCache} from './lib/request-utils/range-request-cache';
+export type {
+  CachedRangeRequest,
+  RangeRequestCacheEvent,
+  RangeRequestCacheProps
+} from './lib/request-utils/range-request-cache';
+
+// LAZ DECODER UTILS
+export {
+  NeedsMoreData,
+  createLAZChunkDecoder,
+  createLAZChunkDecoderCursor,
+  decodeLAZChunkTable,
+  decodeLAZChunk,
+  decodeLAZChunkInBatches,
+  getLAZChunkByteLength,
+  getLAZChunkDeclaredByteLength,
+  getLAZChunkHeaderByteLength
+} from './lib/laz/laz-chunk-decoder';
+export type {
+  FeedableLAZChunkDecoder,
+  LAZChunkDecoderCursor,
+  LAZChunkDecoderOptions,
+  LAZChunkMetadata,
+  LAZChunkTableEntry,
+  LAZPointDataTarget
+} from './lib/laz/laz-chunk-decoder';
+export {
+  createLAZChunkEncoder,
+  encodeLASzipVLR,
+  encodeLAZChunk,
+  encodeLAZChunkTable
+} from './lib/laz/laz-chunk-encoder';
+export type {FeedableLAZChunkEncoder} from './lib/laz/laz-chunk-encoder';
+
+// PATH HELPERS
+export {setPathPrefix, getPathPrefix, resolvePath} from './lib/path-utils/file-aliases';
+export {CachedUriResolver} from './lib/path-utils/cached-uri-resolver';
+export {addAliases as _addAliases} from './lib/path-utils/file-aliases';
+
+// MICRO LOADERS
+export {JSONLoader} from './json-loader';
+
+// NODE support
+
+// Node.js emulation (can be used in browser)
+
+// Avoid direct use of `Buffer` which pulls in 50KB polyfill
+export {
+  isBuffer,
+  toBuffer,
+  toArrayBuffer,
+  toArrayBufferView,
+  copyToArrayBuffer,
+  ensureArrayBuffer
+} from './lib/binary-utils/memory-conversion-utils';
+
+// Note.js wrappers (can be safely imported, but not used in browser)
+
+// Use instead of importing 'util' to avoid node dependencies
+export {promisify1, promisify2} from './lib/node/promisify';
+
+// `path` replacement (avoids bundling big path polyfill)
+import * as path from './lib/path-utils/path';
+export {path};
+
+// Use instead of importing 'stream' to avoid node dependencies`
+import * as stream from './lib/node/stream';
+export {stream};
+
+// EXPERIMENTAL: FILE SYSTEMS
+
+export type {ReadableFile, WritableFile, Stat} from './lib/files/file';
+export {ArrayBufferFile} from './lib/files/array-buffer-file';
+export {BlobFile} from './lib/files/blob-file';
+export {HttpFile} from './lib/files/http-file';
+export type {
+  HttpFileConsistency,
+  HttpFileFetch,
+  HttpFileIdentity,
+  HttpFileOptions,
+  HttpFileTelemetry
+} from './lib/files/http-file';
+export {NodeFileFacade as NodeFile} from './lib/files/node-file-facade';
+
+export type {FileSystem, RandomAccessFileSystem} from './lib/filesystems/filesystem';
+export {NodeFileSystemFacade as NodeFilesystem} from './lib/filesystems/node-filesystem-facade';
+
+// EXPERIMENTAL: DATA SOURCES
+export type {
+  SourceLoader,
+  SourcePropsType,
+  SourceArrayOptionsType,
+  SourceArrayDataSourceType
+} from './source-types';
+export {isSourceLoader} from './source-types';
+
+export type {CoreAPI, DataSourceOptions} from './lib/sources/data-source';
+export {getSourceLoaderOptions} from './lib/sources/data-source';
+export {DataSource} from './lib/sources/data-source';
+export type {
+  ManageableDataSource,
+  DataSourceManagerEntryInfo,
+  DataSourceManagerEntryStatus,
+  DataSourceManagerDiscoveryInfo,
+  DataSourceManagerDiscoveryOptions,
+  DataSourceManagerAddParameters,
+  DataSourceManagerGetOrCreateParameters,
+  DataSourceManagerSubscribeParameters,
+  DataSourceSubscriber
+} from './lib/sources/data-source-manager';
+export {DataSourceManager} from './lib/sources/data-source-manager';
+export {
+  bindColumnarPredicateParameters,
+  copyColumnarPredicate,
+  filterColumnarRowIndices,
+  gatherColumnarColumns,
+  getColumnarPredicateColumns,
+  getColumnarPredicateParameterNames,
+  getColumnarPredicatePath,
+  getColumnarPredicatePaths,
+  isColumnarPredicateParameter,
+  isColumnarPredicateValue,
+  validateColumnarPredicate
+} from './lib/scan-utils/columnar-predicate';
+export {executeScanTasks} from './lib/scan-utils/scan-executor';
+export {
+  executeTableScanBatches,
+  filterTableBatch,
+  makeTableScanBatch,
+  projectTableBatch,
+  projectTableSchema,
+  truncateTableBatch
+} from './lib/scan-utils/table-scan-batch';
+export type {
+  TableScanBatchReader,
+  TableScanBatchOperators
+} from './lib/scan-utils/table-scan-batch';
+export {validateRasterQueryOptions} from './lib/scan-utils/raster-query';
+export type {RasterQueryOptions, RasterQueryCapabilities} from './lib/scan-utils/raster-query';
+export {planRelationalQuery} from './lib/scan-utils/relational-query';
+export type {
+  RelationalAggregate,
+  RelationalChildQuery,
+  RelationalExpression,
+  RelationalOrderKey,
+  RelationalPlanStep,
+  RelationalQueryOptions
+} from './lib/scan-utils/relational-query';
+export {
+  createScanQueryMetadata,
+  emitScanExecutionTelemetry
+} from './lib/scan-utils/scan-query-metadata';
+export {
+  intersectPointCloudBounds,
+  selectPointCloudScanTiles,
+  validatePointCloudQueryOptions
+} from './lib/scan-utils/point-cloud-query';
+export {
+  planTableQuery,
+  validateTableQueryLimit,
+  validateTableQueryOptions
+} from './lib/scan-utils/table-query';
+export {explainTableQuery} from './lib/scan-utils/table-query-explain';
+export type {ScanExecutorOptions, ScanTask} from './lib/scan-utils/scan-executor';
+export type {ScanFragment, ScanFragmentProvider} from './lib/scan-utils/scan-fragments';
+export type {
+  CreateScanQueryMetadataOptions,
+  ScanBounds,
+  ScanColumnMetadata,
+  ScanColumnRole,
+  ScanExecutionMethod,
+  ScanExecutionSupport,
+  ScanQueryCapabilities,
+  ScanQueryMetadata,
+  ScanQueryMetadataOptions,
+  ScanQueryMetadataProvider,
+  ScanExecutionTelemetry,
+  ScanExecutionTelemetryCallback,
+  ScanExecutionTelemetryStatus,
+  ScanSourceExecutionTelemetry,
+  PointCloudScanReadOptions,
+  PointCloudScanSource,
+  TableScanReadOptions,
+  TableScanSource,
+  ScanRasterLevel,
+  ScanSourceStatistics,
+  ScanSpatialMetadata
+} from './lib/scan-utils/scan-query-metadata';
+export type {
+  PointCloudQueryBounds,
+  PointCloudQueryCapabilities,
+  PointCloudQueryOptions,
+  PointCloudScanChildrenLoader,
+  PointCloudScanTile
+} from './lib/scan-utils/point-cloud-query';
+export type {
+  ColumnarComparisonPredicate,
+  ColumnarInPredicate,
+  ColumnarLogicalPredicate,
+  ColumnarNotPredicate,
+  ColumnarNullPredicate,
+  ColumnarPredicate,
+  ColumnarPredicateInputValue,
+  ColumnarPredicateParameter,
+  ColumnarPredicateParameterValues,
+  ColumnarPredicateProperty,
+  ColumnarPredicateValue,
+  ParameterizedColumnarPredicate
+} from './lib/scan-utils/columnar-predicate';
+export type {
+  TableQueryCapabilities,
+  TableQueryFilterStep,
+  TableQueryLimitStep,
+  TableQueryOperatorSupport,
+  TableQueryOptions,
+  TableQueryPlan,
+  TableQueryPlanStep,
+  TableQueryProjectStep,
+  TableQueryScanStep
+} from './lib/scan-utils/table-query';
+export type {
+  TableQueryExplain,
+  TableQueryExplainOperator
+} from './lib/scan-utils/table-query-explain';
+
+export type {CatalogSource, CatalogSourceCapabilities} from './lib/sources/catalog-source';
+export type {GeoServiceType, ServiceCapabilities} from './lib/sources/service-capabilities';
+
+export type {ImageSource} from './lib/sources/image-source';
+export type {ImageType} from './lib/sources/utils/image-type';
+export type {ImageSourceMetadata} from './lib/sources/image-source';
+export type {GetImageParameters} from './lib/sources/image-source';
+
+export type {
+  GetFeaturesParameters,
+  VectorSource,
+  VectorSourceData,
+  VectorSourceLayer,
+  VectorSourceMetadata
+} from './lib/sources/vector-source';
+export {CRSReprojectionError} from './lib/crs/crs-reprojection';
+export type {
+  CRSReprojectionOptions,
+  CRSReprojectionErrorCode
+} from './lib/crs/crs-reprojection';
+
+export type {TileSource, TileGrid, TileGridMatrix, TileGridLimits} from './lib/sources/tile-source';
+export type {TileSourceMetadata, GetTileParameters} from './lib/sources/tile-source';
+export type {GetTileDataBatchResult, GetTileDataParameters} from './lib/sources/tile-source';
+export {getTileDataBatch} from './lib/sources/tile-source-utils';
+export {TileSourceAdapter} from './lib/sources/tile-source-adapter';
+
+export type {ImageTileSource} from './lib/sources/image-tile-source';
+
+export type {VectorTileSource} from './lib/sources/vector-tile-source';
+export type {VectorTile} from './lib/sources/vector-tile-source';
+
+export {getRasterViewportBoundingBox} from './lib/sources/raster-source';
+export type {
+  RasterSource,
+  RasterChannelDataType,
+  RasterBoundingBox,
+  RasterViewport,
+  RasterData,
+  RasterSelection,
+  GetRasterParameters,
+  RasterOverview,
+  RasterSourceMetadata
+} from './lib/sources/raster-source';
+
+export type {
+  RasterAffineTransform,
+  RasterBandMetadata,
+  RasterValidityMask,
+  RasterBandStatistics,
+  RasterDeclaredBandStatistics,
+  RasterRegionParameters,
+  RasterRegionSource
+} from './lib/sources/raster-source';
+export {
+  sampleRaster,
+  rasterCoordinateToPixel,
+  computeRasterStatistics,
+  isRasterSampleValid
+} from './lib/sources/raster-sampling';
+export type {RasterSample, RasterSampleOptions} from './lib/sources/raster-sampling';
+export {canReuseRasterCoverage} from './lib/sources/raster-coverage';
+export type {RasterCoverageIdentity} from './lib/sources/raster-coverage';
+
+export type {RasterMixedData, NumericRasterData} from './lib/sources/raster-source';
+export {waitForPromiseWithSignal, waitForPromiseWithSignals} from './lib/request-utils/abort-utils';
+
+export {validateRasterRegion} from './lib/sources/raster-region';
+export {encodeHex, decodeHex} from './lib/binary-utils/hex-transcoder';
+
+// Shared tile-grid normalization and addressing.
+export {
+  createTileGrid,
+  createTileGridMatrix,
+  getRequestedTileMatrix,
+  getTileGridMatrixId
+} from './lib/sources/tile-grid';
+export type {TileMatrixFields} from './lib/sources/tile-grid';
+
+export {createWebMercatorTileGrid, resolveXYZTileParameters} from './lib/sources/xyz-grid';

package/src/loader-utils/json-loader.ts

@@ -0,0 +1,39 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {LoaderWithParser, LoaderOptions} from './loader-types';
+import type {Table, TableBatch} from '@loaders.gl/core/schema';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type JSONLoaderOptions = LoaderOptions;
+
+/**
+ * A JSON Micro loader (minimal bundle size)
+ * Alternative to `@loaders.gl/json`
+ */
+export const JSONLoader = {
+  dataType: null as unknown as Table,
+  batchType: null as unknown as TableBatch,
+  name: 'JSON',
+  id: 'json',
+  module: 'json',
+  version: VERSION,
+  encoding: 'json',
+  format: 'json',
+  extensions: ['json', 'geojson'],
+  mimeTypes: ['application/json'],
+  category: 'json',
+  text: true,
+  parseTextSync,
+  parse: async arrayBuffer => parseTextSync(new TextDecoder().decode(arrayBuffer)),
+  options: {}
+} as const satisfies LoaderWithParser<Table, TableBatch, JSONLoaderOptions>;
+
+// TODO - Better error handling!
+function parseTextSync(text) {
+  return JSON.parse(text);
+}