@earthmover/icechunk 2.0.0-alpha.11 → 2.0.0-alpha.12

package/README.md

@@ -46,14 +46,30 @@ In-memory storage works on all platforms. The following backends are available o
 - **HTTP** — `Storage.newHttp(baseUrl, config?)`
 - **Local Filesystem** — `Storage.newLocalFilesystem(path)`
 
+### Fetch Storage (read-only, works everywhere)
+
+For read-only access to a publicly hosted Icechunk repository, use the built-in fetch-based storage. This works on both native and WASM builds, making it the easiest way to open a repository in the browser:
+
+```typescript
+import { Repository } from '@earthmover/icechunk'
+import { createFetchStorage } from '@earthmover/icechunk/fetch-storage'
+
+const storage = createFetchStorage('https://my-bucket.s3.us-west-2.amazonaws.com/my-repo.icechunk')
+const repo = await Repository.open(storage)
+const session = await repo.readonlySession({ branch: 'main' })
+const keys = await session.store.list()
+```
+
+The repository must be on S3-compatible storage with public read access and XML listing enabled. `createFetchStorage` uses the browser `fetch` API under the hood, so it works in any environment where `fetch` is available.
+
 ### Custom Storage Backends
 
 For WASM builds (or any environment where the built-in backends aren't suitable), you can provide your own storage implementation in JavaScript using `Storage.newCustom()`:
 
 ```typescript
 const storage = Storage.newCustom({
-  canWrite: async () => true,
-  getObjectRange: async ({ path, rangeStart, rangeEnd }) => {
+  canWrite: async (_err, ) => true,
+  getObjectRange: async (_err, { path, rangeStart, rangeEnd }) => {
     const headers: Record<string, string> = {}
     if (rangeStart != null && rangeEnd != null) {
       headers['Range'] = `bytes=${rangeStart}-${rangeEnd - 1}`
@@ -61,15 +77,17 @@ const storage = Storage.newCustom({
     const resp = await fetch(`https://my-bucket.example.com/${path}`, { headers })
     return { data: new Uint8Array(await resp.arrayBuffer()), version: { etag: resp.headers.get('etag') ?? undefined } }
   },
-  putObject: async ({ path, data, contentType }) => { /* ... */ },
-  copyObject: async ({ from, to }) => { /* ... */ },
-  listObjects: async (prefix) => { /* return [{ id, createdAt, sizeBytes }] */ },
-  deleteBatch: async ({ prefix, batch }) => { /* return { deletedObjects, deletedBytes } */ },
-  getObjectLastModified: async (path) => { /* return Date */ },
-  getObjectConditional: async ({ path, previousVersion }) => { /* ... */ },
+  putObject: async (_err, { path, data, contentType }) => { /* ... */ },
+  copyObject: async (_err, { from, to }) => { /* ... */ },
+  listObjects: async (_err, prefix) => { /* return [{ id, createdAt, sizeBytes }] */ },
+  deleteBatch: async (_err, { prefix, batch }) => { /* return { deletedObjects, deletedBytes } */ },
+  getObjectLastModified: async (_err, path) => { /* return Date */ },
+  getObjectConditional: async (_err, { path, previousVersion }) => { /* ... */ },
 })
 ```
 
+> **Note:** Callbacks use the Node.js error-first convention — the first argument is always `null` (reserved for errors) and the actual arguments follow. Use `_err` to skip it.
+
 This is the primary way to use cloud storage in the browser, where native Rust networking is unavailable. Each callback method maps to an operation on the underlying `Storage` trait. See the exported `Storage*` TypeScript interfaces for the full type signatures.
 
 ### Virtual Chunks
@@ -84,6 +102,18 @@ Install the package with the `--cpu=wasm32` flag to get the WASM binary:
 npm install @earthmover/icechunk --cpu=wasm32
 ```
 
+To open a public repository in the browser, use fetch storage:
+
+```typescript
+import { Repository } from '@earthmover/icechunk'
+import { createFetchStorage } from '@earthmover/icechunk/fetch-storage'
+
+const storage = createFetchStorage('https://my-bucket.s3.us-west-2.amazonaws.com/my-repo.icechunk')
+const repo = await Repository.open(storage)
+```
+
+For more control, use `Storage.newCustom()` to implement your own storage backend with any JS networking library.
+
 The WASM build uses `SharedArrayBuffer` for threading, which requires your server to send these headers:
 
 ```

package/fetch-storage.d.ts

@@ -0,0 +1,20 @@
+import type { Storage } from '@earthmover/icechunk'
+
+/**
+ * Create a read-only storage backend that fetches objects over HTTP.
+ *
+ * Works with any publicly accessible icechunk repository hosted on S3-compatible
+ * storage. Requires the bucket to support anonymous reads and S3 XML listing.
+ *
+ * @param baseUrl - Base URL of the icechunk repository (no trailing slash)
+ *
+ * @example
+ * ```ts
+ * import { Repository } from '@earthmover/icechunk'
+ * import { createFetchStorage } from '@earthmover/icechunk/fetch-storage'
+ *
+ * const storage = createFetchStorage('https://my-bucket.s3.us-west-2.amazonaws.com/path/to/repo.icechunk')
+ * const repo = await Repository.open(storage)
+ * ```
+ */
+export declare function createFetchStorage(baseUrl: string): Storage

package/fetch-storage.js

@@ -0,0 +1,130 @@
+/**
+ * Read-only fetch-based storage backend for Icechunk.
+ *
+ * Usage:
+ *   import { Repository } from '@earthmover/icechunk'
+ *   import { createFetchStorage } from '@earthmover/icechunk/fetch-storage'
+ *
+ *   const storage = createFetchStorage('https://my-bucket.s3.amazonaws.com/my-repo.icechunk')
+ *   const repo = await Repository.open(storage)
+ */
+
+const { Storage } = require('@earthmover/icechunk')
+
+/**
+ * @param {string} baseUrl - Base URL of the icechunk repository (no trailing slash)
+ * @returns {Storage}
+ */
+function createFetchStorage(baseUrl) {
+  // Normalize: strip trailing slash
+  const base = baseUrl.replace(/\/+$/, '')
+
+  function throwForStatus(resp, url) {
+    if (resp.ok) return
+    if (resp.status === 404) {
+      throw new Error(`ObjectNotFound: ${url}`)
+    }
+    throw new Error(`HTTP ${resp.status}: ${url}`)
+  }
+
+  return Storage.newCustom({
+    canWrite: async (_err) => false,
+
+    getObjectRange: async (_err, { path, rangeStart, rangeEnd }) => {
+      const url = `${base}/${path}`
+      const headers = {}
+
+      if (rangeStart != null && rangeEnd != null) {
+        headers['Range'] = `bytes=${rangeStart}-${rangeEnd - 1}`
+      } else if (rangeStart != null) {
+        headers['Range'] = `bytes=${rangeStart}-`
+      }
+
+      const resp = await fetch(url, { headers })
+      throwForStatus(resp, url)
+
+      const data = new Uint8Array(await resp.arrayBuffer())
+      const etag = resp.headers.get('etag') ?? undefined
+      return { data, version: { etag } }
+    },
+
+    putObject: async () => {
+      throw new Error('Read-only storage: putObject not supported')
+    },
+
+    copyObject: async () => {
+      throw new Error('Read-only storage: copyObject not supported')
+    },
+
+    listObjects: async (_err, prefix) => {
+      // Try S3-style XML listing
+      // Derive bucket URL and key prefix from the base URL
+      // e.g. https://bucket.s3.region.amazonaws.com/prefix -> bucket URL + key prefix
+      const url = new URL(base)
+      const keyPrefix = url.pathname.replace(/^\//, '') + '/' + prefix
+      const listUrl = `${url.origin}/?list-type=2&prefix=${encodeURIComponent(keyPrefix)}`
+
+      const resp = await fetch(listUrl)
+      throwForStatus(resp, listUrl)
+
+      const xml = await resp.text()
+      const results = []
+      const contentRegex = /<Contents>([\s\S]*?)<\/Contents>/g
+      let match
+      while ((match = contentRegex.exec(xml)) !== null) {
+        const block = match[1]
+        const key = block.match(/<Key>(.*?)<\/Key>/)?.[1]
+        const lastModified = block.match(/<LastModified>(.*?)<\/LastModified>/)?.[1]
+        const size = block.match(/<Size>(.*?)<\/Size>/)?.[1]
+        if (key && lastModified && size) {
+          const id = key.startsWith(keyPrefix) ? key.slice(keyPrefix.length) : key
+          results.push({
+            id,
+            createdAt: new Date(lastModified),
+            sizeBytes: Number(size),
+          })
+        }
+      }
+
+      return results
+    },
+
+    deleteBatch: async () => {
+      throw new Error('Read-only storage: deleteBatch not supported')
+    },
+
+    getObjectLastModified: async (_err, path) => {
+      const url = `${base}/${path}`
+      const resp = await fetch(url, { method: 'HEAD' })
+      throwForStatus(resp, url)
+      const lastModified = resp.headers.get('last-modified')
+      if (!lastModified) {
+        throw new Error(`No Last-Modified header: ${url}`)
+      }
+      return new Date(lastModified)
+    },
+
+    getObjectConditional: async (_err, { path, previousVersion }) => {
+      const url = `${base}/${path}`
+      const headers = {}
+
+      if (previousVersion?.etag) {
+        headers['If-None-Match'] = previousVersion.etag
+      }
+
+      const resp = await fetch(url, { headers })
+
+      if (resp.status === 304) {
+        return { kind: 'on_latest_version' }
+      }
+
+      throwForStatus(resp, url)
+
+      const data = new Uint8Array(await resp.arrayBuffer())
+      const etag = resp.headers.get('etag') ?? undefined
+      return { kind: 'modified', data, newVersion: { etag } }
+    },
+  })
+}
+
+module.exports = { createFetchStorage }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@earthmover/icechunk",
-  "version": "2.0.0-alpha.11",
+  "version": "2.0.0-alpha.12",
   "description": "JavaScript/TypeScript bindings for Icechunk",
   "main": "index.js",
   "repository": {
@@ -19,10 +19,22 @@
     "science",
     "geospatial"
   ],
+  "exports": {
+    ".": {
+      "browser": "./browser.js",
+      "default": "./index.js"
+    },
+    "./fetch-storage": {
+      "types": "./fetch-storage.d.ts",
+      "default": "./fetch-storage.js"
+    }
+  },
   "files": [
     "index.d.ts",
     "index.js",
-    "browser.js"
+    "browser.js",
+    "fetch-storage.js",
+    "fetch-storage.d.ts"
   ],
   "napi": {
     "binaryName": "icechunk",
@@ -108,9 +120,9 @@
   },
   "packageManager": "yarn@4.12.0",
   "optionalDependencies": {
-    "@earthmover/icechunk-win32-x64-msvc": "2.0.0-alpha.11",
-    "@earthmover/icechunk-linux-x64-gnu": "2.0.0-alpha.11",
-    "@earthmover/icechunk-darwin-arm64": "2.0.0-alpha.11",
-    "@earthmover/icechunk-wasm32-wasi": "2.0.0-alpha.11"
+    "@earthmover/icechunk-win32-x64-msvc": "2.0.0-alpha.12",
+    "@earthmover/icechunk-linux-x64-gnu": "2.0.0-alpha.12",
+    "@earthmover/icechunk-darwin-arm64": "2.0.0-alpha.12",
+    "@earthmover/icechunk-wasm32-wasi": "2.0.0-alpha.12"
   }
 }