@e280/mammoth 0.1.0-2 → 0.1.0-20

package/README.md

@@ -1,13 +1,120 @@
 
-# 🦣 @e280/mammoth
+![](https://i.imgur.com/1TQmFwW.png)
 
-big files. small api.
+# 🦣 mammoth
+> *big files. small api.*
+
+**mammoth is a file storage typescript library.** we use it in our web apps to store files like user uploads. it's content-addressed. files are streamed into a bucket, identified by their blake3 hash, and deduplicated. mammoth works the same whether it's backed by a bucket in memory, on disk, in the cloud, or in the browser's opfs. mammoth uses [kv](https://github.com/e280/kv) for bookkeeping.
+
+```bash
+npm install @e280/mammoth @e280/kv
+```
+
+```ts
+import {Mammoth} from "@e280/mammoth"
+
+const mammoth = new Mammoth()
+```
+
+## 🦣 mammoth stores files.
+- **write a file,** and you get back its blake3 `hash` and `size` in bytes.
+    ```ts
+    const hash = await mammoth.write(blob.stream())
+    ```
+- **read a file,** identified by its hash.
+    ```ts
+    const blob = await mammoth.read(hash)
+    ```
+- **delete a file.**
+    ```ts
+    await mammoth.delete(hash)
+    ```
+
+## 🦣 mammoth is bucket-agnostic.
+- **memory bucket,** for testing.
+  ```ts
+  import {Mammoth, MemoryBucket} from "@e280/mammoth"
+  import {Kv} from "@e280/kv"
+
+  const mammoth = new Mammoth(
+
+    // bucket for blob storage
+    new MemoryBucket(),
+
+    // kv for metadata bookkeeping
+    new Kv(),
+  )
+  ```
+  defaults shown, this is equivalent:
+  ```ts
+  const mammoth = new Mammoth()
+  ```
+- **disk bucket,** for nodejs servers, example using [leveldb](https://github.com/google/leveldb). *(note the import paths)*
+  ```ts
+  import {Mammoth, DiskBucket} from "@e280/mammoth/node"
+  import {Kv, LevelMagazine} from "@e280/kv"
+  import {Level} from "level"
+
+  const mammoth = new Mammoth(
+    new DiskBucket("./data/mammoth/bucket"),
+    new Kv(new LevelMagazine(new Level("./data/mammoth/kv"))),
+  )
+  ```
+- **[opfs](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) bucket and [indexedDB](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API) kv,** for clientside in-browser storage.
+  ```ts
+  import {Mammoth, OpfsBucket} from "@e280/mammoth"
+  import {Kv, idbOpen, IdbMagazine} from "@e280/kv"
+
+  const mammoth = new Mammoth(
+    new OpfsBucket(await navigator.storage.getDirectory()),
+    new Kv(new IdbMagazine(await idbOpen("mammoth"))),
+  )
+  ```
+
+## 🦣 more mammoth methods.
+- **check if a file exists,** get back a boolean.
+    ```ts
+    const exists = await mammoth.has(hash)
+    ```
+- **get file info,** including `size` in bytes, `added` timestamp, and bucket `id`.
+    ```ts
+    const {size, added, id} = await mammoth.info(hash)
+    ```
+- **get stats,** for the whole datalake.
+    ```ts
+    await mammoth.stats()
+      // {count: 123, size: 123456789}
+    ```
+- **loop over every file,** with their hashes and infos.
+    ```ts
+    for await (const [hash, info] of mammoth.entries())
+      console.log(hash, info)
+    ```
+
+## 🦣 mammoth-brained tips.
+- **`analyze` is for hashing files.**
+    ```ts
+    import {analyze} from "@e280/mammoth"
+
+    const {hash, size} = await analyze(blob.stream())
+    ```
+- **`analyze` can help you avoid unnecessary uploads.**
+    ```ts
+    const {hash} = await analyze(blob.stream())
+
+    if (!await mammoth.has(hash))
+      await mammoth.write(blob.stream())
+    ```
+- **`streamify` makes a stream for a Uint8Array.**
+    ```ts
+    import {streamify} from "@e280/mammoth"
+
+    const readable = await streamify(new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF]))
+    ```
 
 
 
 <br/><br/>
 
-## 💖 mammoth is by e280
-reward us with github stars  
-build with us at https://e280.org/ but only if you're cool  
+*https://e280.org/*
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@e280/mammoth",
-	"version": "0.1.0-2",
+	"version": "0.1.0-20",
 	"description": "big file storage",
 	"license": "MIT",
 	"author": "Chase Moskal <chasemoskal@gmail.com>",
@@ -9,7 +9,7 @@
 	"sideEffects": false,
 	"exports": {
 		".": "./x/core/index.js",
-		"./browser": "./x/browser/index.js"
+		"./node": "./x/node/index.js"
 	},
 	"files": [
 		"x",
@@ -21,10 +21,12 @@
 		"test": "node x/test.js",
 		"count": "find s -path '*/_archive' -prune -o -name '*.ts' -exec wc -l {} +"
 	},
+	"peerDependencies": {
+		"@e280/kv": "^0.2.0-5"
+	},
 	"dependencies": {
-		"@e280/kv": "^0.1.5",
-		"@e280/stz": "^0.3.4",
-		"@noble/hashes": "^2.2.0"
+		"@awasm/noble": "^0.1.4",
+		"@e280/stz": "^0.3.6"
 	},
 	"devDependencies": {
 		"@e280/octo": "^0.1.3",
@@ -32,9 +34,6 @@
 		"@types/node": "^26.1.1",
 		"typescript": "^7.0.2"
 	},
-	"allowScripts": {
-		"classic-level@3.0.0": true
-	},
 	"keywords": [],
 	"homepage": "https://github.com/e280/mammoth#readme",
 	"repository": {

package/s/core/{iceberg-memory.ts → buckets/memory-bucket.ts}

@@ -1,19 +1,15 @@
 
 import {bytes, got, nap} from "@e280/stz"
-import {Iceberg, Id} from "./types.js"
-import {relaxer} from "./utils/relaxer.js"
+import {Bucket, Id} from "../types.js"
+import {relaxer} from "../utils/relaxer.js"
 
-export class IcebergMemory implements Iceberg {
+export class MemoryBucket implements Bucket {
 	#map = new Map<string, Uint8Array>()
 
 	async has(id: Id) {
 		return this.#map.has(id)
 	}
 
-	async size(id: Id) {
-		return got(this.#map.get(id)).byteLength
-	}
-
 	async delete(id: Id) {
 		this.#map.delete(id)
 	}

package/s/{browser/iceberg-opfs.ts → core/buckets/opfs-bucket.ts}

@@ -1,8 +1,8 @@
 
-import {Iceberg, Id} from "../core/types.js"
-import {isNotFound} from "./utils/is-not-found.js"
+import {Bucket, Id} from "../types.js"
+import {isNotFoundErr} from "../utils/is-not-found.js"
 
-export class IcebergOpfs implements Iceberg {
+export class OpfsBucket implements Bucket {
 	#directory
 
 	constructor(directory: FileSystemDirectoryHandle) {
@@ -15,23 +15,17 @@ export class IcebergOpfs implements Iceberg {
 			return true
 		}
 		catch (error) {
-			if (isNotFound(error)) return false
+			if (isNotFoundErr(error)) return false
 			throw error
 		}
 	}
 
-	async size(id: Id) {
-		const handle = await this.#directory.getFileHandle(id)
-		const file = await handle.getFile()
-		return file.size
-	}
-
 	async delete(id: Id) {
 		try {
 			await this.#directory.removeEntry(id)
 		}
 		catch (error) {
-			if (!isNotFound(error))
+			if (!isNotFoundErr(error))
 				throw error
 		}
 	}

package/s/core/consts.ts

@@ -0,0 +1,8 @@
+
+import {time} from "@e280/stz"
+
+export const consts = {
+	max_jobs: 64,
+	expiry_time: time.days(7),
+}
+

package/s/core/index.ts

@@ -1,6 +1,10 @@
 
-export * from "./hash-stream.js"
-export * from "./iceberg-memory.js"
+export * from "./buckets/memory-bucket.js"
+export * from "./buckets/opfs-bucket.js"
+
+export * from "./parts/analyze.js"
+export * from "./parts/streamify.js"
+
 export * from "./mammoth.js"
 export * from "./types.js"
 

package/s/core/mammoth.ts

@@ -1,72 +1,80 @@
 
 import {Kv} from "@e280/kv"
-import {got, hex} from "@e280/stz"
-import {blake3} from "@noble/hashes/blake3.js"
+import {lane, queue} from "@e280/stz"
 
+import {consts} from "./consts.js"
+import {save} from "./utils/save.js"
+import {Hash, Bucket} from "./types.js"
+import {cleanup} from "./utils/cleanup.js"
 import {randomId} from "./utils/random-id.js"
-import {Glacier, Hash, Iceberg, Id} from "./types.js"
+import {Manifest} from "./manifest/manifest.js"
+import {commit} from "./manifest/fns/commit.js"
+import {needInfo} from "./manifest/fns/need-info.js"
+import {getStats} from "./manifest/fns/get-stats.js"
+import {MemoryBucket} from "./buckets/memory-bucket.js"
+import {scheduleDeletion} from "./manifest/fns/schedule-deletion.js"
+import {moveWriteToTrash} from "./manifest/fns/move-write-to-trash.js"
 
-export class Mammoth implements Glacier {
-	#index
-	#iceberg
+/** file storage datalake, content-addressed with blake3 hashes. */
+export class Mammoth {
+	#bucket
+	#manifest
+	#wholesome = lane(consts.max_jobs)
 
-	constructor(index: Kv<string>, iceberg: Iceberg) {
-		this.#index = index
-		this.#iceberg = iceberg
+	constructor(bucket: Bucket = new MemoryBucket(), kv = new Kv()) {
+		this.#bucket = bucket
+		this.#manifest = new Manifest(kv)
+	}
+
+	async* entries() {
+		yield* this.#manifest.catalog.entries()
 	}
 
 	async has(hash: Hash) {
-		return this.#index.has(hash)
+		return this.#manifest.catalog.has(hash)
+	}
+
+	async info(hash: Hash) {
+		return needInfo(this.#manifest, hash)
 	}
 
-	async size(hash: Hash) {
-		const id = await this.#needFileId(hash)
-		return this.#iceberg.size(id)
+	async stats() {
+		return getStats(this.#manifest)
 	}
 
 	async read(hash: Hash) {
-		const id = await this.#needFileId(hash)
-		return this.#iceberg.read(id)
+		const {id} = await this.info(hash)
+		return this.#bucket.read(id)
 	}
 
 	async delete(hash: Hash) {
-		const id = await this.#getFileId(hash)
-		if (id) {
-			await this.#index.del(hash)
-			await this.#iceberg.delete(id)
-		}
+		await this.#wholesome(async() => {
+			const info = await this.#manifest.catalog.get(hash)
+			if (info)
+				await scheduleDeletion(this.#manifest, hash, info)
+		})
+		await this.#cleanup()
 	}
 
 	async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
 		const id = randomId()
-		const pipe = new TransformStream()
-		const done = this.#iceberg.write(id, pipe.readable)
-		const writer = pipe.writable.getWriter()
-		const hasher = blake3.create()
-
-		for await (const chunk of readable) {
-			await writer.write(chunk as Uint8Array<ArrayBuffer>)
-			hasher.update(chunk)
+		await this.#manifest.writes.set(id, {created: Date.now()})
+		try {
+			const analysis = await save(this.#bucket, id, readable)
+			const {hash, size} = analysis
+			const info = {id, size, added: Date.now()}
+			await this.#wholesome(() => commit(this.#manifest, hash, info))
+			return analysis.hash
+		}
+		catch (error) {
+			await moveWriteToTrash(this.#manifest, id)
+			throw error
+		}
+		finally {
+			void this.#cleanup().catch(e => console.error("cleanup failed", e))
 		}
-
-		await writer.close()
-		await done
-		const hash = hex.fromBytes(hasher.digest())
-		await this.delete(hash)
-		await this.#index.set(hash, id)
-		return hash
-	}
-
-	async #getFileId(hash: Hash): Promise<Id | undefined> {
-		return this.#index.get<string>(hash)
-	}
-
-	async #needFileId(hash: Hash): Promise<Id> {
-		return got(await this.#getFileId(hash), `file not found by hash "${hash}"`)
 	}
 
-	async* keys() {
-		yield* this.#index.keys()
-	}
+	#cleanup = queue(() => cleanup(this.#bucket, this.#manifest), consts.max_jobs)
 }
 

package/s/core/manifest/fns/commit.ts

@@ -0,0 +1,27 @@
+
+import {getStats} from "./get-stats.js"
+import {Manifest} from "../manifest.js"
+import {Hash, Info} from "../../types.js"
+
+export async function commit(manifest: Manifest, hash: Hash, info: Info) {
+	const {kv, stats, catalog, writes, trash} = manifest
+	const isNewFile = !await catalog.has(hash)
+
+	if (isNewFile) {
+		const s = await getStats(manifest)
+		s.count += 1
+		s.size += info.size
+		await kv.commit([
+			writes.op.delete(info.id),
+			catalog.op.set(hash, info),
+			stats.op.set(s),
+		])
+	}
+	else {
+		await kv.commit([
+			writes.op.delete(info.id),
+			trash.op.set(info.id, true),
+		])
+	}
+}
+

package/s/core/manifest/fns/get-expired-writes.ts

@@ -0,0 +1,11 @@
+
+import {Manifest} from "../manifest.js"
+import {isExpired} from "../../utils/is-expired.js"
+
+export async function* getExpiredWrites({writes}: Manifest) {
+	for await (const [id, w] of writes.entries()) {
+		if (isExpired(w.created))
+			yield id
+	}
+}
+

package/s/core/manifest/fns/get-stats.ts

@@ -0,0 +1,10 @@
+
+import {Manifest} from "../manifest.js"
+
+export async function getStats({stats}: Manifest) {
+	return structuredClone(
+		(await stats.get())
+			?? {count: 0, size: 0}
+	)
+}
+

package/s/core/manifest/fns/move-write-to-trash.ts

@@ -0,0 +1,11 @@
+
+import {Id} from "../../types.js"
+import {Manifest} from "../manifest.js"
+
+export async function moveWriteToTrash({kv, writes, trash}: Manifest, id: Id) {
+	await kv.commit([
+		writes.op.delete(id),
+		trash.op.set(id, true),
+	])
+}
+

package/s/core/manifest/fns/need-info.ts

@@ -0,0 +1,10 @@
+
+import {got} from "@e280/stz"
+import {Hash} from "../../types.js"
+import {Manifest} from "../manifest.js"
+import {notFoundMessage} from "../../utils/not-found.js"
+
+export async function needInfo({catalog}: Manifest, hash: Hash) {
+	return got(await catalog.get(hash), notFoundMessage(hash))
+}
+

package/s/core/manifest/fns/schedule-deletion.ts

@@ -0,0 +1,19 @@
+
+import {getStats} from "./get-stats.js"
+import {Manifest} from "../manifest.js"
+import {Hash, Info} from "../../types.js"
+
+export async function scheduleDeletion(manifest: Manifest, hash: Hash, info: Info) {
+	const {kv, stats, catalog, trash} = manifest
+
+	const s = await getStats(manifest)
+	s.count -= 1
+	s.size -= info.size
+
+	await kv.commit([
+		catalog.op.delete(hash),
+		trash.op.set(info.id, true),
+		stats.op.set(s),
+	])
+}
+

package/s/core/manifest/manifest.ts

@@ -0,0 +1,27 @@
+
+import {Kv} from "@e280/kv"
+import {Write} from "./types.js"
+import {Info, Stats} from "../types.js"
+
+export class Manifest {
+
+	/** statistics about the whole datalake */
+	stats
+
+	/** associates file hashes with bucket ids */
+	catalog
+
+	/** temporary record of writes in-progress */
+	writes
+
+	/** bucket ids that are pending deletion */
+	trash
+
+	constructor(public kv: Kv) {
+		this.stats = kv.store<Stats>("stats")
+		this.catalog = kv.scope<Info>("catalog")
+		this.writes = kv.scope<Write>("writes")
+		this.trash = kv.scope<true>("trash")
+	}
+}
+

package/s/core/manifest/types.ts

@@ -0,0 +1,5 @@
+
+export type Write = {
+	created: number
+}
+

package/s/core/parts/analyze.ts

@@ -0,0 +1,26 @@
+
+import {hex, nap} from "@e280/stz"
+import {blake3} from "@awasm/noble"
+import {Analysis} from "../types.js"
+import {relaxer} from "../utils/relaxer.js"
+
+export async function analyze(
+		readable: ReadableStream<Uint8Array>
+	): Promise<Analysis> {
+
+	let size = 0
+	const hasher = blake3.create()
+	const relax = relaxer()
+
+	for await (const chunk of readable) {
+		hasher.update(chunk)
+		size += chunk.byteLength
+
+		if (relax())
+			await nap()
+	}
+
+	const hash = hex.fromBytes(hasher.digest())
+	return {hash, size}
+}
+

package/s/core/parts/streamify.ts

@@ -0,0 +1,18 @@
+
+export function streamify(bytes: Uint8Array, chunkSize = 64 * 1024) {
+	let offset = 0
+
+	return new ReadableStream<Uint8Array>({
+		pull(controller) {
+			if (offset >= bytes.length) {
+				controller.close()
+				return
+			}
+
+			const end = Math.min(offset + chunkSize, bytes.length)
+			controller.enqueue(bytes.subarray(offset, end))
+			offset = end
+		},
+	})
+}
+

package/s/core/types.ts

@@ -2,20 +2,44 @@
 export type Hash = string
 export type Id = string
 
-export type Iceberg = {
+/** file blob store. */
+export type Bucket = {
 	has(id: Id): Promise<boolean>
-	size(id: Id): Promise<number>
 	delete(id: Id): Promise<void>
 	read(id: Id): Promise<Blob>
 	write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>
 }
 
-export type Glacier = {
-	has(hash: Hash): Promise<boolean>
-	size(hash: Hash): Promise<number>
-	read(hash: Hash): Promise<Blob>
-	delete(hash: Hash): Promise<void>
-	write(readable: ReadableStream<Uint8Array>): Promise<Hash>
-	keys(): AsyncIterable<Hash>
+/** report about a file */
+export type Analysis = {
+
+	/** blake3 hash of the file's contents */
+	hash: Hash
+
+	/** filesize in bytes */
+	size: number
+}
+
+/** metadata for a single file. */
+export type Info = {
+
+	/** bucket id for this file's data. */
+	id: Id
+
+	/** filesize in bytes. */
+	size: number
+
+	/** when this file was added to the datalake. */
+	added: number
+}
+
+/** statistics for the whole datalake. */
+export type Stats = {
+
+	/** total number of bytes in the whole datalake. */
+	size: number
+
+	/** total number of files in the datalake. */
+	count: number
 }
 

package/s/core/utils/cleanup.ts

@@ -0,0 +1,16 @@
+
+import {Bucket} from "../types.js"
+import {Manifest} from "../manifest/manifest.js"
+import {getExpiredWrites} from "../manifest/fns/get-expired-writes.js"
+import {moveWriteToTrash} from "../manifest/fns/move-write-to-trash.js"
+
+export async function cleanup(bucket: Bucket, manifest: Manifest) {
+	for await (const id of getExpiredWrites(manifest))
+		await moveWriteToTrash(manifest, id)
+
+	for await (const id of manifest.trash.keys()) {
+		await bucket.delete(id)
+		await manifest.trash.delete(id)
+	}
+}
+

package/s/core/utils/is-expired.ts

@@ -0,0 +1,9 @@
+
+import {time} from "@e280/stz"
+import {consts} from "../consts.js"
+
+export function isExpired(created: number) {
+	const since = Date.now() - created
+	return since > time.days(consts.expiry_time)
+}
+

package/s/{browser → core}/utils/is-not-found.ts

@@ -1,5 +1,5 @@
 
-export function isNotFound(error: unknown) {
+export function isNotFoundErr(error: unknown) {
 	return error instanceof DOMException
 		&& error.name === "NotFoundError"
 }

package/s/core/utils/not-found.ts

@@ -0,0 +1,7 @@
+
+import {Hash} from "../types.js"
+
+export function notFoundMessage(hash: Hash) {
+	return `file not found by hash "${hash}"`
+}
+