@e280/mammoth 0.1.0-1 → 0.1.0-10

package/README.md

@@ -1,13 +1,122 @@
 
-# 🦣 @e280/mammoth
+![](https://i.imgur.com/1TQmFwW.png)
 
-big files. small api.
+# 🦣 mammoth
+> *big files. small api.*
+
+**mammoth is a file storage typescript library.** we use it in our web apps to store files like user uploads. it's content-addressed. files are streamed into a bucket, identified by their blake3 hash, and deduplicated. mammoth works the same whether it's backed by a bucket in memory, on disk, in the cloud, or in the browser's opfs. mammoth uses [kv](https://github.com/e280/kv) for bookkeeping.
+
+```bash
+npm install @e280/mammoth @e280/kv
+```
+
+```ts
+import {Mammoth} from "@e280/mammoth"
+
+const mammoth = new Mammoth()
+```
+
+## 🦣 mammoth stores files.
+- **write a file,** and you get back its blake3 `hash` and `size` in bytes.
+    ```ts
+    const hash = await mammoth.write(blob.stream())
+    ```
+- **read a file,** identified by its hash.
+    ```ts
+    const blob = await mammoth.read(hash)
+    ```
+- **delete a file.**
+    ```ts
+    await mammoth.delete(hash)
+    ```
+
+## 🦣 mammoth is bucket-agnostic.
+- **memory bucket,** for testing.
+  ```ts
+  import {Mammoth, MemoryBucket} from "@e280/mammoth"
+  import {Kv} from "@e280/kv"
+
+  const mammoth = new Mammoth(
+
+    // bucket for blob storage
+    new MemoryBucket(),
+
+    // kv for metadata bookkeeping
+    new Kv(),
+  )
+  ```
+  defaults shown, this is equivalent:
+  ```ts
+  const mammoth = new Mammoth()
+  ```
+- **disk bucket,** for nodejs servers. *(note the import paths)*
+  ```ts
+  import {Mammoth} from "@e280/mammoth"
+  import {DiskBucket} from "@e280/mammoth/node"
+  import {Kv} from "@e280/kv"
+  import {LevelDriver} from "@e280/kv/level"
+
+  const mammoth = new Mammoth(
+    new DiskBucket("./data/bucket"),
+    new Kv(new LevelDriver("./data/kv")),
+  )
+  ```
+- **opfs bucket,** for local storage in the browser. *(note the import paths)*
+  ```ts
+  import {Mammoth} from "@e280/mammoth"
+  import {OpfsBucket} from "@e280/mammoth/browser"
+  import {Kv, StorageDriver} from "@e280/kv"
+
+  const mammoth = new Mammoth(
+    new OpfsBucket(await navigator.storage.getDirectory()),
+    new Kv(new StorageDriver(localStorage)),
+  )
+  ```
+
+## 🦣 more mammoth methods.
+- **check if a file exists,** get back a boolean.
+    ```ts
+    const exists = await mammoth.has(hash)
+    ```
+- **get file info,** including `size` in bytes, `added` timestamp, and bucket `id`.
+    ```ts
+    const {size, added, id} = await mammoth.info(hash)
+    ```
+- **get stats,** for the whole datalake.
+    ```ts
+    await mammoth.stats()
+      // {count: 123, size: 123456789}
+    ```
+- **loop over all hashes,** for all stored files.
+    ```ts
+    for await (const hash of mammoth.hashes())
+      console.log(hash)
+    ```
+
+## 🦣 mammoth-brained tips.
+- **`analyze` is for hashing files.**
+    ```ts
+    import {analyze} from "@e280/mammoth"
+
+    const {hash, size} = await analyze(blob.stream())
+    ```
+- **`analyze` can help you avoid unnecessary uploads.**
+    ```ts
+    const {hash} = await analyze(blob.stream())
+
+    if (!await mammoth.has(hash))
+      await mammoth.write(blob.stream())
+    ```
+- **`streamify` makes a stream for a Uint8Array.**
+    ```ts
+    import {streamify} from "@e280/mammoth"
+
+    const readable = await streamify(new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF]))
+    ```
 
 
 
 <br/><br/>
 
-## 💖 mammoth is by e280
-reward us with github stars  
-build with us at https://e280.org/ but only if you're cool  
+*https://e280.org/*
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@e280/mammoth",
-	"version": "0.1.0-1",
+	"version": "0.1.0-10",
 	"description": "big file storage",
 	"license": "MIT",
 	"author": "Chase Moskal <chasemoskal@gmail.com>",
@@ -9,7 +9,8 @@
 	"sideEffects": false,
 	"exports": {
 		".": "./x/core/index.js",
-		"./browser": "./x/browser/index.js"
+		"./browser": "./x/browser/index.js",
+		"./node": "./x/node/index.js"
 	},
 	"files": [
 		"x",
@@ -21,9 +22,11 @@
 		"test": "node x/test.js",
 		"count": "find s -path '*/_archive' -prune -o -name '*.ts' -exec wc -l {} +"
 	},
+	"peerDependencies": {
+		"@e280/kv": "^0.1.5"
+	},
 	"dependencies": {
-		"@e280/kv": "^0.1.5",
-		"@e280/stz": "^0.3.4",
+		"@e280/stz": "^0.3.5",
 		"@noble/hashes": "^2.2.0"
 	},
 	"devDependencies": {

package/s/browser/example.ts

@@ -1,26 +1,25 @@
 
 import {Kv, StorageDriver} from "@e280/kv"
 import {Mammoth} from "../core/mammoth.js"
-import {IcebergOpfs} from "./iceberg-opfs.js"
+import {OpfsBucket} from "./opfs-bucket.js"
 
 export async function example() {
 
-	// setup the localstorage kv index
+	// setup the localstorage kv
 	const driver = new StorageDriver(window.localStorage)
-	const kv = new Kv(driver)
-	const index = kv.scope("glacier_index")
+	const kv = new Kv(driver).scope("mammoth")
 
-	// setup the opfs iceberg
+	// setup the opfs file bucket
 	const root = await navigator.storage.getDirectory()
-	const directory = await root.getDirectoryHandle("glacier", {create: true})
-	const iceberg = new IcebergOpfs(directory)
+	const directory = await root.getDirectoryHandle("mammoth", {create: true})
+	const bucket = new OpfsBucket(directory)
 
 	// setup the mammoth
-	const mammoth = new Mammoth(index, iceberg)
-	const bigFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
+	const mammoth = new Mammoth(bucket, kv)
+	const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
 
 	// write a file as an example
-	const hash = await mammoth.write(bigFile.stream())
-	console.log(`mammoth stored file by hash "${hash}"`)
+	const hash = await mammoth.write(myFile.stream())
+	console.log(`mammoth stored file "${hash}"`)
 }
 

package/s/browser/index.ts

@@ -1,4 +1,4 @@
 
 export * from "../core/index.js"
-export * from "./iceberg-opfs.js"
+export * from "./opfs-bucket.js"
 

package/s/browser/{iceberg-opfs.ts → opfs-bucket.ts}

@@ -1,8 +1,8 @@
 
-import {Iceberg, Id} from "../core/types.js"
+import {Bucket, Id} from "../core/types.js"
 import {isNotFound} from "./utils/is-not-found.js"
 
-export class IcebergOpfs implements Iceberg {
+export class OpfsBucket implements Bucket {
 	#directory
 
 	constructor(directory: FileSystemDirectoryHandle) {
@@ -20,12 +20,6 @@ export class IcebergOpfs implements Iceberg {
 		}
 	}
 
-	async size(id: Id) {
-		const handle = await this.#directory.getFileHandle(id)
-		const file = await handle.getFile()
-		return file.size
-	}
-
 	async delete(id: Id) {
 		try {
 			await this.#directory.removeEntry(id)

package/s/core/{hash-stream.ts → analyze.ts}

@@ -1,19 +1,23 @@
 
 import {hex, nap} from "@e280/stz"
 import {blake3} from "@noble/hashes/blake3.js"
+import {Analysis} from "./types.js"
 import {relaxer} from "./utils/relaxer.js"
 
-export async function hashStream(readable: ReadableStream<Uint8Array>) {
+export async function analyze(readable: ReadableStream<Uint8Array>): Promise<Analysis> {
+	let size = 0
 	const hasher = blake3.create()
 	const relax = relaxer()
 
 	for await (const chunk of readable) {
 		hasher.update(chunk)
+		size += chunk.byteLength
 
 		if (relax())
 			await nap()
 	}
 
-	return hex.fromBytes(hasher.digest())
+	const hash = hex.fromBytes(hasher.digest())
+	return {hash, size}
 }
 

package/s/core/consts.ts

@@ -0,0 +1,8 @@
+
+import {time} from "@e280/stz"
+
+export const consts = {
+	max_jobs: 64,
+	expiry_time: time.days(7),
+}
+

package/s/core/index.ts

@@ -1,6 +1,7 @@
 
-export * from "./hash-stream.js"
-export * from "./iceberg-memory.js"
+export * from "./analyze.js"
 export * from "./mammoth.js"
+export * from "./memory-bucket.js"
+export * from "./streamify.js"
 export * from "./types.js"
 

package/s/core/mammoth.ts

@@ -1,72 +1,82 @@
 
 import {Kv} from "@e280/kv"
-import {got, hex} from "@e280/stz"
-import {blake3} from "@noble/hashes/blake3.js"
+import {lane, queue} from "@e280/stz"
 
+import {consts} from "./consts.js"
+import {save} from "./utils/save.js"
+import {Hash, Bucket} from "./types.js"
+import {Manifest} from "./utils/manifest.js"
 import {randomId} from "./utils/random-id.js"
-import {Glacier, Hash, Iceberg, Id} from "./types.js"
+import {MemoryBucket} from "./memory-bucket.js"
 
-export class Mammoth implements Glacier {
-	#index
-	#iceberg
+/** file storage datalake, content-addressed with blake3 hashes. */
+export class Mammoth {
+	#bucket
+	#manifest
+	#wholesome = lane(consts.max_jobs)
 
-	constructor(index: Kv<string>, iceberg: Iceberg) {
-		this.#index = index
-		this.#iceberg = iceberg
+	constructor(bucket: Bucket = new MemoryBucket(), kv = new Kv()) {
+		this.#bucket = bucket
+		this.#manifest = new Manifest(kv)
+	}
+
+	async* hashes() {
+		yield* this.#manifest.hashes()
 	}
 
 	async has(hash: Hash) {
-		return this.#index.has(hash)
+		return this.#manifest.hasHash(hash)
+	}
+
+	async info(hash: Hash) {
+		return this.#manifest.needInfo(hash)
 	}
 
-	async size(hash: Hash) {
-		const id = await this.#needFileId(hash)
-		return this.#iceberg.size(id)
+	async stats() {
+		return this.#manifest.getStats()
 	}
 
 	async read(hash: Hash) {
-		const id = await this.#needFileId(hash)
-		return this.#iceberg.read(id)
+		const {id} = await this.#manifest.needInfo(hash)
+		return this.#bucket.read(id)
 	}
 
 	async delete(hash: Hash) {
-		const id = await this.#getFileId(hash)
-		if (id) {
-			await this.#index.del(hash)
-			await this.#iceberg.delete(id)
-		}
+		await this.#wholesome(async() => {
+			const info = await this.#manifest.getInfo(hash)
+			if (info)
+				await this.#manifest.scheduleDeletion(hash, info)
+		})
+		await this.#cleanup()
 	}
 
 	async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
 		const id = randomId()
-		const pipe = new TransformStream()
-		const done = this.#iceberg.write(id, pipe.readable)
-		const writer = pipe.writable.getWriter()
-		const hasher = blake3.create()
-
-		for await (const chunk of readable) {
-			await writer.write(chunk as Uint8Array<ArrayBuffer>)
-			hasher.update(chunk)
+		await this.#manifest.addWip(id)
+		try {
+			const analysis = await save(this.#bucket, id, readable)
+			const {hash, size} = analysis
+			const info = {id, size, added: Date.now()}
+			await this.#wholesome(() => this.#manifest.commit(hash, info))
+			return analysis.hash
+		}
+		catch (error) {
+			await this.#manifest.moveWipToTrash(id)
+			throw error
+		}
+		finally {
+			void this.#cleanup().catch(e => console.error("cleanup failed", e))
 		}
-
-		await writer.close()
-		await done
-		const hash = hex.fromBytes(hasher.digest())
-		await this.delete(hash)
-		await this.#index.set(hash, id)
-		return hash
 	}
 
-	async #getFileId(hash: Hash): Promise<Id | undefined> {
-		return this.#index.get<string>(hash)
-	}
+	#cleanup = queue(async() => {
+		for await (const id of this.#manifest.getExpiredWipIds())
+			await this.#manifest.moveWipToTrash(id)
 
-	async #needFileId(hash: Hash): Promise<Id> {
-		return got(await this.#getFileId(hash), `file not found by hash "${hash}"`)
-	}
-
-	async* keys() {
-		yield* this.#index.keys()
-	}
+		for await (const id of this.#manifest.listTrashIds()) {
+			await this.#bucket.delete(id)
+			await this.#manifest.dropTrashRecord(id)
+		}
+	}, consts.max_jobs)
 }
 

package/s/core/{iceberg-memory.ts → memory-bucket.ts}

@@ -1,19 +1,15 @@
 
 import {bytes, got, nap} from "@e280/stz"
-import {Iceberg, Id} from "./types.js"
+import {Bucket, Id} from "./types.js"
 import {relaxer} from "./utils/relaxer.js"
 
-export class IcebergMemory implements Iceberg {
+export class MemoryBucket implements Bucket {
 	#map = new Map<string, Uint8Array>()
 
 	async has(id: Id) {
 		return this.#map.has(id)
 	}
 
-	async size(id: Id) {
-		return got(this.#map.get(id)).byteLength
-	}
-
 	async delete(id: Id) {
 		this.#map.delete(id)
 	}

package/s/core/streamify.ts

@@ -0,0 +1,18 @@
+
+export function streamify(bytes: Uint8Array, chunkSize = 64 * 1024) {
+	let offset = 0
+
+	return new ReadableStream<Uint8Array>({
+		pull(controller) {
+			if (offset >= bytes.length) {
+				controller.close()
+				return
+			}
+
+			const end = Math.min(offset + chunkSize, bytes.length)
+			controller.enqueue(bytes.subarray(offset, end))
+			offset = end
+		},
+	})
+}
+

package/s/core/types.ts

@@ -2,20 +2,48 @@
 export type Hash = string
 export type Id = string
 
-export type Iceberg = {
+/** file blob store. */
+export type Bucket = {
 	has(id: Id): Promise<boolean>
-	size(id: Id): Promise<number>
 	delete(id: Id): Promise<void>
 	read(id: Id): Promise<Blob>
 	write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>
 }
 
-export type Glacier = {
-	has(hash: Hash): Promise<boolean>
-	size(hash: Hash): Promise<number>
-	read(hash: Hash): Promise<Blob>
-	delete(hash: Hash): Promise<void>
-	write(readable: ReadableStream<Uint8Array>): Promise<Hash>
-	keys(): AsyncIterable<Hash>
+/** report about a file */
+export type Analysis = {
+
+	/** blake3 hash of the file's contents */
+	hash: Hash
+
+	/** filesize in bytes */
+	size: number
+}
+
+/** metadata for a single file. */
+export type Info = {
+
+	/** bucket id for this file's data. */
+	id: Id
+
+	/** filesize in bytes. */
+	size: number
+
+	/** when this file was added to the datalake. */
+	added: number
+}
+
+/** statistics for the whole datalake. */
+export type Stats = {
+
+	/** total number of bytes in the whole datalake. */
+	size: number
+
+	/** total number of files in the datalake. */
+	count: number
+}
+
+export type Wip = {
+	created: number
 }
 

package/s/core/utils/is-expired.ts

@@ -0,0 +1,10 @@
+
+import {time} from "@e280/stz"
+import {Wip} from "../types.js"
+import {consts} from "../consts.js"
+
+export function isExpired({created}: Wip) {
+	const since = Date.now() - created
+	return since > time.days(consts.expiry_time)
+}
+

package/s/core/utils/manifest.ts

@@ -0,0 +1,113 @@
+
+import {Kv} from "@e280/kv"
+import {got} from "@e280/stz"
+import {notFound} from "./not-found.js"
+import {isExpired} from "./is-expired.js"
+import {Hash, Id, Info, Stats, Wip} from "../types.js"
+
+export class Manifest {
+	#kv
+
+	/** statistics about the whole datalake */
+	#stats
+
+	/** associates file hashes with bucket ids */
+	#info
+
+	/** temporary record of writes in-progress */
+	#wip
+
+	/** statistics about the whole datalake */
+	#trash
+
+	constructor(kv = new Kv()) {
+		this.#kv = kv
+		this.#stats = kv.store<Stats>("stats")
+		this.#info = kv.scope<Info>("info")
+		this.#wip = kv.scope<Wip>("wip")
+		this.#trash = kv.scope<true>("trash")
+	}
+
+	async getStats() {
+		return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
+	}
+
+	async* hashes() {
+		yield* this.#info.keys()
+	}
+
+	async hasHash(hash: Hash) {
+		return this.#info.has(hash)
+	}
+
+	async getInfo(hash: Hash) {
+		return await this.#info.get(hash)
+	}
+
+	async needInfo(hash: Hash) {
+		return got(await this.getInfo(hash), notFound(hash))
+	}
+
+	async addWip(id: Id) {
+		await this.#wip.set(id, {created: Date.now()})
+	}
+
+	async deleteWip(...ids: Id[]) {
+		await this.#wip.del(...ids)
+	}
+
+	async* getExpiredWipIds() {
+		for await (const [id, wip] of this.#wip.entries()) {
+			if (isExpired(wip))
+				yield id
+		}
+	}
+
+	async moveWipToTrash(id: Id) {
+		await this.#kv.transaction(() => [
+			this.#wip.write.del(id),
+			this.#trash.write.set(id, true),
+		])
+	}
+
+	async dropTrashRecord(id: Id) {
+		await this.#trash.del(id)
+	}
+
+	async* listTrashIds() {
+		yield* this.#trash.keys()
+	}
+
+	async scheduleDeletion(hash: Hash, info: Info) {
+		const stats = await this.getStats()
+		stats.count -= 1
+		stats.size -= info.size
+		await this.#kv.transaction(() => [
+			this.#info.write.del(hash),
+			this.#trash.write.set(info.id, true),
+			this.#stats.write.set(stats),
+		])
+	}
+
+	async commit(hash: Hash, info: Info) {
+		const isNewFile = !await this.hasHash(hash)
+
+		if (isNewFile) {
+			const stats = await this.getStats()
+			stats.count += 1
+			stats.size += info.size
+			await this.#kv.transaction(() => [
+				this.#wip.write.del(info.id),
+				this.#info.write.set(hash, info),
+				this.#stats.write.set(stats)
+			])
+		}
+		else {
+			await this.#kv.transaction(() => [
+				this.#wip.write.del(info.id),
+				this.#trash.write.set(info.id, true),
+			])
+		}
+	}
+}
+

package/s/core/utils/not-found.ts

@@ -0,0 +1,7 @@
+
+import {Hash} from "../types.js"
+
+export function notFound(hash: Hash) {
+	return `file not found by hash "${hash}"`
+}
+

package/s/core/utils/save.ts

@@ -0,0 +1,30 @@
+
+import {hex} from "@e280/stz"
+import {blake3} from "@noble/hashes/blake3.js"
+import {Analysis, Bucket, Id} from "../types.js"
+
+export async function save(
+		bucket: Bucket,
+		id: Id,
+		readable: ReadableStream<Uint8Array>,
+	): Promise<Analysis> {
+
+	let size = 0
+	const pipe = new TransformStream()
+	const done = bucket.write(id, pipe.readable)
+	const writer = pipe.writable.getWriter()
+	const hasher = blake3.create()
+
+	for await (const chunk of readable) {
+		await writer.write(chunk as Uint8Array<ArrayBuffer>)
+		hasher.update(chunk)
+		size += chunk.byteLength
+	}
+
+	await writer.close()
+	await done
+
+	const hash = hex.fromBytes(hasher.digest())
+	return {hash, size}
+}
+