@e280/mammoth 0.1.0-5 → 0.1.0-7

package/README.md

@@ -4,7 +4,7 @@
 # 🦣 mammoth
 > *big files. small api.*
 
-**mammoth is a content-addressed file store.** files are streamed into a bucket, identified by their blake3 hash, and automatically deduplicated. mammoth works the same whether it's backed by a bucket in memory, on disk, in the cloud, or in the browser's opfs.
+**mammoth is a file storage typescript library.** we use it in our web apps to store files like user uploads. it's content-addressed. files are streamed into a bucket, identified by their blake3 hash, and deduplicated. mammoth works the same whether it's backed by a bucket in memory, on disk, in the cloud, or in the browser's opfs.
 
 ```bash
 npm install @e280/mammoth
@@ -17,9 +17,9 @@ const mammoth = new Mammoth()
 ```
 
 ### 🦣 mammoth stores files.
-- **write a file,** and you get back its blake3 hash.
+- **write a file,** and you get back its blake3 `hash` and `size` in bytes.
     ```ts
-    const hash = await mammoth.write(blob.stream())
+    const {hash, size} = await mammoth.write(blob.stream())
     ```
 - **read a file,** identified by its hash.
     ```ts
@@ -37,11 +37,15 @@ const mammoth = new Mammoth()
   import {Kv} from "@e280/kv"
 
   const mammoth = new Mammoth(
+
+    // bucket for blob storage
     new MemoryBucket(),
+
+    // kv for metadata bookkeeping
     new Kv(),
   )
   ```
-  defaults shown, so, this is equivalent:
+  defaults shown, this is equivalent:
   ```ts
   const mammoth = new Mammoth()
   ```
@@ -74,9 +78,9 @@ const mammoth = new Mammoth()
     ```ts
     const exists = await mammoth.has(hash)
     ```
-- **get file size,** in bytes.
+- **get file info,** including `size` in bytes, `added` timestamp, and bucket `id`.
     ```ts
-    const size = await mammoth.size(hash)
+    const {size, added, id} = await mammoth.info(hash)
     ```
 - **get stats,** for the whole datalake.
     ```ts

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@e280/mammoth",
-	"version": "0.1.0-5",
+	"version": "0.1.0-7",
 	"description": "big file storage",
 	"license": "MIT",
 	"author": "Chase Moskal <chasemoskal@gmail.com>",
@@ -24,7 +24,7 @@
 	},
 	"dependencies": {
 		"@e280/kv": "^0.1.5",
-		"@e280/stz": "^0.3.4",
+		"@e280/stz": "^0.3.5",
 		"@noble/hashes": "^2.2.0"
 	},
 	"devDependencies": {

package/s/browser/example.ts

@@ -19,7 +19,7 @@ export async function example() {
 	const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
 
 	// write a file as an example
-	const hash = await mammoth.write(myFile.stream())
-	console.log(`mammoth stored file by hash "${hash}"`)
+	const {hash} = await mammoth.write(myFile.stream())
+	console.log(`mammoth stored file "${hash}"`)
 }
 

package/s/browser/opfs-bucket.ts

@@ -20,12 +20,6 @@ export class OpfsBucket implements Bucket {
 		}
 	}
 
-	async size(id: Id) {
-		const handle = await this.#directory.getFileHandle(id)
-		const file = await handle.getFile()
-		return file.size
-	}
-
 	async delete(id: Id) {
 		try {
 			await this.#directory.removeEntry(id)

package/s/core/{hash-stream.ts → analyze.ts}

@@ -1,19 +1,23 @@
 
 import {hex, nap} from "@e280/stz"
 import {blake3} from "@noble/hashes/blake3.js"
+import {Analysis} from "./types.js"
 import {relaxer} from "./utils/relaxer.js"
 
-export async function hashStream(readable: ReadableStream<Uint8Array>) {
+export async function analyze(readable: ReadableStream<Uint8Array>): Promise<Analysis> {
+	let size = 0
 	const hasher = blake3.create()
 	const relax = relaxer()
 
 	for await (const chunk of readable) {
 		hasher.update(chunk)
+		size += chunk.byteLength
 
 		if (relax())
 			await nap()
 	}
 
-	return hex.fromBytes(hasher.digest())
+	const hash = hex.fromBytes(hasher.digest())
+	return {hash, size}
 }
 

package/s/core/consts.ts

@@ -0,0 +1,8 @@
+
+import {time} from "@e280/stz"
+
+export const consts = {
+	max_jobs: 64,
+	expiry_time: time.days(7),
+}
+

package/s/core/index.ts

@@ -1,6 +1,6 @@
 
-export * from "./hash-stream.js"
-export * from "./memory-bucket.js"
+export * from "./analyze.js"
 export * from "./mammoth.js"
+export * from "./memory-bucket.js"
 export * from "./types.js"
 

package/s/core/mammoth.ts

@@ -1,100 +1,82 @@
 
 import {Kv} from "@e280/kv"
-import {collect, got, queue} from "@e280/stz"
+import {lane, queue} from "@e280/stz"
 
+import {consts} from "./consts.js"
+import {save} from "./utils/save.js"
+import {Manifest} from "./utils/manifest.js"
 import {randomId} from "./utils/random-id.js"
-import {notFound} from "./utils/not-found.js"
 import {MemoryBucket} from "./memory-bucket.js"
-import {isExpired} from "./utils/is-expired.js"
-import {saveAndHash} from "./utils/save-and-hash.js"
-import {Hash, Bucket, Id, Stats, Wip} from "./types.js"
+import {Hash, Bucket, Analysis} from "./types.js"
 
 /** file storage datalake, content-addressed with blake3 hashes. */
 export class Mammoth {
-	#ids // associates file hashes with bucket ids
-	#wip // temporary record of writes in-progress
-	#stats // statistics about the whole datalake
-	#bucket // file blobstore
+	#bucket
+	#manifest
+	#wholesome = lane(consts.max_jobs)
 
 	constructor(bucket: Bucket = new MemoryBucket(), kv = new Kv()) {
-		this.#ids = kv.scope<string>("ids")
-		this.#wip = kv.scope<Wip>("wip")
-		this.#stats = kv.store<Stats>("stats")
 		this.#bucket = bucket
+		this.#manifest = new Manifest(kv)
+	}
+
+	async* hashes() {
+		yield* this.#manifest.hashes()
 	}
 
 	async has(hash: Hash) {
-		return this.#ids.has(hash)
+		return this.#manifest.hasHash(hash)
 	}
 
-	async size(hash: Hash) {
-		const id = got(await this.#ids.get(hash), notFound(hash))
-		return this.#bucket.size(id)
+	async info(hash: Hash) {
+		return this.#manifest.needInfo(hash)
+	}
+
+	async stats() {
+		return this.#manifest.getStats()
 	}
 
 	async read(hash: Hash) {
-		const id = got(await this.#ids.get(hash), notFound(hash))
+		const {id} = await this.#manifest.needInfo(hash)
 		return this.#bucket.read(id)
 	}
 
 	async delete(hash: Hash) {
-		const id = await this.#ids.get(hash)
-		if (id) {
-			const size = await this.#bucket.size(id)
-			await this.#ids.del(hash)
-			await this.#bucket.delete(id)
-			await this.#updateStats(stats => {
-				stats.count--
-				stats.size -= size
-			})
-		}
+		await this.#wholesome(async() => {
+			const info = await this.#manifest.getInfo(hash)
+			if (info)
+				await this.#manifest.scheduleDeletion(hash, info)
+		})
+		await this.#cleanup()
 	}
 
-	async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
+	async write(readable: ReadableStream<Uint8Array>): Promise<Analysis> {
 		const id = randomId()
-
-		await this.#wip.set(id, {created: Date.now()})
-		const {hash, size} = await saveAndHash(this.#bucket, id, readable)
-		await this.#wip.del(id)
-
-		await this.#finalizeWrite(hash, id, size)
-		void this.#selfClean().catch(() => {})
-		return hash
-	}
-
-	async* hashes() {
-		yield* this.#ids.keys()
-	}
-
-	async stats() {
-		return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
+		await this.#manifest.addWip(id)
+		try {
+			const analysis = await save(this.#bucket, id, readable)
+			const {hash, size} = analysis
+			const info = {id, size, added: Date.now()}
+			await this.#wholesome(() => this.#manifest.commit(hash, info))
+			return analysis
+		}
+		catch (error) {
+			await this.#manifest.moveWipToTrash(id)
+			throw error
+		}
+		finally {
+			void this.#cleanup().catch(e => console.error("cleanup failed", e))
+		}
 	}
 
-	#updateStats = queue(async(fn: (stats: Stats) => void) => {
-		const stats = await this.stats()
-		fn(stats)
-		await this.#stats.set(stats)
-	})
+	#cleanup = queue(async() => {
+		for await (const id of this.#manifest.getExpiredWipIds())
+			await this.#manifest.moveWipToTrash(id)
 
-	#finalizeWrite = queue(async(hash: Hash, id: Id, size: number) => {
-		if (await this.#ids.has(hash)) {
-			await this.#bucket.delete(id) // forget this new file (we already have it)
-		}
-		else {
-			await this.#ids.set(hash, id)
-			await this.#updateStats(stats => {
-				stats.count++
-				stats.size += size
-			})
+		for await (const id of this.#manifest.listTrashIds()) {
+			await this.#bucket.delete(id)
+			await this.#manifest.dropTrashRecord(id)
 		}
-	})
-
-	#selfClean = queue(async() => {
-		const expiredIds = (await collect(this.#wip.entries({limit: 64})))
-			.filter(([,wip]) => isExpired(wip))
-			.map(([id]) => id)
-		await Promise.all(expiredIds.map(id => this.#bucket.delete(id)))
-		await this.#wip.del(...expiredIds)
-	})
+	}, consts.max_jobs)
 }
 

package/s/core/memory-bucket.ts

@@ -10,10 +10,6 @@ export class MemoryBucket implements Bucket {
 		return this.#map.has(id)
 	}
 
-	async size(id: Id) {
-		return got(this.#map.get(id)).byteLength
-	}
-
 	async delete(id: Id) {
 		this.#map.delete(id)
 	}

package/s/core/types.ts

@@ -5,18 +5,41 @@ export type Id = string
 /** file blob store. */
 export type Bucket = {
 	has(id: Id): Promise<boolean>
-	size(id: Id): Promise<number>
 	delete(id: Id): Promise<void>
 	read(id: Id): Promise<Blob>
 	write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>
 }
 
+/** report about a file */
+export type Analysis = {
+
+	/** blake3 hash of the file's contents */
+	hash: Hash
+
+	/** filesize in bytes */
+	size: number
+}
+
+/** metadata for a single file. */
+export type Info = {
+
+	/** bucket id for this file's data. */
+	id: Id
+
+	/** filesize in bytes. */
+	size: number
+
+	/** when this file was added to the datalake. */
+	added: number
+}
+
+/** statistics for the whole datalake. */
 export type Stats = {
 
 	/** total number of bytes in the whole datalake. */
 	size: number
 
-	/** total number of known files in the datalake. */
+	/** total number of files in the datalake. */
 	count: number
 }
 

package/s/core/utils/is-expired.ts

@@ -1,9 +1,10 @@
 
 import {time} from "@e280/stz"
 import {Wip} from "../types.js"
+import {consts} from "../consts.js"
 
 export function isExpired({created}: Wip) {
 	const since = Date.now() - created
-	return since > time.days(7)
+	return since > time.days(consts.expiry_time)
 }
 

package/s/core/utils/manifest.ts

@@ -0,0 +1,113 @@
+
+import {Kv} from "@e280/kv"
+import {got} from "@e280/stz"
+import {notFound} from "./not-found.js"
+import {isExpired} from "./is-expired.js"
+import {Hash, Id, Info, Stats, Wip} from "../types.js"
+
+export class Manifest {
+	#kv
+
+	/** statistics about the whole datalake */
+	#stats
+
+	/** associates file hashes with bucket ids */
+	#info
+
+	/** temporary record of writes in-progress */
+	#wip
+
+	/** statistics about the whole datalake */
+	#trash
+
+	constructor(kv = new Kv()) {
+		this.#kv = kv
+		this.#stats = kv.store<Stats>("stats")
+		this.#info = kv.scope<Info>("info")
+		this.#wip = kv.scope<Wip>("wip")
+		this.#trash = kv.scope<true>("trash")
+	}
+
+	async getStats() {
+		return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
+	}
+
+	async* hashes() {
+		yield* this.#info.keys()
+	}
+
+	async hasHash(hash: Hash) {
+		return this.#info.has(hash)
+	}
+
+	async getInfo(hash: Hash) {
+		return await this.#info.get(hash)
+	}
+
+	async needInfo(hash: Hash) {
+		return got(await this.getInfo(hash), notFound(hash))
+	}
+
+	async addWip(id: Id) {
+		await this.#wip.set(id, {created: Date.now()})
+	}
+
+	async deleteWip(...ids: Id[]) {
+		await this.#wip.del(...ids)
+	}
+
+	async* getExpiredWipIds() {
+		for await (const [id, wip] of this.#wip.entries()) {
+			if (isExpired(wip))
+				yield id
+		}
+	}
+
+	async moveWipToTrash(id: Id) {
+		await this.#kv.transaction(() => [
+			this.#wip.write.del(id),
+			this.#trash.write.set(id, true),
+		])
+	}
+
+	async dropTrashRecord(id: Id) {
+		await this.#trash.del(id)
+	}
+
+	async* listTrashIds() {
+		yield* this.#trash.keys()
+	}
+
+	async scheduleDeletion(hash: Hash, info: Info) {
+		const stats = await this.getStats()
+		stats.count -= 1
+		stats.size -= info.size
+		await this.#kv.transaction(() => [
+			this.#info.write.del(hash),
+			this.#trash.write.set(info.id, true),
+			this.#stats.write.set(stats),
+		])
+	}
+
+	async commit(hash: Hash, info: Info) {
+		const isNewFile = !await this.hasHash(hash)
+
+		if (isNewFile) {
+			const stats = await this.getStats()
+			stats.count += 1
+			stats.size += info.size
+			await this.#kv.transaction(() => [
+				this.#wip.write.del(info.id),
+				this.#info.write.set(hash, info),
+				this.#stats.write.set(stats)
+			])
+		}
+		else {
+			await this.#kv.transaction(() => [
+				this.#wip.write.del(info.id),
+				this.#trash.write.set(info.id, true),
+			])
+		}
+	}
+}
+

package/s/core/utils/{save-and-hash.ts → save.ts}

@@ -1,9 +1,14 @@
 
 import {hex} from "@e280/stz"
 import {blake3} from "@noble/hashes/blake3.js"
-import {Bucket, Id} from "../types.js"
+import {Analysis, Bucket, Id} from "../types.js"
+
+export async function save(
+		bucket: Bucket,
+		id: Id,
+		readable: ReadableStream<Uint8Array>,
+	): Promise<Analysis> {
 
-export async function saveAndHash(bucket: Bucket, id: Id, readable: ReadableStream<Uint8Array>) {
 	let size = 0
 	const pipe = new TransformStream()
 	const done = bucket.write(id, pipe.readable)

package/s/node/disk-bucket.ts

@@ -26,11 +26,6 @@ export class DiskBucket implements Bucket {
 		}
 	}
 
-	async size(id: Id) {
-		const stats = await stat(this.#path(id))
-		return stats.size
-	}
-
 	async delete(id: Id) {
 		try {
 			await unlink(this.#path(id))

package/s/test.ts

@@ -24,7 +24,7 @@ await science.run({
 		const dataDir = "data"
 		await rm(dataDir, {recursive: true, force: true})
 		const mammoth = new Mammoth(new DiskBucket(dataDir), new Kv())
-		const hash = await mammoth.write(blob().stream())
+		const {hash} = await mammoth.write(blob().stream())
 		const second = await mammoth.read(hash)
 		expect(bytes.eq(await second.bytes(), await blob().bytes()))
 	}),
@@ -32,15 +32,15 @@ await science.run({
 	"mammoth memory": {
 		".write and .read": test(async() => {
 			const {mammoth} = setup()
-			const hash = await mammoth.write(blob().stream())
+			const {hash} = await mammoth.write(blob().stream())
 			const second = await mammoth.read(hash)
 			expect(bytes.eq(await second.bytes(), await blob().bytes()))
 		}),
 
 		".write deduplication": test(async() => {
 			const {mammoth} = setup()
-			const hashA = await mammoth.write(blob().stream())
-			const hashB = await mammoth.write(blob().stream())
+			const {hash: hashA} = await mammoth.write(blob().stream())
+			const {hash: hashB} = await mammoth.write(blob().stream())
 			const keys = await collect(mammoth.hashes())
 			expect(hashA).is(hashB)
 			expect(keys.length).is(1)
@@ -49,8 +49,8 @@ await science.run({
 
 		".write empty file": test(async() => {
 			const {mammoth} = setup()
-			const hash = await mammoth.write(new Blob().stream())
-			expect(await mammoth.size(hash)).is(0)
+			const {hash} = await mammoth.write(new Blob().stream())
+			expect((await mammoth.info(hash)).size).is(0)
 			expect((await mammoth.read(hash)).size).is(0)
 		}),
 
@@ -65,9 +65,9 @@ await science.run({
 		".stats": test(async() => {
 			const {mammoth} = setup()
 			expect(await mammoth.stats()).deep({count: 0, size: 0})
-			const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
+			const {hash: hashA} = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
 			expect(await mammoth.stats()).deep({count: 1, size: 3})
-			const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
+			const {hash: hashB} = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
 			expect(await mammoth.stats()).deep({count: 2, size: 7})
 			await mammoth.delete(hashA)
 			expect(await mammoth.stats()).deep({count: 1, size: 4})
@@ -77,20 +77,21 @@ await science.run({
 
 		".has": test(async() => {
 			const {mammoth} = setup()
-			const hash = await mammoth.write(blob().stream())
+			const {hash} = await mammoth.write(blob().stream())
 			expect(await mammoth.has(hash)).is(true)
 			expect(await mammoth.has(randomId())).is(false)
 		}),
 
 		".size": test(async() => {
 			const {mammoth} = setup()
-			const hash = await mammoth.write(blob().stream())
-			expect(await mammoth.size(hash)).is(4)
+			const b = blob()
+			const {hash} = await mammoth.write(b.stream())
+			expect((await mammoth.info(hash)).size).is(b.size)
 		}),
 
 		".delete": test(async() => {
 			const {mammoth} = setup()
-			const hash = await mammoth.write(blob().stream())
+			const {hash} = await mammoth.write(blob().stream())
 			await mammoth.delete(hash)
 			expect(await mammoth.has(hash)).is(false)
 			expect(await collect(mammoth.hashes())).deep([])
@@ -103,7 +104,7 @@ await science.run({
 
 		".keys": test(async() => {
 			const {mammoth} = setup()
-			const hash = await mammoth.write(blob().stream())
+			const {hash} = await mammoth.write(blob().stream())
 			const keys = await collect(mammoth.hashes())
 			expect(keys.length).is(1)
 			expect(keys[0]).is(hash)

package/x/browser/example.js

@@ -13,7 +13,7 @@ export async function example() {
     const mammoth = new Mammoth(bucket, kv);
     const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
     // write a file as an example
-    const hash = await mammoth.write(myFile.stream());
-    console.log(`mammoth stored file by hash "${hash}"`);
+    const { hash } = await mammoth.write(myFile.stream());
+    console.log(`mammoth stored file "${hash}"`);
 }
 //# sourceMappingURL=example.js.map

package/x/browser/example.js.map

@@ -1 +1 @@
-{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,4BAA4B;IAC5B,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAE1C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;IAExC,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACjD,OAAO,CAAC,GAAG,CAAC,gCAAgC,IAAI,GAAG,CAAC,CAAA;AACrD,CAAC"}
+{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,4BAA4B;IAC5B,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAE1C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;IAExC,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnE,6BAA6B;IAC7B,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACnD,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,CAAC,CAAA;AAC7C,CAAC"}

package/x/browser/opfs-bucket.d.ts

@@ -3,7 +3,6 @@ export declare class OpfsBucket implements Bucket {
     #private;
     constructor(directory: FileSystemDirectoryHandle);
     has(id: Id): Promise<boolean>;
-    size(id: Id): Promise<number>;
     delete(id: Id): Promise<void>;
     read(id: Id): Promise<File>;
     write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;

package/x/browser/opfs-bucket.js

@@ -15,11 +15,6 @@ export class OpfsBucket {
             throw error;
         }
     }
-    async size(id) {
-        const handle = await this.#directory.getFileHandle(id);
-        const file = await handle.getFile();
-        return file.size;
-    }
     async delete(id) {
         try {
             await this.#directory.removeEntry(id);

package/x/browser/opfs-bucket.js.map

@@ -1 +1 @@
-{"version":3,"file":"opfs-bucket.js","sourceRoot":"","sources":["../../s/browser/opfs-bucket.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,UAAU;IACtB,UAAU,CAAA;IAEV,YAAY,SAAoC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;YACvC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAA;QACnC,OAAO,IAAI,CAAC,IAAI,CAAA;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,OAAO,MAAM,CAAC,OAAO,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;QACtE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,EAAE,CAAA;QAC9C,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;CACD"}
+{"version":3,"file":"opfs-bucket.js","sourceRoot":"","sources":["../../s/browser/opfs-bucket.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,UAAU;IACtB,UAAU,CAAA;IAEV,YAAY,SAAoC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;YACvC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,OAAO,MAAM,CAAC,OAAO,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;QACtE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,EAAE,CAAA;QAC9C,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;CACD"}

package/x/core/analyze.d.ts

@@ -0,0 +1,2 @@
+import { Analysis } from "./types.js";
+export declare function analyze(readable: ReadableStream<Uint8Array>): Promise<Analysis>;

package/x/core/{hash-stream.js → analyze.js}

@@ -1,14 +1,17 @@
 import { hex, nap } from "@e280/stz";
 import { blake3 } from "@noble/hashes/blake3.js";
 import { relaxer } from "./utils/relaxer.js";
-export async function hashStream(readable) {
+export async function analyze(readable) {
+    let size = 0;
     const hasher = blake3.create();
     const relax = relaxer();
     for await (const chunk of readable) {
         hasher.update(chunk);
+        size += chunk.byteLength;
         if (relax())
             await nap();
     }
-    return hex.fromBytes(hasher.digest());
+    const hash = hex.fromBytes(hasher.digest());
+    return { hash, size };
 }
-//# sourceMappingURL=hash-stream.js.map
+//# sourceMappingURL=analyze.js.map

package/x/core/analyze.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../s/core/analyze.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAE9C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,QAAoC;IACjE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;QAExB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}