@e280/quay 0.0.0-17 → 0.0.0-19

package/README.md

@@ -124,7 +124,7 @@ In this way, you can setup sophisticated rules about what actions are permitted,
 ```ts
 import {MediaLibrary, brain, register, components} from "@e280/quay"
 
-const media = await MediaLibrary.open("my-project")
+const media = await MediaLibrary.open("my-app")
 brain.setGroup("media", media)
 
 register(components)
@@ -137,13 +137,17 @@ register(components)
 </div>
 ```
 
-The scope passed to `open()` separates media libraries.
+Open a library once, then derive nested scopes as needed.
 
 ```ts
-const projectA = await MediaLibrary.open("project-a")
-const projectB = await MediaLibrary.open("project-b")
+const project = await media.scope("project-a")
+const favorites = await project.scope("favorites")
 ```
 
+Adding media to a scope also adds it to every parent scope. Removing media from
+a scope removes it from that scope and its child scopes. Removing it from the
+root removes it everywhere, including its stored file.
+
 <br/>
 
 ## Styling

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@e280/quay",
-	"version": "0.0.0-17",
+	"version": "0.0.0-19",
 	"description": "File-browser and outliner UI for the web",
 	"author": "Przemysław Gałęzki",
 	"license": "MIT",
@@ -11,18 +11,20 @@
 		"s"
 	],
 	"dependencies": {
-		"@awasm/noble": "^0.1.4",
-		"@e280/kv": "^0.0.0-7",
+		"@e280/comrade": "^0.1.0",
+		"@e280/kv": "^0.2.0-3",
+		"@e280/mammoth": "^0.1.0-19",
 		"@e280/scute": "^0.4.0-next.4",
 		"@e280/sly": "^0.4.0-next.5",
 		"@e280/strata": "^0.4.0-next.8",
-		"@e280/stz": "^0.0.0-22",
+		"@e280/stz": "^0.2.8",
 		"lit": "^3.3.1"
 	},
 	"devDependencies": {
 		"@e280/hottie": "^0.1.1",
 		"@e280/science": "^0.0.5",
 		"@shoelace-style/shoelace": "^2.20.1",
+		"@types/wicg-file-system-access": "^2023.10.7",
 		"npm-run-all": "^4.1.5",
 		"typescript": "^5.8.3"
 	},

package/s/cellar/cask.ts

@@ -1,24 +1,6 @@
 
-import {Hex} from "@e280/stz"
-import {blake3} from "@awasm/noble"
-
 /** File with a hash label */
 export class Cask {
-	static async hash(file: Blob) {
-		const hasher = blake3.create()
-
-		for await (const chunk of file.stream())
-			hasher.update(chunk)
-
-		const digest = hasher.digest()
-		return Hex.fromBytes(digest)
-	}
-
-	static async make(file: Blob) {
-		const hash = await this.hash(file)
-		return new this(hash, file)
-	}
-
 	constructor(public hash: string, public file: Blob) {}
 }
 

package/s/cellar/cellar.test.ts

@@ -2,59 +2,42 @@
 import {Txt} from "@e280/stz"
 import {expect, Science, test} from "@e280/science"
 
-import {Cask} from "./cask.js"
 import {Cellar} from "./cellar.js"
-import {MemoryForklift} from "./memory-forklift.js"
 
 const blob = (text: string) => new Blob([Txt.toBytes(text)])
 
 export default Science.suite({
-	"save + load roundtrip": test(async() => {
+	"write + load roundtrip": test(async() => {
 		const cellar = new Cellar()
 		const text = "hello world"
 		const file = blob(text)
-		await cellar.save(file)
+		const digest = await cellar.write(file.stream())
 
-		const hash = await Cask.hash(file)
-		expect(await cellar.has(hash)).is(true)
+		expect(await cellar.has(digest)).is(true)
 
-		const loaded = await cellar.load(hash)
-		expect(loaded.hash).is(hash)
+		const loaded = await cellar.load(digest)
+		expect(loaded.hash).is(digest)
 		expect(await loaded.file.text()).is(text)
 	}),
 
-	"save is idempotent": test(async() => {
+	"write is idempotent": test(async() => {
 		const cellar = new Cellar()
 		const file = blob("foo bar")
-		await cellar.save(file)
-		await cellar.save(file) // again
+		const digest = await cellar.write(file.stream())
+		expect(await cellar.write(file.stream())).is(digest)
 
-		const hash = await Cask.hash(file)
-		expect(await cellar.has(hash)).is(true)
-	}),
-
-	"throws on corruption": test(async() => {
-		const forklift = new MemoryForklift()
-		const cellar = new Cellar(forklift)
-		const good = blob("valid data")
-		const bad = blob("corrupt data")
-
-		const hash = await Cask.hash(good)
-		await forklift.save(hash, bad)
-
-		await expect(async() => cellar.load(hash)).throwsAsync()
+		expect(await cellar.has(digest)).is(true)
 	}),
 
 	"delete removes file": test(async() => {
 		const cellar = new Cellar()
 		const file = blob("temporary data")
-		await cellar.save(file)
+		const digest = await cellar.write(file.stream())
 
-		const hash = await Cask.hash(file)
-		expect(await cellar.has(hash)).is(true)
+		expect(await cellar.has(digest)).is(true)
 
-		await cellar.delete(hash)
-		expect(await cellar.has(hash)).is(false)
+		await cellar.delete(digest)
+		expect(await cellar.has(digest)).is(false)
 	}),
 
 	"load missing throws": test(async() => {
@@ -69,24 +52,22 @@ export default Science.suite({
 		const texts = ["a", "b", "c"]
 
 		for (const t of texts)
-			await cellar.save(blob(t))
+			await cellar.write(blob(t).stream())
 
 		const hashes = []
 		for await (const hash of cellar.list())
 			hashes.push(hash)
 
 		expect(hashes.length).is(3)
-		for (const t of texts) {
-			const h = await Cask.hash(blob(t))
-			expect(hashes.includes(h)).is(true)
-		}
+		for (const text of texts)
+			expect(hashes.includes(await cellar.write(blob(text).stream()))).is(true)
 	}),
 
 	"clear removes all entries": test(async() => {
 		const cellar = new Cellar()
-		await cellar.save(blob("one"))
-		await cellar.save(blob("two"))
-		await cellar.save(blob("three"))
+		await cellar.write(blob("one").stream())
+		await cellar.write(blob("two").stream())
+		await cellar.write(blob("three").stream())
 
 		let count = 0
 		for await (const _ of cellar.list()) count++

package/s/cellar/cellar.ts

@@ -1,49 +1,45 @@
 
+import {Mammoth} from "@e280/mammoth"
+
 import {Cask} from "./cask.js"
-import {Forklift} from "./forklift.js"
-import {OpfsForklift} from "./opfs-forklift.js"
-import {MemoryForklift} from "./memory-forklift.js"
+import {MammothOpfs} from "./mammoth.js"
 
 /**
  * Cellar is an immutable content-addressable file store
  *  - files are identified by their hashes (duplicates are impossible)
  *  - files cannot be modified (delete it and save another)
  *  - no filenames or metadata (store those somewhere else)
- *  - pass in a forklift, which is the backend that actually stores the data
- *    - MemoryForklift is the default
- *    - OpfsForklift is probably what you really want in the browser
+ *  - new Cellar uses Mammoth's in-memory backend by default
+ *  - Cellar.opfs creates MammothOpfs for persistent browser storage
+ *  - pass in a Mammoth to use a different backend
+ *  - most likely you want to use Cellar.opfs for storing files in the browser
  */
 export class Cellar {
 	static async opfs(dirname = "cellar") {
-		return new this(await OpfsForklift.setup(dirname))
+		return new this(await MammothOpfs.setup(dirname))
 	}
 
-	constructor(private forklift: Forklift = new MemoryForklift()) {}
+	constructor(private mammoth: Mammoth | MammothOpfs = new Mammoth()) {}
 
 	async *list(): AsyncIterable<string> {
-		yield* this.forklift.list()
+		for await (const [hash] of this.mammoth.entries())
+			yield hash
 	}
 
 	async has(hash: string): Promise<boolean> {
-		return this.forklift.has(hash)
+		return this.mammoth.has(hash)
 	}
 
-	async save(file: Blob) {
-		const cask = await Cask.make(file)
-		if (!await this.forklift.has(cask.hash))
-			await this.forklift.save(cask.hash, cask.file)
-		return cask
+	async write(readable: ReadableStream<Uint8Array>) {
+		return this.mammoth.write(readable)
 	}
 
 	async load(hash: string): Promise<Cask> {
-		const bytes = await this.forklift.load(hash)
-		const cask = await Cask.make(bytes)
-		if (cask.hash !== hash) throw new Error(`corruption error, hash mismatch (requested ${hash}) (got ${cask.hash})`)
-		return cask
+		return new Cask(hash, await this.mammoth.read(hash))
 	}
 
 	async delete(hash: string) {
-		await this.forklift.delete(hash)
+		await this.mammoth.delete(hash)
 	}
 
 	async clear() {

package/s/cellar/index.ts

@@ -1,7 +1,3 @@
 
 export * from "./cask.js"
 export * from "./cellar.js"
-export * from "./forklift.js"
-export * from "./memory-forklift.js"
-export * from "./opfs-forklift.js"
-

package/s/cellar/mammoth.ts

@@ -0,0 +1,50 @@
+
+import {Comrade, Thread, tune, AsSchematic} from "@e280/comrade"
+
+export type MammothSchematic = AsSchematic<{
+	work: {
+		setup(dirname: string): Promise<void>
+		write(readable: ReadableStream<Uint8Array>): Promise<string>
+		has(hash: string): Promise<boolean>
+		read(hash: string): Promise<Blob>
+		delete(hash: string): Promise<void>
+		hashes(): Promise<string[]>
+	}
+	host: {}
+}>
+
+/** Worker proxy for Mammoth using an OPFS bucket and IndexedDB manifest. */
+export class MammothOpfs {
+	static async setup(dirname: string) {
+		const thread = await Comrade.thread<MammothSchematic>({
+			workerUrl: "/node_modules/@e280/quay/x/cellar/mammoth.worker.bundle.min.js",
+			setupHost: () => ({}),
+		})
+		const mammoth = new this(thread)
+		await mammoth.thread.work.setup(dirname)
+		return mammoth
+	}
+
+	constructor(private thread: Thread<MammothSchematic>) {}
+
+	async write(readable: ReadableStream<Uint8Array>) {
+		return this.thread.work.write[tune]({transfer: [readable]})(readable)
+	}
+
+	async has(hash: string) {
+		return this.thread.work.has(hash)
+	}
+
+	async read(hash: string) {
+		return this.thread.work.read(hash)
+	}
+
+	async delete(hash: string) {
+		return this.thread.work.delete(hash)
+	}
+
+	async *entries(): AsyncIterable<[string, unknown]> {
+		for (const hash of await this.thread.work.hashes())
+			yield [hash, undefined]
+	}
+}

package/s/cellar/mammoth.worker.bundle.ts

@@ -0,0 +1,30 @@
+
+import {collect} from "@e280/stz"
+import {Comrade} from "@e280/comrade"
+import {Mammoth, OpfsBucket} from "@e280/mammoth"
+import {idbOpen, IdbMagazine, Kv} from "@e280/kv"
+
+import type {MammothSchematic} from "./mammoth.js"
+
+let mammoth: Mammoth
+
+await Comrade.worker<MammothSchematic>(() => ({
+	async setup(dirname) {
+		const root = await navigator.storage.getDirectory()
+		const directory = await root.getDirectoryHandle(dirname, {create: true})
+		mammoth = new Mammoth(
+			new OpfsBucket(directory),
+			new Kv(new IdbMagazine(await idbOpen(`quay.mammoth:${dirname}`))),
+		)
+	},
+
+	write: readable => mammoth.write(readable),
+	has: hash => mammoth.has(hash),
+	read: hash => mammoth.read(hash),
+	delete: hash => mammoth.delete(hash),
+
+	async hashes() {
+		return (await collect(mammoth.entries())).map(([hash]) => hash)
+	},
+}))
+

package/s/genius/parts/hy-tree.ts

@@ -1,5 +1,6 @@
 
-import {Kv, collect} from "@e280/kv"
+import {collect} from "@e280/stz"
+import {Kv} from "@e280/kv"
 
 export type HyId = string
 export type HyItem = {parent: HyId | null, children: HyId[]}
@@ -27,7 +28,7 @@ export class HyTree {
 	}
 
 	async gets(...ids: HyId[]): Promise<(HyItem | undefined)[]> {
-		const records = await this.records.gets(...ids)
+		const records = await this.records.getMany(ids)
 		return records.map(r => {
 			if (!r) return undefined
 			const [parent, children] = r
@@ -36,13 +37,13 @@ export class HyTree {
 	}
 
 	async require(id: HyId): Promise<HyItem> {
-		const record = await this.records.require(id)
+		const record = await this.records.need(id)
 		const [parent, children] = record
 		return {parent, children}
 	}
 
 	async requires(...ids: HyId[]): Promise<HyItem[]> {
-		const records = await this.records.requires(...ids)
+		const records = await this.records.needMany(ids)
 		return records.map(([parent, children]) => ({parent, children}))
 	}
 
@@ -73,10 +74,10 @@ export class HyTree {
 		}
 
 		// save all new children
-		await this.records.sets(...newcomers)
-
-		// save updated parent item
-		await this.records.set(parent, [parentItem.parent, [...siblings]])
+		await this.records.commit([
+			...newcomers.map(([id, record]) => this.records.op.set(id, record)),
+			this.records.op.set(parent, [parentItem.parent, [...siblings]]),
+		])
 	}
 
 	/** detach this id from its parent in the hierarchy */
@@ -102,7 +103,7 @@ export class HyTree {
 		await this.detach(id)
 		const tree = await collect(this.crawl(id))
 		const ids = tree.map(({id}) => id)
-		await this.records.del(...ids)
+		await this.records.commit(ids.map(id => this.records.op.delete(id)))
 	}
 
 	/** delete absolutely everything */

package/s/logic/presets/media/library.test.ts

@@ -34,7 +34,7 @@ export default Science.suite({
 		expect(await recordByLabel(store, "image.png")).ok()
 	}),
 
-	"deletes media records and bytes": test(async() => {
+	"root deletion removes media records and resources": test(async() => {
 		const store = new MediaLibrary()
 		await store.upload([file("delete.png", "delete-image", "image/png")], store.config.root)
 
@@ -43,8 +43,8 @@ export default Science.suite({
 
 		await store.delete(item)
 
-		expect(await store.cellar.has(record.hash)).is(false)
 		expect(store.findByHash(record.hash)).is(undefined)
+		expect(await store.cellar.has(record.hash)).is(false)
 	}),
 
 })
@@ -66,4 +66,3 @@ async function recordByLabel(store: MediaLibrary, label: string) {
 		throw new Error(`expected record "${label}"`)
 	return record
 }
-

package/s/logic/presets/media/library.ts

@@ -1,9 +1,10 @@
 
-import {Kv, StorageDriver} from "@e280/kv"
+import {collect, sub} from "@e280/stz"
+import {Kv, StorageMagazine} from "@e280/kv"
 
 import {MediaGroup} from "./group.js"
 import {Cellar} from "../../../cellar/cellar.js"
-import {MediaFormat, MediaSchema} from "./schema.js"
+import {MediaFormat, MediaProgress, MediaSchema} from "./schema.js"
 import {CodexItem} from "../../aspects/codex/parts/codex-item.js"
 
 export type MediaRecord = {
@@ -21,18 +22,23 @@ export class MediaLibrary extends MediaGroup {
 		const group = new this()
 		group.cellar = await Cellar.opfs("media")
 		group.#index = mediaIndex(scope)
+		group.#lineage = [group.#index]
 		await group.#load()
 		return group
 	}
 
+	#index
+	#lineage
+	#root = this
 	#objectUrls = new Map<string, string>()
-	#index: Kv<MediaRecord>
 
+	progress = sub<[MediaProgress]>()
 	cellar = new Cellar()
 
 	constructor() {
 		super()
 		this.#index = mediaIndex("default")
+		this.#lineage = [this.#index]
 		this.on.upload.sub(({files, target}) => {
 			return this.#upload(files, target)
 		})
@@ -42,6 +48,20 @@ export class MediaLibrary extends MediaGroup {
 		})
 	}
 
+	get #isRoot() {
+		return this === this.#root
+	}
+
+	async scope(name: string) {
+		const child = new MediaLibrary()
+		child.cellar = this.cellar
+		child.#index = this.#index.scope<MediaRecord>(name)
+		child.#lineage = [...this.#lineage, child.#index]
+		child.#root = this.#root
+		await child.#load()
+		return child
+	}
+
 	async *records() {
 		for await (const [, record] of this.#index.entries())
 			yield record
@@ -60,11 +80,42 @@ export class MediaLibrary extends MediaGroup {
 	}
 
 	async #storeFile(file: File, parent: CodexItem<MediaSchema>) {
-		const cask = await this.cellar.save(file)
-		const existing = await this.#index.get(cask.hash)
+		const item = this.#attachPending(file, parent)
+
+		try {
+			const hash = await this.cellar.write(trackProgress(file.stream(), loaded =>
+				this.progress.pub({item, loaded, total: file.size})
+			))
+			const record = await this.#saveRecord(file, hash)
+			item.destroy()
+			this.#attachRecord(record, parent)
+			return record
+		}
+		catch (error) {
+			item.destroy()
+			this.on.refresh.pub({})
+			throw error
+		}
+	}
+
+	#attachPending(file: File, parent: CodexItem<MediaSchema>) {
+		const item = this.config.codex.create("file", {
+			label: file.name,
+			format: mediaFormat(file.type),
+			mime: file.type,
+			size: file.size,
+			previewUrl: null,
+		})
+		parent.attach(item)
+		this.on.refresh.pub({})
+		return item
+	}
+
+	async #saveRecord(file: File, hash: string) {
+		const existing = await this.#index.get(hash) ?? await this.#root.#index.get(hash)
 		const now = Date.now()
 		const record: MediaRecord = {
-			hash: cask.hash,
+			hash,
 			label: existing?.label ?? file.name,
 			format: mediaFormat(file.type),
 			mime: file.type,
@@ -72,10 +123,9 @@ export class MediaLibrary extends MediaGroup {
 			createdAt: existing?.createdAt ?? now,
 			updatedAt: now,
 		}
-		await this.#index.set(record.hash, record)
+		await this.#index.commit(this.#lineage.map(index => index.op.set(hash, record)))
 		if (record.format === "image")
-			this.#setPreview(record.hash, URL.createObjectURL(file))
-		this.#attachRecord(record, parent)
+			this.#setPreview(hash, URL.createObjectURL(file))
 		return record
 	}
 
@@ -86,10 +136,16 @@ export class MediaLibrary extends MediaGroup {
 	}
 
 	async #delete(hashes: string[]) {
+		const index = this.#index.crush()
+		const keys = (await collect(index.entries()))
+			.filter(([, record]) => hashes.includes(record.hash))
+			.map(([key]) => key)
+		await index.commit(keys.map(key => index.op.delete(key)))
+
 		for (const hash of hashes) {
-			await this.#index.del(hash)
-			await this.cellar.delete(hash)
 			this.#revokePreview(hash)
+			if (this.#isRoot)
+				await this.cellar.delete(hash)
 		}
 	}
 
@@ -155,9 +211,10 @@ function mediaFormat(mime: string): MediaFormat {
 function mediaIndex(scope: string) {
 	const storage = globalThis.localStorage
 	if (storage)
-		return new Kv<MediaRecord>(new StorageDriver(storage))
-			.scope("quay.media")
-			.scope(scope)
+		return new Kv<MediaRecord>(new StorageMagazine(storage))
+			.scope("quay")
+			.scope("media")
+			.scope<MediaRecord>(scope)
 
 	const existing = memoryIndexes.get(scope)
 	if (existing)
@@ -168,3 +225,16 @@ function mediaIndex(scope: string) {
 	return index
 }
 
+function trackProgress(
+		readable: ReadableStream<Uint8Array>,
+		onProgress: (loaded: number) => void,
+	) {
+	let loaded = 0
+	return readable.pipeThrough(new TransformStream({
+		transform(chunk, controller) {
+			loaded += chunk.byteLength
+			onProgress(loaded)
+			controller.enqueue(chunk)
+		},
+	}))
+}

package/s/logic/presets/media/schema.ts

@@ -1,6 +1,7 @@
 
 import {Content} from "@e280/sly"
 import {AsSchema} from "../../aspects/codex/parts/types.js"
+import {CodexItem} from "../../aspects/codex/parts/codex-item.js"
 
 export type MediaFormat = "video" | "image" | "audio" | "other"
 
@@ -23,3 +24,8 @@ export type MediaSchema = AsSchema<{
 	}
 }>
 
+export type MediaProgress = {
+	item: CodexItem<MediaSchema>
+	loaded: number
+	total: number
+}

package/x/cellar/cask.d.ts

@@ -2,7 +2,5 @@
 export declare class Cask {
     hash: string;
     file: Blob;
-    static hash(file: Blob): Promise<string>;
-    static make(file: Blob): Promise<Cask>;
     constructor(hash: string, file: Blob);
 }

package/x/cellar/cask.js

@@ -1,20 +1,7 @@
-import { Hex } from "@e280/stz";
-import { blake3 } from "@awasm/noble";
 /** File with a hash label */
 export class Cask {
     hash;
     file;
-    static async hash(file) {
-        const hasher = blake3.create();
-        for await (const chunk of file.stream())
-            hasher.update(chunk);
-        const digest = hasher.digest();
-        return Hex.fromBytes(digest);
-    }
-    static async make(file) {
-        const hash = await this.hash(file);
-        return new this(hash, file);
-    }
     constructor(hash, file) {
         this.hash = hash;
         this.file = file;

package/x/cellar/cask.js.map

@@ -1 +1 @@
-{"version":3,"file":"cask.js","sourceRoot":"","sources":["../../s/cellar/cask.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAEnC,6BAA6B;AAC7B,MAAM,OAAO,IAAI;IAgBG;IAAqB;IAfxC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAU;QAC3B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;QAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;YACtC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAErB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;QAC9B,OAAO,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAU;QAC3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClC,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAC5B,CAAC;IAED,YAAmB,IAAY,EAAS,IAAU;QAA/B,SAAI,GAAJ,IAAI,CAAQ;QAAS,SAAI,GAAJ,IAAI,CAAM;IAAG,CAAC;CACtD"}
+{"version":3,"file":"cask.js","sourceRoot":"","sources":["../../s/cellar/cask.ts"],"names":[],"mappings":"AACA,6BAA6B;AAC7B,MAAM,OAAO,IAAI;IACG;IAAqB;IAAxC,YAAmB,IAAY,EAAS,IAAU;QAA/B,SAAI,GAAJ,IAAI,CAAQ;QAAS,SAAI,GAAJ,IAAI,CAAM;IAAG,CAAC;CACtD"}