chubakabra 0.1.0

package/README.md

@@ -0,0 +1 @@
+# Example package

package/index.d.ts

@@ -0,0 +1,69 @@
+export type ByteRange = [number, number]
+
+export interface KvrHeader {
+	keySize: number
+	indexCount: number
+	indexOffset: number
+}
+
+export interface KvrIndexEntry {
+	key: Uint8Array
+	range: ByteRange
+}
+
+export interface KvrIndex {
+	keySize: number
+	entries: KvrIndexEntry[]
+}
+
+export interface ElementCodec<T = any> {
+	size: number
+	write(buf: Uint8Array, value: T, offset: number): void
+	read(buf: Uint8Array, offset: number): T
+}
+
+export interface KvrWriteEntry<T = any> {
+	key: T[]
+	data: Uint8Array
+}
+
+export function readKvrHeader(
+	urlOrPath: string
+): Promise<KvrHeader> | KvrHeader
+
+export function readKvrIndex(
+	urlOrPath: string
+): Promise<KvrIndex> | KvrIndex
+
+export function readKvrRange(
+	urlOrPath: string,
+	range: ByteRange
+): Promise<Uint8Array> | Uint8Array
+
+export function writeKvrOPFS<T>(
+	filename: string,
+	entries: KvrWriteEntry<T>[],
+	elementCodec: ElementCodec<T>
+): Promise<void>
+
+export function writeKvr<T>(
+	path: string,
+	entries: KvrWriteEntry<T>[],
+	elementCodec: ElementCodec<T>
+): void
+
+export function exportKvr(filename: string): Promise<void>
+
+export function packKey<T>(
+	keyArray: T[],
+	elementCodec: ElementCodec<T>
+): Uint8Array
+
+export function unpackKey<T>(
+	buf: Uint8Array,
+	elementCodec: ElementCodec<T>
+): T[]
+
+export function createElementCodec(
+	type: "int32" | "uint32" | "char"
+): ElementCodec

package/package.json

@@ -0,0 +1,19 @@
+{
+	"name": "chubakabra",
+	"version": "0.1.0",
+	"type": "module",
+	"description": "Browser and Node.js utilities for working with key-addressable byte ranges in large binary files",
+	"exports": {
+		".": {
+			"browser": "./src/browser/index.js",
+			"default": "./src/node/index.js"
+		}
+	},
+	"types": "./index.d.ts",
+	"files": [
+		"src/",
+		"index.d.ts",
+		"README.md"
+	],
+	"license": "MIT"
+}

package/src/browser/index.js

@@ -0,0 +1,196 @@
+const MAGIC = "KVR"
+const HEADER_SIZE = 32
+const RANGE_SIZE = 16
+
+export async function readKvrHeader(url)
+{
+	const buf = await fetchRange(url, 0, HEADER_SIZE - 1)
+	if (readString(buf, 0, 3) !== MAGIC || buf[3] !== 1)
+		throw new Error("Invalid KVR file")
+
+	return {
+		keySize: readUint16LE(buf, 4),
+		indexCount: readUint32LE(buf, 6),
+		indexOffset: Number(readUint64LE(buf, 10))
+	}
+}
+
+export async function readKvrIndex(url)
+{
+	const h = await readKvrHeader(url)
+	const entrySize = h.keySize + RANGE_SIZE
+	const size = entrySize * h.indexCount
+
+	const buf = await fetchRange(
+		url,
+		h.indexOffset,
+		h.indexOffset + size - 1
+	)
+
+	const entries = []
+
+	for (let i = 0; i < h.indexCount; i++)
+	{
+		const o = i * entrySize
+		entries.push({
+			key: buf.slice(o, o + h.keySize),
+			range: [
+				Number(readUint64LE(buf, o + h.keySize)),
+				Number(readUint64LE(buf, o + h.keySize + 8))
+			]
+		})
+	}
+
+	return { keySize: h.keySize, entries }
+}
+
+export async function readKvrRange(url, range)
+{
+	return fetchRange(url, range[0], range[1] - 1)
+}
+
+export async function writeKvrOPFS(filename, entries, codec)
+{
+	const keySize = entries[0].key.length * codec.size
+	const root = await navigator.storage.getDirectory()
+	const fh = await root.getFileHandle(filename, { create: true })
+	const w = await fh.createWritable()
+
+	const header = new Uint8Array(HEADER_SIZE)
+	writeString(header, 0, MAGIC)
+	header[3] = 1
+	writeUint16LE(header, 4, keySize)
+	writeUint32LE(header, 6, entries.length)
+	writeUint64LE(header, 10, 0n)
+	await w.write(header)
+
+	let offset = HEADER_SIZE
+	const index = []
+
+	for (const e of entries)
+	{
+		index.push({
+			key: packKey(e.key, codec),
+			from: offset,
+			to: offset + e.data.byteLength
+		})
+		await w.write(e.data)
+		offset += e.data.byteLength
+	}
+
+	const indexOffset = offset
+
+	for (const i of index)
+	{
+		const buf = new Uint8Array(keySize + RANGE_SIZE)
+		buf.set(i.key, 0)
+		writeUint64LE(buf, keySize, BigInt(i.from))
+		writeUint64LE(buf, keySize + 8, BigInt(i.to))
+		await w.write(buf)
+	}
+
+	await w.seek(10)
+	const p = new Uint8Array(8)
+	writeUint64LE(p, 0, BigInt(indexOffset))
+	await w.write(p)
+	await w.close()
+}
+
+export async function exportKvr(filename)
+{
+	const root = await navigator.storage.getDirectory()
+	const fh = await root.getFileHandle(filename)
+	const file = await fh.getFile()
+	const url = URL.createObjectURL(file)
+	const a = document.createElement("a")
+	a.href = url
+	a.download = filename
+	a.click()
+	URL.revokeObjectURL(url)
+}
+
+export function packKey(arr, codec)
+{
+	const b = new Uint8Array(arr.length * codec.size)
+	for (let i = 0; i < arr.length; i++)
+		codec.write(b, arr[i], i * codec.size)
+	return b
+}
+
+export function unpackKey(buf, codec)
+{
+	const n = buf.length / codec.size
+	const a = new Array(n)
+	for (let i = 0; i < n; i++)
+		a[i] = codec.read(buf, i * codec.size)
+	return a
+}
+
+export function createElementCodec(type)
+{
+	if (type === "int32")
+		return {
+			size: 4,
+			write(b, v, o){ new DataView(b.buffer).setInt32(o, v, true) },
+			read(b, o){ return new DataView(b.buffer).getInt32(o, true) }
+		}
+
+	if (type === "uint32")
+		return {
+			size: 4,
+			write(b, v, o){ new DataView(b.buffer).setUint32(o, v, true) },
+			read(b, o){ return new DataView(b.buffer).getUint32(o, true) }
+		}
+
+	if (type === "char")
+		return {
+			size: 1,
+			write(b, v, o){ b[o] = v.charCodeAt(0) },
+			read(b, o){ return String.fromCharCode(b[o]) }
+		}
+
+	throw new Error("Unsupported codec")
+}
+
+async function fetchRange(url, from, to)
+{
+	const r = await fetch(url, {
+		headers: { Range: `bytes=${from}-${to}` }
+	})
+	if (!r.ok && r.status !== 206)
+		throw new Error("Range failed")
+	return new Uint8Array(await r.arrayBuffer())
+}
+
+function writeString(b, o, s)
+{
+	for (let i = 0; i < s.length; i++)
+		b[o + i] = s.charCodeAt(i)
+}
+
+function readString(b, o, n)
+{
+	return String.fromCharCode(...b.slice(o, o + n))
+}
+
+function writeUint16LE(b, o, v){ new DataView(b.buffer).setUint16(o, v, true) }
+function writeUint32LE(b, o, v){ new DataView(b.buffer).setUint32(o, v, true) }
+function writeUint64LE(b, o, v)
+{
+	let x = BigInt(v)
+	for (let i = 0; i < 8; i++)
+	{
+		b[o + i] = Number(x & 255n)
+		x >>= 8n
+	}
+}
+
+function readUint16LE(b, o){ return b[o] | (b[o+1] << 8) }
+function readUint32LE(b, o){ return b[o] | (b[o+1]<<8) | (b[o+2]<<16) | (b[o+3]<<24) }
+function readUint64LE(b, o)
+{
+	let v = 0n
+	for (let i = 7; i >= 0; i--)
+		v = (v << 8n) | BigInt(b[o + i])
+	return v
+}

package/src/node/index.js

@@ -0,0 +1,146 @@
+import fs from "fs"
+
+const MAGIC = "KVR"
+const HEADER_SIZE = 32
+const RANGE_SIZE = 16
+
+export function readKvrHeader(path)
+{
+	const fd = fs.openSync(path, "r")
+	const b = Buffer.alloc(HEADER_SIZE)
+	fs.readSync(fd, b, 0, HEADER_SIZE, 0)
+	fs.closeSync(fd)
+
+	if (b.toString("ascii", 0, 3) !== MAGIC || b[3] !== 1)
+		throw new Error("Invalid KVR file")
+
+	return {
+		keySize: b.readUInt16LE(4),
+		indexCount: b.readUInt32LE(6),
+		indexOffset: Number(b.readBigUInt64LE(10))
+	}
+}
+
+export function readKvrIndex(path)
+{
+	const h = readKvrHeader(path)
+	const entrySize = h.keySize + RANGE_SIZE
+	const size = entrySize * h.indexCount
+
+	const fd = fs.openSync(path, "r")
+	const buf = Buffer.alloc(size)
+	fs.readSync(fd, buf, 0, size, h.indexOffset)
+	fs.closeSync(fd)
+
+	const entries = []
+
+	for (let i = 0; i < h.indexCount; i++)
+	{
+		const o = i * entrySize
+		entries.push({
+			key: buf.subarray(o, o + h.keySize),
+			range: [
+				Number(buf.readBigUInt64LE(o + h.keySize)),
+				Number(buf.readBigUInt64LE(o + h.keySize + 8))
+			]
+		})
+	}
+
+	return { keySize: h.keySize, entries }
+}
+
+export function readKvrRange(path, range)
+{
+	const fd = fs.openSync(path, "r")
+	const size = range[1] - range[0]
+	const buf = Buffer.alloc(size)
+	fs.readSync(fd, buf, 0, size, range[0])
+	fs.closeSync(fd)
+	return buf
+}
+
+export function writeKvr(path, entries, codec)
+{
+	const keySize = entries[0].key.length * codec.size
+	const fd = fs.openSync(path, "w")
+
+	const header = Buffer.alloc(HEADER_SIZE)
+	header.write(MAGIC, 0)
+	header[3] = 1
+	header.writeUInt16LE(keySize, 4)
+	header.writeUInt32LE(entries.length, 6)
+	header.writeBigUInt64LE(0n, 10)
+	fs.writeSync(fd, header)
+
+	let offset = HEADER_SIZE
+	const index = []
+
+	for (const e of entries)
+	{
+		index.push({
+			key: packKey(e.key, codec),
+			from: offset,
+			to: offset + e.data.length
+		})
+		fs.writeSync(fd, e.data, 0, e.data.length, offset)
+		offset += e.data.length
+	}
+
+	const indexOffset = offset
+
+	for (const i of index)
+	{
+		const b = Buffer.alloc(keySize + RANGE_SIZE)
+		i.key.copy(b, 0)
+		b.writeBigUInt64LE(BigInt(i.from), keySize)
+		b.writeBigUInt64LE(BigInt(i.to), keySize + 8)
+		fs.writeSync(fd, b)
+	}
+
+	header.writeBigUInt64LE(BigInt(indexOffset), 10)
+	fs.writeSync(fd, header, 0, HEADER_SIZE, 0)
+	fs.closeSync(fd)
+}
+
+export function packKey(arr, codec)
+{
+	const b = Buffer.alloc(arr.length * codec.size)
+	for (let i = 0; i < arr.length; i++)
+		codec.write(b, arr[i], i * codec.size)
+	return b
+}
+
+export function unpackKey(buf, codec)
+{
+	const n = buf.length / codec.size
+	const a = new Array(n)
+	for (let i = 0; i < n; i++)
+		a[i] = codec.read(buf, i * codec.size)
+	return a
+}
+
+export function createElementCodec(type)
+{
+	if (type === "int32")
+		return {
+			size: 4,
+			write(b, v, o){ b.writeInt32LE(v, o) },
+			read(b, o){ return b.readInt32LE(o) }
+		}
+
+	if (type === "uint32")
+		return {
+			size: 4,
+			write(b, v, o){ b.writeUInt32LE(v, o) },
+			read(b, o){ return b.readUInt32LE(o) }
+		}
+
+	if (type === "char")
+		return {
+			size: 1,
+			write(b, v, o){ b[o] = v.charCodeAt(0) },
+			read(b, o){ return String.fromCharCode(b[o]) }
+		}
+
+	throw new Error("Unsupported codec")
+}