@frostpillar/frostpillar-storage-engine 0.0.1

package/dist/storage/drivers/syncStorage/syncStorageQuota.d.ts

@@ -0,0 +1,3 @@
+import type { SyncStorageBackendState, SyncStorageManifest } from '../../backend/types.js';
+export declare const isQuotaBrowserError: (error: unknown) => boolean;
+export declare const validateSyncStorageCommitQuota: (state: SyncStorageBackendState, generation: number, chunks: string[], manifest: SyncStorageManifest, resolveChunkKey: (generation: number, index: number) => string, manifestStorageKey: string) => void;

package/dist/storage/drivers/syncStorage/syncStorageQuota.js

@@ -0,0 +1,45 @@
+import { QuotaExceededError, StorageEngineError, } from '../../../errors/index.js';
+const utf8Encoder = new TextEncoder();
+const computeSyncStorageItemBytes = (key, value) => {
+    const valueJson = JSON.stringify(value);
+    if (valueJson === undefined) {
+        throw new StorageEngineError(`syncStorage value for key "${key}" cannot be serialized.`);
+    }
+    return (utf8Encoder.encode(key).byteLength + utf8Encoder.encode(valueJson).byteLength);
+};
+export const isQuotaBrowserError = (error) => {
+    if (error instanceof QuotaExceededError) {
+        return true;
+    }
+    if (!(error instanceof Error)) {
+        return false;
+    }
+    const normalized = `${error.name}:${error.message}`;
+    return /quota|max_items|quota_bytes|quota_bytes_per_item/i.test(normalized);
+};
+export const validateSyncStorageCommitQuota = (state, generation, chunks, manifest, resolveChunkKey, manifestStorageKey) => {
+    const pendingItems = chunks.map((chunkValue, chunkIndex) => {
+        return {
+            key: resolveChunkKey(generation, chunkIndex),
+            value: chunkValue,
+        };
+    });
+    pendingItems.push({
+        key: manifestStorageKey,
+        value: manifest,
+    });
+    if (pendingItems.length > state.maxItems) {
+        throw new QuotaExceededError(`syncStorage snapshot requires ${pendingItems.length} items but maxItems is ${state.maxItems}.`);
+    }
+    let totalBytes = 0;
+    for (const pendingItem of pendingItems) {
+        const itemBytes = computeSyncStorageItemBytes(pendingItem.key, pendingItem.value);
+        if (itemBytes > state.maxItemBytes) {
+            throw new QuotaExceededError(`syncStorage item "${pendingItem.key}" requires ${itemBytes} bytes but maxItemBytes is ${state.maxItemBytes}.`);
+        }
+        totalBytes += itemBytes;
+    }
+    if (totalBytes > state.maxTotalBytes) {
+        throw new QuotaExceededError(`syncStorage snapshot requires ${totalBytes} bytes but maxTotalBytes is ${state.maxTotalBytes}.`);
+    }
+};

package/dist/storage/record/ordering.d.ts

@@ -0,0 +1,3 @@
+import type { KeyedRecord, PersistedRecord } from '../../types.js';
+import type { EntryId } from '../btree/recordKeyIndexBTree.js';
+export declare const toPublicRecord: (entryId: EntryId, key: unknown, record: PersistedRecord) => KeyedRecord<unknown>;

package/dist/storage/record/ordering.js

@@ -0,0 +1,7 @@
+export const toPublicRecord = (entryId, key, record) => {
+    return {
+        _id: entryId,
+        key,
+        payload: record.payload,
+    };
+};

package/dist/types.d.ts

@@ -0,0 +1,125 @@
+import type { BTreeJSON, EntryId } from '@frostpillar/frostpillar-btree';
+export type { BTreeJSON, EntryId };
+export type ByteSizeInput = number | `${number}B` | `${number}KB` | `${number}MB` | `${number}GB` | 'backendLimit';
+export type AutoCommitFrequencyInput = 'immediate' | number | `${number}ms` | `${number}s` | `${number}m` | `${number}h`;
+export type SupportedValue = string | number | boolean | null;
+export type SupportedNestedValue = SupportedValue | {
+    [key in string]: SupportedNestedValue;
+};
+export type RecordPayload = {
+    [key in string]: SupportedNestedValue;
+};
+export interface DatastoreKeyDefinition<TKey = unknown, TInput = TKey> {
+    normalize: (value: TInput, fieldName: string) => TKey;
+    compare: (left: TKey, right: TKey) => number;
+    serialize: (key: TKey) => string;
+    deserialize: (serialized: string) => TKey;
+}
+export interface KeyedRecord<TKey = unknown> {
+    readonly _id: EntryId;
+    readonly key: TKey;
+    readonly payload: RecordPayload;
+}
+export interface PersistedRecord {
+    payload: RecordPayload;
+    sizeBytes: number;
+}
+export interface InputRecord<TKeyInput = unknown> {
+    key: TKeyInput;
+    payload: RecordPayload;
+}
+export interface KeyRangeQuery<TKeyInput = unknown> {
+    start: TKeyInput;
+    end: TKeyInput;
+}
+export type CapacityPolicy = 'strict' | 'turnover';
+export type DuplicateKeyPolicy = 'allow' | 'replace' | 'reject';
+export interface CapacityConfig {
+    maxSize: ByteSizeInput;
+    policy?: CapacityPolicy;
+}
+export interface AutoCommitConfig {
+    frequency?: AutoCommitFrequencyInput;
+    maxPendingBytes?: number;
+}
+export interface DatastoreCommonConfig {
+    key?: DatastoreKeyDefinition<unknown, unknown>;
+    capacity?: CapacityConfig;
+    autoCommit?: AutoCommitConfig;
+    duplicateKeys?: DuplicateKeyPolicy;
+    skipPayloadValidation?: boolean;
+}
+export interface FileTargetByPathConfig {
+    kind: 'path';
+    filePath: string;
+    directory?: never;
+    fileName?: never;
+    filePrefix?: never;
+}
+export interface FileTargetByDirectoryConfig {
+    kind: 'directory';
+    directory: string;
+    fileName?: string;
+    filePrefix?: string;
+    filePath?: never;
+}
+export type FileTargetConfig = FileTargetByPathConfig | FileTargetByDirectoryConfig;
+export interface FileBackendConfig {
+    target?: FileTargetConfig;
+    filePath?: string;
+}
+export interface OpfsConfig {
+    directoryName?: string;
+}
+export interface IndexedDBConfig {
+    databaseName?: string;
+    objectStoreName?: string;
+    version?: number;
+}
+export interface LocalStorageConfig {
+    keyPrefix?: string;
+    databaseKey?: string;
+    maxChunkChars?: number;
+    maxChunks?: number;
+}
+export interface SyncStorageConfig {
+    keyPrefix?: string;
+    databaseKey?: string;
+    maxChunkChars?: number;
+    maxChunks?: number;
+    maxItemBytes?: number;
+    maxTotalBytes?: number;
+    maxItems?: number;
+}
+export interface DatastoreDriverSnapshot {
+    treeJSON: BTreeJSON<unknown, unknown>;
+}
+export interface DatastoreDriverController {
+    handleRecordAppended(encodedBytes: number): Promise<void>;
+    handleCleared(): Promise<void>;
+    commitNow(): Promise<void>;
+    close(): Promise<void>;
+}
+export interface DatastoreDriverInitResult {
+    controller: DatastoreDriverController;
+    initialTreeJSON: BTreeJSON<unknown, unknown> | null;
+    initialCurrentSizeBytes: number;
+}
+export interface DatastoreDriverInitContext {
+    getSnapshot: () => DatastoreDriverSnapshot;
+    autoCommit?: AutoCommitConfig;
+    onAutoCommitError: (error: unknown) => void;
+}
+export interface DatastoreDriver {
+    init(context: DatastoreDriverInitContext): DatastoreDriverInitResult | Promise<DatastoreDriverInitResult>;
+    resolveBackendLimitBytes?: () => number;
+}
+export interface DatastoreConfig extends DatastoreCommonConfig {
+    driver?: DatastoreDriver;
+}
+export interface DatastoreErrorEvent {
+    source: 'autoCommit';
+    error: Error;
+    occurredAt: number;
+}
+export type DatastoreErrorListener = (event: DatastoreErrorEvent) => void | Promise<void>;

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/validation/metadata.d.ts

@@ -0,0 +1 @@
+export declare const parseNonNegativeSafeInteger: (value: unknown, fieldName: string, backendName: string) => number;

package/dist/validation/metadata.js

@@ -0,0 +1,7 @@
+import { StorageEngineError } from '../errors/index.js';
+export const parseNonNegativeSafeInteger = (value, fieldName, backendName) => {
+    if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
+        throw new StorageEngineError(`${backendName} ${fieldName} must be a non-negative safe integer.`);
+    }
+    return value;
+};

package/dist/validation/payload.d.ts

@@ -0,0 +1,7 @@
+import type { RecordPayload } from '../types.js';
+export interface PayloadValidationResult {
+    payload: RecordPayload;
+    sizeBytes: number;
+}
+export declare const deepFreezePayload: (payload: RecordPayload) => RecordPayload;
+export declare const validateAndNormalizePayload: (payload: unknown) => PayloadValidationResult;

package/dist/validation/payload.js

@@ -0,0 +1,135 @@
+import { ValidationError } from '../errors/index.js';
+import { computeUtf8ByteLength, estimateJsonStringBytes } from '../storage/backend/encoding.js';
+const MAX_PAYLOAD_DEPTH = 64;
+const MAX_PAYLOAD_KEY_BYTES = 1024;
+const MAX_PAYLOAD_STRING_BYTES = 65535;
+const MAX_PAYLOAD_KEYS_PER_OBJECT = 256;
+const MAX_PAYLOAD_KEYS_TOTAL = 4096;
+const MAX_PAYLOAD_TOTAL_BYTES = 1048576;
+// JSON-aware byte estimates for non-string primitives.
+const NULL_ESTIMATION_BYTES = 4; // "null"  = 4 bytes
+// JSON structural overhead constants for size estimation.
+const JSON_KEY_COLON_OVERHEAD = 1; // 1 colon per key (quotes handled by estimateJsonStringBytes)
+const JSON_OBJECT_BRACE_OVERHEAD = 2; // {} per object
+// Root [key, {"payload": ...}] wrapper: [ + , + {"payload": + } + ] = 15 bytes
+const JSON_ROOT_WRAPPER_OVERHEAD = 15;
+const isPlainObject = (value) => {
+    if (typeof value !== 'object' || value === null) {
+        return false;
+    }
+    if (Array.isArray(value)) {
+        return false;
+    }
+    const objectValue = value;
+    const prototype = Object.getPrototypeOf(objectValue);
+    return prototype === Object.prototype || prototype === null;
+};
+const addValidationBytes = (state, bytes) => {
+    state.totalValidationBytes += bytes;
+    if (state.totalValidationBytes > MAX_PAYLOAD_TOTAL_BYTES) {
+        throw new ValidationError(`Payload aggregate validation bytes must be <= ${MAX_PAYLOAD_TOTAL_BYTES}.`);
+    }
+};
+const validatePayloadKey = (key, state) => {
+    if (key.trim().length === 0) {
+        throw new ValidationError('Payload keys must be non-empty strings.');
+    }
+    if (key === '__proto__' || key === 'constructor' || key === 'prototype') {
+        throw new ValidationError(`Payload key "${key}" is reserved and not allowed.`);
+    }
+    const keyBytes = computeUtf8ByteLength(key);
+    if (keyBytes > MAX_PAYLOAD_KEY_BYTES) {
+        throw new ValidationError(`Payload key UTF-8 byte length must be <= ${MAX_PAYLOAD_KEY_BYTES}.`);
+    }
+    state.totalKeyCount += 1;
+    if (state.totalKeyCount > MAX_PAYLOAD_KEYS_TOTAL) {
+        throw new ValidationError(`Payload total key count must be <= ${MAX_PAYLOAD_KEYS_TOTAL}.`);
+    }
+    // Add key as JSON string (with escaping + quotes) plus colon.
+    addValidationBytes(state, estimateJsonStringBytes(key) + JSON_KEY_COLON_OVERHEAD);
+};
+export const deepFreezePayload = (payload) => {
+    Object.freeze(payload);
+    for (const value of Object.values(payload)) {
+        if (value !== null && typeof value === 'object' && !Object.isFrozen(value)) {
+            deepFreezePayload(value);
+        }
+    }
+    return payload;
+};
+const validateAndCloneValue = (value, depth, state) => {
+    if (value === null) {
+        addValidationBytes(state, NULL_ESTIMATION_BYTES);
+        return null;
+    }
+    if (typeof value === 'string') {
+        const stringBytes = computeUtf8ByteLength(value);
+        if (stringBytes > MAX_PAYLOAD_STRING_BYTES) {
+            throw new ValidationError(`Payload string UTF-8 byte length must be <= ${MAX_PAYLOAD_STRING_BYTES}.`);
+        }
+        addValidationBytes(state, estimateJsonStringBytes(value));
+        return value;
+    }
+    if (typeof value === 'number') {
+        if (!Number.isFinite(value)) {
+            throw new ValidationError('Payload number values must be finite.');
+        }
+        addValidationBytes(state, String(value).length);
+        return value;
+    }
+    if (typeof value === 'boolean') {
+        addValidationBytes(state, value ? 4 : 5);
+        return value;
+    }
+    if (typeof value === 'bigint') {
+        throw new ValidationError('Payload bigint values are not supported.');
+    }
+    if (typeof value === 'object') {
+        if (Array.isArray(value)) {
+            throw new ValidationError('Payload arrays are not supported.');
+        }
+        if (!isPlainObject(value)) {
+            throw new ValidationError('Payload values must be plain objects.');
+        }
+        return validateAndClonePayloadObject(value, depth + 1, state);
+    }
+    throw new ValidationError('Payload values must be string | number | boolean | null or nested object.');
+};
+const validateAndClonePayloadObject = (payloadObject, depth, state) => {
+    const objectLevel = depth + 1;
+    if (objectLevel > MAX_PAYLOAD_DEPTH) {
+        throw new ValidationError(`Payload nesting depth must be <= ${MAX_PAYLOAD_DEPTH}.`);
+    }
+    if (state.activePath.has(payloadObject)) {
+        throw new ValidationError('Circular payload references are not supported.');
+    }
+    const entries = Object.entries(payloadObject);
+    if (entries.length > MAX_PAYLOAD_KEYS_PER_OBJECT) {
+        throw new ValidationError(`Payload object key count must be <= ${MAX_PAYLOAD_KEYS_PER_OBJECT}.`);
+    }
+    state.activePath.add(payloadObject);
+    // JSON structural overhead: 2 bytes for braces + (N-1) bytes for comma separators.
+    const entryCount = entries.length;
+    const commaBytes = entryCount > 1 ? entryCount - 1 : 0;
+    addValidationBytes(state, JSON_OBJECT_BRACE_OVERHEAD + commaBytes);
+    const copied = {};
+    for (const [key, value] of entries) {
+        validatePayloadKey(key, state);
+        copied[key] = validateAndCloneValue(value, depth, state);
+    }
+    state.activePath.delete(payloadObject);
+    return copied;
+};
+export const validateAndNormalizePayload = (payload) => {
+    if (!isPlainObject(payload)) {
+        throw new ValidationError('payload must be a non-null plain object.');
+    }
+    const state = {
+        activePath: new WeakSet(),
+        totalKeyCount: 0,
+        totalValidationBytes: 0,
+    };
+    const cloned = validateAndClonePayloadObject(payload, 0, state);
+    const sizeBytes = state.totalValidationBytes + JSON_ROOT_WRAPPER_OVERHEAD;
+    return { payload: cloned, sizeBytes };
+};

package/dist/validation/typeGuards.d.ts

@@ -0,0 +1 @@
+export declare const isRecordObject: (value: unknown) => value is Record<string, unknown>;

package/dist/validation/typeGuards.js

@@ -0,0 +1,7 @@
+export const isRecordObject = (value) => {
+    if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+        return false;
+    }
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null;
+};

package/package.json

@@ -0,0 +1,110 @@
+{
+  "name": "@frostpillar/frostpillar-storage-engine",
+  "version": "0.0.1",
+  "description": "Chunk-based storage engine for browsers and Node.js that packs many small key-value entries into a single backing store.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "require": "./dist/index.cjs",
+      "default": "./dist/index.js"
+    },
+    "./drivers/file": {
+      "types": "./dist/drivers/file.d.ts",
+      "require": "./dist/drivers/file.cjs",
+      "default": "./dist/drivers/file.js"
+    },
+    "./drivers/localStorage": {
+      "types": "./dist/drivers/localStorage.d.ts",
+      "require": "./dist/drivers/localStorage.cjs",
+      "default": "./dist/drivers/localStorage.js"
+    },
+    "./drivers/indexedDB": {
+      "types": "./dist/drivers/indexedDB.d.ts",
+      "require": "./dist/drivers/indexedDB.cjs",
+      "default": "./dist/drivers/indexedDB.js"
+    },
+    "./drivers/opfs": {
+      "types": "./dist/drivers/opfs.d.ts",
+      "require": "./dist/drivers/opfs.cjs",
+      "default": "./dist/drivers/opfs.js"
+    },
+    "./drivers/syncStorage": {
+      "types": "./dist/drivers/syncStorage.d.ts",
+      "require": "./dist/drivers/syncStorage.cjs",
+      "default": "./dist/drivers/syncStorage.js"
+    }
+  },
+  "sideEffects": false,
+  "files": [
+    "dist",
+    "README.md",
+    "README-JA.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "storage-engine",
+    "key-value",
+    "typescript",
+    "lightweight",
+    "browser",
+    "nodejs",
+    "database",
+    "indexeddb",
+    "opfs",
+    "embedded"
+  ],
+  "author": "Hajime Sano",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/hjmsano/frostpillar-storage-engine.git"
+  },
+  "homepage": "https://github.com/hjmsano/frostpillar-storage-engine#readme",
+  "bugs": {
+    "url": "https://github.com/hjmsano/frostpillar-storage-engine/issues"
+  },
+  "engines": {
+    "node": ">=24.0.0",
+    "pnpm": ">=10.0.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@types/node": "^24.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.56.1",
+    "@typescript-eslint/parser": "^8.56.1",
+    "esbuild": "^0.27.3",
+    "eslint": "^10.0.3",
+    "eslint-config-prettier": "^10.1.8",
+    "textlint": "^15.5.2",
+    "textlint-rule-preset-ja-technical-writing": "^12.0.2",
+    "prettier": "^3.2.5",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.56.1"
+  },
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org",
+    "access": "public"
+  },
+  "dependencies": {
+    "@frostpillar/frostpillar-btree": "0.2.0"
+  },
+  "scripts": {
+    "clean:build": "rm -rf dist tsconfig.tsbuildinfo",
+    "build": "pnpm clean:build && tsc --build && pnpm build:cjs",
+    "build:cjs": "esbuild src/index.ts src/drivers/file.ts src/drivers/localStorage.ts src/drivers/indexedDB.ts src/drivers/opfs.ts src/drivers/syncStorage.ts --bundle --packages=external --platform=node --format=cjs --target=es2022 --tsconfig=tsconfig.bundle.json --outdir=dist --outbase=src --out-extension:.js=.cjs",
+    "build:bundle": "tsc --project tsconfig.bundle.json --noEmit && esbuild src/index.ts --bundle --minify --target=es2020 --tsconfig=tsconfig.bundle.json --platform=browser --format=iife --global-name=FrostpillarStorageEngine --outfile=dist/frostpillar-storage-engine.min.js",
+    "lint": "eslint .",
+    "format": "prettier --write .",
+    "textlint": "textlint \"**/*.md\"",
+    "textlint:fix": "textlint \"**/*.md\" --fix",
+    "format:md": "pnpm textlint:fix",
+    "check": "pnpm typecheck && pnpm lint && pnpm test && pnpm textlint",
+    "test": "node ./scripts/run-tests.mjs",
+    "test:coverage": "node ./scripts/run-tests.mjs --experimental-test-coverage",
+    "typecheck": "tsc --noEmit",
+    "bench": "node ./scripts/run-benchmarks.mjs"
+  }
+}