@esportsplus/web-storage 0.4.0 → 0.5.1

package/src/index.ts

@@ -1,11 +1,13 @@
-import { decrypt, encrypt } from '@esportsplus/utilities';
-import type { Driver, Filter, GlobalCallback, KeyCallback, MigrationFn, Options, SetOptions, TTLEnvelope } from './types';
+import { encryption } from '@esportsplus/utilities';
 import { DriverType } from './constants';
 import { IndexedDBDriver } from '~/drivers/indexeddb';
 import { LocalStorageDriver } from '~/drivers/localstorage';
 import { MemoryDriver } from '~/drivers/memory';
 import { SessionStorageDriver } from '~/drivers/sessionstorage';
 
+import type { Cipher } from '@esportsplus/utilities';
+import type { Driver, Filter, GlobalCallback, KeyCallback, MigrationFn, Options, SetOptions, TTLEnvelope } from './types';
+
 
 const VERSION_KEY = '__version__';
 
@@ -17,15 +19,14 @@ function isEnvelope<V>(value: unknown): value is TTLEnvelope<V> {
         && '__v' in (value as Record<string, unknown>);
 }
 
-async function deserialize<V>(value: unknown, secret: string | null): Promise<V | undefined> {
+async function deserialize<V>(value: unknown, cipher: Cipher | null): Promise<V | undefined> {
     if (value === undefined || value === null) {
         return undefined;
     }
 
-    if (secret && typeof value === 'string') {
+    if (cipher && typeof value === 'string') {
         try {
-            value = await decrypt(value, secret);
-            value = JSON.parse(value as string);
+            value = await cipher.decrypt(value);
         }
         catch {
             return undefined;
@@ -35,13 +36,13 @@ async function deserialize<V>(value: unknown, secret: string | null): Promise<V
     return value as V;
 }
 
-async function serialize<V>(value: V, secret: string | null): Promise<string | V> {
+async function serialize<V>(value: V, cipher: Cipher | null): Promise<string | V> {
     if (value === undefined || value === null) {
         return value;
     }
 
-    if (secret) {
-        return encrypt(JSON.stringify(value), secret);
+    if (cipher) {
+        return cipher.encrypt(value);
     }
 
     return value;
@@ -101,14 +102,14 @@ async function migrate<T>(driver: Driver<T>, migrations: Record<number, Migratio
 
         let transformed = await migrations[keys[i]]({ all: () => Promise.resolve(data) });
 
-        await driver.clear();
-
         let entries: [keyof T, T[keyof T]][] = [];
 
         for (let key in transformed) {
             entries.push([key as keyof T, transformed[key] as T[keyof T]]);
         }
 
+        await driver.clear();
+
         if (entries.length > 0) {
             await driver.replace(entries);
         }
@@ -120,6 +121,8 @@ async function migrate<T>(driver: Driver<T>, migrations: Record<number, Migratio
 
 class Local<T> {
 
+    private cipher: Cipher | null;
+
     private driver: Driver<T>;
 
     private globals: Set<GlobalCallback<T>>;
@@ -128,15 +131,13 @@ class Local<T> {
 
     private ready: Promise<void>;
 
-    private secret: string | null;
-
     private version: number;
 
 
     constructor(options: Options, secret?: string) {
+        this.cipher = null;
         this.globals = new Set();
         this.listeners = new Map();
-        this.secret = secret || null;
 
         let { migrations, name, version = 1 } = options;
 
@@ -155,11 +156,15 @@ class Local<T> {
             this.driver = new IndexedDBDriver<T>(name, version);
         }
 
+        let init = secret
+            ? encryption(secret).then((c) => { this.cipher = c; })
+            : Promise.resolve();
+
         if (migrations) {
-            this.ready = migrate(this.driver, migrations, version);
+            this.ready = init.then(() => migrate(this.driver, migrations, version));
         }
         else {
-            this.ready = Promise.resolve();
+            this.ready = init;
         }
     }
 
@@ -176,7 +181,7 @@ class Local<T> {
                 continue;
             }
 
-            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw[key], this.secret),
+            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw[key], this.cipher),
                 unwrapped = unwrap<T[keyof T]>(deserialized);
 
             if (deserialized === undefined) {
@@ -192,7 +197,7 @@ class Local<T> {
         }
 
         if (expired.length > 0) {
-            this.driver.delete(expired);
+            await this.driver.delete(expired);
         }
 
         return result;
@@ -223,7 +228,7 @@ class Local<T> {
                 return;
             }
 
-            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.secret),
+            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.cipher),
                 unwrapped = unwrap<T[keyof T]>(deserialized);
 
             if (deserialized !== undefined && unwrapped.expired) {
@@ -244,23 +249,47 @@ class Local<T> {
     async count(): Promise<number> {
         await this.ready;
 
-        let total = await this.driver.count(),
-            raw = await this.driver.get(VERSION_KEY as keyof T);
+        let expired: (keyof T)[] = [],
+            total = 0;
+
+        await this.driver.map(async (raw, key) => {
+            if (key as string === VERSION_KEY) {
+                return;
+            }
 
-        return raw !== undefined ? total - 1 : total;
+            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.cipher),
+                unwrapped = unwrap<T[keyof T]>(deserialized);
+
+            if (deserialized === undefined) {
+                return;
+            }
+
+            if (unwrapped.expired) {
+                expired.push(key);
+                return;
+            }
+
+            total++;
+        });
+
+        if (expired.length > 0) {
+            await this.driver.delete(expired);
+        }
+
+        return total;
     }
 
     async delete(...keys: (keyof T)[]): Promise<void> {
         await this.ready;
 
-        let oldValues = new Map<keyof T, T[keyof T] | undefined>();
+        let oldValues = new Map<keyof T, T[keyof T] | undefined>(),
+            raw = await this.driver.only(keys);
 
-        for (let i = 0, n = keys.length; i < n; i++) {
-            let raw = await this.driver.get(keys[i]),
-                deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.secret),
+        for (let [key, value] of raw) {
+            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(value, this.cipher),
                 unwrapped = unwrap<T[keyof T]>(deserialized);
 
-            oldValues.set(keys[i], deserialized === undefined ? undefined : unwrapped.value);
+            oldValues.set(key, deserialized === undefined ? undefined : unwrapped.value);
         }
 
         await this.driver.delete(keys);
@@ -284,7 +313,7 @@ class Local<T> {
                 return;
             }
 
-            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.secret),
+            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.cipher),
                 unwrapped = unwrap<T[keyof T]>(deserialized);
 
             if (deserialized === undefined) {
@@ -302,7 +331,7 @@ class Local<T> {
         });
 
         if (expired.length > 0) {
-            this.driver.delete(expired);
+            await this.driver.delete(expired);
         }
 
         return result;
@@ -315,7 +344,7 @@ class Local<T> {
 
         let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(
                 await this.driver.get(key),
-                this.secret
+                this.cipher
             ),
             missing = false,
             unwrapped = unwrap<T[keyof T]>(deserialized);
@@ -324,7 +353,7 @@ class Local<T> {
             missing = true;
         }
         else if (unwrapped.expired) {
-            this.driver.delete([key]);
+            await this.driver.delete([key]);
             missing = true;
         }
 
@@ -346,9 +375,34 @@ class Local<T> {
     async keys(): Promise<(keyof T)[]> {
         await this.ready;
 
-        let all = await this.driver.keys();
+        let expired: (keyof T)[] = [],
+            result: (keyof T)[] = [];
+
+        await this.driver.map(async (raw, key) => {
+            if (key as string === VERSION_KEY) {
+                return;
+            }
+
+            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.cipher),
+                unwrapped = unwrap<T[keyof T]>(deserialized);
+
+            if (deserialized === undefined) {
+                return;
+            }
+
+            if (unwrapped.expired) {
+                expired.push(key);
+                return;
+            }
+
+            result.push(key);
+        });
+
+        if (expired.length > 0) {
+            await this.driver.delete(expired);
+        }
 
-        return all.filter((k) => k as string !== VERSION_KEY);
+        return result;
     }
 
     async length(): Promise<number> {
@@ -368,7 +422,7 @@ class Local<T> {
                 return;
             }
 
-            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.secret),
+            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.cipher),
                 unwrapped = unwrap<T[keyof T]>(deserialized);
 
             if (deserialized === undefined) {
@@ -384,7 +438,7 @@ class Local<T> {
         });
 
         if (expired.length > 0) {
-            this.driver.delete(expired);
+            await this.driver.delete(expired);
         }
     }
 
@@ -396,7 +450,7 @@ class Local<T> {
             result = {} as T;
 
         for (let [key, value] of raw) {
-            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(value, this.secret),
+            let deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(value, this.cipher),
                 unwrapped = unwrap<T[keyof T]>(deserialized);
 
             if (deserialized === undefined) {
@@ -412,7 +466,7 @@ class Local<T> {
         }
 
         if (expired.length > 0) {
-            this.driver.delete(expired);
+            await this.driver.delete(expired);
         }
 
         return result;
@@ -422,7 +476,7 @@ class Local<T> {
         await this.ready;
 
         let raw = await this.driver.get(key),
-            deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.secret);
+            deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.cipher);
 
         if (deserialized === undefined) {
             return false;
@@ -431,7 +485,7 @@ class Local<T> {
         let unwrapped = unwrap<T[keyof T]>(deserialized);
 
         if (unwrapped.expired) {
-            this.driver.delete([key]);
+            await this.driver.delete([key]);
             return false;
         }
 
@@ -441,7 +495,7 @@ class Local<T> {
 
         return this.driver.set(
             key,
-            await serialize(unwrapped.value, this.secret) as T[keyof T]
+            await serialize(unwrapped.value, this.cipher) as T[keyof T]
         );
     }
 
@@ -450,11 +504,15 @@ class Local<T> {
 
         let entries: [keyof T, unknown][] = [],
             failed: string[] = [],
-            oldValues = new Map<keyof T, T[keyof T] | undefined>();
-
-        for (let key in values) {
-            let raw = await this.driver.get(key),
-                deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.secret),
+            fetchKeys = Object.keys(values) as (keyof T)[],
+            oldValues = new Map<keyof T, T[keyof T] | undefined>(),
+            raw = await this.driver.only(fetchKeys);
+
+        for (let key of fetchKeys) {
+            let value = raw.get(key),
+                deserialized = value !== undefined
+                    ? await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(value, this.cipher)
+                    : undefined,
                 unwrapped = unwrap<T[keyof T]>(deserialized);
 
             oldValues.set(key, deserialized === undefined ? undefined : unwrapped.value);
@@ -462,11 +520,11 @@ class Local<T> {
             try {
                 entries.push([
                     key,
-                    await serialize(values[key], this.secret)
+                    await serialize(values[key], this.cipher)
                 ]);
             }
             catch {
-                failed.push(key);
+                failed.push(key as string);
             }
         }
 
@@ -488,7 +546,7 @@ class Local<T> {
 
         try {
             let oldRaw = await this.driver.get(key),
-                oldDeserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(oldRaw, this.secret),
+                oldDeserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(oldRaw, this.cipher),
                 oldUnwrapped = unwrap<T[keyof T]>(oldDeserialized),
                 oldValue = oldDeserialized === undefined ? undefined : oldUnwrapped.value,
                 stored: T[keyof T] | string;
@@ -499,10 +557,10 @@ class Local<T> {
                     __v: value
                 };
 
-                stored = await serialize(envelope, this.secret) as T[keyof T];
+                stored = await serialize(envelope, this.cipher) as T[keyof T];
             }
             else {
-                stored = await serialize(value, this.secret) as T[keyof T];
+                stored = await serialize(value, this.cipher) as T[keyof T];
             }
 
             let result = await this.driver.set(key, stored as T[keyof T]);
@@ -545,7 +603,7 @@ class Local<T> {
         await this.ready;
 
         let raw = await this.driver.get(key),
-            deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.secret);
+            deserialized = await deserialize<T[keyof T] | TTLEnvelope<T[keyof T]>>(raw, this.cipher);
 
         if (deserialized === undefined) {
             return -1;
@@ -558,7 +616,7 @@ class Local<T> {
         let remaining = deserialized.__e - Date.now();
 
         if (remaining <= 0) {
-            this.driver.delete([key]);
+            await this.driver.delete([key]);
             return -1;
         }
 

package/src/lz.ts

@@ -0,0 +1,200 @@
+type CompressCtx = { bitsInBuffer: number; buffer: number; numBits: number; output: number[] };
+type DecompressCtx = { bitPos: number; compressed: string; currentValue: number; pos: number };
+
+
+let MAX_DECOMPRESSED_SIZE = 10_485_760;
+
+
+function emitLiteral(ctx: CompressCtx, ch: string) {
+    let code = ch.charCodeAt(0);
+
+    if (code < 256) {
+        writeBits(ctx, ctx.numBits, 0);
+        writeBits(ctx, 8, code);
+    }
+    else {
+        writeBits(ctx, ctx.numBits, 1);
+        writeBits(ctx, 16, code);
+    }
+}
+
+function readBits(ctx: DecompressCtx, n: number): number {
+    let result = 0;
+
+    for (let i = 0; i < n; i++) {
+        if (ctx.bitPos > 15) {
+            if (ctx.pos >= ctx.compressed.length) {
+                throw new Error('LZ: unexpected end of compressed data');
+            }
+
+            ctx.currentValue = ctx.compressed.charCodeAt(ctx.pos++) - 1;
+            ctx.bitPos = 0;
+        }
+
+        result = (result << 1) | ((ctx.currentValue >> (15 - ctx.bitPos)) & 1);
+        ctx.bitPos++;
+    }
+
+    return result;
+}
+
+function writeBits(ctx: CompressCtx, n: number, value: number) {
+    for (let i = n - 1; i >= 0; i--) {
+        ctx.buffer = (ctx.buffer << 1) | ((value >> i) & 1);
+        ctx.bitsInBuffer++;
+
+        if (ctx.bitsInBuffer === 16) {
+            ctx.output.push(ctx.buffer + 1);
+            ctx.buffer = 0;
+            ctx.bitsInBuffer = 0;
+        }
+    }
+}
+
+
+const compress = (input: string): string => {
+    if (!input) {
+        return '';
+    }
+
+    let ctx: CompressCtx = { bitsInBuffer: 0, buffer: 0, numBits: 2, output: [] },
+        dictSize = 3,
+        dictionary = new Map<string, number>(),
+        w = '';
+
+    for (let i = 0, n = input.length; i < n; i++) {
+        let c = input[i],
+            wc = w + c;
+
+        if (dictionary.has(wc)) {
+            w = wc;
+            continue;
+        }
+
+        if (w.length > 0) {
+            if (dictionary.has(w)) {
+                writeBits(ctx, ctx.numBits, dictionary.get(w)!);
+            }
+            else {
+                emitLiteral(ctx, w);
+            }
+
+            dictionary.set(wc, dictSize++);
+
+            if (dictSize > (1 << ctx.numBits)) {
+                ctx.numBits++;
+            }
+        }
+
+        w = c;
+    }
+
+    if (w.length > 0) {
+        if (dictionary.has(w)) {
+            writeBits(ctx, ctx.numBits, dictionary.get(w)!);
+        }
+        else {
+            emitLiteral(ctx, w);
+        }
+    }
+
+    // Trailing dict advance: ensures the decompressor's last placeholder growth
+    // matches (the decompressor will push a placeholder before reading EOF)
+    dictSize++;
+
+    if (dictSize > (1 << ctx.numBits)) {
+        ctx.numBits++;
+    }
+
+    writeBits(ctx, ctx.numBits, 2);
+
+    if (ctx.bitsInBuffer > 0) {
+        ctx.output.push(((ctx.buffer << (16 - ctx.bitsInBuffer)) & 0xFFFF) + 1);
+    }
+
+    ctx.output.push((ctx.bitsInBuffer === 0 ? 16 : ctx.bitsInBuffer) + 1);
+
+    return String.fromCharCode(...ctx.output);
+};
+
+const decompress = (compressed: string): string => {
+    if (!compressed) {
+        return '';
+    }
+
+    let ctx: DecompressCtx = { bitPos: 16, compressed: '', currentValue: 0, pos: 0 },
+        dictSize = 3,
+        dictionary: string[] = [],
+        numBits = 2;
+
+    ctx.compressed = compressed.substring(0, compressed.length - 1);
+
+    let code = readBits(ctx, numBits),
+        entry: string;
+
+    if (code === 0) {
+        entry = String.fromCharCode(readBits(ctx, 8));
+    }
+    else if (code === 1) {
+        entry = String.fromCharCode(readBits(ctx, 16));
+    }
+    else {
+        return '';
+    }
+
+    let parts: string[] = [entry],
+        totalLength = entry.length,
+        w = entry;
+
+    while (true) {
+        // Reserve dict slot BEFORE reading (matches compressor's add-before-next-emit timing)
+        let slotIdx = dictionary.length;
+
+        dictionary.push('');
+        dictSize++;
+
+        if (dictSize > (1 << numBits)) {
+            numBits++;
+        }
+
+        code = readBits(ctx, numBits);
+
+        if (code === 2) {
+            dictionary.pop();
+            break;
+        }
+
+        let slotCode = slotIdx + 3;
+
+        if (code === 0) {
+            entry = String.fromCharCode(readBits(ctx, 8));
+        }
+        else if (code === 1) {
+            entry = String.fromCharCode(readBits(ctx, 16));
+        }
+        else if (code === slotCode) {
+            entry = w + w[0];
+        }
+        else if (code >= 3 && code < slotCode) {
+            entry = dictionary[code - 3];
+        }
+        else {
+            throw new Error('LZ: invalid decompression code');
+        }
+
+        dictionary[slotIdx] = w + entry[0];
+        parts.push(entry);
+        totalLength += entry.length;
+
+        if (totalLength > MAX_DECOMPRESSED_SIZE) {
+            throw new Error('LZ: decompressed output exceeds size limit');
+        }
+
+        w = entry;
+    }
+
+    return parts.join('');
+};
+
+
+export { compress, decompress };

package/src/types.ts

@@ -44,4 +44,4 @@ type GlobalCallback<T> = (key: keyof T, newValue: T[keyof T] | undefined, oldVal
 type KeyCallback<T, K extends keyof T = keyof T> = (newValue: T[K] | undefined, oldValue: T[K] | undefined) => void;
 
 
-export type { Driver, Filter, GlobalCallback, KeyCallback, MigrationContext, MigrationFn, Options, SetOptions, TTLEnvelope };
+export type { Driver, Filter, GlobalCallback, KeyCallback, MigrationFn, Options, SetOptions, TTLEnvelope };

package/tests/drivers/localstorage.ts

@@ -232,6 +232,92 @@ describe('LocalStorageDriver', () => {
     });
 
 
+    describe('compression', () => {
+        type LargeData = { bio: string };
+
+        let largeDriver: LocalStorageDriver<LargeData>,
+            largeValue: string;
+
+        beforeEach(() => {
+            largeDriver = new LocalStorageDriver<LargeData>('lz', 1);
+            largeValue = 'a'.repeat(200);
+        });
+
+        it('stores small values without compression prefix', async () => {
+            await driver.set('name', 'alice');
+
+            let raw = localStorage.getItem('test:1:name')!;
+
+            expect(raw.charCodeAt(0)).not.toBe(1);
+            expect(raw).toBe('"alice"');
+        });
+
+        it('stores large values with \\x01 prefix', async () => {
+            await largeDriver.set('bio', largeValue);
+
+            let raw = localStorage.getItem('lz:1:bio')!;
+
+            expect(raw.charCodeAt(0)).toBe(1);
+        });
+
+        it('round-trips large values through set/get', async () => {
+            await largeDriver.set('bio', largeValue);
+
+            expect(await largeDriver.get('bio')).toBe(largeValue);
+        });
+
+        it('round-trips large values through replace/all', async () => {
+            await largeDriver.replace([['bio', largeValue]]);
+
+            let all = await largeDriver.all();
+
+            expect(all.bio).toBe(largeValue);
+        });
+
+        it('reads existing uncompressed values (backward compat)', async () => {
+            localStorage.setItem('lz:1:bio', JSON.stringify(largeValue));
+
+            expect(await largeDriver.get('bio')).toBe(largeValue);
+        });
+
+        it('compressed output is smaller than raw JSON', async () => {
+            await largeDriver.set('bio', largeValue);
+
+            let compressed = localStorage.getItem('lz:1:bio')!,
+                raw = JSON.stringify(largeValue);
+
+            expect(compressed.length).toBeLessThan(raw.length);
+        });
+
+        it('handles 100-byte boundary correctly', async () => {
+            type BoundaryData = { val: string };
+
+            let boundaryDriver = new LocalStorageDriver<BoundaryData>('bound', 1);
+
+            // JSON.stringify('"' + 'x'.repeat(97) + '"') = 97 chars + 2 quotes = "xxx...x" = 99 chars inside quotes, total 99+2=101? No.
+            // JSON.stringify('x'.repeat(96)) = '"' + 'x'*96 + '"' = 98 bytes < 100 => no compress
+            await boundaryDriver.set('val', 'x'.repeat(96));
+
+            let rawSmall = localStorage.getItem('bound:1:val')!;
+
+            expect(rawSmall.charCodeAt(0)).not.toBe(1);
+
+            // JSON.stringify('x'.repeat(98)) = '"' + 'x'*98 + '"' = 100 bytes >= 100 => compress
+            await boundaryDriver.set('val', 'x'.repeat(98));
+
+            let rawLarge = localStorage.getItem('bound:1:val')!;
+
+            expect(rawLarge.charCodeAt(0)).toBe(1);
+        });
+
+        it('parse returns undefined for corrupted compressed data', async () => {
+            localStorage.setItem('lz:1:bio', '\x01corrupted-data');
+
+            expect(await largeDriver.get('bio')).toBeUndefined();
+        });
+    });
+
+
     describe('set / get', () => {
         it('overwrites existing key', async () => {
             await driver.set('name', 'alice');