@adukiorg/anza 0.2.0

package/src/core/storage/index.js

@@ -0,0 +1,373 @@
+/**
+ * src/core/storage/index.js
+ *
+ * Unified Storage Gateway.
+ * Integrates LRU caching, Cache API, OPFS, and IndexedDB under a unified,
+ * tiered public storage surface.
+ *
+ * Source: doc 22 — Storage Architecture §1, §3
+ */
+
+import { Database } from './idb.js';
+import { opfs } from './opfs.js';
+import { CacheStorage } from './cache.js';
+import { LRUCache, WeakLRUCache } from './lru.js';
+import { quota } from './quota.js';
+import { supports } from '../platform/index.js';
+
+// Default migration: the core key-value store.
+const defaultMigrations = [(db) => db.createObjectStore('keyval')];
+
+// Initialize default storage pool instances (reassignable via storage.configure).
+let idb = new Database('platform-db', 1, defaultMigrations);
+let cache = new CacheStorage('platform-cache');
+let lru = new LRUCache(200);
+
+/**
+ * Normalizes the 3rd argument of get/set/delete: either a tier string or an
+ * options object `{ tier, ttl }`.
+ */
+function resolve(tierOrOptions, ttl = null) {
+  if (tierOrOptions && typeof tierOrOptions === 'object') {
+    return { tier: tierOrOptions.tier ?? 'idb', ttl: tierOrOptions.ttl ?? null };
+  }
+  return { tier: tierOrOptions ?? 'idb', ttl };
+}
+
+// Expiry envelope so idb/opfs honor TTL like memory/cache do.
+function wrapExpiry(value, ttl) {
+  if (!ttl) return value;
+  return { __exp: Date.now() + ttl, __v: value };
+}
+
+function unwrapExpiry(value) {
+  if (value && typeof value === 'object' && typeof value.__exp === 'number') {
+    if (Date.now() > value.__exp) return { expired: true, value: null };
+    return { expired: false, value: value.__v };
+  }
+  return { expired: false, value };
+}
+
+// Helper functions for transparent compression
+async function compress(value) {
+  const isObject = typeof value === 'object' && value !== null;
+  const stringVal = isObject ? JSON.stringify(value) : String(value);
+  const bytes = new TextEncoder().encode(stringVal);
+  const stream = new ReadableStream({
+    start(controller) {
+      controller.enqueue(bytes);
+      controller.close();
+    }
+  }).pipeThrough(new CompressionStream('gzip'));
+  const response = new Response(stream);
+  const buffer = await response.arrayBuffer();
+  return {
+    _compressed: true,
+    _type: isObject ? 'json' : 'string',
+    data: buffer
+  };
+}
+
+async function decompress(envelope) {
+  const stream = new ReadableStream({
+    start(controller) {
+      controller.enqueue(new Uint8Array(envelope.data));
+      controller.close();
+    }
+  }).pipeThrough(new DecompressionStream('gzip'));
+  const response = new Response(stream);
+  const text = await response.text();
+  return envelope._type === 'json' ? JSON.parse(text) : text;
+}
+
+// Replay write journal automatically on startup
+async function replayJournal() {
+  if (typeof localStorage === 'undefined') return;
+  const keysToRemove = [];
+  for (let i = 0; i < localStorage.length; i++) {
+    const k = localStorage.key(i);
+    if (k && k.startsWith('storage:journal:')) {
+      try {
+        const raw = localStorage.getItem(k);
+        if (raw) {
+          const { key, value } = JSON.parse(raw);
+          console.log(`Replaying journal write for key: ${key}`);
+          await storage.set(key, value, 'idb');
+        }
+      } catch (err) {
+        console.error(`Failed to replay journal entry ${k}:`, err);
+      }
+      keysToRemove.push(k);
+    }
+  }
+  for (const k of keysToRemove) {
+    localStorage.removeItem(k);
+  }
+}
+
+if (typeof window !== 'undefined') {
+  Promise.resolve().then(() => replayJournal().catch(console.error));
+}
+
+export const storage = {
+  compressionThreshold: 64 * 1024,
+
+  /**
+   * Reconfigures the storage pool. Call before first use.
+   *
+   * storage.configure({
+   *   idb:   { name, version, migrations },
+   *   lru:   { maxSize },
+   *   cache: { name }
+   * });
+   */
+  configure({ idb: idbOpts, lru: lruOpts, cache: cacheOpts } = {}) {
+    if (idbOpts) {
+      idb = new Database(
+        idbOpts.name ?? 'platform-db',
+        idbOpts.version ?? 1,
+        idbOpts.migrations ?? defaultMigrations
+      );
+    }
+    if (lruOpts) {
+      lru = new LRUCache(lruOpts.maxSize ?? 200);
+    }
+    if (cacheOpts) {
+      cache = new CacheStorage(cacheOpts.name ?? 'platform-cache');
+    }
+    return this;
+  },
+
+  /**
+   * Retrieves an item from the requested tier.
+   * The 2nd argument may be a tier string or `{ tier }`.
+   */
+  async get(key, tierOrOptions = 'idb') {
+    const { tier } = resolve(tierOrOptions);
+
+    if (tier === 'memory') {
+      return lru.get(key);
+    }
+
+    if (tier === 'opfs') {
+      const stored = await opfs.get(key);
+      const ex = unwrapExpiry(stored);
+      if (ex.expired) {
+        await opfs.delete(key);
+        return null;
+      }
+      return ex.value;
+    }
+
+    if (tier === 'cache') {
+      const res = await cache.get(key);
+      if (!res) return null;
+      try {
+        return await res.json();
+      } catch {
+        return await res.text();
+      }
+    }
+
+    // Default 'idb' read with LRU memory caching
+    const cached = lru.get(key);
+    if (cached !== null) return cached;
+
+    let value = await idb.get('keyval', key);
+    if (value !== null) {
+      const ex = unwrapExpiry(value);
+      if (ex.expired) {
+        await idb.delete('keyval', key);
+        return null;
+      }
+      value = ex.value;
+
+      if (value && value._compressed && value.data) {
+        try {
+          value = await decompress(value);
+        } catch (err) {
+          console.error('Decompression failed:', err);
+        }
+      }
+      lru.set(key, value);
+    }
+    return value;
+  },
+
+  /**
+   * Saves an item to the requested tier.
+   * The 2nd argument may be a tier string or `{ tier, ttl }`. TTL is honored across all tiers.
+   */
+  async set(key, value, tierOrOptions = 'idb') {
+    const { tier, ttl } = resolve(tierOrOptions);
+
+    // Quota proactive check before write
+    await quota.check();
+
+    if (tier === 'memory') {
+      lru.set(key, value, ttl);
+      return;
+    }
+
+    if (tier === 'opfs') {
+      await opfs.set(key, wrapExpiry(value, ttl));
+      return;
+    }
+
+    if (tier === 'cache') {
+      const response = new Response(
+        typeof value === 'object' ? JSON.stringify(value) : String(value),
+        {
+          headers: {
+            'content-type': typeof value === 'object' ? 'application/json' : 'text/plain'
+          }
+        }
+      );
+      await cache.set(key, response, ttl);
+      return;
+    }
+
+    // Default: Write to IDB and cache in LRU
+    const journalKey = `storage:journal:${key}`;
+    if (typeof localStorage !== 'undefined') {
+      try {
+        localStorage.setItem(journalKey, JSON.stringify({ key, value }));
+      } catch (err) {
+        console.warn('Failed to write journal:', err);
+      }
+    }
+
+    try {
+      let finalValue = value;
+      if (supports.compression) {
+        const serialized = typeof value === 'object' ? JSON.stringify(value) : String(value);
+        if (serialized.length > this.compressionThreshold) {
+          try {
+            finalValue = await compress(value);
+          } catch (err) {
+            console.error('Compression failed:', err);
+          }
+        }
+      }
+
+      await idb.set('keyval', key, wrapExpiry(finalValue, ttl));
+      lru.set(key, value, ttl);
+    } finally {
+      if (typeof localStorage !== 'undefined') {
+        localStorage.removeItem(journalKey);
+      }
+    }
+  },
+
+  /**
+   * Deletes an item from the storage tiers.
+   */
+  async delete(key, tierOrOptions = 'idb') {
+    const { tier } = resolve(tierOrOptions);
+    lru.delete(key);
+
+    if (tier === 'opfs') {
+      await opfs.delete(key);
+      return;
+    }
+
+    if (tier === 'cache') {
+      await cache.delete(key);
+      return;
+    }
+
+    await idb.delete('keyval', key);
+  },
+
+  /**
+   * Performs advanced index/cursor queries on the IDB store.
+   */
+  async query(storeName, queryOpts) {
+    return idb.query(storeName, queryOpts);
+  },
+
+  /**
+   * Run a callback inside a multi-store transaction context.
+   */
+  transaction(storeNames, mode, fn) {
+    return idb.transaction(storeNames, mode, fn);
+  },
+
+  /**
+   * Lists keys available in the store.
+   */
+  async list(tier = 'idb') {
+    if (tier === 'opfs') {
+      return opfs.list();
+    }
+
+    if (tier === 'cache') {
+      const c = await cache.open();
+      const keys = await c.keys();
+      return keys.map((req) => req.url);
+    }
+
+    return idb.keys('keyval');
+  },
+
+  /**
+   * Clears the storage pool for a specific tier or all.
+   */
+  async clear(tier = 'all') {
+    lru.clear();
+
+    if (tier === 'all' || tier === 'opfs') {
+      await opfs.clear();
+    }
+
+    if (tier === 'all' || tier === 'cache') {
+      await cache.clear();
+    }
+
+    if (tier === 'all' || tier === 'idb') {
+      await idb.clear('keyval');
+    }
+  },
+
+  /**
+   * Retrieves storage quotas.
+   */
+  async estimate() {
+    const est = await quota.estimate();
+    const persisted = await this.persisted();
+    return {
+      quota: est.quota,
+      usage: est.usage,
+      persisted
+    };
+  },
+
+  /**
+   * Requests storage persistence.
+   */
+  persist() {
+    return quota.persist();
+  },
+
+  /**
+   * Checks if storage is persisted.
+   */
+  async persisted() {
+    if (typeof navigator !== 'undefined' && navigator.storage?.persisted) {
+      return navigator.storage.persisted();
+    }
+    return false;
+  },
+
+  /**
+   * Registers a callback for storage quota warning events.
+   */
+  onQuotaWarning(handler) {
+    return quota.onQuotaWarning(handler);
+  }
+};
+
+
+// Named class exports — lets consumers instantiate adapters directly:
+// import { Database, LRUCache } from '@adukiorg/anza/storage';
+export { Database, LRUCache, WeakLRUCache };

package/src/core/storage/lru.js

@@ -0,0 +1,107 @@
+/**
+ * src/core/storage/lru.js
+ *
+ * Bounded Least-Recently-Used (LRU) Cache.
+ * Provides Map-based LRU and WeakRef-based GC-eligible bounded caches
+ * supporting custom TTL values.
+ *
+ * Source: doc 22 — Storage Architecture §2
+ */
+
+export class LRUCache {
+  constructor(maxSize = 100) {
+    this.maxSize = maxSize;
+    this.cache = new Map();
+  }
+
+  get(key) {
+    if (!this.cache.has(key)) return null;
+
+    const entry = this.cache.get(key);
+    if (entry.expires && Date.now() > entry.expires) {
+      this.cache.delete(key);
+      return null;
+    }
+
+    // Refresh access position by moving key to the end
+    this.cache.delete(key);
+    this.cache.set(key, entry);
+    return entry.value;
+  }
+
+  set(key, value, ttlMs) {
+    if (this.cache.has(key)) {
+      this.cache.delete(key);
+    } else if (this.cache.size >= this.maxSize) {
+      // Map keys iterator guarantees insertion-order; first item is least recently used
+      const oldest = this.cache.keys().next().value;
+      this.cache.delete(oldest);
+    }
+
+    const expires = ttlMs ? Date.now() + ttlMs : null;
+    this.cache.set(key, { value, expires });
+  }
+
+  delete(key) {
+    return this.cache.delete(key);
+  }
+
+  clear() {
+    this.cache.clear();
+  }
+}
+
+/**
+ * WeakRef-based bounded LRU cache.
+ * Yields elements to the Garbage Collector if memory pressure requires.
+ */
+export class WeakLRUCache {
+  constructor(maxSize = 100) {
+    this.maxSize = maxSize;
+    this.cache = new Map();
+  }
+
+  get(key) {
+    if (!this.cache.has(key)) return null;
+
+    const entry = this.cache.get(key);
+    const value = entry.ref.deref();
+
+    // Clean up if GC collected the reference or TTL expired
+    if (value === undefined || (entry.expires && Date.now() > entry.expires)) {
+      this.cache.delete(key);
+      return null;
+    }
+
+    this.cache.delete(key);
+    this.cache.set(key, entry);
+    return value;
+  }
+
+  set(key, value, ttlMs) {
+    if (typeof value !== 'object' && typeof value !== 'function') {
+      throw new TypeError('WeakLRUCache values must be objects or functions');
+    }
+
+    if (this.cache.has(key)) {
+      this.cache.delete(key);
+    } else if (this.cache.size >= this.maxSize) {
+      const oldest = this.cache.keys().next().value;
+      this.cache.delete(oldest);
+    }
+
+    const expires = ttlMs ? Date.now() + ttlMs : null;
+    this.cache.set(key, {
+      ref: new WeakRef(value),
+      expires
+    });
+  }
+
+  delete(key) {
+    return this.cache.delete(key);
+  }
+
+  clear() {
+    this.cache.clear();
+  }
+}

package/src/core/storage/missing.md

@@ -0,0 +1,165 @@
+# Storage Missing Support
+
+This document tracks remaining support, runtime gaps, concurrency constraints, serialization enhancements, and performance optimization work for `src/core/storage`. Implemented behavior belongs in `usage.md` (or core documentation); unsupported, under-tested, or performance-gated behavior belongs here until it is built and verified.
+
+---
+
+## 0. Toolchain and Naming Rules
+
+Heavy compilation, static analysis, and code generation belong in `tools/`, not in browser runtime modules.
+
+Current toolchain storage support requirements:
+
+- Analyze storage tier utilization and statically compile Dedicated Web Worker scripts to avoid runtime ObjectURL blobs.
+- Detect synchronous storage access patterns (like `localStorage` or `sessionStorage` usage) outside designated bootstrap paths.
+- Statically validate migration schema progression profiles.
+
+Naming rules for storage support:
+
+- Prefer one-word files: `idb.js`, `lru.js`, `opfs.js`, `quota.js`, `cache.js`, `index.js`.
+- Plural folders: `tests/`, `migrations/`.
+- Single-word methods: `get`, `set`, `delete`, `query`, `list`, `clear`, `estimate`, `persist`, `persisted`, `transaction`.
+- Single-word events: `warning`, `quota`, `blocked`.
+- Avoid compound names where single words carry full meaning in context.
+
+---
+
+## 1. Critical Runtime and Type Gaps
+
+### 1.1. Missing Unified Gateway Methods
+
+- **Status**: Runtime gap.
+- **Files**:
+  - `src/core/storage/index.js`
+- **Problem**:
+  The Storage Architecture specification (§15) defines `storage.transaction(stores, mode, fn)`, `storage.persisted()`, and `storage.onQuotaWarning(handler)` as part of the Unified Storage Gateway interface. Currently, these methods are not implemented on the exported `storage` facade.
+- **Expected Support**:
+  - Implement `storage.transaction(stores, mode, fn)` to execute multi-store operations inside a single transactional block.
+  - Implement `storage.persisted()` wrapping `navigator.storage.persisted()`.
+  - Implement `storage.onQuotaWarning(handler)` to subscribe to quota limit checks.
+
+### 1.2. Storage Type Mismatches
+
+- **Status**: Type mismatch.
+- **Files**:
+  - `types/core/storage/index.d.ts`
+- **Problem**:
+  The type declaration file does not align with the public specifications. It is missing declarations for `persisted()`, `transaction()`, and the proper returned structure from `estimate()` (including the `persisted` flag).
+- **Expected Support**:
+  - Align `types/core/storage/index.d.ts` to expose the true interface.
+
+---
+
+## 2. Concurrency & Durability Gaps
+
+### 2.1. Web Locks Coordination for OPFS
+
+- **Status**: Critical concurrency gap.
+- **Files**:
+  - `src/core/storage/opfs.js`
+- **Problem**:
+  The OPFS manager does not coordinate file access across different tabs. Concurrent reads and writes to the same virtual file will throw errors or lead to raw binary data corruption.
+- **Expected Support**:
+  - Wrap OPFS operations (`get`, `set`, `delete`) in an exclusive Web Lock (`workers.lock` or `navigator.locks.request`) scoped to the filename to serialize cross-tab operations.
+
+### 2.2. IndexedDB Upgrade Blocking Coordination
+
+- **Status**: Runtime gap.
+- **Files**:
+  - `src/core/storage/idb.js`
+- **Problem**:
+  If a user has multiple tabs open, an IndexedDB database schema upgrade (triggered by opening with a higher version) is blocked indefinitely because the old tabs hold active connections. Currently, the database class does not listen to `onblocked` or `onversionchange` events.
+- **Expected Support**:
+  - Handle the `blocked` event on the database open request, dispatching an event to warn the system of connection locks.
+  - Listen to the `versionchange` event on the active database connection. If triggered, gracefully close the database immediately so that other tabs can proceed with schema upgrades.
+
+### 2.3. Write Journal for Durability
+
+- **Status**: Planned but missing.
+- **Files**:
+  - `src/core/storage/index.js`
+- **Problem**:
+  Transactions can fail to write to disk due to abrupt tab crashes or sudden power losses, leaving memory and disk states inconsistent.
+- **Expected Support**:
+  - Introduce a write journal using crash-durable `localStorage` to buffer pending keys before committing them to IndexedDB.
+  - Verify and replay uncommitted journal operations automatically during client boot.
+
+---
+
+## 3. Serialization & Performance Gaps
+
+### 3.1. Transparent Compression for Large Records
+
+- **Status**: Performance enhancement.
+- **Files**:
+  - `src/core/storage/index.js`
+- **Problem**:
+  Writing large objects (>64KB) directly to IndexedDB consumes excess quota, increases serialization/deserialization delay, and degrades storage bandwidth.
+- **Expected Support**:
+  - Implement transparent compression using the browser's native Compression Streams API (`new CompressionStream('gzip')`) when writing values exceeding 64KB.
+  - Inject a metadata envelope (`{ value, compressed: true }`) to identify compressed payloads and decompress them automatically on read.
+
+### 3.2. Worker Script Optimization
+
+- **Status**: Optimization.
+- **Files**:
+  - `src/core/storage/opfs.js`
+- **Problem**:
+  The OPFS manager instantiates its Dedicated Worker using a Blob URL wrapper on an inline string (`URL.createObjectURL(new Blob(...))`). This slows down initial load, blocks minification, and triggers Content Security Policy (CSP) security warnings.
+- **Expected Support**:
+  - Offload worker script extraction to the build-time compiler (`tools/`) to generate a static asset or optimize string injection.
+
+---
+
+## 4. Test Coverage Gaps
+
+### 4.1. OPFS Integration Tests
+
+- **Status**: Missing coverage.
+- **Files**:
+  - `tests/core/storage/opfs.test.js` [NEW]
+- **Needed Coverage**:
+  - Dedicated Worker execution offloading.
+  - Parallel reads/writes and verification of data persistence.
+  - Cross-tab change invalidation alerts over BroadcastChannel.
+
+### 4.2. Cache API and TTL Verification
+
+- **Status**: Missing coverage.
+- **Files**:
+  - `tests/core/storage/cache.test.js` [NEW]
+- **Needed Coverage**:
+  - Cache matches and correct header insertion for `x-expires-at`.
+  - Automatic eviction of expired items.
+
+### 4.3. Quota Warning and Eviction Tests
+
+- **Status**: Missing coverage.
+- **Files**:
+  - `tests/core/storage/quota.test.js` [NEW]
+- **Needed Coverage**:
+  - Mocking quota estimate queries.
+  - Verification that warnings fire when threshold usage exceeds 80%.
+
+---
+
+## 5. Suggested Implementation Order
+
+1. Align `types/core/storage/index.d.ts` with the full unified storage API.
+2. Add comprehensive tests for `opfs.js`, `cache.js`, and `quota.js`.
+3. Implement Web Locks for OPFS file synchronization.
+4. Implement database blocking and connection version upgrade handlers.
+5. Implement transparent Compression Streams for large values (>64KB).
+6. Implement write journaling for transactional durability.
+
+---
+
+## 6. Done Criteria
+
+This missing-support list is complete when:
+
+- TypeScript declarations map the complete unified storage API.
+- OPFS, Cache TTL, and Quota warning components have complete integration tests.
+- Web Locks synchronize multi-tab file writes.
+- DB upgrades are not blocked by connections in sibling tabs.
+- Large records (>64KB) are compressed transparently using native streams.