@metamask-previews/storage-service 1.0.0-preview-e493d3e8

package/CHANGELOG.md

@@ -0,0 +1,22 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- Initial release of `@metamask/storage-service`
+- Add `StorageService` class for platform-agnostic storage
+- Add `StorageAdapter` interface for platform-specific implementations
+- Add `InMemoryStorageAdapter` as default storage (for tests/dev)
+- Add namespace-based key isolation
+- Add support for `setItem`, `getItem`, `removeItem`, `getAllKeys`, and `clear` operations
+- Add messenger integration for cross-controller communication
+- Add `STORAGE_KEY_PREFIX` constant for consistent key prefixing across adapters
+- Add comprehensive test coverage
+
+[Unreleased]: https://github.com/MetaMask/core/

package/LICENSE

@@ -0,0 +1,20 @@
+MIT License
+
+Copyright (c) 2025 MetaMask
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

package/README.md

@@ -0,0 +1,377 @@
+# `@metamask/storage-service`
+
+A platform-agnostic service for storing large, infrequently accessed controller data outside of memory.
+
+## Problem
+
+Controllers store large, infrequently-accessed data in Redux state, causing:
+- **State bloat**: 10.79 MB total, with 9.94 MB (92%) in just 2 controllers
+- **Slow app startup**: Parsing 10.79 MB on every launch
+- **High memory usage**: All data loaded, even if rarely accessed
+- **Slow persist operations**: Up to 5.95 MB written per controller change
+
+**Production measurements** (MetaMask Mobile):
+- SnapController sourceCode: 5.95 MB (55% of state)
+- TokenListController cache: 3.99 MB (37% of state)
+- **Combined**: 9.94 MB in just 2 controllers
+
+## Solution
+
+`StorageService` provides a messenger-based API for controllers to store large data on disk instead of in memory. Data is loaded lazily only when needed.
+
+## Installation
+
+`yarn add @metamask/storage-service`
+
+or
+
+`npm install @metamask/storage-service`
+
+## Architecture
+
+The service is **platform-agnostic** and accepts an optional `StorageAdapter`:
+
+- **With Adapter** (Production): Client provides platform-specific storage
+  - Mobile: FilesystemStorage adapter → Data persists
+  - Extension: IndexedDB adapter → Data persists
+  
+- **Without Adapter** (Default): Uses in-memory storage
+  - Testing: No setup needed, isolated tests
+  - Development: Quick start, no config
+  - ⚠️ Data lost on restart
+
+## Events
+
+StorageService publishes events when data changes, enabling reactive patterns:
+
+**Events published**:
+- `StorageService:itemSet:{namespace}` - When data is stored
+  - Payload: `[value, key]`
+- `StorageService:itemRemoved:{namespace}` - When data is removed
+  - Payload: `[key]`
+
+**Example - Subscribe to changes**:
+```typescript
+// In another controller
+this.messenger.subscribe(
+  'StorageService:itemSet:ControllerA',
+  (value, key) => {
+    console.log(`ControllerA stored data: ${key}`);
+    // React to changes without coupling
+  },
+);
+```
+
+## Usage
+
+### Via Messenger (Recommended)
+
+The service is designed to be used via a messenger, allowing controllers to access storage without direct dependencies.
+
+#### 1. Controller Setup
+
+```typescript
+import type {
+  StorageServiceSetItemAction,
+  StorageServiceGetItemAction,
+  StorageServiceRemoveItemAction,
+} from '@metamask/storage-service';
+
+// Grant access to storage actions
+type AllowedActions =
+  | StorageServiceSetItemAction
+  | StorageServiceGetItemAction
+  | StorageServiceRemoveItemAction;
+
+type SnapControllerMessenger = Messenger<
+  'SnapController',
+  SnapControllerActions | AllowedActions,
+  SnapControllerEvents
+>;
+
+class SnapController extends BaseController<
+  'SnapController',
+  SnapControllerState,
+  SnapControllerMessenger
+> {
+  async storeSnapSourceCode(snapId: string, sourceCode: string) {
+    // Store 3.86 MB of source code on disk, not in state
+    await this.messenger.call(
+      'StorageService:setItem',
+      'SnapController',
+      `${snapId}:sourceCode`,
+      sourceCode,
+    );
+  }
+
+  async getSnapSourceCode(snapId: string): Promise<string | null> {
+    // Load source code only when snap needs to execute
+    return await this.messenger.call(
+      'StorageService:getItem',
+      'SnapController',
+      `${snapId}:sourceCode`,
+    );
+  }
+}
+```
+
+#### 2. Service Initialization (Client)
+
+**Mobile:**
+
+```typescript
+import {
+  StorageService,
+  type StorageAdapter,
+  STORAGE_KEY_PREFIX,
+} from '@metamask/storage-service';
+import FilesystemStorage from 'redux-persist-filesystem-storage';
+
+// Adapters handle key building and serialization
+const mobileStorageAdapter: StorageAdapter = {
+  async getItem(namespace: string, key: string) {
+    const fullKey = `${STORAGE_KEY_PREFIX}${namespace}:${key}`;
+    const serialized = await FilesystemStorage.getItem(fullKey);
+    if (!serialized) return null;
+    const wrapper = JSON.parse(serialized);
+    return wrapper.data;
+  },
+  async setItem(namespace: string, key: string, value: unknown) {
+    const fullKey = `${STORAGE_KEY_PREFIX}${namespace}:${key}`;
+    const wrapper = { timestamp: Date.now(), data: value };
+    await FilesystemStorage.setItem(fullKey, JSON.stringify(wrapper), Device.isIos());
+  },
+  async removeItem(namespace: string, key: string) {
+    const fullKey = `${STORAGE_KEY_PREFIX}${namespace}:${key}`;
+    await FilesystemStorage.removeItem(fullKey);
+  },
+  async getAllKeys(namespace: string) {
+    const prefix = `${STORAGE_KEY_PREFIX}${namespace}:`;
+    const allKeys = await FilesystemStorage.getAllKeys();
+    return allKeys
+      .filter((k: string) => k.startsWith(prefix))
+      .map((k: string) => k.slice(prefix.length));
+  },
+  async clear(namespace: string) {
+    const keys = await this.getAllKeys(namespace);
+    await Promise.all(keys.map((k) => this.removeItem(namespace, k)));
+  },
+};
+
+// Initialize service
+const service = new StorageService({
+  messenger: storageServiceMessenger,
+  storage: mobileStorageAdapter,
+});
+```
+
+**Extension:**
+
+```typescript
+import {
+  StorageService,
+  type StorageAdapter,
+  STORAGE_KEY_PREFIX,
+} from '@metamask/storage-service';
+
+// Adapters handle key building and serialization
+const extensionStorageAdapter: StorageAdapter = {
+  async getItem(namespace: string, key: string) {
+    const fullKey = `${STORAGE_KEY_PREFIX}${namespace}:${key}`;
+    const db = await openDB();
+    const serialized = await db.get('storage-service', fullKey);
+    if (!serialized) return null;
+    const wrapper = JSON.parse(serialized);
+    return wrapper.data;
+  },
+  async setItem(namespace: string, key: string, value: unknown) {
+    const fullKey = `${STORAGE_KEY_PREFIX}${namespace}:${key}`;
+    const wrapper = { timestamp: Date.now(), data: value };
+    const db = await openDB();
+    await db.put('storage-service', JSON.stringify(wrapper), fullKey);
+  },
+  async removeItem(namespace: string, key: string) {
+    const fullKey = `${STORAGE_KEY_PREFIX}${namespace}:${key}`;
+    const db = await openDB();
+    await db.delete('storage-service', fullKey);
+  },
+  async getAllKeys(namespace: string) {
+    const prefix = `${STORAGE_KEY_PREFIX}${namespace}:`;
+    const db = await openDB();
+    const allKeys = await db.getAllKeys('storage-service');
+    return allKeys
+      .filter((k: string) => k.startsWith(prefix))
+      .map((k: string) => k.slice(prefix.length));
+  },
+  async clear(namespace: string) {
+    const keys = await this.getAllKeys(namespace);
+    await Promise.all(keys.map((k) => this.removeItem(namespace, k)));
+  },
+};
+
+// Initialize service
+const service = new StorageService({
+  messenger: storageServiceMessenger,
+  storage: extensionStorageAdapter,
+});
+```
+
+**Testing:**
+
+```typescript
+import { StorageService } from '@metamask/storage-service';
+
+// No storage adapter needed - uses in-memory by default
+const service = new StorageService({
+  messenger: testMessenger,
+  // storage: undefined, // Optional - defaults to InMemoryStorageAdapter
+});
+
+// Works immediately, data isolated per test
+await service.setItem('TestController', 'key', 'value');
+```
+
+#### 3. Delegate Actions to Controllers
+
+```typescript
+rootMessenger.delegate({
+  actions: [
+    'StorageService:setItem',
+    'StorageService:getItem',
+    'StorageService:removeItem',
+  ],
+  messenger: snapControllerMessenger,
+});
+```
+
+### Direct Usage
+
+You can also use the service directly without a messenger:
+
+```typescript
+import { StorageService, InMemoryStorageAdapter } from '@metamask/storage-service';
+
+const service = new StorageService({
+  messenger: myMessenger,
+  storage: new InMemoryStorageAdapter(),
+});
+
+await service.setItem('MyController', 'myKey', { data: 'value' });
+const data = await service.getItem('MyController', 'myKey');
+```
+
+## API
+
+### `StorageService`
+
+#### `setItem<T>(namespace: string, key: string, value: T): Promise<void>`
+
+Store data in storage.
+
+- `namespace` - Controller namespace (e.g., 'SnapController')
+- `key` - Storage key (e.g., 'npm:@metamask/bitcoin-wallet-snap:sourceCode')
+- `value` - Data to store (will be JSON stringified)
+
+```typescript
+await service.setItem('SnapController', 'snap-id:sourceCode', sourceCode);
+```
+
+#### `getItem<T>(namespace: string, key: string): Promise<T | null>`
+
+Retrieve data from storage.
+
+- `namespace` - Controller namespace
+- `key` - Storage key
+- **Returns**: Parsed data or null if not found
+
+```typescript
+const sourceCode = await service.getItem('SnapController', 'snap-id:sourceCode');
+```
+
+#### `removeItem(namespace: string, key: string): Promise<void>`
+
+Remove data from storage.
+
+```typescript
+await service.removeItem('SnapController', 'snap-id:sourceCode');
+```
+
+#### `getAllKeys(namespace: string): Promise<string[]>`
+
+Get all keys for a namespace (without prefix).
+
+```typescript
+const keys = await service.getAllKeys('SnapController');
+// Returns: ['snap-id-1:sourceCode', 'snap-id-2:sourceCode', ...]
+```
+
+#### `clear(namespace: string): Promise<void>`
+
+Clear all data for a namespace.
+
+```typescript
+await service.clear('SnapController');
+```
+
+## StorageAdapter Interface
+
+Implement this interface to provide platform-specific storage. Adapters are responsible for:
+- Building the full storage key (e.g., `storageService:namespace:key`)
+- Wrapping data with metadata (timestamp) before serialization
+- Serializing/deserializing data (JSON.stringify/parse)
+
+```typescript
+export type StorageAdapter = {
+  getItem(namespace: string, key: string): Promise<unknown>;
+  setItem(namespace: string, key: string, value: unknown): Promise<void>;
+  removeItem(namespace: string, key: string): Promise<void>;
+  getAllKeys(namespace: string): Promise<string[]>;
+  clear(namespace: string): Promise<void>;
+};
+```
+
+## When to Use
+
+✅ **Use StorageService for:**
+- Large data (> 100 KB)
+- Infrequently accessed data
+- Data that doesn't need to be in Redux state
+- Examples: Snap source code (6 MB), cached API responses (4 MB)
+
+❌ **Don't use for:**
+- Frequently accessed data (use controller state)
+- Small data (< 10 KB - overhead not worth it)
+- Data needed for UI rendering
+- Critical data that must be in Redux
+
+## Storage Key Format
+
+Adapters build keys with prefix: `storageService:{namespace}:{key}`
+
+Examples:
+- `storageService:SnapController:npm:@metamask/bitcoin-wallet-snap:sourceCode`
+- `storageService:TokenListController:cache:0x1`
+
+This provides:
+- Namespace isolation (prevents collisions)
+- Easy debugging (clear key format)
+- Scoped clearing (clear removes all keys for controller)
+
+## Real-World Impact
+
+**Production measurements** (MetaMask Mobile):
+
+**Per-controller**:
+- SnapController: 5.95 MB sourceCode → 166 KB metadata (97% reduction)
+- TokenListController: 3.99 MB cache → 61 bytes metadata (99.9% reduction)
+
+**Combined**:
+- Total state: 10.79 MB → 0.85 MB (**92% reduction**)
+- App startup: 92% less data to parse
+- Memory freed: 9.94 MB
+- Disk I/O: Up to 9.94 MB less per persist operation
+
+## Contributing
+
+This package is part of a monorepo. Instructions for contributing can be found in the [monorepo README](https://github.com/MetaMask/core#readme).
+

package/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@metamask-previews/storage-service",
+  "version": "1.0.0-preview-e493d3e8",
+  "description": "Platform-agnostic service for storing large, infrequently accessed controller data",
+  "keywords": [
+    "MetaMask",
+    "Ethereum",
+    "Storage",
+    "Service"
+  ],
+  "homepage": "https://github.com/MetaMask/core/tree/main/packages/storage-service#readme",
+  "bugs": {
+    "url": "https://github.com/MetaMask/core/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/MetaMask/core.git"
+  },
+  "license": "MIT",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.mts",
+        "default": "./dist/index.mjs"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "main": "./dist/index.cjs",
+  "types": "./dist/index.d.cts",
+  "files": [
+    "dist/"
+  ],
+  "scripts": {
+    "build": "ts-bridge --project tsconfig.build.json --verbose --clean --no-references",
+    "build:all": "ts-bridge --project tsconfig.build.json --verbose --clean",
+    "build:docs": "typedoc",
+    "changelog:update": "../../scripts/update-changelog.sh @metamask/storage-service",
+    "changelog:validate": "../../scripts/validate-changelog.sh @metamask/storage-service",
+    "publish:preview": "yarn npm publish --tag preview",
+    "since-latest-release": "../../scripts/since-latest-release.sh",
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest --reporters=jest-silent-reporter",
+    "test:clean": "NODE_OPTIONS=--experimental-vm-modules jest --clearCache",
+    "test:verbose": "NODE_OPTIONS=--experimental-vm-modules jest --verbose",
+    "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch"
+  },
+  "dependencies": {
+    "@metamask/messenger": "^0.3.0"
+  },
+  "devDependencies": {
+    "@metamask/auto-changelog": "^3.4.4",
+    "@ts-bridge/cli": "^0.6.4",
+    "@types/jest": "^27.4.1",
+    "deepmerge": "^4.2.2",
+    "jest": "^27.5.1",
+    "ts-jest": "^27.1.4",
+    "typedoc": "^0.24.8",
+    "typedoc-plugin-missing-exports": "^2.0.0",
+    "typescript": "~5.3.3"
+  },
+  "engines": {
+    "node": "^18.18 || >=20"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  }
+}