@casfa/storage-s3 0.1.0

package/README.md

@@ -0,0 +1,182 @@
+# @casfa/storage-s3
+
+S3 storage provider for CAS.
+
+## Installation
+
+```bash
+bun add @casfa/storage-s3
+```
+
+## Overview
+
+An S3-backed storage provider for CAS (Content-Addressable Storage). Suitable for cloud deployments with high availability and durability requirements.
+
+## Usage
+
+### Basic Usage
+
+```typescript
+import { createS3Storage } from '@casfa/storage-s3';
+
+const storage = createS3Storage({
+  bucket: 'my-cas-bucket',
+  region: 'us-east-1',
+  prefix: 'cas/',  // Optional key prefix
+});
+
+// Store data
+await storage.put('node:abcd1234...', data);
+
+// Retrieve data
+const data = await storage.get('node:abcd1234...');
+
+// Check existence
+const exists = await storage.has('node:abcd1234...');
+
+// Delete
+const deleted = await storage.delete('node:abcd1234...');
+```
+
+### With LRU Cache
+
+```typescript
+import { createS3StorageWithCache } from '@casfa/storage-s3';
+
+const storage = createS3StorageWithCache({
+  bucket: 'my-cas-bucket',
+  region: 'us-east-1',
+  cacheSize: 1000,      // Max items in cache
+  cacheMaxAge: 300000,  // 5 minutes TTL
+});
+
+// Cache reduces S3 API calls for repeated reads
+const data = await storage.get('node:abcd1234...');
+```
+
+### With Custom S3 Client
+
+```typescript
+import { S3Client } from '@aws-sdk/client-s3';
+import { createS3Storage } from '@casfa/storage-s3';
+
+const s3Client = new S3Client({
+  region: 'us-east-1',
+  credentials: {
+    accessKeyId: 'AKIA...',
+    secretAccessKey: '...',
+  },
+});
+
+const storage = createS3Storage({
+  bucket: 'my-cas-bucket',
+  client: s3Client,
+});
+```
+
+## Configuration
+
+```typescript
+interface S3StorageConfig {
+  // Required: S3 bucket name
+  bucket: string;
+  
+  // AWS region (required if not using custom client)
+  region?: string;
+  
+  // Optional: Key prefix for all objects
+  prefix?: string;
+  
+  // Optional: Custom S3 client
+  client?: S3Client;
+  
+  // Optional: Storage class for new objects
+  storageClass?: 'STANDARD' | 'STANDARD_IA' | 'GLACIER' | 'DEEP_ARCHIVE';
+}
+```
+
+### Cache Options
+
+```typescript
+interface S3StorageWithCacheConfig extends S3StorageConfig {
+  // Max items in LRU cache (default: 1000)
+  cacheSize?: number;
+  
+  // Cache TTL in milliseconds (default: 60000)
+  cacheMaxAge?: number;
+}
+```
+
+## S3 Key Structure
+
+Objects are stored with sharded prefixes for better S3 performance:
+
+```
+{prefix}ab/cd/abcd1234...
+```
+
+Where `ab` and `cd` are the first 4 characters of the hash, providing good distribution across S3 partitions.
+
+## API Reference
+
+### Functions
+
+- `createS3Storage(config)` - Create S3 storage without cache
+- `createS3StorageWithCache(config)` - Create S3 storage with LRU cache
+
+### StorageProvider Interface
+
+```typescript
+interface StorageProvider {
+  get(key: string): Promise<Uint8Array | null>;
+  put(key: string, data: Uint8Array): Promise<void>;
+  has(key: string): Promise<boolean>;
+  delete(key: string): Promise<boolean>;
+}
+```
+
+## AWS Permissions
+
+Required IAM permissions for the S3 bucket:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetObject",
+        "s3:PutObject",
+        "s3:DeleteObject",
+        "s3:HeadObject"
+      ],
+      "Resource": "arn:aws:s3:::my-cas-bucket/*"
+    }
+  ]
+}
+```
+
+## Performance Tips
+
+1. **Use caching** to reduce S3 API calls and latency
+2. **Use S3 Transfer Acceleration** for global access
+3. **Choose appropriate storage class** based on access patterns
+4. **Enable S3 Intelligent-Tiering** for cost optimization
+5. **Use regional endpoints** to minimize latency
+
+## Cost Considerations
+
+- CAS data is immutable, so versioning is not needed
+- Consider S3 Intelligent-Tiering for infrequently accessed data
+- Monitor PUT/GET request costs for high-throughput workloads
+
+## Related Packages
+
+- `@casfa/storage-core` - Core types and utilities
+- `@casfa/storage-fs` - File system storage (for local deployment)
+- `@casfa/storage-memory` - In-memory storage (for testing)
+
+## License
+
+MIT

package/README.zh-CN.md

@@ -0,0 +1,182 @@
+# @casfa/storage-s3
+
+基于 S3 的 CAS 存储提供者。
+
+## 安装
+
+```bash
+bun add @casfa/storage-s3
+```
+
+## 概述
+
+基于 S3 的 CAS（内容寻址存储）存储提供者。适用于对高可用性和数据持久性有要求的云端部署。
+
+## 使用方法
+
+### 基本用法
+
+```typescript
+import { createS3Storage } from '@casfa/storage-s3';
+
+const storage = createS3Storage({
+  bucket: 'my-cas-bucket',
+  region: 'us-east-1',
+  prefix: 'cas/',  // 可选的键前缀
+});
+
+// 存储数据
+await storage.put('node:abcd1234...', data);
+
+// 检索数据
+const data = await storage.get('node:abcd1234...');
+
+// 检查是否存在
+const exists = await storage.has('node:abcd1234...');
+
+// 删除
+const deleted = await storage.delete('node:abcd1234...');
+```
+
+### 带 LRU 缓存
+
+```typescript
+import { createS3StorageWithCache } from '@casfa/storage-s3';
+
+const storage = createS3StorageWithCache({
+  bucket: 'my-cas-bucket',
+  region: 'us-east-1',
+  cacheSize: 1000,      // 缓存最大条目数
+  cacheMaxAge: 300000,  // 5 分钟 TTL
+});
+
+// 缓存减少 S3 API 调用，降低重复读取延迟
+const data = await storage.get('node:abcd1234...');
+```
+
+### 使用自定义 S3 客户端
+
+```typescript
+import { S3Client } from '@aws-sdk/client-s3';
+import { createS3Storage } from '@casfa/storage-s3';
+
+const s3Client = new S3Client({
+  region: 'us-east-1',
+  credentials: {
+    accessKeyId: 'AKIA...',
+    secretAccessKey: '...',
+  },
+});
+
+const storage = createS3Storage({
+  bucket: 'my-cas-bucket',
+  client: s3Client,
+});
+```
+
+## 配置
+
+```typescript
+interface S3StorageConfig {
+  // 必需：S3 桶名称
+  bucket: string;
+  
+  // AWS 区域（不使用自定义客户端时必需）
+  region?: string;
+  
+  // 可选：所有对象的键前缀
+  prefix?: string;
+  
+  // 可选：自定义 S3 客户端
+  client?: S3Client;
+  
+  // 可选：新对象的存储类别
+  storageClass?: 'STANDARD' | 'STANDARD_IA' | 'GLACIER' | 'DEEP_ARCHIVE';
+}
+```
+
+### 缓存选项
+
+```typescript
+interface S3StorageWithCacheConfig extends S3StorageConfig {
+  // LRU 缓存最大条目数（默认: 1000）
+  cacheSize?: number;
+  
+  // 缓存 TTL，单位毫秒（默认: 60000）
+  cacheMaxAge?: number;
+}
+```
+
+## S3 键结构
+
+对象使用分片前缀存储，以获得更好的 S3 性能：
+
+```
+{prefix}ab/cd/abcd1234...
+```
+
+其中 `ab` 和 `cd` 是哈希的前 4 个字符，可在 S3 分区间提供良好的分布。
+
+## API 参考
+
+### 函数
+
+- `createS3Storage(config)` - 创建不带缓存的 S3 存储
+- `createS3StorageWithCache(config)` - 创建带 LRU 缓存的 S3 存储
+
+### StorageProvider 接口
+
+```typescript
+interface StorageProvider {
+  get(key: string): Promise<Uint8Array | null>;
+  put(key: string, data: Uint8Array): Promise<void>;
+  has(key: string): Promise<boolean>;
+  delete(key: string): Promise<boolean>;
+}
+```
+
+## AWS 权限
+
+S3 桶所需的 IAM 权限：
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:GetObject",
+        "s3:PutObject",
+        "s3:DeleteObject",
+        "s3:HeadObject"
+      ],
+      "Resource": "arn:aws:s3:::my-cas-bucket/*"
+    }
+  ]
+}
+```
+
+## 性能建议
+
+1. **使用缓存** 以减少 S3 API 调用和延迟
+2. **使用 S3 Transfer Acceleration** 以实现全球访问
+3. **根据访问模式选择合适的存储类别**
+4. **启用 S3 Intelligent-Tiering** 以优化成本
+5. **使用区域端点** 以最小化延迟
+
+## 成本考虑
+
+- CAS 数据是不可变的，因此不需要版本控制
+- 对于不常访问的数据，考虑使用 S3 Intelligent-Tiering
+- 对于高吞吐量工作负载，注意监控 PUT/GET 请求费用
+
+## 相关包
+
+- `@casfa/storage-core` - 核心类型与工具
+- `@casfa/storage-fs` - 文件系统存储（用于本地部署）
+- `@casfa/storage-memory` - 内存存储（用于测试）
+
+## 许可证
+
+MIT

package/dist/index.d.ts

@@ -0,0 +1,42 @@
+import { S3Client } from '@aws-sdk/client-s3';
+import { StorageProvider } from '@casfa/storage-core';
+
+/**
+ * S3 Storage Provider for CAS
+ *
+ * Implements StorageProvider with:
+ * - LRU cache for key existence checks
+ * - S3 backend storage
+ */
+
+/**
+ * S3 Storage configuration
+ */
+type S3StorageConfig = {
+    /** S3 bucket name */
+    bucket: string;
+    /** Optional S3 client (for testing or custom config) */
+    client?: S3Client;
+    /** LRU cache size for key existence (default: 10000) */
+    cacheSize?: number;
+    /** Key prefix in S3 (default: "cas/sha256/") */
+    prefix?: string;
+};
+/**
+ * Create an S3-backed storage provider
+ */
+declare const createS3Storage: (config: S3StorageConfig) => StorageProvider;
+/**
+ * Create S3 storage with cache control methods (for testing)
+ */
+declare const createS3StorageWithCache: (config: S3StorageConfig) => {
+    clearCache: () => void;
+    getCacheStats: () => {
+        size: number;
+    };
+    has: (key: string) => Promise<boolean>;
+    get: (key: string) => Promise<Uint8Array | null>;
+    put: (key: string, value: Uint8Array) => Promise<void>;
+};
+
+export { type S3StorageConfig, createS3Storage, createS3StorageWithCache };

package/dist/index.js

@@ -0,0 +1,97 @@
+// src/s3-storage.ts
+import {
+  GetObjectCommand,
+  HeadObjectCommand,
+  PutObjectCommand,
+  S3Client
+} from "@aws-sdk/client-s3";
+import {
+  createLRUCache,
+  DEFAULT_CACHE_SIZE,
+  toStoragePath
+} from "@casfa/storage-core";
+var createS3Storage = (config) => {
+  const client = config.client ?? new S3Client({});
+  const bucket = config.bucket;
+  const prefix = config.prefix ?? "cas/sha256/";
+  const existsCache = createLRUCache(config.cacheSize ?? DEFAULT_CACHE_SIZE);
+  const toS3Key = (casKey) => toStoragePath(casKey, prefix);
+  const has = async (key) => {
+    const cached = existsCache.get(key);
+    if (cached !== void 0) {
+      return cached;
+    }
+    try {
+      await client.send(
+        new HeadObjectCommand({
+          Bucket: bucket,
+          Key: toS3Key(key)
+        })
+      );
+      existsCache.set(key, true);
+      return true;
+    } catch (error) {
+      const err = error;
+      if (err.name === "NotFound" || err.$metadata?.httpStatusCode === 404) {
+        return false;
+      }
+      throw error;
+    }
+  };
+  const get = async (key) => {
+    try {
+      const result = await client.send(
+        new GetObjectCommand({
+          Bucket: bucket,
+          Key: toS3Key(key)
+        })
+      );
+      const bytes = await result.Body.transformToByteArray();
+      existsCache.set(key, true);
+      return new Uint8Array(bytes);
+    } catch (error) {
+      const err = error;
+      if (err.name === "NoSuchKey" || err.$metadata?.httpStatusCode === 404) {
+        return null;
+      }
+      throw error;
+    }
+  };
+  const put = async (key, value) => {
+    if (existsCache.get(key)) {
+      return;
+    }
+    const exists = await has(key);
+    if (exists) {
+      return;
+    }
+    await client.send(
+      new PutObjectCommand({
+        Bucket: bucket,
+        Key: toS3Key(key),
+        Body: value,
+        ContentType: "application/octet-stream"
+      })
+    );
+    existsCache.set(key, true);
+  };
+  return { has, get, put };
+};
+var createS3StorageWithCache = (config) => {
+  const client = config.client ?? new S3Client({});
+  const _bucket = config.bucket;
+  const prefix = config.prefix ?? "cas/sha256/";
+  const existsCache = createLRUCache(config.cacheSize ?? DEFAULT_CACHE_SIZE);
+  const _toS3Key = (casKey) => toStoragePath(casKey, prefix);
+  const storage = createS3Storage({ ...config, client });
+  return {
+    ...storage,
+    clearCache: () => existsCache.clear(),
+    getCacheStats: () => ({ size: existsCache.size() })
+  };
+};
+export {
+  createS3Storage,
+  createS3StorageWithCache
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/s3-storage.ts"],"sourcesContent":["/**\n * S3 Storage Provider for CAS\n *\n * Implements StorageProvider with:\n * - LRU cache for key existence checks\n * - S3 backend storage\n */\n\nimport {\n  GetObjectCommand,\n  HeadObjectCommand,\n  PutObjectCommand,\n  S3Client,\n} from \"@aws-sdk/client-s3\";\nimport {\n  createLRUCache,\n  DEFAULT_CACHE_SIZE,\n  type StorageProvider,\n  toStoragePath,\n} from \"@casfa/storage-core\";\n\n/**\n * S3 Storage configuration\n */\nexport type S3StorageConfig = {\n  /** S3 bucket name */\n  bucket: string;\n  /** Optional S3 client (for testing or custom config) */\n  client?: S3Client;\n  /** LRU cache size for key existence (default: 10000) */\n  cacheSize?: number;\n  /** Key prefix in S3 (default: \"cas/sha256/\") */\n  prefix?: string;\n};\n\n/**\n * Create an S3-backed storage provider\n */\nexport const createS3Storage = (config: S3StorageConfig): StorageProvider => {\n  const client = config.client ?? new S3Client({});\n  const bucket = config.bucket;\n  const prefix = config.prefix ?? \"cas/sha256/\";\n  const existsCache = createLRUCache<string, boolean>(config.cacheSize ?? DEFAULT_CACHE_SIZE);\n\n  const toS3Key = (casKey: string): string => toStoragePath(casKey, prefix);\n\n  const has = async (key: string): Promise<boolean> => {\n    // Check cache first\n    const cached = existsCache.get(key);\n    if (cached !== undefined) {\n      return cached;\n    }\n\n    // Check S3\n    try {\n      await client.send(\n        new HeadObjectCommand({\n          Bucket: bucket,\n          Key: toS3Key(key),\n        })\n      );\n      existsCache.set(key, true);\n      return true;\n    } catch (error: unknown) {\n      const err = error as { name?: string; $metadata?: { httpStatusCode?: number } };\n      if (err.name === \"NotFound\" || err.$metadata?.httpStatusCode === 404) {\n        // Don't cache non-existence (it might be uploaded later)\n        return false;\n      }\n      throw error;\n    }\n  };\n\n  const get = async (key: string): Promise<Uint8Array | null> => {\n    try {\n      const result = await client.send(\n        new GetObjectCommand({\n          Bucket: bucket,\n          Key: toS3Key(key),\n        })\n      );\n\n      const bytes = await result.Body!.transformToByteArray();\n\n      // Mark as existing in cache\n      existsCache.set(key, true);\n\n      return new Uint8Array(bytes);\n    } catch (error: unknown) {\n      const err = error as { name?: string; $metadata?: { httpStatusCode?: number } };\n      if (err.name === \"NoSuchKey\" || err.$metadata?.httpStatusCode === 404) {\n        return null;\n      }\n      throw error;\n    }\n  };\n\n  const put = async (key: string, value: Uint8Array): Promise<void> => {\n    // Check cache first (avoid redundant writes)\n    if (existsCache.get(key)) {\n      return;\n    }\n\n    // Check if already exists in S3\n    const exists = await has(key);\n    if (exists) {\n      return;\n    }\n\n    // Upload to S3\n    await client.send(\n      new PutObjectCommand({\n        Bucket: bucket,\n        Key: toS3Key(key),\n        Body: value,\n        ContentType: \"application/octet-stream\",\n      })\n    );\n\n    // Mark as existing\n    existsCache.set(key, true);\n  };\n\n  return { has, get, put };\n};\n\n/**\n * Create S3 storage with cache control methods (for testing)\n */\nexport const createS3StorageWithCache = (config: S3StorageConfig) => {\n  const client = config.client ?? new S3Client({});\n  const _bucket = config.bucket;\n  const prefix = config.prefix ?? \"cas/sha256/\";\n  const existsCache = createLRUCache<string, boolean>(config.cacheSize ?? DEFAULT_CACHE_SIZE);\n\n  const _toS3Key = (casKey: string): string => toStoragePath(casKey, prefix);\n\n  const storage = createS3Storage({ ...config, client });\n\n  return {\n    ...storage,\n    clearCache: () => existsCache.clear(),\n    getCacheStats: () => ({ size: existsCache.size() }),\n  };\n};\n"],"mappings":";AAQA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EAEA;AAAA,OACK;AAmBA,IAAM,kBAAkB,CAAC,WAA6C;AAC3E,QAAM,SAAS,OAAO,UAAU,IAAI,SAAS,CAAC,CAAC;AAC/C,QAAM,SAAS,OAAO;AACtB,QAAM,SAAS,OAAO,UAAU;AAChC,QAAM,cAAc,eAAgC,OAAO,aAAa,kBAAkB;AAE1F,QAAM,UAAU,CAAC,WAA2B,cAAc,QAAQ,MAAM;AAExE,QAAM,MAAM,OAAO,QAAkC;AAEnD,UAAM,SAAS,YAAY,IAAI,GAAG;AAClC,QAAI,WAAW,QAAW;AACxB,aAAO;AAAA,IACT;AAGA,QAAI;AACF,YAAM,OAAO;AAAA,QACX,IAAI,kBAAkB;AAAA,UACpB,QAAQ;AAAA,UACR,KAAK,QAAQ,GAAG;AAAA,QAClB,CAAC;AAAA,MACH;AACA,kBAAY,IAAI,KAAK,IAAI;AACzB,aAAO;AAAA,IACT,SAAS,OAAgB;AACvB,YAAM,MAAM;AACZ,UAAI,IAAI,SAAS,cAAc,IAAI,WAAW,mBAAmB,KAAK;AAEpE,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,QAAM,MAAM,OAAO,QAA4C;AAC7D,QAAI;AACF,YAAM,SAAS,MAAM,OAAO;AAAA,QAC1B,IAAI,iBAAiB;AAAA,UACnB,QAAQ;AAAA,UACR,KAAK,QAAQ,GAAG;AAAA,QAClB,CAAC;AAAA,MACH;AAEA,YAAM,QAAQ,MAAM,OAAO,KAAM,qBAAqB;AAGtD,kBAAY,IAAI,KAAK,IAAI;AAEzB,aAAO,IAAI,WAAW,KAAK;AAAA,IAC7B,SAAS,OAAgB;AACvB,YAAM,MAAM;AACZ,UAAI,IAAI,SAAS,eAAe,IAAI,WAAW,mBAAmB,KAAK;AACrE,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,QAAM,MAAM,OAAO,KAAa,UAAqC;AAEnE,QAAI,YAAY,IAAI,GAAG,GAAG;AACxB;AAAA,IACF;AAGA,UAAM,SAAS,MAAM,IAAI,GAAG;AAC5B,QAAI,QAAQ;AACV;AAAA,IACF;AAGA,UAAM,OAAO;AAAA,MACX,IAAI,iBAAiB;AAAA,QACnB,QAAQ;AAAA,QACR,KAAK,QAAQ,GAAG;AAAA,QAChB,MAAM;AAAA,QACN,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AAGA,gBAAY,IAAI,KAAK,IAAI;AAAA,EAC3B;AAEA,SAAO,EAAE,KAAK,KAAK,IAAI;AACzB;AAKO,IAAM,2BAA2B,CAAC,WAA4B;AACnE,QAAM,SAAS,OAAO,UAAU,IAAI,SAAS,CAAC,CAAC;AAC/C,QAAM,UAAU,OAAO;AACvB,QAAM,SAAS,OAAO,UAAU;AAChC,QAAM,cAAc,eAAgC,OAAO,aAAa,kBAAkB;AAE1F,QAAM,WAAW,CAAC,WAA2B,cAAc,QAAQ,MAAM;AAEzE,QAAM,UAAU,gBAAgB,EAAE,GAAG,QAAQ,OAAO,CAAC;AAErD,SAAO;AAAA,IACL,GAAG;AAAA,IACH,YAAY,MAAM,YAAY,MAAM;AAAA,IACpC,eAAe,OAAO,EAAE,MAAM,YAAY,KAAK,EAAE;AAAA,EACnD;AACF;","names":[]}

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@casfa/storage-s3",
+  "version": "0.1.0",
+  "description": "S3 storage provider for CAS",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check .",
+    "lint:fix": "biome check --write .",
+    "prepublishOnly": "bun run build"
+  },
+  "dependencies": {
+    "@casfa/storage-core": "workspace:*",
+    "@aws-sdk/client-s3": "^3.975.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.7.2"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "MIT"
+}