@casfa/storage-s3 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/README.md +0 -182
- package/README.zh-CN.md +0 -182
package/package.json
CHANGED
package/README.md
DELETED
|
@@ -1,182 +0,0 @@
|
|
|
1
|
-
# @casfa/storage-s3
|
|
2
|
-
|
|
3
|
-
S3 storage provider for CAS.
|
|
4
|
-
|
|
5
|
-
## Installation
|
|
6
|
-
|
|
7
|
-
```bash
|
|
8
|
-
bun add @casfa/storage-s3
|
|
9
|
-
```
|
|
10
|
-
|
|
11
|
-
## Overview
|
|
12
|
-
|
|
13
|
-
An S3-backed storage provider for CAS (Content-Addressable Storage). Suitable for cloud deployments with high availability and durability requirements.
|
|
14
|
-
|
|
15
|
-
## Usage
|
|
16
|
-
|
|
17
|
-
### Basic Usage
|
|
18
|
-
|
|
19
|
-
```typescript
|
|
20
|
-
import { createS3Storage } from '@casfa/storage-s3';
|
|
21
|
-
|
|
22
|
-
const storage = createS3Storage({
|
|
23
|
-
bucket: 'my-cas-bucket',
|
|
24
|
-
region: 'us-east-1',
|
|
25
|
-
prefix: 'cas/', // Optional key prefix
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
// Store data
|
|
29
|
-
await storage.put('node:abcd1234...', data);
|
|
30
|
-
|
|
31
|
-
// Retrieve data
|
|
32
|
-
const data = await storage.get('node:abcd1234...');
|
|
33
|
-
|
|
34
|
-
// Check existence
|
|
35
|
-
const exists = await storage.has('node:abcd1234...');
|
|
36
|
-
|
|
37
|
-
// Delete
|
|
38
|
-
const deleted = await storage.delete('node:abcd1234...');
|
|
39
|
-
```
|
|
40
|
-
|
|
41
|
-
### With LRU Cache
|
|
42
|
-
|
|
43
|
-
```typescript
|
|
44
|
-
import { createS3StorageWithCache } from '@casfa/storage-s3';
|
|
45
|
-
|
|
46
|
-
const storage = createS3StorageWithCache({
|
|
47
|
-
bucket: 'my-cas-bucket',
|
|
48
|
-
region: 'us-east-1',
|
|
49
|
-
cacheSize: 1000, // Max items in cache
|
|
50
|
-
cacheMaxAge: 300000, // 5 minutes TTL
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
// Cache reduces S3 API calls for repeated reads
|
|
54
|
-
const data = await storage.get('node:abcd1234...');
|
|
55
|
-
```
|
|
56
|
-
|
|
57
|
-
### With Custom S3 Client
|
|
58
|
-
|
|
59
|
-
```typescript
|
|
60
|
-
import { S3Client } from '@aws-sdk/client-s3';
|
|
61
|
-
import { createS3Storage } from '@casfa/storage-s3';
|
|
62
|
-
|
|
63
|
-
const s3Client = new S3Client({
|
|
64
|
-
region: 'us-east-1',
|
|
65
|
-
credentials: {
|
|
66
|
-
accessKeyId: 'AKIA...',
|
|
67
|
-
secretAccessKey: '...',
|
|
68
|
-
},
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
const storage = createS3Storage({
|
|
72
|
-
bucket: 'my-cas-bucket',
|
|
73
|
-
client: s3Client,
|
|
74
|
-
});
|
|
75
|
-
```
|
|
76
|
-
|
|
77
|
-
## Configuration
|
|
78
|
-
|
|
79
|
-
```typescript
|
|
80
|
-
interface S3StorageConfig {
|
|
81
|
-
// Required: S3 bucket name
|
|
82
|
-
bucket: string;
|
|
83
|
-
|
|
84
|
-
// AWS region (required if not using custom client)
|
|
85
|
-
region?: string;
|
|
86
|
-
|
|
87
|
-
// Optional: Key prefix for all objects
|
|
88
|
-
prefix?: string;
|
|
89
|
-
|
|
90
|
-
// Optional: Custom S3 client
|
|
91
|
-
client?: S3Client;
|
|
92
|
-
|
|
93
|
-
// Optional: Storage class for new objects
|
|
94
|
-
storageClass?: 'STANDARD' | 'STANDARD_IA' | 'GLACIER' | 'DEEP_ARCHIVE';
|
|
95
|
-
}
|
|
96
|
-
```
|
|
97
|
-
|
|
98
|
-
### Cache Options
|
|
99
|
-
|
|
100
|
-
```typescript
|
|
101
|
-
interface S3StorageWithCacheConfig extends S3StorageConfig {
|
|
102
|
-
// Max items in LRU cache (default: 1000)
|
|
103
|
-
cacheSize?: number;
|
|
104
|
-
|
|
105
|
-
// Cache TTL in milliseconds (default: 60000)
|
|
106
|
-
cacheMaxAge?: number;
|
|
107
|
-
}
|
|
108
|
-
```
|
|
109
|
-
|
|
110
|
-
## S3 Key Structure
|
|
111
|
-
|
|
112
|
-
Objects are stored with sharded prefixes for better S3 performance:
|
|
113
|
-
|
|
114
|
-
```
|
|
115
|
-
{prefix}ab/cd/abcd1234...
|
|
116
|
-
```
|
|
117
|
-
|
|
118
|
-
Where `ab` and `cd` are the first 4 characters of the hash, providing good distribution across S3 partitions.
|
|
119
|
-
|
|
120
|
-
## API Reference
|
|
121
|
-
|
|
122
|
-
### Functions
|
|
123
|
-
|
|
124
|
-
- `createS3Storage(config)` - Create S3 storage without cache
|
|
125
|
-
- `createS3StorageWithCache(config)` - Create S3 storage with LRU cache
|
|
126
|
-
|
|
127
|
-
### StorageProvider Interface
|
|
128
|
-
|
|
129
|
-
```typescript
|
|
130
|
-
interface StorageProvider {
|
|
131
|
-
get(key: string): Promise<Uint8Array | null>;
|
|
132
|
-
put(key: string, data: Uint8Array): Promise<void>;
|
|
133
|
-
has(key: string): Promise<boolean>;
|
|
134
|
-
delete(key: string): Promise<boolean>;
|
|
135
|
-
}
|
|
136
|
-
```
|
|
137
|
-
|
|
138
|
-
## AWS Permissions
|
|
139
|
-
|
|
140
|
-
Required IAM permissions for the S3 bucket:
|
|
141
|
-
|
|
142
|
-
```json
|
|
143
|
-
{
|
|
144
|
-
"Version": "2012-10-17",
|
|
145
|
-
"Statement": [
|
|
146
|
-
{
|
|
147
|
-
"Effect": "Allow",
|
|
148
|
-
"Action": [
|
|
149
|
-
"s3:GetObject",
|
|
150
|
-
"s3:PutObject",
|
|
151
|
-
"s3:DeleteObject",
|
|
152
|
-
"s3:HeadObject"
|
|
153
|
-
],
|
|
154
|
-
"Resource": "arn:aws:s3:::my-cas-bucket/*"
|
|
155
|
-
}
|
|
156
|
-
]
|
|
157
|
-
}
|
|
158
|
-
```
|
|
159
|
-
|
|
160
|
-
## Performance Tips
|
|
161
|
-
|
|
162
|
-
1. **Use caching** to reduce S3 API calls and latency
|
|
163
|
-
2. **Use S3 Transfer Acceleration** for global access
|
|
164
|
-
3. **Choose appropriate storage class** based on access patterns
|
|
165
|
-
4. **Enable S3 Intelligent-Tiering** for cost optimization
|
|
166
|
-
5. **Use regional endpoints** to minimize latency
|
|
167
|
-
|
|
168
|
-
## Cost Considerations
|
|
169
|
-
|
|
170
|
-
- CAS data is immutable, so versioning is not needed
|
|
171
|
-
- Consider S3 Intelligent-Tiering for infrequently accessed data
|
|
172
|
-
- Monitor PUT/GET request costs for high-throughput workloads
|
|
173
|
-
|
|
174
|
-
## Related Packages
|
|
175
|
-
|
|
176
|
-
- `@casfa/storage-core` - Core types and utilities
|
|
177
|
-
- `@casfa/storage-fs` - File system storage (for local deployment)
|
|
178
|
-
- `@casfa/storage-memory` - In-memory storage (for testing)
|
|
179
|
-
|
|
180
|
-
## License
|
|
181
|
-
|
|
182
|
-
MIT
|
package/README.zh-CN.md
DELETED
|
@@ -1,182 +0,0 @@
|
|
|
1
|
-
# @casfa/storage-s3
|
|
2
|
-
|
|
3
|
-
基于 S3 的 CAS 存储提供者。
|
|
4
|
-
|
|
5
|
-
## 安装
|
|
6
|
-
|
|
7
|
-
```bash
|
|
8
|
-
bun add @casfa/storage-s3
|
|
9
|
-
```
|
|
10
|
-
|
|
11
|
-
## 概述
|
|
12
|
-
|
|
13
|
-
基于 S3 的 CAS(内容寻址存储)存储提供者。适用于对高可用性和数据持久性有要求的云端部署。
|
|
14
|
-
|
|
15
|
-
## 使用方法
|
|
16
|
-
|
|
17
|
-
### 基本用法
|
|
18
|
-
|
|
19
|
-
```typescript
|
|
20
|
-
import { createS3Storage } from '@casfa/storage-s3';
|
|
21
|
-
|
|
22
|
-
const storage = createS3Storage({
|
|
23
|
-
bucket: 'my-cas-bucket',
|
|
24
|
-
region: 'us-east-1',
|
|
25
|
-
prefix: 'cas/', // 可选的键前缀
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
// 存储数据
|
|
29
|
-
await storage.put('node:abcd1234...', data);
|
|
30
|
-
|
|
31
|
-
// 检索数据
|
|
32
|
-
const data = await storage.get('node:abcd1234...');
|
|
33
|
-
|
|
34
|
-
// 检查是否存在
|
|
35
|
-
const exists = await storage.has('node:abcd1234...');
|
|
36
|
-
|
|
37
|
-
// 删除
|
|
38
|
-
const deleted = await storage.delete('node:abcd1234...');
|
|
39
|
-
```
|
|
40
|
-
|
|
41
|
-
### 带 LRU 缓存
|
|
42
|
-
|
|
43
|
-
```typescript
|
|
44
|
-
import { createS3StorageWithCache } from '@casfa/storage-s3';
|
|
45
|
-
|
|
46
|
-
const storage = createS3StorageWithCache({
|
|
47
|
-
bucket: 'my-cas-bucket',
|
|
48
|
-
region: 'us-east-1',
|
|
49
|
-
cacheSize: 1000, // 缓存最大条目数
|
|
50
|
-
cacheMaxAge: 300000, // 5 分钟 TTL
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
// 缓存减少 S3 API 调用,降低重复读取延迟
|
|
54
|
-
const data = await storage.get('node:abcd1234...');
|
|
55
|
-
```
|
|
56
|
-
|
|
57
|
-
### 使用自定义 S3 客户端
|
|
58
|
-
|
|
59
|
-
```typescript
|
|
60
|
-
import { S3Client } from '@aws-sdk/client-s3';
|
|
61
|
-
import { createS3Storage } from '@casfa/storage-s3';
|
|
62
|
-
|
|
63
|
-
const s3Client = new S3Client({
|
|
64
|
-
region: 'us-east-1',
|
|
65
|
-
credentials: {
|
|
66
|
-
accessKeyId: 'AKIA...',
|
|
67
|
-
secretAccessKey: '...',
|
|
68
|
-
},
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
const storage = createS3Storage({
|
|
72
|
-
bucket: 'my-cas-bucket',
|
|
73
|
-
client: s3Client,
|
|
74
|
-
});
|
|
75
|
-
```
|
|
76
|
-
|
|
77
|
-
## 配置
|
|
78
|
-
|
|
79
|
-
```typescript
|
|
80
|
-
interface S3StorageConfig {
|
|
81
|
-
// 必需:S3 桶名称
|
|
82
|
-
bucket: string;
|
|
83
|
-
|
|
84
|
-
// AWS 区域(不使用自定义客户端时必需)
|
|
85
|
-
region?: string;
|
|
86
|
-
|
|
87
|
-
// 可选:所有对象的键前缀
|
|
88
|
-
prefix?: string;
|
|
89
|
-
|
|
90
|
-
// 可选:自定义 S3 客户端
|
|
91
|
-
client?: S3Client;
|
|
92
|
-
|
|
93
|
-
// 可选:新对象的存储类别
|
|
94
|
-
storageClass?: 'STANDARD' | 'STANDARD_IA' | 'GLACIER' | 'DEEP_ARCHIVE';
|
|
95
|
-
}
|
|
96
|
-
```
|
|
97
|
-
|
|
98
|
-
### 缓存选项
|
|
99
|
-
|
|
100
|
-
```typescript
|
|
101
|
-
interface S3StorageWithCacheConfig extends S3StorageConfig {
|
|
102
|
-
// LRU 缓存最大条目数(默认: 1000)
|
|
103
|
-
cacheSize?: number;
|
|
104
|
-
|
|
105
|
-
// 缓存 TTL,单位毫秒(默认: 60000)
|
|
106
|
-
cacheMaxAge?: number;
|
|
107
|
-
}
|
|
108
|
-
```
|
|
109
|
-
|
|
110
|
-
## S3 键结构
|
|
111
|
-
|
|
112
|
-
对象使用分片前缀存储,以获得更好的 S3 性能:
|
|
113
|
-
|
|
114
|
-
```
|
|
115
|
-
{prefix}ab/cd/abcd1234...
|
|
116
|
-
```
|
|
117
|
-
|
|
118
|
-
其中 `ab` 和 `cd` 是哈希的前 4 个字符,可在 S3 分区间提供良好的分布。
|
|
119
|
-
|
|
120
|
-
## API 参考
|
|
121
|
-
|
|
122
|
-
### 函数
|
|
123
|
-
|
|
124
|
-
- `createS3Storage(config)` - 创建不带缓存的 S3 存储
|
|
125
|
-
- `createS3StorageWithCache(config)` - 创建带 LRU 缓存的 S3 存储
|
|
126
|
-
|
|
127
|
-
### StorageProvider 接口
|
|
128
|
-
|
|
129
|
-
```typescript
|
|
130
|
-
interface StorageProvider {
|
|
131
|
-
get(key: string): Promise<Uint8Array | null>;
|
|
132
|
-
put(key: string, data: Uint8Array): Promise<void>;
|
|
133
|
-
has(key: string): Promise<boolean>;
|
|
134
|
-
delete(key: string): Promise<boolean>;
|
|
135
|
-
}
|
|
136
|
-
```
|
|
137
|
-
|
|
138
|
-
## AWS 权限
|
|
139
|
-
|
|
140
|
-
S3 桶所需的 IAM 权限:
|
|
141
|
-
|
|
142
|
-
```json
|
|
143
|
-
{
|
|
144
|
-
"Version": "2012-10-17",
|
|
145
|
-
"Statement": [
|
|
146
|
-
{
|
|
147
|
-
"Effect": "Allow",
|
|
148
|
-
"Action": [
|
|
149
|
-
"s3:GetObject",
|
|
150
|
-
"s3:PutObject",
|
|
151
|
-
"s3:DeleteObject",
|
|
152
|
-
"s3:HeadObject"
|
|
153
|
-
],
|
|
154
|
-
"Resource": "arn:aws:s3:::my-cas-bucket/*"
|
|
155
|
-
}
|
|
156
|
-
]
|
|
157
|
-
}
|
|
158
|
-
```
|
|
159
|
-
|
|
160
|
-
## 性能建议
|
|
161
|
-
|
|
162
|
-
1. **使用缓存** 以减少 S3 API 调用和延迟
|
|
163
|
-
2. **使用 S3 Transfer Acceleration** 以实现全球访问
|
|
164
|
-
3. **根据访问模式选择合适的存储类别**
|
|
165
|
-
4. **启用 S3 Intelligent-Tiering** 以优化成本
|
|
166
|
-
5. **使用区域端点** 以最小化延迟
|
|
167
|
-
|
|
168
|
-
## 成本考虑
|
|
169
|
-
|
|
170
|
-
- CAS 数据是不可变的,因此不需要版本控制
|
|
171
|
-
- 对于不常访问的数据,考虑使用 S3 Intelligent-Tiering
|
|
172
|
-
- 对于高吞吐量工作负载,注意监控 PUT/GET 请求费用
|
|
173
|
-
|
|
174
|
-
## 相关包
|
|
175
|
-
|
|
176
|
-
- `@casfa/storage-core` - 核心类型与工具
|
|
177
|
-
- `@casfa/storage-fs` - 文件系统存储(用于本地部署)
|
|
178
|
-
- `@casfa/storage-memory` - 内存存储(用于测试)
|
|
179
|
-
|
|
180
|
-
## 许可证
|
|
181
|
-
|
|
182
|
-
MIT
|