fsd-tos 0.15.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Liang Xingchen https://github.com/liangxingchen
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, destribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/index.d.ts ADDED
@@ -0,0 +1,228 @@
1
+ import { Adapter } from 'fsd';
2
+
3
+ /**
4
+ * TOSAdapter 配置选项
5
+ *
6
+ * 火山引擎 TOS 对象存储适配器的初始化配置。
7
+ *
8
+ * @example
9
+ * ```typescript
10
+ * const adapter = new TOSAdapter({
11
+ * accessKeyId: 'your-access-key-id',
12
+ * accessKeySecret: 'your-access-key-secret',
13
+ * region: 'cn-beijing',
14
+ * bucket: 'my-bucket'
15
+ * });
16
+ * ```
17
+ */
18
+ export interface TOSAdapterOptions {
19
+ /**
20
+ * TOS 访问 Key ID(必需)
21
+ *
22
+ * 火山引擎的 Access Key ID。
23
+ */
24
+ accessKeyId: string;
25
+
26
+ /**
27
+ * TOS 访问 Key Secret(必需)
28
+ *
29
+ * 火山引擎的 Access Key Secret。
30
+ *
31
+ * @remarks
32
+ * 请妥善保管此密钥,不要提交到代码仓库。
33
+ * 建议使用环境变量存储。
34
+ */
35
+ accessKeySecret: string;
36
+
37
+ /**
38
+ * TOS 区域(必需)
39
+ *
40
+ * 火山引擎 TOS 的区域,如 'cn-beijing'。
41
+ *
42
+ * @example
43
+ * ```typescript
44
+ * region: 'cn-beijing', // 华北-北京
45
+ * region: 'cn-guangzhou', // 华南-广州
46
+ * region: 'cn-shanghai' // 华东-上海
47
+ * ```
48
+ */
49
+ region: string;
50
+
51
+ /**
52
+ * TOS Bucket 名称(可选)
53
+ *
54
+ * 如果不指定,需要在上传时在 path 中包含 bucket 名称。
55
+ */
56
+ bucket?: string;
57
+
58
+ /**
59
+ * TOS Endpoint(可选)
60
+ *
61
+ * 默认根据 region 自动生成。
62
+ * 如需自定义可手动指定,如 'tos-cn-beijing.volces.com'。
63
+ */
64
+ endpoint?: string;
65
+
66
+ /**
67
+ * TOS 存储根路径(可选)
68
+ *
69
+ * 以 TOS 子目录作为存储根路径。
70
+ * 所有文件操作都会在此路径下进行。
71
+ *
72
+ * @defaultValue '/'
73
+ */
74
+ root?: string;
75
+
76
+ /**
77
+ * URL 前缀(可选)
78
+ *
79
+ * 用于生成访问链接时添加前缀。
80
+ * 通常配合 CDN 或反向代理使用。
81
+ */
82
+ urlPrefix?: string;
83
+
84
+ /**
85
+ * 是否公共读(可选)
86
+ *
87
+ * - `true`: 公共 Bucket,生成直接访问 URL(无需签名)
88
+ * - `false`: 私有 Bucket,生成带签名的临时 URL(默认值)
89
+ */
90
+ publicRead?: boolean;
91
+
92
+ /**
93
+ * 请求超时时间(可选)
94
+ *
95
+ * 单位:毫秒。
96
+ */
97
+ timeout?: number;
98
+
99
+ /**
100
+ * STS 临时 Token(可选)
101
+ *
102
+ * 用于临时凭证访问。
103
+ */
104
+ stsToken?: string;
105
+
106
+ /**
107
+ * 火山引擎账号 ID(可选)
108
+ *
109
+ * 用于 STS 角色扮演,生成边缘上传的临时凭证。
110
+ *
111
+ * @example
112
+ * ```typescript
113
+ * accountId: '2000000001',
114
+ * roleName: 'TOSUploadRole'
115
+ * ```
116
+ *
117
+ * @remarks
118
+ * 当同时提供 `accountId` 和 `roleName` 时,适配器会生成 STS 临时凭证。
119
+ * 临时凭证用于客户端直接上传到 TOS,无需通过服务器中转。
120
+ */
121
+ accountId?: string;
122
+
123
+ /**
124
+ * 火山引擎角色名称(可选)
125
+ *
126
+ * 用于 STS 角色扮演,配合 `accountId` 使用。
127
+ *
128
+ * @example
129
+ * ```typescript
130
+ * accountId: '2000000001',
131
+ * roleName: 'TOSUploadRole'
132
+ * ```
133
+ */
134
+ roleName?: string;
135
+
136
+ /**
137
+ * 上传回调 URL(可选)
138
+ *
139
+ * 文件上传完成后,TOS 会调用此 URL 通知应用。
140
+ *
141
+ * @example
142
+ * ```typescript
143
+ * callbackUrl: 'https://api.example.com/tos/callback'
144
+ * ```
145
+ */
146
+ callbackUrl?: string;
147
+ }
148
+
149
+ /**
150
+ * 火山引擎 TOS 适配器
151
+ *
152
+ * 提供对火山引擎 TOS 对象存储的访问能力。
153
+ *
154
+ * @remarks
155
+ * ### 核心特性
156
+ * - 完整的文件 CRUD 操作
157
+ * - 原生追加上传(AppendObject)
158
+ * - 分段上传(大文件优化)
159
+ * - 原生重命名(RenameObject)
160
+ * - 预签名 URL
161
+ *
162
+ * ### 不支持的操作
163
+ * - `createWriteStream({ start })` - 不支持 start 选项
164
+ *
165
+ * @example
166
+ * ```typescript
167
+ * import TOSAdapter from 'fsd-tos';
168
+ * import FSD from 'fsd';
169
+ *
170
+ * const adapter = new TOSAdapter({
171
+ * accessKeyId: process.env.TOS_ACCESS_KEY_ID,
172
+ * accessKeySecret: process.env.TOS_ACCESS_KEY_SECRET,
173
+ * region: 'cn-beijing',
174
+ * bucket: process.env.TOS_BUCKET
175
+ * });
176
+ *
177
+ * const fsd = FSD({ adapter });
178
+ * await fsd('/uploads/file.jpg').write(buffer);
179
+ * const url = await fsd('/uploads/file.jpg').createUrl({ expires: 3600 });
180
+ * ```
181
+ */
182
+ /**
183
+ * 上传凭证
184
+ *
185
+ * 包含临时访问凭证的返回值。
186
+ */
187
+ export interface UploadToken {
188
+ auth: {
189
+ accessKeyId: string;
190
+ accessKeySecret: string;
191
+ stsToken: string;
192
+ bucket: string;
193
+ endpoint: string;
194
+ };
195
+ path: string;
196
+ expiration: string;
197
+ callback?: any;
198
+ }
199
+
200
+ /**
201
+ * 带自动刷新的上传凭证
202
+ */
203
+ export interface UploadTokenWithAutoRefresh {
204
+ auth: {
205
+ accessKeyId: string;
206
+ accessKeySecret: string;
207
+ stsToken: string;
208
+ bucket: string;
209
+ endpoint: string;
210
+ refreshSTSToken: () => Promise<{
211
+ accessKeyId: string;
212
+ accessKeySecret: string;
213
+ stsToken: string;
214
+ }>;
215
+ };
216
+ path: string;
217
+ expiration: string;
218
+ callback?: any;
219
+ }
220
+
221
+ export default class TOSAdapter extends Adapter<TOSAdapterOptions> {
222
+ createUploadToken: (path: string, meta?: any, durationSeconds?: number) => Promise<UploadToken>;
223
+ createUploadTokenWithAutoRefresh: (
224
+ path: string,
225
+ meta?: any,
226
+ durationSeconds?: number
227
+ ) => Promise<UploadTokenWithAutoRefresh>;
228
+ }
package/lib/index.js ADDED
@@ -0,0 +1,429 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const tslib_1 = require("tslib");
4
+ const path_1 = tslib_1.__importDefault(require("path"));
5
+ const slash_1 = tslib_1.__importDefault(require("slash"));
6
+ const minimatch_1 = require("minimatch");
7
+ const debug_1 = tslib_1.__importDefault(require("debug"));
8
+ const tos_sdk_1 = require("@volcengine/tos-sdk");
9
+ const openapi_1 = require("@volcengine/openapi");
10
+ const stream_1 = require("stream");
11
+ const debug = (0, debug_1.default)('fsd-tos');
12
+ class TOSAdapter {
13
+ constructor(options) {
14
+ this.instanceOfFSDAdapter = true;
15
+ this.name = 'TOSAdapter';
16
+ this.needEnsureDir = false;
17
+ if (!options.accessKeyId)
18
+ throw new Error('option accessKeyId is required for fsd-tos');
19
+ if (!options.accessKeySecret)
20
+ throw new Error('option accessKeySecret is required for fsd-tos');
21
+ if (!options.region)
22
+ throw new Error('option region is required for fsd-tos');
23
+ options = Object.assign({}, options, { root: options.root || '/' });
24
+ if (options.root[0] !== '/') {
25
+ options.root = `/${options.root}`;
26
+ }
27
+ this._options = options;
28
+ this._tos = new tos_sdk_1.TosClient({
29
+ accessKeyId: options.accessKeyId,
30
+ accessKeySecret: options.accessKeySecret,
31
+ region: options.region,
32
+ bucket: options.bucket,
33
+ endpoint: options.endpoint,
34
+ stsToken: options.stsToken,
35
+ secure: true,
36
+ requestTimeout: options.timeout
37
+ });
38
+ if (options.accountId && options.roleName) {
39
+ this._sts = new openapi_1.sts.StsService();
40
+ this._sts.setAccessKeyId(options.accessKeyId);
41
+ this._sts.setSecretKey(options.accessKeySecret);
42
+ }
43
+ this.createUploadToken = async (path, meta, durationSeconds) => {
44
+ if (!options.accountId || !options.roleName)
45
+ throw new Error('Can not create sts token, missing options: accountId and roleName!');
46
+ path = (0, slash_1.default)(path_1.default.join(options.root, path)).substring(1);
47
+ let params = {
48
+ RoleTrn: `trn:iam::${options.accountId}:role/${options.roleName}`,
49
+ RoleSessionName: 'fsd',
50
+ Policy: JSON.stringify({
51
+ Version: '1',
52
+ Statement: [
53
+ {
54
+ Effect: 'Allow',
55
+ Action: ['tos:PutObject'],
56
+ Resource: [`trn:tos:*:*:${options.bucket}/${path}`]
57
+ }
58
+ ]
59
+ }),
60
+ DurationSeconds: durationSeconds || 3600
61
+ };
62
+ let result = await this._sts.AssumeRole(params);
63
+ if (result.ResponseMetadata?.Error) {
64
+ throw new Error(result.ResponseMetadata.Error.Message);
65
+ }
66
+ let creds = result.Result.Credentials;
67
+ let token = {
68
+ auth: {
69
+ accessKeyId: creds.AccessKeyId,
70
+ accessKeySecret: creds.SecretAccessKey,
71
+ stsToken: creds.SessionToken,
72
+ bucket: options.bucket,
73
+ endpoint: `tos-${options.region}.volces.com`
74
+ },
75
+ path,
76
+ expiration: creds.ExpiredTime
77
+ };
78
+ if (options.callbackUrl) {
79
+ token.callback = {
80
+ url: options.callbackUrl,
81
+ body: Object.keys(meta || {})
82
+ .map((key) => `${key}=\${x:${key}}`)
83
+ .join('&'),
84
+ contentType: 'application/x-www-form-urlencoded',
85
+ customValue: meta
86
+ };
87
+ }
88
+ return token;
89
+ };
90
+ this.createUploadTokenWithAutoRefresh = async (path, meta, durationSeconds) => {
91
+ let token = await this.createUploadToken(path, meta, durationSeconds);
92
+ let auth = token.auth;
93
+ auth = Object.assign({}, auth, {
94
+ refreshSTSToken: async () => {
95
+ let t = await this.createUploadToken(path, meta, durationSeconds);
96
+ return t.auth;
97
+ }
98
+ });
99
+ return Object.assign({}, token, { auth });
100
+ };
101
+ }
102
+ async append(path, data) {
103
+ debug('append %s', path);
104
+ const { root } = this._options;
105
+ let p = (0, slash_1.default)(path_1.default.join(root, path)).substring(1);
106
+ if (typeof data === 'string') {
107
+ data = Buffer.from(data);
108
+ }
109
+ if (Buffer.isBuffer(data)) {
110
+ let position = 0;
111
+ try {
112
+ position = await this.size(path);
113
+ }
114
+ catch (_e) { }
115
+ await this._tos.appendObject({ key: p, body: data, offset: position });
116
+ }
117
+ else {
118
+ let chunks = [];
119
+ await new Promise((resolve, reject) => {
120
+ data.on('data', (chunk) => chunks.push(chunk));
121
+ data.on('end', resolve);
122
+ data.on('error', reject);
123
+ });
124
+ let buf = Buffer.concat(chunks);
125
+ let position = 0;
126
+ try {
127
+ position = await this.size(path);
128
+ }
129
+ catch (_e) { }
130
+ await this._tos.appendObject({ key: p, body: buf, offset: position });
131
+ }
132
+ }
133
+ async createReadStream(path, options) {
134
+ debug('createReadStream %s options: %o', path, options);
135
+ const { root } = this._options;
136
+ let p = (0, slash_1.default)(path_1.default.join(root, path)).substring(1);
137
+ let input = { key: p };
138
+ if (options) {
139
+ if (typeof options.start === 'number') {
140
+ input.rangeStart = options.start;
141
+ }
142
+ if (typeof options.end === 'number') {
143
+ input.rangeEnd = options.end;
144
+ }
145
+ }
146
+ let result = await this._tos.getObjectV2(input);
147
+ return result.data.content;
148
+ }
149
+ async createWriteStream(path, options) {
150
+ debug('createWriteStream %s', path);
151
+ if (options?.start)
152
+ throw new Error('fsd-tos write stream does not support start options');
153
+ const { root } = this._options;
154
+ let p = (0, slash_1.default)(path_1.default.join(root, path)).substring(1);
155
+ let stream = new stream_1.PassThrough();
156
+ stream.promise = this._tos.putObject({ key: p, body: stream });
157
+ return stream;
158
+ }
159
+ async unlink(path) {
160
+ debug('unlink %s', path);
161
+ const { root } = this._options;
162
+ let p = (0, slash_1.default)(path_1.default.join(root, path)).substring(1);
163
+ if (path.endsWith('/')) {
164
+ let continuationToken = '';
165
+ do {
166
+ let result = await this._tos.listObjectsType2({
167
+ prefix: p,
168
+ continuationToken,
169
+ maxKeys: 1000
170
+ });
171
+ debug('unlink list: %O', result.data);
172
+ let list = result.data;
173
+ continuationToken = list.NextContinuationToken || '';
174
+ if (list.Contents?.length) {
175
+ let objects = list.Contents.map((o) => ({ key: o.Key }));
176
+ await this._tos.deleteMultiObjects({ objects, quiet: true });
177
+ }
178
+ } while (continuationToken);
179
+ }
180
+ else {
181
+ await this._tos.deleteObject({ key: p });
182
+ }
183
+ }
184
+ async mkdir(path, recursive) {
185
+ debug('mkdir %s', path);
186
+ let parent = path_1.default.dirname(path);
187
+ if (recursive && parent !== '/') {
188
+ parent += '/';
189
+ if (!(await this.exists(parent))) {
190
+ debug('mkdir prefix: %s', parent);
191
+ this.mkdir(parent, true);
192
+ }
193
+ }
194
+ const { root } = this._options;
195
+ let p = (0, slash_1.default)(path_1.default.join(root, path)).substring(1);
196
+ await this._tos.putObject({ key: p, body: Buffer.from('') });
197
+ }
198
+ async readdir(path, recursion) {
199
+ debug('readdir %s', path);
200
+ let delimiter = recursion ? '' : '/';
201
+ let pattern = '';
202
+ if (recursion === true) {
203
+ pattern = '**/*';
204
+ }
205
+ else if (recursion) {
206
+ pattern = recursion;
207
+ }
208
+ const { root } = this._options;
209
+ let p = (0, slash_1.default)(path_1.default.join(root, path)).substring(1);
210
+ let results = Object.create(null);
211
+ let continuationToken = '';
212
+ let hasContents = false;
213
+ let hasCommonPrefixes = false;
214
+ do {
215
+ let result = await this._tos.listObjectsType2({
216
+ prefix: p,
217
+ delimiter,
218
+ continuationToken,
219
+ maxKeys: 1000
220
+ });
221
+ let list = result.data;
222
+ debug('list: %O', list);
223
+ continuationToken = list.NextContinuationToken || '';
224
+ if (list.Contents) {
225
+ hasContents = true;
226
+ list.Contents.forEach((object) => {
227
+ let relative = (0, slash_1.default)(path_1.default.relative(p, object.Key));
228
+ if (!relative)
229
+ return;
230
+ if (object.Key.endsWith('/'))
231
+ relative += '/';
232
+ if (pattern && pattern !== '**/*' && !(0, minimatch_1.minimatch)(relative, pattern))
233
+ return;
234
+ results[relative] = {
235
+ name: relative,
236
+ metadata: {
237
+ size: object.Size,
238
+ lastModified: new Date(object.LastModified)
239
+ }
240
+ };
241
+ });
242
+ }
243
+ if (list.CommonPrefixes) {
244
+ hasCommonPrefixes = true;
245
+ list.CommonPrefixes.forEach((item) => {
246
+ let relative = (0, slash_1.default)(path_1.default.relative(p, item.Prefix));
247
+ if (!relative)
248
+ return;
249
+ relative += '/';
250
+ results[relative] = {
251
+ name: relative
252
+ };
253
+ });
254
+ }
255
+ } while (continuationToken);
256
+ if (hasContents && hasCommonPrefixes) {
257
+ return Object.keys(results)
258
+ .sort()
259
+ .map((key) => results[key]);
260
+ }
261
+ return Object.values(results);
262
+ }
263
+ async createUrl(path, options) {
264
+ debug('createUrl %s', path);
265
+ options = Object.assign({}, options);
266
+ const { root, urlPrefix, publicRead } = this._options;
267
+ let p = (0, slash_1.default)(path_1.default.join(root, path));
268
+ if (urlPrefix && publicRead) {
269
+ return urlPrefix + p;
270
+ }
271
+ let url = this._tos.getPreSignedUrl({
272
+ method: 'GET',
273
+ key: p.substring(1),
274
+ expires: options?.expires || 3600,
275
+ response: options?.response
276
+ });
277
+ if (urlPrefix) {
278
+ url = url.replace(/https?:\/\/[^/]+/, urlPrefix);
279
+ }
280
+ return url;
281
+ }
282
+ async copy(path, dest) {
283
+ debug('copy %s to %s', path, dest);
284
+ if (!(await this.exists(path)))
285
+ throw new Error('The source path is not exists!');
286
+ const { root, bucket } = this._options;
287
+ let from = (0, slash_1.default)(path_1.default.join(root, path)).substring(1);
288
+ let to = (0, slash_1.default)(path_1.default.join(root, dest)).substring(1);
289
+ if (path.endsWith('/')) {
290
+ debug('copy directory %s -> %s', from, to);
291
+ let continuationToken = '';
292
+ do {
293
+ let result = await this._tos.listObjectsType2({
294
+ prefix: from,
295
+ continuationToken,
296
+ maxKeys: 1000
297
+ });
298
+ let list = result.data;
299
+ debug('list result: %O', list);
300
+ continuationToken = list.NextContinuationToken || '';
301
+ if (list.Contents?.length) {
302
+ for (let object of list.Contents) {
303
+ debug(' -> copy %s', object.Key);
304
+ let relative = (0, slash_1.default)(path_1.default.relative(from, object.Key));
305
+ let target = (0, slash_1.default)(path_1.default.join(to, relative));
306
+ await this._tos.copyObject({
307
+ key: target,
308
+ srcBucket: bucket,
309
+ srcKey: object.Key
310
+ });
311
+ }
312
+ }
313
+ } while (continuationToken);
314
+ }
315
+ else {
316
+ debug('copy file %s -> %s', from, to);
317
+ await this._tos.copyObject({
318
+ key: to,
319
+ srcBucket: bucket,
320
+ srcKey: from
321
+ });
322
+ }
323
+ }
324
+ async rename(path, dest) {
325
+ debug('rename %s to %s', path, dest);
326
+ if (!(await this.exists(path)))
327
+ throw new Error('Source path not found');
328
+ if (await this.exists(dest))
329
+ throw new Error('Target path already exists');
330
+ await this.copy(path, dest);
331
+ await this.unlink(path);
332
+ }
333
+ async exists(path) {
334
+ debug('check exists %s', path);
335
+ const { root } = this._options;
336
+ let p = (0, slash_1.default)(path_1.default.join(root, path)).substring(1);
337
+ if (path.endsWith('/')) {
338
+ let result = await this._tos.listObjectsType2({
339
+ prefix: p,
340
+ maxKeys: 1
341
+ });
342
+ return result.data.Contents !== null && result.data.Contents.length > 0;
343
+ }
344
+ try {
345
+ await this._tos.headObject({ key: p });
346
+ return true;
347
+ }
348
+ catch (_e) {
349
+ return false;
350
+ }
351
+ }
352
+ async isFile(path) {
353
+ debug('check is file %s', path);
354
+ const { root } = this._options;
355
+ let p = (0, slash_1.default)(path_1.default.join(root, path)).substring(1);
356
+ try {
357
+ await this._tos.headObject({ key: p });
358
+ return true;
359
+ }
360
+ catch (_e) {
361
+ return false;
362
+ }
363
+ }
364
+ async isDirectory(path) {
365
+ debug('check is directory %s', path);
366
+ let p = (0, slash_1.default)(path_1.default.join(this._options.root, path)).substring(1);
367
+ try {
368
+ await this._tos.headObject({ key: p });
369
+ return true;
370
+ }
371
+ catch (_e) {
372
+ return false;
373
+ }
374
+ }
375
+ async size(path) {
376
+ debug('get file size %s', path);
377
+ let p = (0, slash_1.default)(path_1.default.join(this._options.root, path)).substring(1);
378
+ let result = await this._tos.headObject({ key: p });
379
+ return parseInt(result.data['content-length']) || 0;
380
+ }
381
+ async lastModified(path) {
382
+ debug('get file lastModified %s', path);
383
+ let p = (0, slash_1.default)(path_1.default.join(this._options.root, path)).substring(1);
384
+ let result = await this._tos.headObject({ key: p });
385
+ return new Date(result.data['last-modified']);
386
+ }
387
+ async initMultipartUpload(path, partCount) {
388
+ debug('initMultipartUpload %s, partCount: %d', path, partCount);
389
+ let p = (0, slash_1.default)(path_1.default.join(this._options.root, path)).substring(1);
390
+ let result = await this._tos.createMultipartUpload({ key: p });
391
+ let uploadId = result.data.UploadId;
392
+ let files = [];
393
+ for (let i = 1; i <= partCount; i += 1) {
394
+ files.push(`task://${uploadId}?${i}`);
395
+ }
396
+ return files;
397
+ }
398
+ async writePart(path, partTask, data, _size) {
399
+ debug('writePart %s, task: %s', path, partTask);
400
+ let p = (0, slash_1.default)(path_1.default.join(this._options.root, path)).substring(1);
401
+ if (!partTask.startsWith('task://'))
402
+ throw new Error('Invalid part task id');
403
+ let [uploadId, no] = partTask.replace('task://', '').split('?');
404
+ let result = await this._tos.uploadPart({
405
+ key: p,
406
+ uploadId,
407
+ partNumber: parseInt(no),
408
+ body: data
409
+ });
410
+ let etag = result.data.ETag;
411
+ return `${partTask.replace('task://', 'part://')}#${etag}`;
412
+ }
413
+ async completeMultipartUpload(path, parts) {
414
+ debug('completeMultipartUpload %s', path);
415
+ let uploadId = parts[0].replace('part://', '').split('?')[0];
416
+ let p = (0, slash_1.default)(path_1.default.join(this._options.root, path)).substring(1);
417
+ debug('upload id: %s, target: %s', uploadId, p);
418
+ let mappedParts = parts.map((item, key) => ({
419
+ eTag: item.split('#')[1],
420
+ partNumber: key + 1
421
+ }));
422
+ await this._tos.completeMultipartUpload({
423
+ key: p,
424
+ uploadId,
425
+ parts: mappedParts
426
+ });
427
+ }
428
+ }
429
+ exports.default = TOSAdapter;
package/package.json ADDED
@@ -0,0 +1,26 @@
1
+ {
2
+ "name": "fsd-tos",
3
+ "version": "0.15.2",
4
+ "description": "Volcengine TOS adapter for fsd",
5
+ "main": "lib/index.js",
6
+ "types": "index.d.ts",
7
+ "scripts": {
8
+ "build": "tsc",
9
+ "prepublish": "npm run build"
10
+ },
11
+ "repository": "https://github.com/liangxingchen/fsd/tree/master/packages/fsd-tos",
12
+ "author": {
13
+ "name": "Liang",
14
+ "email": "liang@miaomo.cn",
15
+ "url": "https://github.com/liangxingchen"
16
+ },
17
+ "license": "MIT",
18
+ "dependencies": {
19
+ "@volcengine/openapi": "^1.36.0",
20
+ "@volcengine/tos-sdk": "^2.9.1",
21
+ "debug": "^4.4.3",
22
+ "minimatch": "^9.0.5",
23
+ "slash": "^3.0.0"
24
+ },
25
+ "gitHead": "f65390dbe513ce81cd033ece4607aec024a88e48"
26
+ }
package/src/index.ts ADDED
@@ -0,0 +1,484 @@
1
+ import Path from 'path';
2
+ import slash from 'slash';
3
+ import { minimatch } from 'minimatch';
4
+ import Debugger from 'debug';
5
+ import { TosClient } from '@volcengine/tos-sdk';
6
+ import { sts as volcSTS } from '@volcengine/openapi';
7
+ import { PassThrough } from 'stream';
8
+ import type {
9
+ ReadStreamOptions,
10
+ WriteStreamOptions,
11
+ Task,
12
+ Part,
13
+ FileMetadata,
14
+ CreateUrlOptions,
15
+ WithPromise
16
+ } from 'fsd';
17
+ import type { TOSAdapterOptions, UploadToken, UploadTokenWithAutoRefresh } from '..';
18
+
19
+ const debug = Debugger('fsd-tos');
20
+
21
+ export default class TOSAdapter {
22
+ instanceOfFSDAdapter: true;
23
+ name: string;
24
+ needEnsureDir: boolean;
25
+ _options: TOSAdapterOptions;
26
+ _tos: TosClient;
27
+ _sts: volcSTS.StsService;
28
+ createUploadToken: (path: string, meta?: any, durationSeconds?: number) => Promise<UploadToken>;
29
+ createUploadTokenWithAutoRefresh: (
30
+ path: string,
31
+ meta?: any,
32
+ durationSeconds?: number
33
+ ) => Promise<UploadTokenWithAutoRefresh>;
34
+
35
+ constructor(options: TOSAdapterOptions) {
36
+ this.instanceOfFSDAdapter = true;
37
+ this.name = 'TOSAdapter';
38
+ this.needEnsureDir = false;
39
+ /* istanbul ignore if */
40
+ if (!options.accessKeyId) throw new Error('option accessKeyId is required for fsd-tos');
41
+ /* istanbul ignore if */
42
+ if (!options.accessKeySecret) throw new Error('option accessKeySecret is required for fsd-tos');
43
+ /* istanbul ignore if */
44
+ if (!options.region) throw new Error('option region is required for fsd-tos');
45
+ options = Object.assign({}, options, { root: options.root || '/' });
46
+ if (options.root[0] !== '/') {
47
+ options.root = `/${options.root}`;
48
+ }
49
+ this._options = options;
50
+ this._tos = new TosClient({
51
+ accessKeyId: options.accessKeyId,
52
+ accessKeySecret: options.accessKeySecret,
53
+ region: options.region,
54
+ bucket: options.bucket,
55
+ endpoint: options.endpoint,
56
+ stsToken: options.stsToken,
57
+ secure: true,
58
+ requestTimeout: options.timeout
59
+ });
60
+
61
+ if (options.accountId && options.roleName) {
62
+ this._sts = new volcSTS.StsService();
63
+ this._sts.setAccessKeyId(options.accessKeyId);
64
+ this._sts.setSecretKey(options.accessKeySecret);
65
+ }
66
+
67
+ this.createUploadToken = async (path: string, meta?: any, durationSeconds?: number) => {
68
+ if (!options.accountId || !options.roleName)
69
+ throw new Error('Can not create sts token, missing options: accountId and roleName!');
70
+
71
+ path = slash(Path.join(options.root, path)).substring(1);
72
+ let params = {
73
+ RoleTrn: `trn:iam::${options.accountId}:role/${options.roleName}`,
74
+ RoleSessionName: 'fsd',
75
+ Policy: JSON.stringify({
76
+ Version: '1',
77
+ Statement: [
78
+ {
79
+ Effect: 'Allow',
80
+ Action: ['tos:PutObject'],
81
+ Resource: [`trn:tos:*:*:${options.bucket}/${path}`]
82
+ }
83
+ ]
84
+ }),
85
+ DurationSeconds: durationSeconds || 3600
86
+ };
87
+ let result = await this._sts.AssumeRole(params);
88
+ if (result.ResponseMetadata?.Error) {
89
+ throw new Error(result.ResponseMetadata.Error.Message);
90
+ }
91
+ let creds = result.Result!.Credentials;
92
+
93
+ let token: UploadToken = {
94
+ auth: {
95
+ accessKeyId: creds.AccessKeyId,
96
+ accessKeySecret: creds.SecretAccessKey,
97
+ stsToken: creds.SessionToken,
98
+ bucket: options.bucket,
99
+ endpoint: `tos-${options.region}.volces.com`
100
+ },
101
+ path,
102
+ expiration: creds.ExpiredTime
103
+ };
104
+
105
+ if (options.callbackUrl) {
106
+ token.callback = {
107
+ url: options.callbackUrl,
108
+ body: Object.keys(meta || {})
109
+ .map((key) => `${key}=\${x:${key}}`)
110
+ .join('&'),
111
+ contentType: 'application/x-www-form-urlencoded',
112
+ customValue: meta
113
+ };
114
+ }
115
+
116
+ return token;
117
+ };
118
+
119
+ this.createUploadTokenWithAutoRefresh = async (
120
+ path: string,
121
+ meta?: any,
122
+ durationSeconds?: number
123
+ ) => {
124
+ let token = await this.createUploadToken(path, meta, durationSeconds);
125
+ let auth = token.auth;
126
+ auth = Object.assign({}, auth, {
127
+ refreshSTSToken: async () => {
128
+ let t = await this.createUploadToken(path, meta, durationSeconds);
129
+ return t.auth;
130
+ }
131
+ });
132
+ return Object.assign({}, token, { auth }) as UploadTokenWithAutoRefresh;
133
+ };
134
+ }
135
+
136
+ async append(path: string, data: string | Buffer | NodeJS.ReadableStream): Promise<void> {
137
+ debug('append %s', path);
138
+ const { root } = this._options;
139
+ let p = slash(Path.join(root, path)).substring(1);
140
+ if (typeof data === 'string') {
141
+ data = Buffer.from(data);
142
+ }
143
+ if (Buffer.isBuffer(data)) {
144
+ let position = 0;
145
+ try {
146
+ position = await this.size(path);
147
+ } catch (_e) {}
148
+ await this._tos.appendObject({ key: p, body: data, offset: position });
149
+ } else {
150
+ let chunks: Buffer[] = [];
151
+ await new Promise<void>((resolve, reject) => {
152
+ data.on('data', (chunk: Buffer) => chunks.push(chunk));
153
+ data.on('end', resolve);
154
+ data.on('error', reject);
155
+ });
156
+ let buf = Buffer.concat(chunks);
157
+ let position = 0;
158
+ try {
159
+ position = await this.size(path);
160
+ } catch (_e) {}
161
+ await this._tos.appendObject({ key: p, body: buf, offset: position });
162
+ }
163
+ }
164
+
165
+ async createReadStream(
166
+ path: string,
167
+ options?: ReadStreamOptions
168
+ ): Promise<NodeJS.ReadableStream> {
169
+ debug('createReadStream %s options: %o', path, options);
170
+ const { root } = this._options;
171
+ let p = slash(Path.join(root, path)).substring(1);
172
+ let input: any = { key: p };
173
+ if (options) {
174
+ if (typeof options.start === 'number') {
175
+ input.rangeStart = options.start;
176
+ }
177
+ if (typeof options.end === 'number') {
178
+ input.rangeEnd = options.end;
179
+ }
180
+ }
181
+ // Default dataType='stream' returns GetObjectV2OutputStream with content: NodeJS.ReadableStream
182
+ let result: any = await this._tos.getObjectV2(input);
183
+ return result.data.content;
184
+ }
185
+
186
+ async createWriteStream(
187
+ path: string,
188
+ options?: WriteStreamOptions
189
+ ): Promise<NodeJS.WritableStream & WithPromise> {
190
+ debug('createWriteStream %s', path);
191
+ if (options?.start) throw new Error('fsd-tos write stream does not support start options');
192
+ const { root } = this._options;
193
+ let p = slash(Path.join(root, path)).substring(1);
194
+ let stream: NodeJS.WritableStream & WithPromise = new PassThrough();
195
+ stream.promise = this._tos.putObject({ key: p, body: stream as any });
196
+ return stream;
197
+ }
198
+
199
+ async unlink(path: string): Promise<void> {
200
+ debug('unlink %s', path);
201
+ const { root } = this._options;
202
+ let p = slash(Path.join(root, path)).substring(1);
203
+ if (path.endsWith('/')) {
204
+ let continuationToken = '';
205
+ do {
206
+ let result = await this._tos.listObjectsType2({
207
+ prefix: p,
208
+ continuationToken,
209
+ maxKeys: 1000
210
+ });
211
+ debug('unlink list: %O', result.data);
212
+ let list = result.data;
213
+ continuationToken = list.NextContinuationToken || '';
214
+ if (list.Contents?.length) {
215
+ let objects = list.Contents.map((o) => ({ key: o.Key }));
216
+ await this._tos.deleteMultiObjects({ objects, quiet: true });
217
+ }
218
+ } while (continuationToken);
219
+ } else {
220
+ await this._tos.deleteObject({ key: p });
221
+ }
222
+ }
223
+
224
+ async mkdir(path: string, recursive?: boolean): Promise<void> {
225
+ debug('mkdir %s', path);
226
+ let parent = Path.dirname(path);
227
+ if (recursive && parent !== '/') {
228
+ parent += '/';
229
+ if (!(await this.exists(parent))) {
230
+ debug('mkdir prefix: %s', parent);
231
+ this.mkdir(parent, true);
232
+ }
233
+ }
234
+ const { root } = this._options;
235
+ let p = slash(Path.join(root, path)).substring(1);
236
+ await this._tos.putObject({ key: p, body: Buffer.from('') });
237
+ }
238
+
239
+ async readdir(
240
+ path: string,
241
+ recursion?: true | string
242
+ ): Promise<Array<{ name: string; metadata?: FileMetadata }>> {
243
+ debug('readdir %s', path);
244
+ let delimiter = recursion ? '' : '/';
245
+ let pattern = '';
246
+ if (recursion === true) {
247
+ pattern = '**/*';
248
+ } else if (recursion) {
249
+ pattern = recursion;
250
+ }
251
+
252
+ const { root } = this._options;
253
+ let p = slash(Path.join(root, path)).substring(1);
254
+
255
+ let results: Record<string, { name: string; metadata?: FileMetadata }> = Object.create(null);
256
+ let continuationToken = '';
257
+ let hasContents = false;
258
+ let hasCommonPrefixes = false;
259
+ do {
260
+ let result = await this._tos.listObjectsType2({
261
+ prefix: p,
262
+ delimiter,
263
+ continuationToken,
264
+ maxKeys: 1000
265
+ });
266
+ let list = result.data;
267
+ debug('list: %O', list);
268
+ continuationToken = list.NextContinuationToken || '';
269
+ if (list.Contents) {
270
+ hasContents = true;
271
+ list.Contents.forEach((object) => {
272
+ let relative = slash(Path.relative(p, object.Key));
273
+ if (!relative) return;
274
+ if (object.Key.endsWith('/')) relative += '/';
275
+ if (pattern && pattern !== '**/*' && !minimatch(relative, pattern)) return;
276
+ results[relative] = {
277
+ name: relative,
278
+ metadata: {
279
+ size: object.Size,
280
+ lastModified: new Date(object.LastModified)
281
+ }
282
+ };
283
+ });
284
+ }
285
+ if (list.CommonPrefixes) {
286
+ hasCommonPrefixes = true;
287
+ list.CommonPrefixes.forEach((item) => {
288
+ let relative = slash(Path.relative(p, item.Prefix));
289
+ if (!relative) return;
290
+ relative += '/';
291
+ results[relative] = {
292
+ name: relative
293
+ };
294
+ });
295
+ }
296
+ } while (continuationToken);
297
+ if (hasContents && hasCommonPrefixes) {
298
+ return Object.keys(results)
299
+ .sort()
300
+ .map((key) => results[key]);
301
+ }
302
+ return Object.values(results);
303
+ }
304
+
305
+ async createUrl(path: string, options?: CreateUrlOptions): Promise<string> {
306
+ debug('createUrl %s', path);
307
+ options = Object.assign({}, options);
308
+ const { root, urlPrefix, publicRead } = this._options;
309
+ let p = slash(Path.join(root, path));
310
+ if (urlPrefix && publicRead) {
311
+ return urlPrefix + p;
312
+ }
313
+ let url = this._tos.getPreSignedUrl({
314
+ method: 'GET',
315
+ key: p.substring(1),
316
+ expires: options?.expires || 3600,
317
+ response: options?.response as any
318
+ });
319
+ if (urlPrefix) {
320
+ url = url.replace(/https?:\/\/[^/]+/, urlPrefix);
321
+ }
322
+ return url;
323
+ }
324
+
325
+ async copy(path: string, dest: string): Promise<void> {
326
+ debug('copy %s to %s', path, dest);
327
+ /* istanbul ignore if */
328
+ if (!(await this.exists(path))) throw new Error('The source path is not exists!');
329
+
330
+ const { root, bucket } = this._options;
331
+ let from = slash(Path.join(root, path)).substring(1);
332
+ let to = slash(Path.join(root, dest)).substring(1);
333
+
334
+ if (path.endsWith('/')) {
335
+ debug('copy directory %s -> %s', from, to);
336
+ let continuationToken = '';
337
+ do {
338
+ let result = await this._tos.listObjectsType2({
339
+ prefix: from,
340
+ continuationToken,
341
+ maxKeys: 1000
342
+ });
343
+ let list = result.data;
344
+ debug('list result: %O', list);
345
+ continuationToken = list.NextContinuationToken || '';
346
+ if (list.Contents?.length) {
347
+ for (let object of list.Contents) {
348
+ debug(' -> copy %s', object.Key);
349
+ let relative = slash(Path.relative(from, object.Key));
350
+ let target = slash(Path.join(to, relative));
351
+ await this._tos.copyObject({
352
+ key: target,
353
+ srcBucket: bucket,
354
+ srcKey: object.Key
355
+ });
356
+ }
357
+ }
358
+ } while (continuationToken);
359
+ } else {
360
+ debug('copy file %s -> %s', from, to);
361
+ await this._tos.copyObject({
362
+ key: to,
363
+ srcBucket: bucket,
364
+ srcKey: from
365
+ });
366
+ }
367
+ }
368
+
369
+ async rename(path: string, dest: string): Promise<void> {
370
+ debug('rename %s to %s', path, dest);
371
+ /* istanbul ignore if */
372
+ if (!(await this.exists(path))) throw new Error('Source path not found');
373
+ /* istanbul ignore if */
374
+ if (await this.exists(dest)) throw new Error('Target path already exists');
375
+ await this.copy(path, dest);
376
+ await this.unlink(path);
377
+ }
378
+
379
+ async exists(path: string): Promise<boolean> {
380
+ debug('check exists %s', path);
381
+ const { root } = this._options;
382
+ let p = slash(Path.join(root, path)).substring(1);
383
+ if (path.endsWith('/')) {
384
+ let result = await this._tos.listObjectsType2({
385
+ prefix: p,
386
+ maxKeys: 1
387
+ });
388
+ return result.data.Contents !== null && result.data.Contents.length > 0;
389
+ }
390
+ try {
391
+ await this._tos.headObject({ key: p });
392
+ return true;
393
+ } catch (_e) {
394
+ return false;
395
+ }
396
+ }
397
+
398
+ async isFile(path: string): Promise<boolean> {
399
+ debug('check is file %s', path);
400
+ const { root } = this._options;
401
+ let p = slash(Path.join(root, path)).substring(1);
402
+ try {
403
+ await this._tos.headObject({ key: p });
404
+ return true;
405
+ } catch (_e) {
406
+ return false;
407
+ }
408
+ }
409
+
410
+ async isDirectory(path: string): Promise<boolean> {
411
+ debug('check is directory %s', path);
412
+ let p = slash(Path.join(this._options.root, path)).substring(1);
413
+ try {
414
+ await this._tos.headObject({ key: p });
415
+ return true;
416
+ } catch (_e) {
417
+ return false;
418
+ }
419
+ }
420
+
421
+ async size(path: string): Promise<number> {
422
+ debug('get file size %s', path);
423
+ let p = slash(Path.join(this._options.root, path)).substring(1);
424
+ let result = await this._tos.headObject({ key: p });
425
+ return parseInt(result.data['content-length']) || 0;
426
+ }
427
+
428
+ async lastModified(path: string): Promise<Date> {
429
+ debug('get file lastModified %s', path);
430
+ let p = slash(Path.join(this._options.root, path)).substring(1);
431
+ let result = await this._tos.headObject({ key: p });
432
+ return new Date(result.data['last-modified']);
433
+ }
434
+
435
+ async initMultipartUpload(path: string, partCount: number): Promise<Task[]> {
436
+ debug('initMultipartUpload %s, partCount: %d', path, partCount);
437
+ let p = slash(Path.join(this._options.root, path)).substring(1);
438
+ let result = await this._tos.createMultipartUpload({ key: p });
439
+ let uploadId = result.data.UploadId;
440
+ let files = [];
441
+ for (let i = 1; i <= partCount; i += 1) {
442
+ files.push(`task://${uploadId}?${i}`);
443
+ }
444
+ return files;
445
+ }
446
+
447
+ async writePart(
448
+ path: string,
449
+ partTask: Task,
450
+ data: NodeJS.ReadableStream,
451
+ _size: number
452
+ ): Promise<Part> {
453
+ debug('writePart %s, task: %s', path, partTask);
454
+ let p = slash(Path.join(this._options.root, path)).substring(1);
455
+
456
+ if (!partTask.startsWith('task://')) throw new Error('Invalid part task id');
457
+
458
+ let [uploadId, no] = partTask.replace('task://', '').split('?');
459
+ let result = await this._tos.uploadPart({
460
+ key: p,
461
+ uploadId,
462
+ partNumber: parseInt(no),
463
+ body: data
464
+ });
465
+ let etag = result.data.ETag;
466
+ return `${partTask.replace('task://', 'part://')}#${etag}`;
467
+ }
468
+
469
+ async completeMultipartUpload(path: string, parts: Part[]): Promise<void> {
470
+ debug('completeMultipartUpload %s', path);
471
+ let uploadId = parts[0].replace('part://', '').split('?')[0];
472
+ let p = slash(Path.join(this._options.root, path)).substring(1);
473
+ debug('upload id: %s, target: %s', uploadId, p);
474
+ let mappedParts = parts.map((item, key) => ({
475
+ eTag: item.split('#')[1],
476
+ partNumber: key + 1
477
+ }));
478
+ await this._tos.completeMultipartUpload({
479
+ key: p,
480
+ uploadId,
481
+ parts: mappedParts
482
+ });
483
+ }
484
+ }
package/tsconfig.json ADDED
@@ -0,0 +1,39 @@
1
+ {
2
+ "compilerOptions": {
3
+ // "strict": true,
4
+ "noImplicitAny": true,
5
+ // "strictNullChecks": true,
6
+ // "strictFunctionTypes": true,
7
+ // "strictBindCallApply": true,
8
+ // "strictPropertyInitialization": true,
9
+ // "noImplicitThis": true,
10
+ "alwaysStrict": true,
11
+ "newLine": "lf",
12
+ "moduleResolution": "node",
13
+ "module": "CommonJS",
14
+ "importHelpers": true,
15
+ "allowSyntheticDefaultImports": true,
16
+ "esModuleInterop": true,
17
+ "removeComments": true,
18
+ "preserveConstEnums": true,
19
+ "outDir": "lib",
20
+ "baseUrl": ".",
21
+ "paths": {
22
+ "*": [
23
+ "../../packages/*",
24
+ "../../typings/*"
25
+ ]
26
+ },
27
+ "useDefineForClassFields": false,
28
+ "target": "ES2023",
29
+ "lib": [
30
+ "ES2023"
31
+ ]
32
+ },
33
+ "include": [
34
+ "src/**/*"
35
+ ],
36
+ "exclude": [
37
+ "node_modules"
38
+ ]
39
+ }