ai-yuca 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.ts +7 -1
- package/dist/bin/cli.js +6 -2
- package/dist/package.json +3 -2
- package/dist/src/cache.d.ts +57 -0
- package/dist/src/cache.js +148 -0
- package/dist/src/uploadWithConfig.d.ts +4 -0
- package/dist/src/uploadWithConfig.js +125 -3
- package/package.json +3 -2
- package/src/cache.ts +128 -0
- package/src/uploadWithConfig.ts +110 -3
- package/vs.config.json +4 -0
- package/out/test.txt +0 -1
package/bin/cli.ts
CHANGED
|
@@ -157,6 +157,8 @@ program
|
|
|
157
157
|
.option('-d, --destination <path>', '自定义目标路径(覆盖配置文件中的目标路径)')
|
|
158
158
|
.option('--no-recursive', '禁用递归上传目录中的文件')
|
|
159
159
|
.option('--no-compression', '禁用GZIP压缩')
|
|
160
|
+
.option('--no-cache', '禁用文件缓存功能')
|
|
161
|
+
.option('--cache-file <path>', '指定缓存文件路径(默认为.cdn.cache.json)')
|
|
160
162
|
.option('--show-config', '仅显示配置信息,不执行上传')
|
|
161
163
|
.action(async (options: any) => {
|
|
162
164
|
try {
|
|
@@ -167,6 +169,8 @@ program
|
|
|
167
169
|
destination: customDestination,
|
|
168
170
|
recursive = true,
|
|
169
171
|
compression = true,
|
|
172
|
+
cache = true,
|
|
173
|
+
cacheFile,
|
|
170
174
|
showConfig = false
|
|
171
175
|
} = options;
|
|
172
176
|
|
|
@@ -192,7 +196,9 @@ program
|
|
|
192
196
|
customSourcePath,
|
|
193
197
|
customDestination,
|
|
194
198
|
recursive,
|
|
195
|
-
enableCompression: compression
|
|
199
|
+
enableCompression: compression,
|
|
200
|
+
enableCache: cache,
|
|
201
|
+
cacheFilePath: cacheFile
|
|
196
202
|
};
|
|
197
203
|
|
|
198
204
|
const result = await uploadFilesWithConfig(uploadOptions);
|
package/dist/bin/cli.js
CHANGED
|
@@ -171,10 +171,12 @@ program
|
|
|
171
171
|
.option('-d, --destination <path>', '自定义目标路径(覆盖配置文件中的目标路径)')
|
|
172
172
|
.option('--no-recursive', '禁用递归上传目录中的文件')
|
|
173
173
|
.option('--no-compression', '禁用GZIP压缩')
|
|
174
|
+
.option('--no-cache', '禁用文件缓存功能')
|
|
175
|
+
.option('--cache-file <path>', '指定缓存文件路径(默认为.cdn.cache.json)')
|
|
174
176
|
.option('--show-config', '仅显示配置信息,不执行上传')
|
|
175
177
|
.action(async (options) => {
|
|
176
178
|
try {
|
|
177
|
-
const { config: configPath, keyFile, source: customSourcePath, destination: customDestination, recursive = true, compression = true, showConfig = false } = options;
|
|
179
|
+
const { config: configPath, keyFile, source: customSourcePath, destination: customDestination, recursive = true, compression = true, cache = true, cacheFile, showConfig = false } = options;
|
|
178
180
|
// 如果只是显示配置信息
|
|
179
181
|
if (showConfig) {
|
|
180
182
|
try {
|
|
@@ -197,7 +199,9 @@ program
|
|
|
197
199
|
customSourcePath,
|
|
198
200
|
customDestination,
|
|
199
201
|
recursive,
|
|
200
|
-
enableCompression: compression
|
|
202
|
+
enableCompression: compression,
|
|
203
|
+
enableCache: cache,
|
|
204
|
+
cacheFilePath: cacheFile
|
|
201
205
|
};
|
|
202
206
|
const result = await (0, uploadWithConfig_1.uploadFilesWithConfig)(uploadOptions);
|
|
203
207
|
if (result.success.length > 0) {
|
package/dist/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-yuca",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"description": "一个实用的AI辅助工具",
|
|
5
5
|
"main": "dist/src/index.js",
|
|
6
6
|
"types": "dist/src/index.d.ts",
|
|
@@ -31,7 +31,8 @@
|
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"@google-cloud/storage": "^7.17.1",
|
|
34
|
-
"commander": "^10.0.0"
|
|
34
|
+
"commander": "^10.0.0",
|
|
35
|
+
"md5-file": "^5.0.0"
|
|
35
36
|
},
|
|
36
37
|
"devDependencies": {
|
|
37
38
|
"@types/chai": "^5.2.2",
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 缓存文件接口
|
|
3
|
+
*/
|
|
4
|
+
export interface CacheData {
|
|
5
|
+
[md5Hash: string]: string;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* 缓存管理类
|
|
9
|
+
*/
|
|
10
|
+
export declare class CacheManager {
|
|
11
|
+
private cacheFilePath;
|
|
12
|
+
private cache;
|
|
13
|
+
constructor(cacheFilePath?: string);
|
|
14
|
+
/**
|
|
15
|
+
* 加载缓存文件
|
|
16
|
+
*/
|
|
17
|
+
private loadCache;
|
|
18
|
+
/**
|
|
19
|
+
* 保存缓存到文件
|
|
20
|
+
*/
|
|
21
|
+
private saveCache;
|
|
22
|
+
/**
|
|
23
|
+
* 获取文件的MD5哈希值
|
|
24
|
+
*/
|
|
25
|
+
getFileHash(filePath: string): string;
|
|
26
|
+
/**
|
|
27
|
+
* 检查文件是否已经上传过
|
|
28
|
+
*/
|
|
29
|
+
isFileUploaded(filePath: string): boolean;
|
|
30
|
+
/**
|
|
31
|
+
* 获取已上传文件的URL
|
|
32
|
+
*/
|
|
33
|
+
getUploadedUrl(filePath: string): string | null;
|
|
34
|
+
/**
|
|
35
|
+
* 记录文件上传成功
|
|
36
|
+
*/
|
|
37
|
+
recordUpload(filePath: string, uploadedUrl: string): void;
|
|
38
|
+
/**
|
|
39
|
+
* 清除缓存
|
|
40
|
+
*/
|
|
41
|
+
clearCache(): void;
|
|
42
|
+
/**
|
|
43
|
+
* 获取缓存统计信息
|
|
44
|
+
*/
|
|
45
|
+
getCacheStats(): {
|
|
46
|
+
totalFiles: number;
|
|
47
|
+
cacheFilePath: string;
|
|
48
|
+
};
|
|
49
|
+
/**
|
|
50
|
+
* 删除特定文件的缓存记录
|
|
51
|
+
*/
|
|
52
|
+
removeFromCache(filePath: string): boolean;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* 创建默认的缓存管理器实例
|
|
56
|
+
*/
|
|
57
|
+
export declare function createCacheManager(cacheFilePath?: string): CacheManager;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.CacheManager = void 0;
|
|
37
|
+
exports.createCacheManager = createCacheManager;
|
|
38
|
+
const fs = __importStar(require("fs"));
|
|
39
|
+
const path = __importStar(require("path"));
|
|
40
|
+
const md5File = __importStar(require("md5-file"));
|
|
41
|
+
/**
|
|
42
|
+
* 缓存管理类
|
|
43
|
+
*/
|
|
44
|
+
class CacheManager {
|
|
45
|
+
constructor(cacheFilePath = '.cdn.cache.json') {
|
|
46
|
+
this.cacheFilePath = path.resolve(cacheFilePath);
|
|
47
|
+
this.cache = this.loadCache();
|
|
48
|
+
// 确保缓存文件存在,如果不存在则创建空文件
|
|
49
|
+
if (!fs.existsSync(this.cacheFilePath)) {
|
|
50
|
+
this.saveCache();
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* 加载缓存文件
|
|
55
|
+
*/
|
|
56
|
+
loadCache() {
|
|
57
|
+
try {
|
|
58
|
+
if (fs.existsSync(this.cacheFilePath)) {
|
|
59
|
+
const content = fs.readFileSync(this.cacheFilePath, 'utf-8');
|
|
60
|
+
return JSON.parse(content);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
console.warn(`警告: 无法读取缓存文件 ${this.cacheFilePath}:`, error);
|
|
65
|
+
}
|
|
66
|
+
return {};
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* 保存缓存到文件
|
|
70
|
+
*/
|
|
71
|
+
saveCache() {
|
|
72
|
+
try {
|
|
73
|
+
const content = JSON.stringify(this.cache, null, 2);
|
|
74
|
+
fs.writeFileSync(this.cacheFilePath, content, 'utf-8');
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
console.error(`错误: 无法保存缓存文件 ${this.cacheFilePath}:`, error);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* 获取文件的MD5哈希值
|
|
82
|
+
*/
|
|
83
|
+
getFileHash(filePath) {
|
|
84
|
+
try {
|
|
85
|
+
return md5File.sync(filePath);
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
throw new Error(`无法计算文件 ${filePath} 的MD5哈希值: ${error}`);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* 检查文件是否已经上传过
|
|
93
|
+
*/
|
|
94
|
+
isFileUploaded(filePath) {
|
|
95
|
+
const hash = this.getFileHash(filePath);
|
|
96
|
+
return hash in this.cache;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* 获取已上传文件的URL
|
|
100
|
+
*/
|
|
101
|
+
getUploadedUrl(filePath) {
|
|
102
|
+
const hash = this.getFileHash(filePath);
|
|
103
|
+
return this.cache[hash] || null;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* 记录文件上传成功
|
|
107
|
+
*/
|
|
108
|
+
recordUpload(filePath, uploadedUrl) {
|
|
109
|
+
const hash = this.getFileHash(filePath);
|
|
110
|
+
this.cache[hash] = uploadedUrl;
|
|
111
|
+
this.saveCache();
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* 清除缓存
|
|
115
|
+
*/
|
|
116
|
+
clearCache() {
|
|
117
|
+
this.cache = {};
|
|
118
|
+
this.saveCache();
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* 获取缓存统计信息
|
|
122
|
+
*/
|
|
123
|
+
getCacheStats() {
|
|
124
|
+
return {
|
|
125
|
+
totalFiles: Object.keys(this.cache).length,
|
|
126
|
+
cacheFilePath: this.cacheFilePath
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* 删除特定文件的缓存记录
|
|
131
|
+
*/
|
|
132
|
+
removeFromCache(filePath) {
|
|
133
|
+
const hash = this.getFileHash(filePath);
|
|
134
|
+
if (hash in this.cache) {
|
|
135
|
+
delete this.cache[hash];
|
|
136
|
+
this.saveCache();
|
|
137
|
+
return true;
|
|
138
|
+
}
|
|
139
|
+
return false;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
exports.CacheManager = CacheManager;
|
|
143
|
+
/**
|
|
144
|
+
* 创建默认的缓存管理器实例
|
|
145
|
+
*/
|
|
146
|
+
function createCacheManager(cacheFilePath) {
|
|
147
|
+
return new CacheManager(cacheFilePath);
|
|
148
|
+
}
|
|
@@ -15,6 +15,10 @@ export interface UploadWithConfigOptions {
|
|
|
15
15
|
customSourcePath?: string;
|
|
16
16
|
/** 自定义目标路径,如果提供则覆盖配置文件中的目标路径 */
|
|
17
17
|
customDestination?: string;
|
|
18
|
+
/** 是否启用缓存,默认为true */
|
|
19
|
+
enableCache?: boolean;
|
|
20
|
+
/** 缓存文件路径,默认为.cdn.cache.json */
|
|
21
|
+
cacheFilePath?: string;
|
|
18
22
|
}
|
|
19
23
|
/**
|
|
20
24
|
* 基于配置文件批量上传文件
|
|
@@ -1,4 +1,37 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
36
|
exports.uploadFilesWithConfig = uploadFilesWithConfig;
|
|
4
37
|
exports.getConfigSummary = getConfigSummary;
|
|
@@ -6,8 +39,37 @@ exports.getConfigSummary = getConfigSummary;
|
|
|
6
39
|
* 基于配置文件的GCP上传功能模块
|
|
7
40
|
*/
|
|
8
41
|
const storage_1 = require("@google-cloud/storage");
|
|
42
|
+
const path = __importStar(require("path"));
|
|
9
43
|
const upload_1 = require("./upload");
|
|
10
44
|
const config_1 = require("./config");
|
|
45
|
+
const cache_1 = require("./cache");
|
|
46
|
+
const readdir = require('util').promisify(require('fs').readdir);
|
|
47
|
+
const stat = require('util').promisify(require('fs').stat);
|
|
48
|
+
/**
|
|
49
|
+
* 获取需要处理的文件列表
|
|
50
|
+
*/
|
|
51
|
+
async function getFilesToProcess(sourcePath, recursive) {
|
|
52
|
+
const files = [];
|
|
53
|
+
const stats = await stat(sourcePath);
|
|
54
|
+
if (stats.isFile()) {
|
|
55
|
+
files.push(sourcePath);
|
|
56
|
+
}
|
|
57
|
+
else if (stats.isDirectory()) {
|
|
58
|
+
const dirFiles = await readdir(sourcePath);
|
|
59
|
+
for (const file of dirFiles) {
|
|
60
|
+
const filePath = path.join(sourcePath, file);
|
|
61
|
+
const fileStats = await stat(filePath);
|
|
62
|
+
if (fileStats.isFile()) {
|
|
63
|
+
files.push(filePath);
|
|
64
|
+
}
|
|
65
|
+
else if (recursive && fileStats.isDirectory()) {
|
|
66
|
+
const subFiles = await getFilesToProcess(filePath, recursive);
|
|
67
|
+
files.push(...subFiles);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return files;
|
|
72
|
+
}
|
|
11
73
|
/**
|
|
12
74
|
* 创建存储客户端
|
|
13
75
|
* @param options - 客户端配置选项
|
|
@@ -37,7 +99,7 @@ function createStorageClientFromConfig(options = {}) {
|
|
|
37
99
|
* @returns 上传结果
|
|
38
100
|
*/
|
|
39
101
|
async function uploadFilesWithConfig(options = {}) {
|
|
40
|
-
const { storageClientOptions = {}, configPath, recursive = true, enableCompression = true, customSourcePath, customDestination } = options;
|
|
102
|
+
const { storageClientOptions = {}, configPath, recursive = true, enableCompression = true, customSourcePath, customDestination, enableCache = true, cacheFilePath = '.cdn.cache.json' } = options;
|
|
41
103
|
try {
|
|
42
104
|
// 加载配置文件
|
|
43
105
|
const config = (0, config_1.loadConfig)(configPath);
|
|
@@ -51,16 +113,76 @@ async function uploadFilesWithConfig(options = {}) {
|
|
|
51
113
|
console.log(`源路径: ${sourcePath}`);
|
|
52
114
|
console.log(`目标桶: ${bucketName}`);
|
|
53
115
|
console.log(`目标路径: ${destination}`);
|
|
116
|
+
// 初始化缓存管理器
|
|
117
|
+
let cacheManager = null;
|
|
118
|
+
if (enableCache) {
|
|
119
|
+
cacheManager = (0, cache_1.createCacheManager)(cacheFilePath);
|
|
120
|
+
const cacheStats = cacheManager.getCacheStats();
|
|
121
|
+
console.log(`缓存已启用,当前缓存文件数: ${cacheStats.totalFiles}`);
|
|
122
|
+
}
|
|
123
|
+
// 如果启用了缓存,先检查哪些文件需要上传
|
|
124
|
+
let filesToUpload = [];
|
|
125
|
+
let skippedFiles = [];
|
|
126
|
+
if (enableCache && cacheManager) {
|
|
127
|
+
// 获取所有需要处理的文件
|
|
128
|
+
const allFiles = await getFilesToProcess(sourcePath, recursive);
|
|
129
|
+
for (const filePath of allFiles) {
|
|
130
|
+
if (cacheManager.isFileUploaded(filePath)) {
|
|
131
|
+
const cachedUrl = cacheManager.getUploadedUrl(filePath);
|
|
132
|
+
console.log(`跳过已上传文件: ${filePath} -> ${cachedUrl}`);
|
|
133
|
+
skippedFiles.push(filePath);
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
filesToUpload.push(filePath);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
console.log(`总文件数: ${allFiles.length}, 需要上传: ${filesToUpload.length}, 跳过: ${skippedFiles.length}`);
|
|
140
|
+
// 如果没有文件需要上传,直接返回
|
|
141
|
+
if (filesToUpload.length === 0) {
|
|
142
|
+
console.log('所有文件都已上传,无需重复上传。');
|
|
143
|
+
return {
|
|
144
|
+
success: skippedFiles.map(file => ({
|
|
145
|
+
success: true,
|
|
146
|
+
file,
|
|
147
|
+
url: cacheManager.getUploadedUrl(file),
|
|
148
|
+
size: 0,
|
|
149
|
+
contentType: 'application/octet-stream',
|
|
150
|
+
timeCreated: new Date().toISOString()
|
|
151
|
+
})),
|
|
152
|
+
failed: []
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
}
|
|
54
156
|
// 执行批量上传
|
|
157
|
+
const uploadSource = enableCache && filesToUpload.length > 0 ? filesToUpload : sourcePath;
|
|
55
158
|
const result = await (0, upload_1.uploadFiles)({
|
|
56
159
|
bucketName,
|
|
57
|
-
sourcePath,
|
|
160
|
+
sourcePath: uploadSource,
|
|
58
161
|
destination,
|
|
59
162
|
storageClient,
|
|
60
163
|
recursive,
|
|
61
164
|
enableCompression
|
|
62
165
|
});
|
|
63
|
-
|
|
166
|
+
// 如果启用了缓存,记录上传成功的文件
|
|
167
|
+
if (enableCache && cacheManager) {
|
|
168
|
+
for (const successItem of result.success) {
|
|
169
|
+
cacheManager.recordUpload(successItem.file, successItem.url);
|
|
170
|
+
}
|
|
171
|
+
console.log(`已更新缓存,新增 ${result.success.length} 个文件记录`);
|
|
172
|
+
}
|
|
173
|
+
console.log(`上传完成! 成功: ${result.success.length}, 失败: ${result.failed.length}, 跳过: ${skippedFiles.length}`);
|
|
174
|
+
// 合并跳过的文件到成功结果中
|
|
175
|
+
if (skippedFiles.length > 0 && cacheManager) {
|
|
176
|
+
const skippedResults = skippedFiles.map(file => ({
|
|
177
|
+
success: true,
|
|
178
|
+
file,
|
|
179
|
+
url: cacheManager.getUploadedUrl(file),
|
|
180
|
+
size: 0,
|
|
181
|
+
contentType: 'application/octet-stream',
|
|
182
|
+
timeCreated: new Date().toISOString()
|
|
183
|
+
}));
|
|
184
|
+
result.success.push(...skippedResults);
|
|
185
|
+
}
|
|
64
186
|
return result;
|
|
65
187
|
}
|
|
66
188
|
catch (error) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-yuca",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"description": "一个实用的AI辅助工具",
|
|
5
5
|
"main": "dist/src/index.js",
|
|
6
6
|
"types": "dist/src/index.d.ts",
|
|
@@ -31,7 +31,8 @@
|
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"@google-cloud/storage": "^7.17.1",
|
|
34
|
-
"commander": "^10.0.0"
|
|
34
|
+
"commander": "^10.0.0",
|
|
35
|
+
"md5-file": "^5.0.0"
|
|
35
36
|
},
|
|
36
37
|
"devDependencies": {
|
|
37
38
|
"@types/chai": "^5.2.2",
|
package/src/cache.ts
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import * as fs from 'fs';
|
|
2
|
+
import * as path from 'path';
|
|
3
|
+
import * as md5File from 'md5-file';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* 缓存文件接口
|
|
7
|
+
*/
|
|
8
|
+
export interface CacheData {
|
|
9
|
+
[md5Hash: string]: string; // MD5哈希值 -> 上传后的URL
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* 缓存管理类
|
|
14
|
+
*/
|
|
15
|
+
export class CacheManager {
|
|
16
|
+
private cacheFilePath: string;
|
|
17
|
+
private cache: CacheData;
|
|
18
|
+
|
|
19
|
+
constructor(cacheFilePath: string = '.cdn.cache.json') {
|
|
20
|
+
this.cacheFilePath = path.resolve(cacheFilePath);
|
|
21
|
+
this.cache = this.loadCache();
|
|
22
|
+
// 确保缓存文件存在,如果不存在则创建空文件
|
|
23
|
+
if (!fs.existsSync(this.cacheFilePath)) {
|
|
24
|
+
this.saveCache();
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* 加载缓存文件
|
|
30
|
+
*/
|
|
31
|
+
private loadCache(): CacheData {
|
|
32
|
+
try {
|
|
33
|
+
if (fs.existsSync(this.cacheFilePath)) {
|
|
34
|
+
const content = fs.readFileSync(this.cacheFilePath, 'utf-8');
|
|
35
|
+
return JSON.parse(content);
|
|
36
|
+
}
|
|
37
|
+
} catch (error) {
|
|
38
|
+
console.warn(`警告: 无法读取缓存文件 ${this.cacheFilePath}:`, error);
|
|
39
|
+
}
|
|
40
|
+
return {};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* 保存缓存到文件
|
|
45
|
+
*/
|
|
46
|
+
private saveCache(): void {
|
|
47
|
+
try {
|
|
48
|
+
const content = JSON.stringify(this.cache, null, 2);
|
|
49
|
+
fs.writeFileSync(this.cacheFilePath, content, 'utf-8');
|
|
50
|
+
} catch (error) {
|
|
51
|
+
console.error(`错误: 无法保存缓存文件 ${this.cacheFilePath}:`, error);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* 获取文件的MD5哈希值
|
|
57
|
+
*/
|
|
58
|
+
public getFileHash(filePath: string): string {
|
|
59
|
+
try {
|
|
60
|
+
return md5File.sync(filePath);
|
|
61
|
+
} catch (error) {
|
|
62
|
+
throw new Error(`无法计算文件 ${filePath} 的MD5哈希值: ${error}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* 检查文件是否已经上传过
|
|
68
|
+
*/
|
|
69
|
+
public isFileUploaded(filePath: string): boolean {
|
|
70
|
+
const hash = this.getFileHash(filePath);
|
|
71
|
+
return hash in this.cache;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* 获取已上传文件的URL
|
|
76
|
+
*/
|
|
77
|
+
public getUploadedUrl(filePath: string): string | null {
|
|
78
|
+
const hash = this.getFileHash(filePath);
|
|
79
|
+
return this.cache[hash] || null;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* 记录文件上传成功
|
|
84
|
+
*/
|
|
85
|
+
public recordUpload(filePath: string, uploadedUrl: string): void {
|
|
86
|
+
const hash = this.getFileHash(filePath);
|
|
87
|
+
this.cache[hash] = uploadedUrl;
|
|
88
|
+
this.saveCache();
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* 清除缓存
|
|
93
|
+
*/
|
|
94
|
+
public clearCache(): void {
|
|
95
|
+
this.cache = {};
|
|
96
|
+
this.saveCache();
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* 获取缓存统计信息
|
|
101
|
+
*/
|
|
102
|
+
public getCacheStats(): { totalFiles: number; cacheFilePath: string } {
|
|
103
|
+
return {
|
|
104
|
+
totalFiles: Object.keys(this.cache).length,
|
|
105
|
+
cacheFilePath: this.cacheFilePath
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* 删除特定文件的缓存记录
|
|
111
|
+
*/
|
|
112
|
+
public removeFromCache(filePath: string): boolean {
|
|
113
|
+
const hash = this.getFileHash(filePath);
|
|
114
|
+
if (hash in this.cache) {
|
|
115
|
+
delete this.cache[hash];
|
|
116
|
+
this.saveCache();
|
|
117
|
+
return true;
|
|
118
|
+
}
|
|
119
|
+
return false;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* 创建默认的缓存管理器实例
|
|
125
|
+
*/
|
|
126
|
+
export function createCacheManager(cacheFilePath?: string): CacheManager {
|
|
127
|
+
return new CacheManager(cacheFilePath);
|
|
128
|
+
}
|
package/src/uploadWithConfig.ts
CHANGED
|
@@ -3,9 +3,43 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { Storage } from '@google-cloud/storage';
|
|
5
5
|
import * as path from 'path';
|
|
6
|
+
import * as fs from 'fs';
|
|
6
7
|
import { StorageClientOptions, UploadFilesResult } from './types/upload';
|
|
7
8
|
import { uploadFiles } from './upload';
|
|
8
9
|
import { loadConfig, getBucketName, getUploadDestination, getUploadSourcePath, VSConfig } from './config';
|
|
10
|
+
import { CacheManager, createCacheManager } from './cache';
|
|
11
|
+
|
|
12
|
+
const readdir = require('util').promisify(require('fs').readdir);
|
|
13
|
+
const stat = require('util').promisify(require('fs').stat);
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* 获取需要处理的文件列表
|
|
17
|
+
*/
|
|
18
|
+
async function getFilesToProcess(sourcePath: string, recursive: boolean): Promise<string[]> {
|
|
19
|
+
const files: string[] = [];
|
|
20
|
+
|
|
21
|
+
const stats = await stat(sourcePath);
|
|
22
|
+
|
|
23
|
+
if (stats.isFile()) {
|
|
24
|
+
files.push(sourcePath);
|
|
25
|
+
} else if (stats.isDirectory()) {
|
|
26
|
+
const dirFiles = await readdir(sourcePath);
|
|
27
|
+
|
|
28
|
+
for (const file of dirFiles) {
|
|
29
|
+
const filePath = path.join(sourcePath, file);
|
|
30
|
+
const fileStats = await stat(filePath);
|
|
31
|
+
|
|
32
|
+
if (fileStats.isFile()) {
|
|
33
|
+
files.push(filePath);
|
|
34
|
+
} else if (recursive && fileStats.isDirectory()) {
|
|
35
|
+
const subFiles = await getFilesToProcess(filePath, recursive);
|
|
36
|
+
files.push(...subFiles);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return files;
|
|
42
|
+
}
|
|
9
43
|
|
|
10
44
|
/**
|
|
11
45
|
* 基于配置文件的上传选项
|
|
@@ -23,6 +57,10 @@ export interface UploadWithConfigOptions {
|
|
|
23
57
|
customSourcePath?: string;
|
|
24
58
|
/** 自定义目标路径,如果提供则覆盖配置文件中的目标路径 */
|
|
25
59
|
customDestination?: string;
|
|
60
|
+
/** 是否启用缓存,默认为true */
|
|
61
|
+
enableCache?: boolean;
|
|
62
|
+
/** 缓存文件路径,默认为.cdn.cache.json */
|
|
63
|
+
cacheFilePath?: string;
|
|
26
64
|
}
|
|
27
65
|
|
|
28
66
|
/**
|
|
@@ -64,7 +102,9 @@ export async function uploadFilesWithConfig(options: UploadWithConfigOptions = {
|
|
|
64
102
|
recursive = true,
|
|
65
103
|
enableCompression = true,
|
|
66
104
|
customSourcePath,
|
|
67
|
-
customDestination
|
|
105
|
+
customDestination,
|
|
106
|
+
enableCache = true,
|
|
107
|
+
cacheFilePath = '.cdn.cache.json'
|
|
68
108
|
} = options;
|
|
69
109
|
|
|
70
110
|
try {
|
|
@@ -84,17 +124,84 @@ export async function uploadFilesWithConfig(options: UploadWithConfigOptions = {
|
|
|
84
124
|
console.log(`目标桶: ${bucketName}`);
|
|
85
125
|
console.log(`目标路径: ${destination}`);
|
|
86
126
|
|
|
127
|
+
// 初始化缓存管理器
|
|
128
|
+
let cacheManager: CacheManager | null = null;
|
|
129
|
+
if (enableCache) {
|
|
130
|
+
cacheManager = createCacheManager(cacheFilePath);
|
|
131
|
+
const cacheStats = cacheManager.getCacheStats();
|
|
132
|
+
console.log(`缓存已启用,当前缓存文件数: ${cacheStats.totalFiles}`);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// 如果启用了缓存,先检查哪些文件需要上传
|
|
136
|
+
let filesToUpload: string[] = [];
|
|
137
|
+
let skippedFiles: string[] = [];
|
|
138
|
+
|
|
139
|
+
if (enableCache && cacheManager) {
|
|
140
|
+
// 获取所有需要处理的文件
|
|
141
|
+
const allFiles = await getFilesToProcess(sourcePath, recursive);
|
|
142
|
+
|
|
143
|
+
for (const filePath of allFiles) {
|
|
144
|
+
if (cacheManager.isFileUploaded(filePath)) {
|
|
145
|
+
const cachedUrl = cacheManager.getUploadedUrl(filePath);
|
|
146
|
+
console.log(`跳过已上传文件: ${filePath} -> ${cachedUrl}`);
|
|
147
|
+
skippedFiles.push(filePath);
|
|
148
|
+
} else {
|
|
149
|
+
filesToUpload.push(filePath);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
console.log(`总文件数: ${allFiles.length}, 需要上传: ${filesToUpload.length}, 跳过: ${skippedFiles.length}`);
|
|
154
|
+
|
|
155
|
+
// 如果没有文件需要上传,直接返回
|
|
156
|
+
if (filesToUpload.length === 0) {
|
|
157
|
+
console.log('所有文件都已上传,无需重复上传。');
|
|
158
|
+
return {
|
|
159
|
+
success: skippedFiles.map(file => ({
|
|
160
|
+
success: true as const,
|
|
161
|
+
file,
|
|
162
|
+
url: cacheManager!.getUploadedUrl(file)!,
|
|
163
|
+
size: 0,
|
|
164
|
+
contentType: 'application/octet-stream',
|
|
165
|
+
timeCreated: new Date().toISOString()
|
|
166
|
+
})),
|
|
167
|
+
failed: []
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
87
172
|
// 执行批量上传
|
|
173
|
+
const uploadSource = enableCache && filesToUpload.length > 0 ? filesToUpload : sourcePath;
|
|
88
174
|
const result = await uploadFiles({
|
|
89
175
|
bucketName,
|
|
90
|
-
sourcePath,
|
|
176
|
+
sourcePath: uploadSource,
|
|
91
177
|
destination,
|
|
92
178
|
storageClient,
|
|
93
179
|
recursive,
|
|
94
180
|
enableCompression
|
|
95
181
|
});
|
|
96
182
|
|
|
97
|
-
|
|
183
|
+
// 如果启用了缓存,记录上传成功的文件
|
|
184
|
+
if (enableCache && cacheManager) {
|
|
185
|
+
for (const successItem of result.success) {
|
|
186
|
+
cacheManager.recordUpload(successItem.file, successItem.url);
|
|
187
|
+
}
|
|
188
|
+
console.log(`已更新缓存,新增 ${result.success.length} 个文件记录`);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
console.log(`上传完成! 成功: ${result.success.length}, 失败: ${result.failed.length}, 跳过: ${skippedFiles.length}`);
|
|
192
|
+
|
|
193
|
+
// 合并跳过的文件到成功结果中
|
|
194
|
+
if (skippedFiles.length > 0 && cacheManager) {
|
|
195
|
+
const skippedResults = skippedFiles.map(file => ({
|
|
196
|
+
success: true as const,
|
|
197
|
+
file,
|
|
198
|
+
url: cacheManager!.getUploadedUrl(file)!,
|
|
199
|
+
size: 0,
|
|
200
|
+
contentType: 'application/octet-stream',
|
|
201
|
+
timeCreated: new Date().toISOString()
|
|
202
|
+
}));
|
|
203
|
+
result.success.push(...skippedResults);
|
|
204
|
+
}
|
|
98
205
|
|
|
99
206
|
return result;
|
|
100
207
|
} catch (error) {
|
package/vs.config.json
CHANGED
package/out/test.txt
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
Hello World from config upload test
|