@sinoia/hubdoc-tools 1.3.2 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/plugins/alfresco/plugin.json +12 -0
- package/dist/plugins/aws-s3/plugin.json +12 -0
- package/dist/plugins/azure-blob/plugin.json +12 -0
- package/dist/plugins/box/plugin.json +12 -0
- package/dist/plugins/core/index.d.ts +25 -0
- package/dist/plugins/core/index.d.ts.map +1 -0
- package/dist/plugins/core/index.js +400 -0
- package/dist/plugins/core/index.js.map +1 -0
- package/dist/plugins/core/plugin.json +26 -0
- package/dist/plugins/dropbox/plugin.json +12 -0
- package/dist/plugins/filesystem/index.d.ts +22 -0
- package/dist/plugins/filesystem/index.d.ts.map +1 -0
- package/dist/plugins/filesystem/index.js +306 -0
- package/dist/plugins/filesystem/index.js.map +1 -0
- package/dist/plugins/filesystem/plugin.json +12 -0
- package/dist/plugins/googledrive/plugin.json +12 -0
- package/dist/plugins/nuxeo/plugin.json +12 -0
- package/dist/plugins/onedrive/plugin.json +12 -0
- package/dist/plugins/opentext/plugin.json +12 -0
- package/dist/plugins/sharepoint/plugin.json +12 -0
- package/dist/services/hubdoc-api.d.ts +1 -1
- package/dist/services/hubdoc-api.js +1 -1
- package/dist/services/oauth-token-service.d.ts +1 -1
- package/dist/services/oauth-token-service.js +2 -2
- package/dist/services/permission-manager.d.ts +1 -1
- package/dist/services/permission-manager.js +1 -1
- package/dist/src/types/plugins.d.ts +111 -0
- package/dist/src/types/plugins.d.ts.map +1 -0
- package/dist/src/types/plugins.js +3 -0
- package/dist/src/types/plugins.js.map +1 -0
- package/dist/src/utils/concurrent-processor.d.ts +63 -0
- package/dist/src/utils/concurrent-processor.d.ts.map +1 -0
- package/dist/src/utils/concurrent-processor.js +240 -0
- package/dist/src/utils/concurrent-processor.js.map +1 -0
- package/dist/src/utils/xml-metadata.d.ts +47 -0
- package/dist/src/utils/xml-metadata.d.ts.map +1 -0
- package/dist/src/utils/xml-metadata.js +200 -0
- package/dist/src/utils/xml-metadata.js.map +1 -0
- package/dist/types/index.d.ts +1 -1
- package/package.json +6 -2
- package/plugins/alfresco/index.ts +518 -0
- package/plugins/alfresco/plugin.json +12 -0
- package/plugins/aws-s3/index.ts +471 -0
- package/plugins/aws-s3/plugin.json +12 -0
- package/plugins/azure-blob/index.ts +420 -0
- package/plugins/azure-blob/plugin.json +12 -0
- package/plugins/box/index.ts +495 -0
- package/plugins/box/plugin.json +12 -0
- package/plugins/core/README.md +122 -0
- package/plugins/core/TESTING.md +155 -0
- package/plugins/core/index.ts +510 -0
- package/plugins/core/plugin.json +26 -0
- package/plugins/dropbox/index.ts +451 -0
- package/plugins/dropbox/plugin.json +12 -0
- package/plugins/filesystem/index.ts +360 -0
- package/plugins/filesystem/plugin.json +12 -0
- package/plugins/googledrive/index.ts +463 -0
- package/plugins/googledrive/plugin.json +12 -0
- package/plugins/nuxeo/index.ts +512 -0
- package/plugins/nuxeo/plugin.json +12 -0
- package/plugins/onedrive/TESTING.md +197 -0
- package/plugins/onedrive/index.ts +447 -0
- package/plugins/onedrive/plugin.json +12 -0
- package/plugins/opentext/index.ts +542 -0
- package/plugins/opentext/plugin.json +12 -0
- package/plugins/sharepoint/index.ts +509 -0
- package/plugins/sharepoint/plugin.json +12 -0
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
7
|
+
const path_1 = __importDefault(require("path"));
|
|
8
|
+
const glob_1 = require("glob");
|
|
9
|
+
const xml_metadata_1 = require("../../src/utils/xml-metadata");
|
|
10
|
+
class FileSystemPlugin {
|
|
11
|
+
constructor() {
|
|
12
|
+
this.name = 'filesystem';
|
|
13
|
+
this.version = '1.0.0';
|
|
14
|
+
this.description = 'Local filesystem document source';
|
|
15
|
+
this.supportedOperations = ['import', 'export', 'both'];
|
|
16
|
+
}
|
|
17
|
+
async testConnection(config) {
|
|
18
|
+
const fsConfig = config;
|
|
19
|
+
try {
|
|
20
|
+
const stats = await fs_extra_1.default.stat(fsConfig.basePath);
|
|
21
|
+
return stats.isDirectory();
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
async scan(config, options) {
|
|
28
|
+
const fsConfig = config;
|
|
29
|
+
const sources = [];
|
|
30
|
+
const errors = [];
|
|
31
|
+
try {
|
|
32
|
+
let pattern = '**/*';
|
|
33
|
+
if (options?.filters?.path) {
|
|
34
|
+
pattern = path_1.default.join(options.filters.path, '**/*');
|
|
35
|
+
}
|
|
36
|
+
const globOptions = {
|
|
37
|
+
cwd: fsConfig.basePath,
|
|
38
|
+
absolute: true,
|
|
39
|
+
dot: fsConfig.includeHidden || false,
|
|
40
|
+
followSymbolicLinks: fsConfig.followSymlinks || false
|
|
41
|
+
};
|
|
42
|
+
const files = await (0, glob_1.glob)(pattern, globOptions);
|
|
43
|
+
let totalSize = 0;
|
|
44
|
+
let processedCount = 0;
|
|
45
|
+
console.log(`📁 Found ${files.length} files${options?.limit ? ` (limit: ${options.limit})` : ''}`);
|
|
46
|
+
for (const filePath of files) {
|
|
47
|
+
// Check limit
|
|
48
|
+
if (options?.limit && processedCount >= options.limit) {
|
|
49
|
+
console.log(`📏 Reached limit of ${options.limit} files`);
|
|
50
|
+
break;
|
|
51
|
+
}
|
|
52
|
+
try {
|
|
53
|
+
const stats = await fs_extra_1.default.stat(filePath);
|
|
54
|
+
if (!stats.isFile())
|
|
55
|
+
continue;
|
|
56
|
+
// Apply filters
|
|
57
|
+
if (options?.filters?.maxSize && stats.size > options.filters.maxSize) {
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
if (options?.filters?.dateRange) {
|
|
61
|
+
const { from, to } = options.filters.dateRange;
|
|
62
|
+
if (from && stats.mtime < from)
|
|
63
|
+
continue;
|
|
64
|
+
if (to && stats.mtime > to)
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
const relativePath = path_1.default.relative(fsConfig.basePath, filePath);
|
|
68
|
+
const mimeType = this.getMimeType(filePath);
|
|
69
|
+
if (options?.filters?.mimeTypes && !options.filters.mimeTypes.includes(mimeType)) {
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
// Skip metadata files from being processed as documents
|
|
73
|
+
if (this.isMetadataFile(filePath)) {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
// Build base metadata
|
|
77
|
+
const baseMetadata = {
|
|
78
|
+
fullPath: filePath,
|
|
79
|
+
directory: path_1.default.dirname(relativePath),
|
|
80
|
+
extension: path_1.default.extname(filePath)
|
|
81
|
+
};
|
|
82
|
+
// Try to find and parse XML metadata if enabled
|
|
83
|
+
let xmlMetadata = {};
|
|
84
|
+
if (fsConfig.useXmlMetadata !== false) { // Default to true unless explicitly disabled
|
|
85
|
+
try {
|
|
86
|
+
const metadataFile = xml_metadata_1.XmlMetadataParser.findMetadataFile(filePath);
|
|
87
|
+
if (metadataFile) {
|
|
88
|
+
const parsedXmlMetadata = await xml_metadata_1.XmlMetadataParser.parseMetadataFile(metadataFile);
|
|
89
|
+
xmlMetadata = xml_metadata_1.XmlMetadataParser.getHubDocMetadata(parsedXmlMetadata);
|
|
90
|
+
console.log(`📋 Found metadata for ${path_1.default.basename(filePath)}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
console.warn(`⚠️ Failed to parse metadata for ${filePath}: ${error}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
sources.push({
|
|
98
|
+
id: filePath,
|
|
99
|
+
// Normalize filename to NFC to handle accented characters consistently across platforms
|
|
100
|
+
name: path_1.default.basename(filePath).normalize('NFC'),
|
|
101
|
+
path: relativePath,
|
|
102
|
+
size: stats.size,
|
|
103
|
+
mimeType,
|
|
104
|
+
lastModified: stats.mtime,
|
|
105
|
+
metadata: {
|
|
106
|
+
...baseMetadata,
|
|
107
|
+
...xmlMetadata
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
totalSize += stats.size;
|
|
111
|
+
processedCount++;
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
errors.push(`Error reading ${filePath}: ${error.message}`);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return {
|
|
118
|
+
sources,
|
|
119
|
+
totalCount: sources.length,
|
|
120
|
+
totalSize,
|
|
121
|
+
errors
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
return {
|
|
126
|
+
sources: [],
|
|
127
|
+
totalCount: 0,
|
|
128
|
+
totalSize: 0,
|
|
129
|
+
errors: [`Scan failed: ${error.message}`]
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
async import(config, sources, targetDir, options) {
|
|
134
|
+
const results = [];
|
|
135
|
+
const batchSize = options?.batchSize || 10;
|
|
136
|
+
// Process in batches
|
|
137
|
+
for (let i = 0; i < sources.length; i += batchSize) {
|
|
138
|
+
const batch = sources.slice(i, i + batchSize);
|
|
139
|
+
const batchPromises = batch.map(source => this.importSingle(source, targetDir));
|
|
140
|
+
const batchResults = await Promise.allSettled(batchPromises);
|
|
141
|
+
for (const result of batchResults) {
|
|
142
|
+
if (result.status === 'fulfilled') {
|
|
143
|
+
results.push(result.value);
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
results.push({
|
|
147
|
+
success: false,
|
|
148
|
+
source: batch[results.length % batch.length],
|
|
149
|
+
error: result.reason?.message || 'Unknown error'
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return results;
|
|
155
|
+
}
|
|
156
|
+
async importSingle(source, targetDir) {
|
|
157
|
+
try {
|
|
158
|
+
const targetPath = path_1.default.join(targetDir, source.path);
|
|
159
|
+
const targetDirectory = path_1.default.dirname(targetPath);
|
|
160
|
+
await fs_extra_1.default.ensureDir(targetDirectory);
|
|
161
|
+
await fs_extra_1.default.copy(source.id, targetPath);
|
|
162
|
+
const stats = await fs_extra_1.default.stat(targetPath);
|
|
163
|
+
return {
|
|
164
|
+
success: true,
|
|
165
|
+
source,
|
|
166
|
+
localPath: targetPath,
|
|
167
|
+
bytesTransferred: stats.size
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
return {
|
|
172
|
+
success: false,
|
|
173
|
+
source,
|
|
174
|
+
error: error.message
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
async export(config, localSources, options) {
|
|
179
|
+
const fsConfig = config;
|
|
180
|
+
const results = [];
|
|
181
|
+
for (const source of localSources) {
|
|
182
|
+
try {
|
|
183
|
+
let targetPath;
|
|
184
|
+
if (options?.preserveStructure) {
|
|
185
|
+
targetPath = path_1.default.join(fsConfig.basePath, options.targetPath, source.path);
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
targetPath = path_1.default.join(fsConfig.basePath, options?.targetPath || '', source.name);
|
|
189
|
+
}
|
|
190
|
+
const targetDir = path_1.default.dirname(targetPath);
|
|
191
|
+
await fs_extra_1.default.ensureDir(targetDir);
|
|
192
|
+
if (!options?.overwrite && await fs_extra_1.default.pathExists(targetPath)) {
|
|
193
|
+
results.push({
|
|
194
|
+
success: false,
|
|
195
|
+
targetPath,
|
|
196
|
+
source,
|
|
197
|
+
error: 'File already exists and overwrite is disabled'
|
|
198
|
+
});
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
await fs_extra_1.default.copy(source.id, targetPath);
|
|
202
|
+
const stats = await fs_extra_1.default.stat(targetPath);
|
|
203
|
+
results.push({
|
|
204
|
+
success: true,
|
|
205
|
+
targetPath,
|
|
206
|
+
source,
|
|
207
|
+
bytesTransferred: stats.size
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
catch (error) {
|
|
211
|
+
results.push({
|
|
212
|
+
success: false,
|
|
213
|
+
targetPath: '',
|
|
214
|
+
source,
|
|
215
|
+
error: error.message
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return results;
|
|
220
|
+
}
|
|
221
|
+
getConfigSchema() {
|
|
222
|
+
return {
|
|
223
|
+
type: 'object',
|
|
224
|
+
properties: {
|
|
225
|
+
basePath: {
|
|
226
|
+
type: 'string',
|
|
227
|
+
description: 'Base directory path to scan',
|
|
228
|
+
required: true
|
|
229
|
+
},
|
|
230
|
+
includeHidden: {
|
|
231
|
+
type: 'boolean',
|
|
232
|
+
description: 'Include hidden files and directories',
|
|
233
|
+
default: false
|
|
234
|
+
},
|
|
235
|
+
followSymlinks: {
|
|
236
|
+
type: 'boolean',
|
|
237
|
+
description: 'Follow symbolic links',
|
|
238
|
+
default: false
|
|
239
|
+
},
|
|
240
|
+
useXmlMetadata: {
|
|
241
|
+
type: 'boolean',
|
|
242
|
+
description: 'Parse XML metadata files (e.g., *_metadata.xml)',
|
|
243
|
+
default: true
|
|
244
|
+
},
|
|
245
|
+
xmlMetadataPattern: {
|
|
246
|
+
type: 'string',
|
|
247
|
+
description: 'Pattern for metadata files',
|
|
248
|
+
default: '*_metadata.xml'
|
|
249
|
+
}
|
|
250
|
+
},
|
|
251
|
+
required: ['basePath']
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
async initialize(config) {
|
|
255
|
+
this.config = config;
|
|
256
|
+
// Validate base path exists
|
|
257
|
+
if (!await fs_extra_1.default.pathExists(this.config.basePath)) {
|
|
258
|
+
throw new Error(`Base path does not exist: ${this.config.basePath}`);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
async destroy() {
|
|
262
|
+
this.config = undefined;
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Check if a file is a metadata file and should be skipped from document processing
|
|
266
|
+
*/
|
|
267
|
+
isMetadataFile(filePath) {
|
|
268
|
+
const fileName = path_1.default.basename(filePath).toLowerCase();
|
|
269
|
+
// Common metadata file patterns
|
|
270
|
+
const metadataPatterns = [
|
|
271
|
+
/_metadata\.xml$/,
|
|
272
|
+
/\.metadata\.xml$/,
|
|
273
|
+
/_meta\.xml$/,
|
|
274
|
+
/^metadata_.*\.xml$/
|
|
275
|
+
];
|
|
276
|
+
return metadataPatterns.some(pattern => pattern.test(fileName));
|
|
277
|
+
}
|
|
278
|
+
getMimeType(filePath) {
|
|
279
|
+
const ext = path_1.default.extname(filePath).toLowerCase();
|
|
280
|
+
const mimeTypes = {
|
|
281
|
+
'.pdf': 'application/pdf',
|
|
282
|
+
'.doc': 'application/msword',
|
|
283
|
+
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
284
|
+
'.xls': 'application/vnd.ms-excel',
|
|
285
|
+
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
286
|
+
'.ppt': 'application/vnd.ms-powerpoint',
|
|
287
|
+
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
288
|
+
'.txt': 'text/plain',
|
|
289
|
+
'.csv': 'text/csv',
|
|
290
|
+
'.json': 'application/json',
|
|
291
|
+
'.xml': 'application/xml',
|
|
292
|
+
'.jpg': 'image/jpeg',
|
|
293
|
+
'.jpeg': 'image/jpeg',
|
|
294
|
+
'.png': 'image/png',
|
|
295
|
+
'.gif': 'image/gif',
|
|
296
|
+
'.bmp': 'image/bmp',
|
|
297
|
+
'.tiff': 'image/tiff',
|
|
298
|
+
'.zip': 'application/zip',
|
|
299
|
+
'.rar': 'application/vnd.rar',
|
|
300
|
+
'.7z': 'application/x-7z-compressed'
|
|
301
|
+
};
|
|
302
|
+
return mimeTypes[ext] || 'application/octet-stream';
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
exports.default = FileSystemPlugin;
|
|
306
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../plugins/filesystem/index.ts"],"names":[],"mappings":";;;;;AAAA,wDAA0B;AAC1B,gDAAwB;AACxB,+BAA4B;AAW5B,+DAAiE;AAUjE,MAAqB,gBAAgB;IAArC;QACW,SAAI,GAAG,YAAY,CAAC;QACpB,YAAO,GAAG,OAAO,CAAC;QAClB,gBAAW,GAAG,kCAAkC,CAAC;QACjD,wBAAmB,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAU,CAAC;IA4UvE,CAAC;IAxUC,KAAK,CAAC,cAAc,CAAC,MAAoB;QACvC,MAAM,QAAQ,GAAG,MAA0B,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC/C,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAoB,EAAE,OAA6B;QAC5D,MAAM,QAAQ,GAAG,MAA0B,CAAC;QAC5C,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,MAAM,CAAC;YACrB,IAAI,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;gBAC3B,OAAO,GAAG,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACpD,CAAC;YAED,MAAM,WAAW,GAAG;gBAClB,GAAG,EAAE,QAAQ,CAAC,QAAQ;gBACtB,QAAQ,EAAE,IAAI;gBACd,GAAG,EAAE,QAAQ,CAAC,aAAa,IAAI,KAAK;gBACpC,mBAAmB,EAAE,QAAQ,CAAC,cAAc,IAAI,KAAK;aACtD,CAAC;YAEF,MAAM,KAAK,GAAG,MAAM,IAAA,WAAI,EAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAC/C,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,cAAc,GAAG,CAAC,CAAC;YAEvB,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,MAAM,SAAS,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAEnG,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,cAAc;gBACd,IAAI,OAAO,EAAE,KAAK,IAAI,cAAc,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBACtD,OAAO,CAAC,GAAG,CAAC,uBAAuB,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;oBAC1D,MAAM;gBACR,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAEtC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;wBAAE,SAAS;oBAE9B,gBAAgB;oBAChB,IAAI,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,KAAK,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;wBACtE,SAAS;oBACX,CAAC;oBAED,IAAI,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;wBAChC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;wBAC/C,IAAI,IAAI,IAAI,KAAK,CAAC,KAAK,GAAG,IAAI;4BAAE,SAAS;wBACzC,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,GAAG,EAAE;4BAAE,SAAS;oBACvC,CAAC;oBAED,MAAM,YAAY,GAAG,cAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;oBAE5C,IAAI,OAAO,EAAE,OAAO,EAAE,SAAS,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACjF,SAAS;oBACX,CAAC;oBAED,wDAAwD;oBACxD,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAClC,SAAS;oBACX,CAAC;oBAED,sBAAsB;oBACtB,MAAM,YAAY,GAAG;wBACnB,QAAQ,EAAE,QAAQ;wBAClB,SAAS,EAAE,cAAI,CAAC,OAAO,CAAC,YAAY,CAAC;wBACrC,SAAS,EAAE,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;qBAClC,CAAC;oBAEF,gDAAgD;oBAChD,IAAI,WAAW,GAAG,EAAE,CAAC;oBACrB,IAAI,QAAQ,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC,CAAC,6CAA6C;wBACpF,IAAI,CAAC;4BACH,MAAM,YAAY,GAAG,gCAAiB,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;4BAClE,IAAI,YAAY,EAAE,CAAC;gCACjB,MAAM,iBAAiB,GAAG,MAAM,gCAAiB,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;gCAClF,WAAW,GAAG,gCAAiB,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;gCACrE,OAAO,CAAC,GAAG,CAAC,yBAAyB,cAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;4BAClE,CAAC;wBACH,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,OAAO,CAAC,IAAI,CAAC,oCAAoC,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;wBACzE,CAAC;oBACH,CAAC;oBAED,OAAO,CAAC,IAAI,CAAC;wBACX,EAAE,EAAE,QAAQ;wBACZ,wFAAwF;wBACxF,IAAI,EAAE,cAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC;wBAC9C,IAAI,EAAE,YAAY;wBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,QAAQ;wBACR,YAAY,EAAE,KAAK,CAAC,KAAK;wBACzB,QAAQ,EAAE;4BACR,GAAG,YAAY;4BACf,GAAG,WAAW;yBACf;qBACF,CAAC,CAAC;oBAEH,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC;oBACxB,cAAc,EAAE,CAAC;gBACnB,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,MAAM,CAAC,IAAI,CAAC,iBAAiB,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO;gBACP,UAAU,EAAE,OAAO,CAAC,MAAM;gBAC1B,SAAS;gBACT,MAAM;aACP,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC,gBAAgB,KAAK,CAAC,OAAO,EAAE,CAAC;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CACV,MAAoB,EACpB,OAAyB,EACzB,SAAiB,EACjB,OAA6B;QAE7B,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;QAE3C,qBAAqB;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;YAEhF,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;YAE7D,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;gBAClC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,IAAI,CAAC;wBACX,OAAO,EAAE,KAAK;wBACd,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;wBAC5C,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,eAAe;qBACjD,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,MAAsB,EAAE,SAAiB;QAClE,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,eAAe,GAAG,cAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAEjD,MAAM,kBAAE,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YACpC,MAAM,kBAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAErC,MAAM,KAAK,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAExC,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,MAAM;gBACN,SAAS,EAAE,UAAU;gBACrB,gBAAgB,EAAE,KAAK,CAAC,IAAI;aAC7B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,MAAM;gBACN,KAAK,EAAE,KAAK,CAAC,OAAO;aACrB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CACV,MAAoB,EACpB,YAA8B,EAC9B,OAA6B;QAE7B,MAAM,QAAQ,GAAG,MAA0B,CAAC;QAC5C,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,IAAI,UAAkB,CAAC;gBAEvB,IAAI,OAAO,EAAE,iBAAiB,EAAE,CAAC;oBAC/B,UAAU,GAAG,cAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC7E,CAAC;qBAAM,CAAC;oBACN,UAAU,GAAG,cAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBACpF,CAAC;gBAED,MAAM,SAAS,GAAG,cAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBAC3C,MAAM,kBAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAE9B,IAAI,CAAC,OAAO,EAAE,SAAS,IAAI,MAAM,kBAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC3D,OAAO,CAAC,IAAI,CAAC;wBACX,OAAO,EAAE,KAAK;wBACd,UAAU;wBACV,MAAM;wBACN,KAAK,EAAE,+CAA+C;qBACvD,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBAED,MAAM,kBAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;gBACrC,MAAM,KAAK,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAExC,OAAO,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,IAAI;oBACb,UAAU;oBACV,MAAM;oBACN,gBAAgB,EAAE,KAAK,CAAC,IAAI;iBAC7B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,EAAE;oBACd,MAAM;oBACN,KAAK,EAAE,KAAK,CAAC,OAAO;iBACrB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,eAAe;QACb,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,6BAA6B;oBAC1C,QAAQ,EAAE,IAAI;iBACf;gBACD,aAAa,EAAE;oBACb,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,sCAAsC;oBACnD,OAAO,EAAE,KAAK;iBACf;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,uBAAuB;oBACpC,OAAO,EAAE,KAAK;iBACf;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,iDAAiD;oBAC9D,OAAO,EAAE,IAAI;iBACd;gBACD,kBAAkB,EAAE;oBAClB,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,4BAA4B;oBACzC,OAAO,EAAE,gBAAgB;iBAC1B;aACF;YACD,QAAQ,EAAE,CAAC,UAAU,CAAC;SACvB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAoB;QACnC,IAAI,CAAC,MAAM,GAAG,MAA0B,CAAC;QAEzC,4BAA4B;QAC5B,IAAI,CAAC,MAAM,kBAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,QAAgB;QACrC,MAAM,QAAQ,GAAG,cAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QAEvD,gCAAgC;QAChC,MAAM,gBAAgB,GAAG;YACvB,iBAAiB;YACjB,kBAAkB;YAClB,aAAa;YACb,oBAAoB;SACrB,CAAC;QAEF,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClE,CAAC;IAEO,WAAW,CAAC,QAAgB;QAClC,MAAM,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QACjD,MAAM,SAAS,GAA2B;YACxC,MAAM,EAAE,iBAAiB;YACzB,MAAM,EAAE,oBAAoB;YAC5B,OAAO,EAAE,yEAAyE;YAClF,MAAM,EAAE,0BAA0B;YAClC,OAAO,EAAE,mEAAmE;YAC5E,MAAM,EAAE,+BAA+B;YACvC,OAAO,EAAE,2EAA2E;YACpF,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,UAAU;YAClB,OAAO,EAAE,kBAAkB;YAC3B,MAAM,EAAE,iBAAiB;YACzB,MAAM,EAAE,YAAY;YACpB,OAAO,EAAE,YAAY;YACrB,MAAM,EAAE,WAAW;YACnB,MAAM,EAAE,WAAW;YACnB,MAAM,EAAE,WAAW;YACnB,OAAO,EAAE,YAAY;YACrB,MAAM,EAAE,iBAAiB;YACzB,MAAM,EAAE,qBAAqB;YAC7B,KAAK,EAAE,6BAA6B;SACrC,CAAC;QAEF,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,0BAA0B,CAAC;IACtD,CAAC;CACF;AAhVD,mCAgVC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "filesystem",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Local filesystem document source plugin",
|
|
5
|
+
"author": "HubDoc Tools",
|
|
6
|
+
"main": "index.js",
|
|
7
|
+
"hubdocToolVersion": "^1.0.0",
|
|
8
|
+
"dependencies": {
|
|
9
|
+
"fs-extra": "^11.1.0",
|
|
10
|
+
"glob": "^10.3.0"
|
|
11
|
+
}
|
|
12
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "googledrive",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Google Drive document source plugin",
|
|
5
|
+
"author": "HubDoc Tools",
|
|
6
|
+
"main": "index.js",
|
|
7
|
+
"hubdocToolVersion": "^1.0.0",
|
|
8
|
+
"dependencies": {
|
|
9
|
+
"googleapis": "^128.0.0",
|
|
10
|
+
"fs-extra": "^11.1.0"
|
|
11
|
+
}
|
|
12
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "onedrive",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Microsoft OneDrive document source plugin",
|
|
5
|
+
"author": "HubDoc Tools",
|
|
6
|
+
"main": "index.js",
|
|
7
|
+
"hubdocToolVersion": "^1.0.0",
|
|
8
|
+
"dependencies": {
|
|
9
|
+
"axios": "^1.5.0",
|
|
10
|
+
"fs-extra": "^11.1.0"
|
|
11
|
+
}
|
|
12
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "sharepoint",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "SharePoint document source plugin",
|
|
5
|
+
"author": "HubDoc Tools",
|
|
6
|
+
"main": "index.ts",
|
|
7
|
+
"hubdocToolVersion": "^1.0.0",
|
|
8
|
+
"dependencies": {
|
|
9
|
+
"@pnp/sp": "^3.0.0",
|
|
10
|
+
"fs-extra": "^11.1.0"
|
|
11
|
+
}
|
|
12
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Workspace } from '
|
|
1
|
+
import { Workspace } from '@api/models';
|
|
2
2
|
import { DocumentMapping, HubDocBulkUpload, HubDocConfig, HubDocDocument, HubDocFolder, ImportResult } from '../types';
|
|
3
3
|
import { ChunkedUploadProgress } from './chunked-uploader';
|
|
4
4
|
export declare class HubDocApiService {
|
|
@@ -4,7 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.HubDocApiService = void 0;
|
|
7
|
-
const api_1 = require("
|
|
7
|
+
const api_1 = require("@api/api");
|
|
8
8
|
const chalk_1 = __importDefault(require("chalk"));
|
|
9
9
|
const form_data_1 = __importDefault(require("form-data"));
|
|
10
10
|
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
@@ -9,8 +9,8 @@ const child_process_1 = require("child_process");
|
|
|
9
9
|
const http_1 = __importDefault(require("http"));
|
|
10
10
|
const url_1 = require("url");
|
|
11
11
|
const util_1 = require("util");
|
|
12
|
-
const api_1 = require("
|
|
13
|
-
const configuration_1 = require("
|
|
12
|
+
const api_1 = require("@api/api");
|
|
13
|
+
const configuration_1 = require("@api/configuration");
|
|
14
14
|
const axios_1 = __importDefault(require("axios"));
|
|
15
15
|
const execAsync = (0, util_1.promisify)(child_process_1.exec);
|
|
16
16
|
/**
|
|
@@ -4,7 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.PermissionManager = void 0;
|
|
7
|
-
const api_1 = require("
|
|
7
|
+
const api_1 = require("@api/api");
|
|
8
8
|
const chalk_1 = __importDefault(require("chalk"));
|
|
9
9
|
class PermissionManager {
|
|
10
10
|
constructor(config, client) {
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
export interface DocumentSource {
|
|
2
|
+
id: string;
|
|
3
|
+
name: string;
|
|
4
|
+
path: string;
|
|
5
|
+
size: number;
|
|
6
|
+
mimeType: string;
|
|
7
|
+
lastModified: Date;
|
|
8
|
+
metadata?: Record<string, any>;
|
|
9
|
+
}
|
|
10
|
+
export interface PluginConfig {
|
|
11
|
+
[key: string]: any;
|
|
12
|
+
}
|
|
13
|
+
export interface SourceConnection {
|
|
14
|
+
id: string;
|
|
15
|
+
name: string;
|
|
16
|
+
type: string;
|
|
17
|
+
config: PluginConfig;
|
|
18
|
+
isConnected: boolean;
|
|
19
|
+
}
|
|
20
|
+
export interface ScanResult {
|
|
21
|
+
sources: DocumentSource[];
|
|
22
|
+
totalCount: number;
|
|
23
|
+
totalSize: number;
|
|
24
|
+
errors: string[];
|
|
25
|
+
}
|
|
26
|
+
export interface PluginImportOptions {
|
|
27
|
+
filters?: {
|
|
28
|
+
path?: string;
|
|
29
|
+
mimeTypes?: string[];
|
|
30
|
+
maxSize?: number;
|
|
31
|
+
dateRange?: {
|
|
32
|
+
from?: Date;
|
|
33
|
+
to?: Date;
|
|
34
|
+
};
|
|
35
|
+
};
|
|
36
|
+
batchSize?: number;
|
|
37
|
+
concurrent?: number;
|
|
38
|
+
limit?: number;
|
|
39
|
+
}
|
|
40
|
+
export interface PluginExportOptions {
|
|
41
|
+
targetPath: string;
|
|
42
|
+
overwrite?: boolean;
|
|
43
|
+
preserveStructure?: boolean;
|
|
44
|
+
metadata?: boolean;
|
|
45
|
+
}
|
|
46
|
+
export interface ImportResult {
|
|
47
|
+
success: boolean;
|
|
48
|
+
source: DocumentSource;
|
|
49
|
+
localPath?: string;
|
|
50
|
+
error?: string;
|
|
51
|
+
bytesTransferred?: number;
|
|
52
|
+
}
|
|
53
|
+
export interface ExportResult {
|
|
54
|
+
success: boolean;
|
|
55
|
+
targetPath: string;
|
|
56
|
+
source: DocumentSource;
|
|
57
|
+
error?: string;
|
|
58
|
+
bytesTransferred?: number;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Base interface for all document source plugins
|
|
62
|
+
*/
|
|
63
|
+
export interface DocumentSourcePlugin {
|
|
64
|
+
readonly name: string;
|
|
65
|
+
readonly version: string;
|
|
66
|
+
readonly description: string;
|
|
67
|
+
readonly supportedOperations: readonly ('import' | 'export' | 'both')[];
|
|
68
|
+
/**
|
|
69
|
+
* Test if the plugin can connect with given configuration
|
|
70
|
+
*/
|
|
71
|
+
testConnection(config: PluginConfig): Promise<boolean>;
|
|
72
|
+
/**
|
|
73
|
+
* Scan the external source for available documents
|
|
74
|
+
*/
|
|
75
|
+
scan(config: PluginConfig, options?: PluginImportOptions): Promise<ScanResult>;
|
|
76
|
+
/**
|
|
77
|
+
* Import documents from external source to local storage
|
|
78
|
+
*/
|
|
79
|
+
import(config: PluginConfig, sources: DocumentSource[], targetDir: string, options?: PluginImportOptions): Promise<ImportResult[]>;
|
|
80
|
+
/**
|
|
81
|
+
* Export documents to external source (if supported)
|
|
82
|
+
*/
|
|
83
|
+
export?(config: PluginConfig, localSources: DocumentSource[], options?: PluginExportOptions): Promise<ExportResult[]>;
|
|
84
|
+
/**
|
|
85
|
+
* Get configuration schema for this plugin
|
|
86
|
+
*/
|
|
87
|
+
getConfigSchema(): Record<string, any>;
|
|
88
|
+
/**
|
|
89
|
+
* Initialize plugin with configuration
|
|
90
|
+
*/
|
|
91
|
+
initialize(config: PluginConfig): Promise<void>;
|
|
92
|
+
/**
|
|
93
|
+
* Cleanup plugin resources
|
|
94
|
+
*/
|
|
95
|
+
destroy(): Promise<void>;
|
|
96
|
+
}
|
|
97
|
+
export interface PluginManifest {
|
|
98
|
+
name: string;
|
|
99
|
+
version: string;
|
|
100
|
+
description: string;
|
|
101
|
+
author?: string;
|
|
102
|
+
main: string;
|
|
103
|
+
dependencies?: Record<string, string>;
|
|
104
|
+
hubdocToolVersion: string;
|
|
105
|
+
}
|
|
106
|
+
export interface RegisteredPlugin {
|
|
107
|
+
manifest: PluginManifest;
|
|
108
|
+
plugin: DocumentSourcePlugin;
|
|
109
|
+
enabled: boolean;
|
|
110
|
+
}
|
|
111
|
+
//# sourceMappingURL=plugins.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plugins.d.ts","sourceRoot":"","sources":["../../../src/types/plugins.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,IAAI,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE;YACV,IAAI,CAAC,EAAE,IAAI,CAAC;YACZ,EAAE,CAAC,EAAE,IAAI,CAAC;SACX,CAAC;KACH,CAAC;IACF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,cAAc,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,cAAc,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,SAAS,CAAC,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;IAExE;;OAEG;IACH,cAAc,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEvD;;OAEG;IACH,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAE/E;;OAEG;IACH,MAAM,CACJ,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,cAAc,EAAE,EACzB,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAE3B;;OAEG;IACH,MAAM,CAAC,CACL,MAAM,EAAE,YAAY,EACpB,YAAY,EAAE,cAAc,EAAE,EAC9B,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAE3B;;OAEG;IACH,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAEvC;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhD;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,OAAO,EAAE,OAAO,CAAC;CAClB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plugins.js","sourceRoot":"","sources":["../../../src/types/plugins.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
export interface ConcurrentProcessorOptions {
|
|
2
|
+
concurrency: number;
|
|
3
|
+
onProgress?: (completed: number, total: number, current?: string) => void;
|
|
4
|
+
onError?: (error: Error, item: any, index: number) => void;
|
|
5
|
+
}
|
|
6
|
+
export interface ProcessingResult<T, R> {
|
|
7
|
+
results: R[];
|
|
8
|
+
errors: Array<{
|
|
9
|
+
item: T;
|
|
10
|
+
index: number;
|
|
11
|
+
error: Error;
|
|
12
|
+
}>;
|
|
13
|
+
totalProcessed: number;
|
|
14
|
+
successCount: number;
|
|
15
|
+
errorCount: number;
|
|
16
|
+
processingTime: number;
|
|
17
|
+
}
|
|
18
|
+
export declare class ConcurrentProcessor {
|
|
19
|
+
/**
|
|
20
|
+
* Process an array of items concurrently with limited parallelism
|
|
21
|
+
*/
|
|
22
|
+
static processInBatches<T, R>(items: T[], processor: (item: T, index: number) => Promise<R>, options: ConcurrentProcessorOptions): Promise<ProcessingResult<T, R>>;
|
|
23
|
+
/**
|
|
24
|
+
* Process items with automatic progress display
|
|
25
|
+
*/
|
|
26
|
+
static processWithProgress<T, R>(items: T[], processor: (item: T, index: number) => Promise<R>, options: {
|
|
27
|
+
concurrency: number;
|
|
28
|
+
operation: string;
|
|
29
|
+
itemName?: string;
|
|
30
|
+
}): Promise<ProcessingResult<T, R>>;
|
|
31
|
+
/**
|
|
32
|
+
* Create batches from an array
|
|
33
|
+
*/
|
|
34
|
+
static createBatches<T>(items: T[], batchSize: number): T[][];
|
|
35
|
+
/**
|
|
36
|
+
* Process items in sequential batches with concurrency within each batch
|
|
37
|
+
*/
|
|
38
|
+
static processInSequentialBatches<T, R>(items: T[], processor: (item: T, index: number) => Promise<R>, options: {
|
|
39
|
+
batchSize: number;
|
|
40
|
+
concurrencyPerBatch: number;
|
|
41
|
+
onBatchComplete?: (batchIndex: number, totalBatches: number) => void;
|
|
42
|
+
onProgress?: (completed: number, total: number) => void;
|
|
43
|
+
onError?: (error: Error, item: T, index: number) => void;
|
|
44
|
+
}): Promise<ProcessingResult<T, R>>;
|
|
45
|
+
/**
|
|
46
|
+
* Get a string description of an item for logging
|
|
47
|
+
*/
|
|
48
|
+
private static getItemDescription;
|
|
49
|
+
/**
|
|
50
|
+
* Utility to delay execution (for rate limiting)
|
|
51
|
+
*/
|
|
52
|
+
static delay(ms: number): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Process with automatic retry on failure
|
|
55
|
+
*/
|
|
56
|
+
static processWithRetry<T, R>(items: T[], processor: (item: T, index: number, attempt: number) => Promise<R>, options: {
|
|
57
|
+
concurrency: number;
|
|
58
|
+
maxRetries?: number;
|
|
59
|
+
retryDelay?: number;
|
|
60
|
+
onProgress?: (completed: number, total: number) => void;
|
|
61
|
+
}): Promise<ProcessingResult<T, R>>;
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=concurrent-processor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"concurrent-processor.d.ts","sourceRoot":"","sources":["../../../src/utils/concurrent-processor.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,0BAA0B;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAC1E,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAC5D;AAED,MAAM,WAAW,gBAAgB,CAAC,CAAC,EAAE,CAAC;IACpC,OAAO,EAAE,CAAC,EAAE,CAAC;IACb,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,CAAC,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAA;KAAE,CAAC,CAAC;IACxD,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,mBAAmB;IAC9B;;OAEG;WACU,gBAAgB,CAAC,CAAC,EAAE,CAAC,EAChC,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,OAAO,EAAE,0BAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAuElC;;OAEG;WACU,mBAAmB,CAAC,CAAC,EAAE,CAAC,EACnC,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,OAAO,EAAE;QACP,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GACA,OAAO,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAsDlC;;OAEG;IACH,MAAM,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,EAAE;IAU7D;;OAEG;WACU,0BAA0B,CAAC,CAAC,EAAE,CAAC,EAC1C,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,OAAO,EAAE;QACP,SAAS,EAAE,MAAM,CAAC;QAClB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,eAAe,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,IAAI,CAAC;QACrE,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;QACxD,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KAC1D,GACA,OAAO,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAiElC;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAejC;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvC;;OAEG;WACU,gBAAgB,CAAC,CAAC,EAAE,CAAC,EAChC,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EAClE,OAAO,EAAE;QACP,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KACzD,GACA,OAAO,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;CA0BnC"}
|