@arke-institute/sdk 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index.d.mts → client-dAk3E64p.d.cts} +1 -7
- package/dist/client-dAk3E64p.d.ts +183 -0
- package/dist/{index.mjs → collections/index.cjs} +34 -5
- package/dist/collections/index.cjs.map +1 -0
- package/dist/collections/index.d.cts +9 -0
- package/dist/collections/index.d.ts +9 -1
- package/dist/collections/index.js +5 -32
- package/dist/collections/index.js.map +1 -1
- package/dist/content/index.cjs +506 -0
- package/dist/content/index.cjs.map +1 -0
- package/dist/content/index.d.cts +403 -0
- package/dist/content/index.d.ts +403 -0
- package/dist/content/index.js +473 -0
- package/dist/content/index.js.map +1 -0
- package/dist/edit/index.cjs +1029 -0
- package/dist/edit/index.cjs.map +1 -0
- package/dist/edit/index.d.cts +78 -0
- package/dist/edit/index.d.ts +78 -0
- package/dist/edit/index.js +983 -0
- package/dist/edit/index.js.map +1 -0
- package/dist/errors-3L7IiHcr.d.cts +480 -0
- package/dist/errors-B82BMmRP.d.cts +343 -0
- package/dist/errors-B82BMmRP.d.ts +343 -0
- package/dist/errors-BTe8GKRQ.d.ts +480 -0
- package/dist/graph/index.cjs +433 -0
- package/dist/graph/index.cjs.map +1 -0
- package/dist/graph/index.d.cts +456 -0
- package/dist/graph/index.d.ts +456 -0
- package/dist/graph/index.js +402 -0
- package/dist/graph/index.js.map +1 -0
- package/dist/index.cjs +3761 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +7 -0
- package/dist/index.d.ts +7 -189
- package/dist/index.js +3502 -30
- package/dist/index.js.map +1 -1
- package/dist/query/index.cjs +289 -0
- package/dist/query/index.cjs.map +1 -0
- package/dist/query/index.d.cts +541 -0
- package/dist/query/index.d.ts +541 -0
- package/dist/query/index.js +261 -0
- package/dist/query/index.js.map +1 -0
- package/dist/upload/index.cjs +1634 -0
- package/dist/upload/index.cjs.map +1 -0
- package/dist/upload/index.d.cts +150 -0
- package/dist/upload/index.d.ts +150 -0
- package/dist/upload/index.js +1597 -0
- package/dist/upload/index.js.map +1 -0
- package/package.json +43 -8
- package/dist/collections/index.d.mts +0 -1
- package/dist/collections/index.mjs +0 -204
- package/dist/collections/index.mjs.map +0 -1
- package/dist/index.mjs.map +0 -1
|
@@ -0,0 +1,1597 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
3
|
+
var __esm = (fn, res) => function __init() {
|
|
4
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
5
|
+
};
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
// src/upload/utils/errors.ts
|
|
12
|
+
function isRetryableError(error) {
|
|
13
|
+
if (error instanceof NetworkError) {
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
if (error instanceof WorkerAPIError) {
|
|
17
|
+
return error.statusCode ? error.statusCode >= 500 : false;
|
|
18
|
+
}
|
|
19
|
+
if (error instanceof UploadError) {
|
|
20
|
+
if (error.statusCode) {
|
|
21
|
+
return error.statusCode >= 500 || error.statusCode === 429;
|
|
22
|
+
}
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
if (error.code === "ECONNRESET" || error.code === "ETIMEDOUT" || error.code === "ENOTFOUND" || error.code === "ECONNREFUSED") {
|
|
26
|
+
return true;
|
|
27
|
+
}
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
var WorkerAPIError, UploadError, ValidationError, NetworkError, ScanError;
|
|
31
|
+
var init_errors = __esm({
|
|
32
|
+
"src/upload/utils/errors.ts"() {
|
|
33
|
+
"use strict";
|
|
34
|
+
WorkerAPIError = class extends Error {
|
|
35
|
+
constructor(message, statusCode, details) {
|
|
36
|
+
super(message);
|
|
37
|
+
this.statusCode = statusCode;
|
|
38
|
+
this.details = details;
|
|
39
|
+
this.name = "WorkerAPIError";
|
|
40
|
+
Error.captureStackTrace(this, this.constructor);
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
UploadError = class extends Error {
|
|
44
|
+
constructor(message, fileName, statusCode, cause) {
|
|
45
|
+
super(message);
|
|
46
|
+
this.fileName = fileName;
|
|
47
|
+
this.statusCode = statusCode;
|
|
48
|
+
this.cause = cause;
|
|
49
|
+
this.name = "UploadError";
|
|
50
|
+
Error.captureStackTrace(this, this.constructor);
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
ValidationError = class extends Error {
|
|
54
|
+
constructor(message, field) {
|
|
55
|
+
super(message);
|
|
56
|
+
this.field = field;
|
|
57
|
+
this.name = "ValidationError";
|
|
58
|
+
Error.captureStackTrace(this, this.constructor);
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
NetworkError = class extends Error {
|
|
62
|
+
constructor(message, cause) {
|
|
63
|
+
super(message);
|
|
64
|
+
this.cause = cause;
|
|
65
|
+
this.name = "NetworkError";
|
|
66
|
+
Error.captureStackTrace(this, this.constructor);
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
ScanError = class extends Error {
|
|
70
|
+
constructor(message, path2) {
|
|
71
|
+
super(message);
|
|
72
|
+
this.path = path2;
|
|
73
|
+
this.name = "ScanError";
|
|
74
|
+
Error.captureStackTrace(this, this.constructor);
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
// src/upload/platforms/common.ts
|
|
81
|
+
function detectPlatform() {
|
|
82
|
+
if (typeof process !== "undefined" && process.versions != null && process.versions.node != null) {
|
|
83
|
+
return "node";
|
|
84
|
+
}
|
|
85
|
+
if (typeof window !== "undefined" && typeof document !== "undefined") {
|
|
86
|
+
return "browser";
|
|
87
|
+
}
|
|
88
|
+
return "unknown";
|
|
89
|
+
}
|
|
90
|
+
function normalizePath(p) {
|
|
91
|
+
return p.replace(/\\/g, "/");
|
|
92
|
+
}
|
|
93
|
+
function getExtension(filename) {
|
|
94
|
+
const lastDot = filename.lastIndexOf(".");
|
|
95
|
+
return lastDot === -1 ? "" : filename.slice(lastDot + 1).toLowerCase();
|
|
96
|
+
}
|
|
97
|
+
function getMimeType(filename) {
|
|
98
|
+
const ext = getExtension(filename);
|
|
99
|
+
const mimeTypes = {
|
|
100
|
+
// Images
|
|
101
|
+
"jpg": "image/jpeg",
|
|
102
|
+
"jpeg": "image/jpeg",
|
|
103
|
+
"png": "image/png",
|
|
104
|
+
"gif": "image/gif",
|
|
105
|
+
"webp": "image/webp",
|
|
106
|
+
"tif": "image/tiff",
|
|
107
|
+
"tiff": "image/tiff",
|
|
108
|
+
"bmp": "image/bmp",
|
|
109
|
+
"svg": "image/svg+xml",
|
|
110
|
+
// Documents
|
|
111
|
+
"pdf": "application/pdf",
|
|
112
|
+
"txt": "text/plain",
|
|
113
|
+
"json": "application/json",
|
|
114
|
+
"xml": "application/xml",
|
|
115
|
+
"html": "text/html",
|
|
116
|
+
"htm": "text/html",
|
|
117
|
+
"css": "text/css",
|
|
118
|
+
"js": "application/javascript",
|
|
119
|
+
// Archives
|
|
120
|
+
"zip": "application/zip",
|
|
121
|
+
"tar": "application/x-tar",
|
|
122
|
+
"gz": "application/gzip",
|
|
123
|
+
// Audio
|
|
124
|
+
"mp3": "audio/mpeg",
|
|
125
|
+
"wav": "audio/wav",
|
|
126
|
+
"ogg": "audio/ogg",
|
|
127
|
+
// Video
|
|
128
|
+
"mp4": "video/mp4",
|
|
129
|
+
"webm": "video/webm",
|
|
130
|
+
"mov": "video/quicktime"
|
|
131
|
+
};
|
|
132
|
+
return mimeTypes[ext] || "application/octet-stream";
|
|
133
|
+
}
|
|
134
|
+
var init_common = __esm({
|
|
135
|
+
"src/upload/platforms/common.ts"() {
|
|
136
|
+
"use strict";
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
// src/upload/lib/validation.ts
|
|
141
|
+
function validateFileSize(size) {
|
|
142
|
+
if (size <= 0) {
|
|
143
|
+
throw new ValidationError("File size must be greater than 0");
|
|
144
|
+
}
|
|
145
|
+
if (size > MAX_FILE_SIZE) {
|
|
146
|
+
throw new ValidationError(
|
|
147
|
+
`File size (${formatBytes(size)}) exceeds maximum allowed size (${formatBytes(MAX_FILE_SIZE)})`
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
function validateBatchSize(totalSize) {
|
|
152
|
+
if (totalSize > MAX_BATCH_SIZE) {
|
|
153
|
+
throw new ValidationError(
|
|
154
|
+
`Total batch size (${formatBytes(totalSize)}) exceeds maximum allowed size (${formatBytes(MAX_BATCH_SIZE)})`
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
function validateLogicalPath(path2) {
|
|
159
|
+
if (!path2.startsWith("/")) {
|
|
160
|
+
throw new ValidationError("Logical path must start with /", "path");
|
|
161
|
+
}
|
|
162
|
+
if (INVALID_PATH_CHARS.test(path2)) {
|
|
163
|
+
throw new ValidationError(
|
|
164
|
+
"Logical path contains invalid characters",
|
|
165
|
+
"path"
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
const segments = path2.split("/").filter((s) => s.length > 0);
|
|
169
|
+
if (segments.length === 0 && path2 !== "/") {
|
|
170
|
+
throw new ValidationError("Logical path cannot be empty", "path");
|
|
171
|
+
}
|
|
172
|
+
for (const segment of segments) {
|
|
173
|
+
if (segment === "." || segment === "..") {
|
|
174
|
+
throw new ValidationError(
|
|
175
|
+
"Logical path cannot contain . or .. segments",
|
|
176
|
+
"path"
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
function validateRefJson(content, fileName, logger) {
|
|
182
|
+
let parsed;
|
|
183
|
+
try {
|
|
184
|
+
parsed = JSON.parse(content);
|
|
185
|
+
} catch (error) {
|
|
186
|
+
throw new ValidationError(
|
|
187
|
+
`Invalid JSON in ${fileName}: ${error.message}`,
|
|
188
|
+
"ref"
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
if (typeof parsed !== "object" || Array.isArray(parsed) || parsed === null) {
|
|
192
|
+
throw new ValidationError(
|
|
193
|
+
`${fileName} must contain a JSON object`,
|
|
194
|
+
"ref"
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
if (!parsed.url || typeof parsed.url !== "string") {
|
|
198
|
+
throw new ValidationError(
|
|
199
|
+
`${fileName} must contain a 'url' field with a string value`,
|
|
200
|
+
"ref"
|
|
201
|
+
);
|
|
202
|
+
}
|
|
203
|
+
try {
|
|
204
|
+
const url = new URL(parsed.url);
|
|
205
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
206
|
+
throw new Error("URL must use HTTP or HTTPS protocol");
|
|
207
|
+
}
|
|
208
|
+
} catch (error) {
|
|
209
|
+
throw new ValidationError(
|
|
210
|
+
`Invalid URL in ${fileName}: ${error.message}`,
|
|
211
|
+
"ref"
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
if (!parsed.type) {
|
|
215
|
+
if (logger) {
|
|
216
|
+
logger.warn(`${fileName}: Missing 'type' field (optional but recommended)`);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (parsed.type && OCR_PROCESSABLE_TYPES.includes(parsed.type)) {
|
|
220
|
+
const typeToExt = {
|
|
221
|
+
"image/jpeg": ".jpg",
|
|
222
|
+
"image/png": ".png",
|
|
223
|
+
"image/webp": ".webp"
|
|
224
|
+
};
|
|
225
|
+
const expectedExt = typeToExt[parsed.type];
|
|
226
|
+
if (expectedExt && !fileName.includes(`${expectedExt}.ref.json`)) {
|
|
227
|
+
if (logger) {
|
|
228
|
+
logger.warn(
|
|
229
|
+
`${fileName}: Type is '${parsed.type}' but filename doesn't include '${expectedExt}.ref.json' pattern. This file may not be processed by OCR. Consider renaming to include the extension (e.g., 'photo${expectedExt}.ref.json').`
|
|
230
|
+
);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
function formatBytes(bytes) {
|
|
236
|
+
if (bytes === 0) return "0 B";
|
|
237
|
+
const k = 1024;
|
|
238
|
+
const sizes = ["B", "KB", "MB", "GB", "TB"];
|
|
239
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
240
|
+
return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`;
|
|
241
|
+
}
|
|
242
|
+
function validateCustomPrompts(prompts) {
|
|
243
|
+
if (!prompts) return;
|
|
244
|
+
const MAX_LENGTH = 5e4;
|
|
245
|
+
const MAX_TOTAL_LENGTH = 75e3;
|
|
246
|
+
const fields = [
|
|
247
|
+
"general",
|
|
248
|
+
"reorganization",
|
|
249
|
+
"pinax",
|
|
250
|
+
"description",
|
|
251
|
+
"cheimarros"
|
|
252
|
+
];
|
|
253
|
+
let totalLength = 0;
|
|
254
|
+
for (const field of fields) {
|
|
255
|
+
const value = prompts[field];
|
|
256
|
+
if (value) {
|
|
257
|
+
if (value.length > MAX_LENGTH) {
|
|
258
|
+
throw new ValidationError(
|
|
259
|
+
`Custom prompt '${field}' exceeds maximum length of ${MAX_LENGTH} characters (current: ${value.length})`,
|
|
260
|
+
"customPrompts"
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
totalLength += value.length;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if (totalLength > MAX_TOTAL_LENGTH) {
|
|
267
|
+
throw new ValidationError(
|
|
268
|
+
`Total custom prompts length (${totalLength}) exceeds maximum of ${MAX_TOTAL_LENGTH} characters`,
|
|
269
|
+
"customPrompts"
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
function validateCustomPromptsLocation(processingConfig) {
|
|
274
|
+
if (!processingConfig) return;
|
|
275
|
+
if ("customPrompts" in processingConfig) {
|
|
276
|
+
throw new ValidationError(
|
|
277
|
+
"customPrompts must be a top-level field in UploaderConfig, not inside the processing config. Use: new ArkeUploader({ customPrompts: {...}, processing: {...} }) NOT: new ArkeUploader({ processing: { customPrompts: {...} } })",
|
|
278
|
+
"processing"
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
var MAX_FILE_SIZE, MAX_BATCH_SIZE, INVALID_PATH_CHARS, OCR_PROCESSABLE_TYPES;
|
|
283
|
+
var init_validation = __esm({
|
|
284
|
+
"src/upload/lib/validation.ts"() {
|
|
285
|
+
"use strict";
|
|
286
|
+
init_errors();
|
|
287
|
+
MAX_FILE_SIZE = 5 * 1024 * 1024 * 1024;
|
|
288
|
+
MAX_BATCH_SIZE = 100 * 1024 * 1024 * 1024;
|
|
289
|
+
INVALID_PATH_CHARS = /[<>:"|?*\x00-\x1f]/;
|
|
290
|
+
OCR_PROCESSABLE_TYPES = [
|
|
291
|
+
"image/jpeg",
|
|
292
|
+
"image/png",
|
|
293
|
+
"image/webp"
|
|
294
|
+
];
|
|
295
|
+
}
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
// src/upload/utils/hash.ts
|
|
299
|
+
import { CID } from "multiformats/cid";
|
|
300
|
+
import * as raw from "multiformats/codecs/raw";
|
|
301
|
+
import { sha256 } from "multiformats/hashes/sha2";
|
|
302
|
+
async function computeFileCID(filePath) {
|
|
303
|
+
const fs2 = await import("fs/promises");
|
|
304
|
+
try {
|
|
305
|
+
const fileBuffer = await fs2.readFile(filePath);
|
|
306
|
+
const hash = await sha256.digest(fileBuffer);
|
|
307
|
+
const cid = CID.create(1, raw.code, hash);
|
|
308
|
+
return cid.toString();
|
|
309
|
+
} catch (error) {
|
|
310
|
+
throw new Error(`CID computation failed: ${error.message}`);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
async function computeCIDFromBuffer(data) {
|
|
314
|
+
const hash = await sha256.digest(data);
|
|
315
|
+
const cid = CID.create(1, raw.code, hash);
|
|
316
|
+
return cid.toString();
|
|
317
|
+
}
|
|
318
|
+
var init_hash = __esm({
|
|
319
|
+
"src/upload/utils/hash.ts"() {
|
|
320
|
+
"use strict";
|
|
321
|
+
}
|
|
322
|
+
});
|
|
323
|
+
|
|
324
|
+
// src/upload/types/processing.ts
|
|
325
|
+
var DEFAULT_PROCESSING_CONFIG;
|
|
326
|
+
var init_processing = __esm({
|
|
327
|
+
"src/upload/types/processing.ts"() {
|
|
328
|
+
"use strict";
|
|
329
|
+
DEFAULT_PROCESSING_CONFIG = {
|
|
330
|
+
ocr: true,
|
|
331
|
+
describe: true,
|
|
332
|
+
pinax: true
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
// src/upload/platforms/node.ts
|
|
338
|
+
var node_exports = {};
|
|
339
|
+
__export(node_exports, {
|
|
340
|
+
NodeScanner: () => NodeScanner
|
|
341
|
+
});
|
|
342
|
+
import fs from "fs/promises";
|
|
343
|
+
import path from "path";
|
|
344
|
+
var NodeScanner;
|
|
345
|
+
var init_node = __esm({
|
|
346
|
+
"src/upload/platforms/node.ts"() {
|
|
347
|
+
"use strict";
|
|
348
|
+
init_errors();
|
|
349
|
+
init_validation();
|
|
350
|
+
init_hash();
|
|
351
|
+
init_processing();
|
|
352
|
+
init_common();
|
|
353
|
+
NodeScanner = class {
|
|
354
|
+
/**
|
|
355
|
+
* Scan directory recursively and collect file metadata
|
|
356
|
+
*/
|
|
357
|
+
async scanFiles(source, options) {
|
|
358
|
+
const dirPath = Array.isArray(source) ? source[0] : source;
|
|
359
|
+
if (!dirPath || typeof dirPath !== "string") {
|
|
360
|
+
throw new ScanError("Node.js scanner requires a directory path", "");
|
|
361
|
+
}
|
|
362
|
+
const files = [];
|
|
363
|
+
try {
|
|
364
|
+
const stats = await fs.stat(dirPath);
|
|
365
|
+
if (!stats.isDirectory()) {
|
|
366
|
+
throw new ScanError(`Path is not a directory: ${dirPath}`, dirPath);
|
|
367
|
+
}
|
|
368
|
+
} catch (error) {
|
|
369
|
+
if (error.code === "ENOENT") {
|
|
370
|
+
throw new ScanError(`Directory not found: ${dirPath}`, dirPath);
|
|
371
|
+
}
|
|
372
|
+
throw new ScanError(`Cannot access directory: ${error.message}`, dirPath);
|
|
373
|
+
}
|
|
374
|
+
validateLogicalPath(options.rootPath);
|
|
375
|
+
const globalProcessingConfig = options.defaultProcessingConfig || DEFAULT_PROCESSING_CONFIG;
|
|
376
|
+
async function loadDirectoryProcessingConfig(dirPath2) {
|
|
377
|
+
const configPath = path.join(dirPath2, ".arke-process.json");
|
|
378
|
+
try {
|
|
379
|
+
const content = await fs.readFile(configPath, "utf-8");
|
|
380
|
+
return JSON.parse(content);
|
|
381
|
+
} catch (error) {
|
|
382
|
+
if (error.code !== "ENOENT") {
|
|
383
|
+
console.warn(`Error reading processing config ${configPath}: ${error.message}`);
|
|
384
|
+
}
|
|
385
|
+
return null;
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
function mergeProcessingConfig(defaults, override) {
|
|
389
|
+
if (!override) return defaults;
|
|
390
|
+
return {
|
|
391
|
+
ocr: override.ocr ?? defaults.ocr,
|
|
392
|
+
describe: override.describe ?? defaults.describe,
|
|
393
|
+
pinax: override.pinax ?? defaults.pinax
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
async function walk(currentPath, relativePath = "") {
|
|
397
|
+
const dirConfigOverride = await loadDirectoryProcessingConfig(currentPath);
|
|
398
|
+
const currentProcessingConfig = mergeProcessingConfig(
|
|
399
|
+
globalProcessingConfig,
|
|
400
|
+
dirConfigOverride
|
|
401
|
+
);
|
|
402
|
+
let entries;
|
|
403
|
+
try {
|
|
404
|
+
entries = await fs.readdir(currentPath, { withFileTypes: true });
|
|
405
|
+
} catch (error) {
|
|
406
|
+
console.warn(`Cannot read directory: ${currentPath}`, error.message);
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
for (const entry of entries) {
|
|
410
|
+
const fullPath = path.join(currentPath, entry.name);
|
|
411
|
+
const relPath = path.join(relativePath, entry.name);
|
|
412
|
+
try {
|
|
413
|
+
if (entry.isSymbolicLink()) {
|
|
414
|
+
if (!options.followSymlinks) {
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
const stats = await fs.stat(fullPath);
|
|
418
|
+
if (stats.isDirectory()) {
|
|
419
|
+
await walk(fullPath, relPath);
|
|
420
|
+
} else if (stats.isFile()) {
|
|
421
|
+
await processFile(fullPath, relPath, stats.size, currentProcessingConfig);
|
|
422
|
+
}
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
if (entry.isDirectory()) {
|
|
426
|
+
await walk(fullPath, relPath);
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
if (entry.isFile()) {
|
|
430
|
+
const stats = await fs.stat(fullPath);
|
|
431
|
+
await processFile(fullPath, relPath, stats.size, currentProcessingConfig);
|
|
432
|
+
}
|
|
433
|
+
} catch (error) {
|
|
434
|
+
if (error instanceof ScanError && error.message.includes(".ref.json")) {
|
|
435
|
+
throw error;
|
|
436
|
+
}
|
|
437
|
+
console.warn(`Error processing ${fullPath}: ${error.message}`);
|
|
438
|
+
continue;
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
async function processFile(fullPath, relativePath, size, processingConfig) {
|
|
443
|
+
const fileName = path.basename(fullPath);
|
|
444
|
+
if (fileName === ".arke-process.json") {
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
if (fileName.endsWith(".ref.json")) {
|
|
448
|
+
try {
|
|
449
|
+
const content = await fs.readFile(fullPath, "utf-8");
|
|
450
|
+
validateRefJson(content, fileName, console);
|
|
451
|
+
} catch (error) {
|
|
452
|
+
throw new ScanError(
|
|
453
|
+
`Invalid .ref.json file: ${fileName} - ${error.message}`,
|
|
454
|
+
fullPath
|
|
455
|
+
);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
try {
|
|
459
|
+
validateFileSize(size);
|
|
460
|
+
} catch (error) {
|
|
461
|
+
console.warn(`Skipping file that exceeds size limit: ${fileName}`, error.message);
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
const normalizedRelPath = normalizePath(relativePath);
|
|
465
|
+
const logicalPath = path.posix.join(options.rootPath, normalizedRelPath);
|
|
466
|
+
try {
|
|
467
|
+
validateLogicalPath(logicalPath);
|
|
468
|
+
} catch (error) {
|
|
469
|
+
console.warn(`Skipping file with invalid logical path: ${logicalPath}`, error.message);
|
|
470
|
+
return;
|
|
471
|
+
}
|
|
472
|
+
const contentType = getMimeType(fileName);
|
|
473
|
+
try {
|
|
474
|
+
await fs.access(fullPath, fs.constants.R_OK);
|
|
475
|
+
} catch (error) {
|
|
476
|
+
console.warn(`Skipping unreadable file: ${fullPath}`);
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
let cid;
|
|
480
|
+
try {
|
|
481
|
+
cid = await computeFileCID(fullPath);
|
|
482
|
+
} catch (error) {
|
|
483
|
+
console.warn(`Warning: CID computation failed for ${fullPath}, continuing without CID:`, error.message);
|
|
484
|
+
cid = void 0;
|
|
485
|
+
}
|
|
486
|
+
files.push({
|
|
487
|
+
localPath: fullPath,
|
|
488
|
+
logicalPath,
|
|
489
|
+
fileName,
|
|
490
|
+
size,
|
|
491
|
+
contentType,
|
|
492
|
+
cid,
|
|
493
|
+
processingConfig
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
await walk(dirPath);
|
|
497
|
+
files.sort((a, b) => a.size - b.size);
|
|
498
|
+
return files;
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* Read file contents as ArrayBuffer
|
|
502
|
+
*/
|
|
503
|
+
async readFile(file) {
|
|
504
|
+
const buffer = await fs.readFile(file.localPath);
|
|
505
|
+
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
|
|
506
|
+
}
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
// src/upload/platforms/browser.ts
|
|
512
|
+
var browser_exports = {};
|
|
513
|
+
__export(browser_exports, {
|
|
514
|
+
BrowserScanner: () => BrowserScanner
|
|
515
|
+
});
|
|
516
|
+
var BrowserScanner;
|
|
517
|
+
var init_browser = __esm({
|
|
518
|
+
"src/upload/platforms/browser.ts"() {
|
|
519
|
+
"use strict";
|
|
520
|
+
init_errors();
|
|
521
|
+
init_validation();
|
|
522
|
+
init_hash();
|
|
523
|
+
init_processing();
|
|
524
|
+
init_common();
|
|
525
|
+
BrowserScanner = class {
|
|
526
|
+
/**
|
|
527
|
+
* Scan files from File or FileList
|
|
528
|
+
*/
|
|
529
|
+
async scanFiles(source, options) {
|
|
530
|
+
const fileList = Array.isArray(source) ? source : [source];
|
|
531
|
+
if (fileList.length === 0) {
|
|
532
|
+
throw new ScanError("No files provided", "");
|
|
533
|
+
}
|
|
534
|
+
validateLogicalPath(options.rootPath);
|
|
535
|
+
const globalProcessingConfig = options.defaultProcessingConfig || DEFAULT_PROCESSING_CONFIG;
|
|
536
|
+
const files = [];
|
|
537
|
+
for (const file of fileList) {
|
|
538
|
+
try {
|
|
539
|
+
const fileInfo = await this.processFile(file, options.rootPath, globalProcessingConfig);
|
|
540
|
+
if (fileInfo) {
|
|
541
|
+
files.push(fileInfo);
|
|
542
|
+
}
|
|
543
|
+
} catch (error) {
|
|
544
|
+
console.warn(`Error processing ${file.name}: ${error.message}`);
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
files.sort((a, b) => a.size - b.size);
|
|
549
|
+
return files;
|
|
550
|
+
}
|
|
551
|
+
/**
|
|
552
|
+
* Process a single File object
|
|
553
|
+
*/
|
|
554
|
+
async processFile(file, rootPath, processingConfig) {
|
|
555
|
+
const fileName = file.name;
|
|
556
|
+
const size = file.size;
|
|
557
|
+
if (fileName.startsWith(".")) {
|
|
558
|
+
return null;
|
|
559
|
+
}
|
|
560
|
+
const skipFiles = ["Thumbs.db", "desktop.ini", "__MACOSX"];
|
|
561
|
+
if (skipFiles.includes(fileName)) {
|
|
562
|
+
return null;
|
|
563
|
+
}
|
|
564
|
+
if (fileName === ".arke-process.json") {
|
|
565
|
+
return null;
|
|
566
|
+
}
|
|
567
|
+
try {
|
|
568
|
+
validateFileSize(size);
|
|
569
|
+
} catch (error) {
|
|
570
|
+
console.warn(`Skipping file that exceeds size limit: ${fileName}`, error.message);
|
|
571
|
+
return null;
|
|
572
|
+
}
|
|
573
|
+
let relativePath = "";
|
|
574
|
+
if ("webkitRelativePath" in file && file.webkitRelativePath) {
|
|
575
|
+
const parts = file.webkitRelativePath.split("/");
|
|
576
|
+
if (parts.length > 1) {
|
|
577
|
+
relativePath = parts.slice(1).join("/");
|
|
578
|
+
} else {
|
|
579
|
+
relativePath = fileName;
|
|
580
|
+
}
|
|
581
|
+
} else {
|
|
582
|
+
relativePath = fileName;
|
|
583
|
+
}
|
|
584
|
+
const normalizedRelPath = normalizePath(relativePath);
|
|
585
|
+
const logicalPath = `${rootPath}/${normalizedRelPath}`.replace(/\/+/g, "/");
|
|
586
|
+
try {
|
|
587
|
+
validateLogicalPath(logicalPath);
|
|
588
|
+
} catch (error) {
|
|
589
|
+
console.warn(`Skipping file with invalid logical path: ${logicalPath}`, error.message);
|
|
590
|
+
return null;
|
|
591
|
+
}
|
|
592
|
+
const contentType = file.type || getMimeType(fileName);
|
|
593
|
+
let cid;
|
|
594
|
+
try {
|
|
595
|
+
const buffer = await file.arrayBuffer();
|
|
596
|
+
cid = await computeCIDFromBuffer(new Uint8Array(buffer));
|
|
597
|
+
} catch (error) {
|
|
598
|
+
console.warn(`Warning: CID computation failed for ${fileName}, continuing without CID:`, error.message);
|
|
599
|
+
cid = void 0;
|
|
600
|
+
}
|
|
601
|
+
return {
|
|
602
|
+
localPath: `__browser_file__${fileName}`,
|
|
603
|
+
// Special marker for browser files
|
|
604
|
+
logicalPath,
|
|
605
|
+
fileName,
|
|
606
|
+
size,
|
|
607
|
+
contentType,
|
|
608
|
+
cid,
|
|
609
|
+
processingConfig
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
/**
|
|
613
|
+
* Read file contents as ArrayBuffer
|
|
614
|
+
* Note: In browser context, the File object should be passed directly
|
|
615
|
+
*/
|
|
616
|
+
async readFile(file) {
|
|
617
|
+
throw new Error("Browser scanner requires File objects to be provided directly during upload");
|
|
618
|
+
}
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
});
|
|
622
|
+
|
|
623
|
+
// src/upload/lib/worker-client-fetch.ts
|
|
624
|
+
init_errors();
|
|
625
|
+
|
|
626
|
+
// src/upload/utils/retry.ts
|
|
627
|
+
init_errors();
|
|
628
|
+
var DEFAULT_OPTIONS = {
|
|
629
|
+
maxRetries: 3,
|
|
630
|
+
initialDelay: 1e3,
|
|
631
|
+
// 1 second
|
|
632
|
+
maxDelay: 3e4,
|
|
633
|
+
// 30 seconds
|
|
634
|
+
shouldRetry: isRetryableError,
|
|
635
|
+
jitter: true
|
|
636
|
+
};
|
|
637
|
+
async function retryWithBackoff(fn, options = {}) {
|
|
638
|
+
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
639
|
+
let lastError;
|
|
640
|
+
for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
|
|
641
|
+
try {
|
|
642
|
+
return await fn();
|
|
643
|
+
} catch (error) {
|
|
644
|
+
lastError = error;
|
|
645
|
+
if (attempt >= opts.maxRetries) {
|
|
646
|
+
throw error;
|
|
647
|
+
}
|
|
648
|
+
if (opts.shouldRetry && !opts.shouldRetry(error)) {
|
|
649
|
+
throw error;
|
|
650
|
+
}
|
|
651
|
+
let delay;
|
|
652
|
+
if (error.statusCode === 429 && error.retryAfter) {
|
|
653
|
+
delay = Math.min(error.retryAfter * 1e3, opts.maxDelay);
|
|
654
|
+
} else {
|
|
655
|
+
delay = Math.min(
|
|
656
|
+
opts.initialDelay * Math.pow(2, attempt),
|
|
657
|
+
opts.maxDelay
|
|
658
|
+
);
|
|
659
|
+
}
|
|
660
|
+
if (opts.jitter) {
|
|
661
|
+
const jitterAmount = delay * 0.25;
|
|
662
|
+
delay = delay + (Math.random() * jitterAmount * 2 - jitterAmount);
|
|
663
|
+
}
|
|
664
|
+
await sleep(Math.floor(delay));
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
throw lastError;
|
|
668
|
+
}
|
|
669
|
+
function sleep(ms) {
|
|
670
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// src/upload/lib/worker-client-fetch.ts
|
|
674
|
+
var WorkerClient = class {
|
|
675
|
+
constructor(config) {
|
|
676
|
+
this.baseUrl = config.baseUrl.replace(/\/$/, "");
|
|
677
|
+
this.authToken = config.authToken;
|
|
678
|
+
this.timeout = config.timeout ?? 3e4;
|
|
679
|
+
this.maxRetries = config.maxRetries ?? 3;
|
|
680
|
+
this.retryInitialDelay = config.retryInitialDelay ?? 1e3;
|
|
681
|
+
this.retryMaxDelay = config.retryMaxDelay ?? 3e4;
|
|
682
|
+
this.retryJitter = config.retryJitter ?? true;
|
|
683
|
+
this.debug = config.debug ?? false;
|
|
684
|
+
}
|
|
685
|
+
setAuthToken(token) {
|
|
686
|
+
this.authToken = token;
|
|
687
|
+
}
|
|
688
|
+
/**
|
|
689
|
+
* Make HTTP request with fetch
|
|
690
|
+
*/
|
|
691
|
+
async request(method, path2, body) {
|
|
692
|
+
const url = `${this.baseUrl}${path2}`;
|
|
693
|
+
if (this.debug) {
|
|
694
|
+
console.log(`HTTP Request: ${method} ${url}`, body);
|
|
695
|
+
}
|
|
696
|
+
try {
|
|
697
|
+
const controller = new AbortController();
|
|
698
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
699
|
+
const headers = {
|
|
700
|
+
"Content-Type": "application/json"
|
|
701
|
+
};
|
|
702
|
+
if (this.authToken) {
|
|
703
|
+
headers["Authorization"] = `Bearer ${this.authToken}`;
|
|
704
|
+
}
|
|
705
|
+
const response = await fetch(url, {
|
|
706
|
+
method,
|
|
707
|
+
headers,
|
|
708
|
+
body: body ? JSON.stringify(body) : void 0,
|
|
709
|
+
signal: controller.signal
|
|
710
|
+
});
|
|
711
|
+
clearTimeout(timeoutId);
|
|
712
|
+
const data = await response.json();
|
|
713
|
+
if (this.debug) {
|
|
714
|
+
console.log(`HTTP Response: ${response.status}`, data);
|
|
715
|
+
}
|
|
716
|
+
if (!response.ok) {
|
|
717
|
+
const errorData = data;
|
|
718
|
+
throw new WorkerAPIError(
|
|
719
|
+
errorData.error || "Request failed",
|
|
720
|
+
response.status,
|
|
721
|
+
errorData.details
|
|
722
|
+
);
|
|
723
|
+
}
|
|
724
|
+
return data;
|
|
725
|
+
} catch (error) {
|
|
726
|
+
if (error instanceof WorkerAPIError) {
|
|
727
|
+
throw error;
|
|
728
|
+
}
|
|
729
|
+
if (error.name === "AbortError") {
|
|
730
|
+
throw new NetworkError(`Request timeout after ${this.timeout}ms`);
|
|
731
|
+
}
|
|
732
|
+
throw new NetworkError(`Network request failed: ${error.message}`);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
/**
|
|
736
|
+
* Initialize a new batch upload
|
|
737
|
+
*/
|
|
738
|
+
async initBatch(params) {
|
|
739
|
+
return retryWithBackoff(
|
|
740
|
+
() => this.request("POST", "/ingest/batches/init", params),
|
|
741
|
+
{
|
|
742
|
+
maxRetries: this.maxRetries,
|
|
743
|
+
initialDelay: this.retryInitialDelay,
|
|
744
|
+
maxDelay: this.retryMaxDelay,
|
|
745
|
+
jitter: this.retryJitter
|
|
746
|
+
}
|
|
747
|
+
);
|
|
748
|
+
}
|
|
749
|
+
/**
|
|
750
|
+
* Request presigned URLs for a file upload
|
|
751
|
+
*/
|
|
752
|
+
async startFileUpload(batchId, params) {
|
|
753
|
+
return retryWithBackoff(
|
|
754
|
+
() => this.request(
|
|
755
|
+
"POST",
|
|
756
|
+
`/ingest/batches/${batchId}/files/start`,
|
|
757
|
+
params
|
|
758
|
+
),
|
|
759
|
+
{
|
|
760
|
+
maxRetries: this.maxRetries,
|
|
761
|
+
initialDelay: this.retryInitialDelay,
|
|
762
|
+
maxDelay: this.retryMaxDelay,
|
|
763
|
+
jitter: this.retryJitter
|
|
764
|
+
}
|
|
765
|
+
);
|
|
766
|
+
}
|
|
767
|
+
/**
|
|
768
|
+
* Mark a file upload as complete
|
|
769
|
+
*/
|
|
770
|
+
async completeFileUpload(batchId, params) {
|
|
771
|
+
return retryWithBackoff(
|
|
772
|
+
() => this.request(
|
|
773
|
+
"POST",
|
|
774
|
+
`/ingest/batches/${batchId}/files/complete`,
|
|
775
|
+
params
|
|
776
|
+
),
|
|
777
|
+
{
|
|
778
|
+
maxRetries: this.maxRetries,
|
|
779
|
+
initialDelay: this.retryInitialDelay,
|
|
780
|
+
maxDelay: this.retryMaxDelay,
|
|
781
|
+
jitter: this.retryJitter
|
|
782
|
+
}
|
|
783
|
+
);
|
|
784
|
+
}
|
|
785
|
+
/**
|
|
786
|
+
* Finalize the batch after all files are uploaded
|
|
787
|
+
* Returns root_pi immediately for small batches, or status='discovery' for large batches
|
|
788
|
+
*/
|
|
789
|
+
async finalizeBatch(batchId) {
|
|
790
|
+
return retryWithBackoff(
|
|
791
|
+
() => this.request(
|
|
792
|
+
"POST",
|
|
793
|
+
`/ingest/batches/${batchId}/finalize`,
|
|
794
|
+
{}
|
|
795
|
+
),
|
|
796
|
+
{
|
|
797
|
+
maxRetries: this.maxRetries,
|
|
798
|
+
initialDelay: this.retryInitialDelay,
|
|
799
|
+
maxDelay: this.retryMaxDelay,
|
|
800
|
+
jitter: this.retryJitter
|
|
801
|
+
}
|
|
802
|
+
);
|
|
803
|
+
}
|
|
804
|
+
/**
|
|
805
|
+
* Get current batch status (used for polling during async discovery)
|
|
806
|
+
*/
|
|
807
|
+
async getBatchStatus(batchId) {
|
|
808
|
+
return retryWithBackoff(
|
|
809
|
+
() => this.request(
|
|
810
|
+
"GET",
|
|
811
|
+
`/ingest/batches/${batchId}/status`
|
|
812
|
+
),
|
|
813
|
+
{
|
|
814
|
+
maxRetries: this.maxRetries,
|
|
815
|
+
initialDelay: this.retryInitialDelay,
|
|
816
|
+
maxDelay: this.retryMaxDelay,
|
|
817
|
+
jitter: this.retryJitter
|
|
818
|
+
}
|
|
819
|
+
);
|
|
820
|
+
}
|
|
821
|
+
};
|
|
822
|
+
|
|
823
|
+
// src/upload/uploader.ts
|
|
824
|
+
init_common();
|
|
825
|
+
|
|
826
|
+
// src/upload/lib/simple-fetch.ts
|
|
827
|
+
init_errors();
|
|
828
|
+
async function uploadSimple(fileData, presignedUrl, contentType, options = {}) {
|
|
829
|
+
const { maxRetries = 3, retryInitialDelay, retryMaxDelay, retryJitter } = options;
|
|
830
|
+
await retryWithBackoff(
|
|
831
|
+
async () => {
|
|
832
|
+
let response;
|
|
833
|
+
try {
|
|
834
|
+
response = await fetch(presignedUrl, {
|
|
835
|
+
method: "PUT",
|
|
836
|
+
body: fileData,
|
|
837
|
+
headers: {
|
|
838
|
+
...contentType ? { "Content-Type": contentType } : {}
|
|
839
|
+
}
|
|
840
|
+
});
|
|
841
|
+
} catch (error) {
|
|
842
|
+
throw new UploadError(`Upload failed: ${error.message}`, void 0, void 0, error);
|
|
843
|
+
}
|
|
844
|
+
if (!response.ok) {
|
|
845
|
+
const retryAfter = response.headers.get("retry-after");
|
|
846
|
+
const error = new UploadError(
|
|
847
|
+
`Upload failed with status ${response.status}: ${response.statusText}`,
|
|
848
|
+
void 0,
|
|
849
|
+
response.status
|
|
850
|
+
);
|
|
851
|
+
if (retryAfter && response.status === 429) {
|
|
852
|
+
error.retryAfter = parseInt(retryAfter, 10);
|
|
853
|
+
}
|
|
854
|
+
throw error;
|
|
855
|
+
}
|
|
856
|
+
},
|
|
857
|
+
{
|
|
858
|
+
maxRetries,
|
|
859
|
+
initialDelay: retryInitialDelay,
|
|
860
|
+
maxDelay: retryMaxDelay,
|
|
861
|
+
jitter: retryJitter
|
|
862
|
+
}
|
|
863
|
+
);
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
// src/upload/lib/multipart-fetch.ts
|
|
867
|
+
init_errors();
|
|
868
|
+
var DEFAULT_PART_SIZE = 10 * 1024 * 1024;
|
|
869
|
+
async function uploadMultipart(fileData, presignedUrls, concurrency = 3, options = {}) {
|
|
870
|
+
const totalSize = fileData.byteLength;
|
|
871
|
+
const partSize = Math.ceil(totalSize / presignedUrls.length);
|
|
872
|
+
const parts = [];
|
|
873
|
+
const queue = [];
|
|
874
|
+
const { maxRetries = 3, retryInitialDelay, retryMaxDelay, retryJitter } = options;
|
|
875
|
+
for (let i = 0; i < presignedUrls.length; i++) {
|
|
876
|
+
const partNumber = i + 1;
|
|
877
|
+
const start = i * partSize;
|
|
878
|
+
const end = Math.min(start + partSize, totalSize);
|
|
879
|
+
const partData = fileData.slice(start, end);
|
|
880
|
+
const url = presignedUrls[i];
|
|
881
|
+
queue.push(async () => {
|
|
882
|
+
const etag = await uploadPart(partData, url, partNumber, maxRetries, {
|
|
883
|
+
initialDelay: retryInitialDelay,
|
|
884
|
+
maxDelay: retryMaxDelay,
|
|
885
|
+
jitter: retryJitter
|
|
886
|
+
});
|
|
887
|
+
parts.push({ part_number: partNumber, etag });
|
|
888
|
+
});
|
|
889
|
+
}
|
|
890
|
+
await executeWithConcurrency(queue, concurrency);
|
|
891
|
+
parts.sort((a, b) => a.part_number - b.part_number);
|
|
892
|
+
return parts;
|
|
893
|
+
}
|
|
894
|
+
async function uploadPart(partData, presignedUrl, partNumber, maxRetries = 3, retryOptions = {}) {
|
|
895
|
+
return retryWithBackoff(
|
|
896
|
+
async () => {
|
|
897
|
+
let response;
|
|
898
|
+
try {
|
|
899
|
+
response = await fetch(presignedUrl, {
|
|
900
|
+
method: "PUT",
|
|
901
|
+
body: partData
|
|
902
|
+
});
|
|
903
|
+
} catch (error) {
|
|
904
|
+
throw new UploadError(
|
|
905
|
+
`Part ${partNumber} upload failed: ${error.message}`,
|
|
906
|
+
void 0,
|
|
907
|
+
void 0,
|
|
908
|
+
error
|
|
909
|
+
);
|
|
910
|
+
}
|
|
911
|
+
if (!response.ok) {
|
|
912
|
+
const retryAfter = response.headers.get("retry-after");
|
|
913
|
+
const error = new UploadError(
|
|
914
|
+
`Part ${partNumber} upload failed with status ${response.status}: ${response.statusText}`,
|
|
915
|
+
void 0,
|
|
916
|
+
response.status
|
|
917
|
+
);
|
|
918
|
+
if (retryAfter && response.status === 429) {
|
|
919
|
+
error.retryAfter = parseInt(retryAfter, 10);
|
|
920
|
+
}
|
|
921
|
+
throw error;
|
|
922
|
+
}
|
|
923
|
+
const etag = response.headers.get("etag");
|
|
924
|
+
if (!etag) {
|
|
925
|
+
throw new UploadError(
|
|
926
|
+
`Part ${partNumber} upload succeeded but no ETag returned`,
|
|
927
|
+
void 0,
|
|
928
|
+
response.status
|
|
929
|
+
);
|
|
930
|
+
}
|
|
931
|
+
return etag.replace(/"/g, "");
|
|
932
|
+
},
|
|
933
|
+
{
|
|
934
|
+
maxRetries,
|
|
935
|
+
initialDelay: retryOptions.initialDelay,
|
|
936
|
+
maxDelay: retryOptions.maxDelay,
|
|
937
|
+
jitter: retryOptions.jitter
|
|
938
|
+
}
|
|
939
|
+
);
|
|
940
|
+
}
|
|
941
|
+
async function executeWithConcurrency(tasks, concurrency) {
|
|
942
|
+
const queue = [...tasks];
|
|
943
|
+
const workers = [];
|
|
944
|
+
const processNext = async () => {
|
|
945
|
+
while (queue.length > 0) {
|
|
946
|
+
const task = queue.shift();
|
|
947
|
+
await task();
|
|
948
|
+
}
|
|
949
|
+
};
|
|
950
|
+
for (let i = 0; i < Math.min(concurrency, tasks.length); i++) {
|
|
951
|
+
workers.push(processNext());
|
|
952
|
+
}
|
|
953
|
+
await Promise.all(workers);
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
// src/upload/uploader.ts
|
|
957
|
+
init_errors();
|
|
958
|
+
init_validation();
|
|
959
|
+
var MULTIPART_THRESHOLD = 5 * 1024 * 1024;
|
|
960
|
+
var ArkeUploader = class {
|
|
961
|
+
constructor(config) {
|
|
962
|
+
this.scanner = null;
|
|
963
|
+
validateCustomPromptsLocation(config.processing);
|
|
964
|
+
this.config = {
|
|
965
|
+
rootPath: "/uploads",
|
|
966
|
+
// Must have at least one segment (not just '/')
|
|
967
|
+
parallelUploads: 5,
|
|
968
|
+
parallelParts: 3,
|
|
969
|
+
...config
|
|
970
|
+
};
|
|
971
|
+
this.workerClient = new WorkerClient({
|
|
972
|
+
baseUrl: config.gatewayUrl,
|
|
973
|
+
authToken: config.authToken,
|
|
974
|
+
timeout: config.timeout,
|
|
975
|
+
maxRetries: config.maxRetries,
|
|
976
|
+
retryInitialDelay: config.retryInitialDelay,
|
|
977
|
+
retryMaxDelay: config.retryMaxDelay,
|
|
978
|
+
retryJitter: config.retryJitter,
|
|
979
|
+
debug: false
|
|
980
|
+
});
|
|
981
|
+
this.platform = detectPlatform();
|
|
982
|
+
}
|
|
983
|
+
/**
|
|
984
|
+
* Get platform-specific scanner
|
|
985
|
+
*/
|
|
986
|
+
async getScanner() {
|
|
987
|
+
if (this.scanner) {
|
|
988
|
+
return this.scanner;
|
|
989
|
+
}
|
|
990
|
+
if (this.platform === "node") {
|
|
991
|
+
const { NodeScanner: NodeScanner2 } = await Promise.resolve().then(() => (init_node(), node_exports));
|
|
992
|
+
this.scanner = new NodeScanner2();
|
|
993
|
+
} else if (this.platform === "browser") {
|
|
994
|
+
const { BrowserScanner: BrowserScanner2 } = await Promise.resolve().then(() => (init_browser(), browser_exports));
|
|
995
|
+
this.scanner = new BrowserScanner2();
|
|
996
|
+
} else {
|
|
997
|
+
throw new ValidationError("Unsupported platform");
|
|
998
|
+
}
|
|
999
|
+
return this.scanner;
|
|
1000
|
+
}
|
|
1001
|
+
/**
|
|
1002
|
+
* Upload a batch of files
|
|
1003
|
+
* @param source - Directory path (Node.js) or File[]/FileList (browser)
|
|
1004
|
+
* @param options - Upload options
|
|
1005
|
+
*/
|
|
1006
|
+
async uploadBatch(source, options = {}) {
|
|
1007
|
+
const startTime = Date.now();
|
|
1008
|
+
const { onProgress, dryRun = false } = options;
|
|
1009
|
+
this.reportProgress(onProgress, {
|
|
1010
|
+
phase: "scanning",
|
|
1011
|
+
filesTotal: 0,
|
|
1012
|
+
filesUploaded: 0,
|
|
1013
|
+
bytesTotal: 0,
|
|
1014
|
+
bytesUploaded: 0,
|
|
1015
|
+
percentComplete: 0
|
|
1016
|
+
});
|
|
1017
|
+
const scanner = await this.getScanner();
|
|
1018
|
+
const files = await scanner.scanFiles(source, {
|
|
1019
|
+
rootPath: this.config.rootPath || "/",
|
|
1020
|
+
followSymlinks: true,
|
|
1021
|
+
defaultProcessingConfig: this.config.processing
|
|
1022
|
+
});
|
|
1023
|
+
if (files.length === 0) {
|
|
1024
|
+
throw new ValidationError("No files found to upload");
|
|
1025
|
+
}
|
|
1026
|
+
const totalSize = files.reduce((sum, f) => sum + f.size, 0);
|
|
1027
|
+
validateBatchSize(totalSize);
|
|
1028
|
+
if (this.config.customPrompts) {
|
|
1029
|
+
validateCustomPrompts(this.config.customPrompts);
|
|
1030
|
+
const promptFields = Object.keys(this.config.customPrompts).filter(
|
|
1031
|
+
(key) => this.config.customPrompts[key]
|
|
1032
|
+
);
|
|
1033
|
+
console.log(`[Arke Upload SDK] Custom prompts configured: ${promptFields.join(", ")}`);
|
|
1034
|
+
}
|
|
1035
|
+
if (dryRun) {
|
|
1036
|
+
return {
|
|
1037
|
+
batchId: "dry-run",
|
|
1038
|
+
rootPi: "dry-run",
|
|
1039
|
+
filesUploaded: files.length,
|
|
1040
|
+
bytesUploaded: totalSize,
|
|
1041
|
+
durationMs: Date.now() - startTime
|
|
1042
|
+
};
|
|
1043
|
+
}
|
|
1044
|
+
const { batch_id } = await this.workerClient.initBatch({
|
|
1045
|
+
uploader: this.config.uploader,
|
|
1046
|
+
root_path: this.config.rootPath || "/",
|
|
1047
|
+
parent_pi: this.config.parentPi || "",
|
|
1048
|
+
metadata: this.config.metadata,
|
|
1049
|
+
file_count: files.length,
|
|
1050
|
+
total_size: totalSize,
|
|
1051
|
+
custom_prompts: this.config.customPrompts
|
|
1052
|
+
});
|
|
1053
|
+
if (this.config.customPrompts) {
|
|
1054
|
+
console.log(`[Arke Upload SDK] Custom prompts sent to worker for batch ${batch_id}`);
|
|
1055
|
+
}
|
|
1056
|
+
this.reportProgress(onProgress, {
|
|
1057
|
+
phase: "uploading",
|
|
1058
|
+
filesTotal: files.length,
|
|
1059
|
+
filesUploaded: 0,
|
|
1060
|
+
bytesTotal: totalSize,
|
|
1061
|
+
bytesUploaded: 0,
|
|
1062
|
+
percentComplete: 0
|
|
1063
|
+
});
|
|
1064
|
+
let filesUploaded = 0;
|
|
1065
|
+
let bytesUploaded = 0;
|
|
1066
|
+
const { failedFiles } = await this.uploadFilesWithConcurrency(
|
|
1067
|
+
batch_id,
|
|
1068
|
+
files,
|
|
1069
|
+
source,
|
|
1070
|
+
this.config.parallelUploads || 5,
|
|
1071
|
+
(file, bytes) => {
|
|
1072
|
+
filesUploaded++;
|
|
1073
|
+
bytesUploaded += bytes;
|
|
1074
|
+
this.reportProgress(onProgress, {
|
|
1075
|
+
phase: "uploading",
|
|
1076
|
+
filesTotal: files.length,
|
|
1077
|
+
filesUploaded,
|
|
1078
|
+
bytesTotal: totalSize,
|
|
1079
|
+
bytesUploaded,
|
|
1080
|
+
currentFile: file.fileName,
|
|
1081
|
+
percentComplete: Math.round(bytesUploaded / totalSize * 100)
|
|
1082
|
+
});
|
|
1083
|
+
}
|
|
1084
|
+
);
|
|
1085
|
+
if (failedFiles.length === files.length) {
|
|
1086
|
+
throw new ValidationError(
|
|
1087
|
+
`All ${files.length} files failed to upload. First error: ${failedFiles[0]?.error || "Unknown"}`
|
|
1088
|
+
);
|
|
1089
|
+
}
|
|
1090
|
+
if (failedFiles.length > 0) {
|
|
1091
|
+
console.warn(
|
|
1092
|
+
`Warning: ${failedFiles.length} of ${files.length} files failed to upload:`,
|
|
1093
|
+
failedFiles.map((f) => `${f.file.fileName}: ${f.error}`).join(", ")
|
|
1094
|
+
);
|
|
1095
|
+
}
|
|
1096
|
+
this.reportProgress(onProgress, {
|
|
1097
|
+
phase: "finalizing",
|
|
1098
|
+
filesTotal: files.length,
|
|
1099
|
+
filesUploaded,
|
|
1100
|
+
bytesTotal: totalSize,
|
|
1101
|
+
bytesUploaded,
|
|
1102
|
+
percentComplete: 95
|
|
1103
|
+
});
|
|
1104
|
+
const finalizeResult = await this.workerClient.finalizeBatch(batch_id);
|
|
1105
|
+
let rootPi;
|
|
1106
|
+
if (finalizeResult.root_pi) {
|
|
1107
|
+
rootPi = finalizeResult.root_pi;
|
|
1108
|
+
} else if (finalizeResult.status === "discovery") {
|
|
1109
|
+
this.reportProgress(onProgress, {
|
|
1110
|
+
phase: "discovery",
|
|
1111
|
+
filesTotal: files.length,
|
|
1112
|
+
filesUploaded,
|
|
1113
|
+
bytesTotal: totalSize,
|
|
1114
|
+
bytesUploaded,
|
|
1115
|
+
percentComplete: 97
|
|
1116
|
+
});
|
|
1117
|
+
rootPi = await this.pollForRootPi(batch_id, onProgress, files.length, totalSize, bytesUploaded);
|
|
1118
|
+
} else {
|
|
1119
|
+
throw new ValidationError(
|
|
1120
|
+
`Finalization returned unexpected status: ${finalizeResult.status} without root_pi`
|
|
1121
|
+
);
|
|
1122
|
+
}
|
|
1123
|
+
this.reportProgress(onProgress, {
|
|
1124
|
+
phase: "complete",
|
|
1125
|
+
filesTotal: files.length,
|
|
1126
|
+
filesUploaded,
|
|
1127
|
+
bytesTotal: totalSize,
|
|
1128
|
+
bytesUploaded,
|
|
1129
|
+
percentComplete: 100
|
|
1130
|
+
});
|
|
1131
|
+
return {
|
|
1132
|
+
batchId: batch_id,
|
|
1133
|
+
rootPi,
|
|
1134
|
+
filesUploaded,
|
|
1135
|
+
bytesUploaded,
|
|
1136
|
+
durationMs: Date.now() - startTime
|
|
1137
|
+
};
|
|
1138
|
+
}
|
|
1139
|
+
/**
|
|
1140
|
+
* Poll for root_pi during async discovery
|
|
1141
|
+
*/
|
|
1142
|
+
async pollForRootPi(batchId, onProgress, filesTotal, bytesTotal, bytesUploaded) {
|
|
1143
|
+
const POLL_INTERVAL_MS = 2e3;
|
|
1144
|
+
const MAX_POLL_TIME_MS = 30 * 60 * 1e3;
|
|
1145
|
+
const startTime = Date.now();
|
|
1146
|
+
while (Date.now() - startTime < MAX_POLL_TIME_MS) {
|
|
1147
|
+
const status = await this.workerClient.getBatchStatus(batchId);
|
|
1148
|
+
if (status.root_pi) {
|
|
1149
|
+
return status.root_pi;
|
|
1150
|
+
}
|
|
1151
|
+
if (status.status === "failed") {
|
|
1152
|
+
throw new ValidationError(`Batch discovery failed`);
|
|
1153
|
+
}
|
|
1154
|
+
if (status.discovery_progress && onProgress) {
|
|
1155
|
+
const { total, published } = status.discovery_progress;
|
|
1156
|
+
const discoveryPercent = total > 0 ? Math.round(published / total * 100) : 0;
|
|
1157
|
+
this.reportProgress(onProgress, {
|
|
1158
|
+
phase: "discovery",
|
|
1159
|
+
filesTotal,
|
|
1160
|
+
filesUploaded: filesTotal,
|
|
1161
|
+
bytesTotal,
|
|
1162
|
+
bytesUploaded,
|
|
1163
|
+
percentComplete: 95 + Math.round(discoveryPercent * 0.04)
|
|
1164
|
+
// 95-99%
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
|
1168
|
+
}
|
|
1169
|
+
throw new ValidationError(`Discovery timed out after ${MAX_POLL_TIME_MS / 1e3} seconds`);
|
|
1170
|
+
}
|
|
1171
|
+
/**
|
|
1172
|
+
* Upload files with controlled concurrency
|
|
1173
|
+
*/
|
|
1174
|
+
async uploadFilesWithConcurrency(batchId, files, source, concurrency, onFileComplete) {
|
|
1175
|
+
const queue = [...files];
|
|
1176
|
+
const workers = [];
|
|
1177
|
+
const failedFiles = [];
|
|
1178
|
+
const processNext = async () => {
|
|
1179
|
+
while (queue.length > 0) {
|
|
1180
|
+
const file = queue.shift();
|
|
1181
|
+
try {
|
|
1182
|
+
await this.uploadSingleFile(batchId, file, source);
|
|
1183
|
+
onFileComplete(file, file.size);
|
|
1184
|
+
} catch (error) {
|
|
1185
|
+
const errorMessage = error.message || "Unknown error";
|
|
1186
|
+
console.error(`Failed to upload ${file.fileName}: ${errorMessage}`);
|
|
1187
|
+
failedFiles.push({ file, error: errorMessage });
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
};
|
|
1191
|
+
for (let i = 0; i < Math.min(concurrency, files.length); i++) {
|
|
1192
|
+
workers.push(processNext());
|
|
1193
|
+
}
|
|
1194
|
+
await Promise.all(workers);
|
|
1195
|
+
return { failedFiles };
|
|
1196
|
+
}
|
|
1197
|
+
/**
|
|
1198
|
+
* Upload a single file
|
|
1199
|
+
*/
|
|
1200
|
+
async uploadSingleFile(batchId, file, source) {
|
|
1201
|
+
const uploadInfo = await this.workerClient.startFileUpload(batchId, {
|
|
1202
|
+
file_name: file.fileName,
|
|
1203
|
+
file_size: file.size,
|
|
1204
|
+
logical_path: file.logicalPath,
|
|
1205
|
+
content_type: file.contentType,
|
|
1206
|
+
cid: file.cid,
|
|
1207
|
+
processing_config: file.processingConfig
|
|
1208
|
+
});
|
|
1209
|
+
const fileData = await this.getFileData(file, source);
|
|
1210
|
+
const retryOptions = {
|
|
1211
|
+
maxRetries: this.config.maxRetries,
|
|
1212
|
+
retryInitialDelay: this.config.retryInitialDelay,
|
|
1213
|
+
retryMaxDelay: this.config.retryMaxDelay,
|
|
1214
|
+
retryJitter: this.config.retryJitter
|
|
1215
|
+
};
|
|
1216
|
+
if (uploadInfo.upload_type === "simple") {
|
|
1217
|
+
await uploadSimple(fileData, uploadInfo.presigned_url, file.contentType, retryOptions);
|
|
1218
|
+
} else {
|
|
1219
|
+
const partUrls = uploadInfo.presigned_urls.map((p) => p.url);
|
|
1220
|
+
const parts = await uploadMultipart(
|
|
1221
|
+
fileData,
|
|
1222
|
+
partUrls,
|
|
1223
|
+
this.config.parallelParts || 3,
|
|
1224
|
+
retryOptions
|
|
1225
|
+
);
|
|
1226
|
+
await this.workerClient.completeFileUpload(batchId, {
|
|
1227
|
+
r2_key: uploadInfo.r2_key,
|
|
1228
|
+
upload_id: uploadInfo.upload_id,
|
|
1229
|
+
parts
|
|
1230
|
+
});
|
|
1231
|
+
return;
|
|
1232
|
+
}
|
|
1233
|
+
await this.workerClient.completeFileUpload(batchId, {
|
|
1234
|
+
r2_key: uploadInfo.r2_key
|
|
1235
|
+
});
|
|
1236
|
+
}
|
|
1237
|
+
/**
|
|
1238
|
+
* Get file data based on platform
|
|
1239
|
+
*/
|
|
1240
|
+
async getFileData(file, source) {
|
|
1241
|
+
if (this.platform === "node") {
|
|
1242
|
+
const fs2 = await import("fs/promises");
|
|
1243
|
+
const buffer = await fs2.readFile(file.localPath);
|
|
1244
|
+
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
|
|
1245
|
+
} else if (this.platform === "browser") {
|
|
1246
|
+
const files = Array.isArray(source) ? source : [source];
|
|
1247
|
+
const browserFile = files.find(
|
|
1248
|
+
(f) => f instanceof File && f.name === file.fileName
|
|
1249
|
+
);
|
|
1250
|
+
if (!browserFile) {
|
|
1251
|
+
throw new Error(`Could not find browser File object for ${file.fileName}`);
|
|
1252
|
+
}
|
|
1253
|
+
return browserFile.arrayBuffer();
|
|
1254
|
+
}
|
|
1255
|
+
throw new Error("Unsupported platform for file reading");
|
|
1256
|
+
}
|
|
1257
|
+
/**
|
|
1258
|
+
* Report progress to callback
|
|
1259
|
+
*/
|
|
1260
|
+
reportProgress(callback, progress) {
|
|
1261
|
+
if (callback) {
|
|
1262
|
+
callback(progress);
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
};
|
|
1266
|
+
|
|
1267
|
+
// src/collections/errors.ts
|
|
1268
|
+
var CollectionsError = class extends Error {
|
|
1269
|
+
constructor(message, code2 = "UNKNOWN_ERROR", details) {
|
|
1270
|
+
super(message);
|
|
1271
|
+
this.code = code2;
|
|
1272
|
+
this.details = details;
|
|
1273
|
+
this.name = "CollectionsError";
|
|
1274
|
+
}
|
|
1275
|
+
};
|
|
1276
|
+
|
|
1277
|
+
// src/collections/client.ts
|
|
1278
|
+
var CollectionsClient = class {
|
|
1279
|
+
constructor(config) {
|
|
1280
|
+
this.baseUrl = config.gatewayUrl.replace(/\/$/, "");
|
|
1281
|
+
this.authToken = config.authToken;
|
|
1282
|
+
this.fetchImpl = config.fetchImpl ?? fetch;
|
|
1283
|
+
}
|
|
1284
|
+
setAuthToken(token) {
|
|
1285
|
+
this.authToken = token;
|
|
1286
|
+
}
|
|
1287
|
+
// ---------------------------------------------------------------------------
|
|
1288
|
+
// Request helpers
|
|
1289
|
+
// ---------------------------------------------------------------------------
|
|
1290
|
+
buildUrl(path2, query) {
|
|
1291
|
+
const url = new URL(`${this.baseUrl}${path2}`);
|
|
1292
|
+
if (query) {
|
|
1293
|
+
Object.entries(query).forEach(([key, value]) => {
|
|
1294
|
+
if (value !== void 0 && value !== null) {
|
|
1295
|
+
url.searchParams.set(key, String(value));
|
|
1296
|
+
}
|
|
1297
|
+
});
|
|
1298
|
+
}
|
|
1299
|
+
return url.toString();
|
|
1300
|
+
}
|
|
1301
|
+
getHeaders(authRequired) {
|
|
1302
|
+
const headers = { "Content-Type": "application/json" };
|
|
1303
|
+
if (authRequired || this.authToken) {
|
|
1304
|
+
if (!this.authToken && authRequired) {
|
|
1305
|
+
throw new CollectionsError("Authentication required for this operation", "AUTH_REQUIRED");
|
|
1306
|
+
}
|
|
1307
|
+
if (this.authToken) {
|
|
1308
|
+
headers["Authorization"] = `Bearer ${this.authToken}`;
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
return headers;
|
|
1312
|
+
}
|
|
1313
|
+
async request(path2, options = {}) {
|
|
1314
|
+
const authRequired = options.authRequired ?? false;
|
|
1315
|
+
const url = this.buildUrl(path2, options.query);
|
|
1316
|
+
const headers = new Headers(this.getHeaders(authRequired));
|
|
1317
|
+
if (options.headers) {
|
|
1318
|
+
Object.entries(options.headers).forEach(([k, v]) => {
|
|
1319
|
+
if (v !== void 0) headers.set(k, v);
|
|
1320
|
+
});
|
|
1321
|
+
}
|
|
1322
|
+
const response = await this.fetchImpl(url, { ...options, headers });
|
|
1323
|
+
if (response.ok) {
|
|
1324
|
+
if (response.status === 204) {
|
|
1325
|
+
return void 0;
|
|
1326
|
+
}
|
|
1327
|
+
const contentType = response.headers.get("content-type") || "";
|
|
1328
|
+
if (contentType.includes("application/json")) {
|
|
1329
|
+
return await response.json();
|
|
1330
|
+
}
|
|
1331
|
+
return await response.text();
|
|
1332
|
+
}
|
|
1333
|
+
let body;
|
|
1334
|
+
const text = await response.text();
|
|
1335
|
+
try {
|
|
1336
|
+
body = JSON.parse(text);
|
|
1337
|
+
} catch {
|
|
1338
|
+
body = text;
|
|
1339
|
+
}
|
|
1340
|
+
const message = body?.error && typeof body.error === "string" ? body.error : `Request failed with status ${response.status}`;
|
|
1341
|
+
throw new CollectionsError(message, "HTTP_ERROR", {
|
|
1342
|
+
status: response.status,
|
|
1343
|
+
body
|
|
1344
|
+
});
|
|
1345
|
+
}
|
|
1346
|
+
// ---------------------------------------------------------------------------
|
|
1347
|
+
// Collections
|
|
1348
|
+
// ---------------------------------------------------------------------------
|
|
1349
|
+
async listCollections(params) {
|
|
1350
|
+
return this.request("/collections", {
|
|
1351
|
+
method: "GET",
|
|
1352
|
+
query: { limit: params?.limit, offset: params?.offset }
|
|
1353
|
+
});
|
|
1354
|
+
}
|
|
1355
|
+
async getCollection(id) {
|
|
1356
|
+
return this.request(`/collections/${id}`, { method: "GET" });
|
|
1357
|
+
}
|
|
1358
|
+
async getCollectionRoot(id) {
|
|
1359
|
+
return this.request(`/collections/${id}/root`, { method: "GET" });
|
|
1360
|
+
}
|
|
1361
|
+
async getMyAccess(id) {
|
|
1362
|
+
return this.request(`/collections/${id}/my-access`, { method: "GET", authRequired: true });
|
|
1363
|
+
}
|
|
1364
|
+
async createCollection(payload) {
|
|
1365
|
+
return this.request("/collections", {
|
|
1366
|
+
method: "POST",
|
|
1367
|
+
authRequired: true,
|
|
1368
|
+
body: JSON.stringify(payload)
|
|
1369
|
+
});
|
|
1370
|
+
}
|
|
1371
|
+
async registerRoot(payload) {
|
|
1372
|
+
return this.request("/collections/register-root", {
|
|
1373
|
+
method: "POST",
|
|
1374
|
+
authRequired: true,
|
|
1375
|
+
body: JSON.stringify(payload)
|
|
1376
|
+
});
|
|
1377
|
+
}
|
|
1378
|
+
async updateCollection(id, payload) {
|
|
1379
|
+
return this.request(`/collections/${id}`, {
|
|
1380
|
+
method: "PATCH",
|
|
1381
|
+
authRequired: true,
|
|
1382
|
+
body: JSON.stringify(payload)
|
|
1383
|
+
});
|
|
1384
|
+
}
|
|
1385
|
+
async changeRoot(id, payload) {
|
|
1386
|
+
return this.request(`/collections/${id}/change-root`, {
|
|
1387
|
+
method: "PATCH",
|
|
1388
|
+
authRequired: true,
|
|
1389
|
+
body: JSON.stringify(payload)
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
async deleteCollection(id) {
|
|
1393
|
+
return this.request(`/collections/${id}`, {
|
|
1394
|
+
method: "DELETE",
|
|
1395
|
+
authRequired: true
|
|
1396
|
+
});
|
|
1397
|
+
}
|
|
1398
|
+
// ---------------------------------------------------------------------------
|
|
1399
|
+
// Members
|
|
1400
|
+
// ---------------------------------------------------------------------------
|
|
1401
|
+
async listMembers(collectionId) {
|
|
1402
|
+
return this.request(`/collections/${collectionId}/members`, { method: "GET" });
|
|
1403
|
+
}
|
|
1404
|
+
async updateMemberRole(collectionId, userId, role) {
|
|
1405
|
+
return this.request(`/collections/${collectionId}/members/${userId}`, {
|
|
1406
|
+
method: "PATCH",
|
|
1407
|
+
authRequired: true,
|
|
1408
|
+
body: JSON.stringify({ role })
|
|
1409
|
+
});
|
|
1410
|
+
}
|
|
1411
|
+
async removeMember(collectionId, userId) {
|
|
1412
|
+
return this.request(`/collections/${collectionId}/members/${userId}`, {
|
|
1413
|
+
method: "DELETE",
|
|
1414
|
+
authRequired: true
|
|
1415
|
+
});
|
|
1416
|
+
}
|
|
1417
|
+
// ---------------------------------------------------------------------------
|
|
1418
|
+
// Invitations
|
|
1419
|
+
// ---------------------------------------------------------------------------
|
|
1420
|
+
async createInvitation(collectionId, email, role) {
|
|
1421
|
+
return this.request(`/collections/${collectionId}/invitations`, {
|
|
1422
|
+
method: "POST",
|
|
1423
|
+
authRequired: true,
|
|
1424
|
+
body: JSON.stringify({ email, role })
|
|
1425
|
+
});
|
|
1426
|
+
}
|
|
1427
|
+
async listInvitations(collectionId) {
|
|
1428
|
+
return this.request(`/collections/${collectionId}/invitations`, {
|
|
1429
|
+
method: "GET",
|
|
1430
|
+
authRequired: true
|
|
1431
|
+
});
|
|
1432
|
+
}
|
|
1433
|
+
async acceptInvitation(invitationId) {
|
|
1434
|
+
return this.request(`/invitations/${invitationId}/accept`, {
|
|
1435
|
+
method: "POST",
|
|
1436
|
+
authRequired: true
|
|
1437
|
+
});
|
|
1438
|
+
}
|
|
1439
|
+
async declineInvitation(invitationId) {
|
|
1440
|
+
return this.request(`/invitations/${invitationId}/decline`, {
|
|
1441
|
+
method: "POST",
|
|
1442
|
+
authRequired: true
|
|
1443
|
+
});
|
|
1444
|
+
}
|
|
1445
|
+
async revokeInvitation(invitationId) {
|
|
1446
|
+
return this.request(`/invitations/${invitationId}`, {
|
|
1447
|
+
method: "DELETE",
|
|
1448
|
+
authRequired: true
|
|
1449
|
+
});
|
|
1450
|
+
}
|
|
1451
|
+
// ---------------------------------------------------------------------------
|
|
1452
|
+
// Current user
|
|
1453
|
+
// ---------------------------------------------------------------------------
|
|
1454
|
+
async getMyCollections() {
|
|
1455
|
+
return this.request("/me/collections", { method: "GET", authRequired: true });
|
|
1456
|
+
}
|
|
1457
|
+
async getMyInvitations() {
|
|
1458
|
+
return this.request("/me/invitations", { method: "GET", authRequired: true });
|
|
1459
|
+
}
|
|
1460
|
+
// ---------------------------------------------------------------------------
|
|
1461
|
+
// PI permissions
|
|
1462
|
+
// ---------------------------------------------------------------------------
|
|
1463
|
+
async getPiPermissions(pi) {
|
|
1464
|
+
return this.request(`/pi/${pi}/permissions`, { method: "GET" });
|
|
1465
|
+
}
|
|
1466
|
+
};
|
|
1467
|
+
|
|
1468
|
+
// src/upload/client.ts
|
|
1469
|
+
function getUserIdFromToken(token) {
|
|
1470
|
+
try {
|
|
1471
|
+
const parts = token.split(".");
|
|
1472
|
+
if (parts.length !== 3) return null;
|
|
1473
|
+
const payload = parts[1].replace(/-/g, "+").replace(/_/g, "/");
|
|
1474
|
+
let decoded;
|
|
1475
|
+
if (typeof atob === "function") {
|
|
1476
|
+
decoded = atob(payload);
|
|
1477
|
+
} else {
|
|
1478
|
+
decoded = Buffer.from(payload, "base64").toString("utf-8");
|
|
1479
|
+
}
|
|
1480
|
+
const data = JSON.parse(decoded);
|
|
1481
|
+
return data.sub || null;
|
|
1482
|
+
} catch {
|
|
1483
|
+
return null;
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
var UploadClient = class {
|
|
1487
|
+
constructor(config) {
|
|
1488
|
+
const uploader = config.uploader || getUserIdFromToken(config.authToken) || "unknown";
|
|
1489
|
+
this.config = { ...config, uploader };
|
|
1490
|
+
this.collectionsClient = new CollectionsClient({
|
|
1491
|
+
gatewayUrl: config.gatewayUrl,
|
|
1492
|
+
authToken: config.authToken,
|
|
1493
|
+
fetchImpl: config.fetchImpl
|
|
1494
|
+
});
|
|
1495
|
+
}
|
|
1496
|
+
/**
|
|
1497
|
+
* Update the auth token (e.g., after token refresh)
|
|
1498
|
+
*/
|
|
1499
|
+
setAuthToken(token) {
|
|
1500
|
+
this.config = { ...this.config, authToken: token };
|
|
1501
|
+
this.collectionsClient.setAuthToken(token);
|
|
1502
|
+
}
|
|
1503
|
+
/**
|
|
1504
|
+
* Create a new collection and upload files to it
|
|
1505
|
+
*
|
|
1506
|
+
* Anyone authenticated can create a new collection.
|
|
1507
|
+
* The root PI of the uploaded files becomes the collection's root.
|
|
1508
|
+
*/
|
|
1509
|
+
async createCollection(options) {
|
|
1510
|
+
const { files, collectionMetadata, customPrompts, processing, onProgress, dryRun } = options;
|
|
1511
|
+
const metadata = {
|
|
1512
|
+
...collectionMetadata,
|
|
1513
|
+
visibility: collectionMetadata.visibility || "public"
|
|
1514
|
+
};
|
|
1515
|
+
const uploader = new ArkeUploader({
|
|
1516
|
+
gatewayUrl: this.config.gatewayUrl,
|
|
1517
|
+
authToken: this.config.authToken,
|
|
1518
|
+
uploader: this.config.uploader,
|
|
1519
|
+
customPrompts,
|
|
1520
|
+
processing
|
|
1521
|
+
});
|
|
1522
|
+
const batchResult = await uploader.uploadBatch(files, {
|
|
1523
|
+
onProgress,
|
|
1524
|
+
dryRun
|
|
1525
|
+
});
|
|
1526
|
+
if (dryRun) {
|
|
1527
|
+
return {
|
|
1528
|
+
...batchResult,
|
|
1529
|
+
collection: {
|
|
1530
|
+
id: "dry-run",
|
|
1531
|
+
title: metadata.title,
|
|
1532
|
+
slug: metadata.slug,
|
|
1533
|
+
description: metadata.description,
|
|
1534
|
+
visibility: metadata.visibility,
|
|
1535
|
+
rootPi: "dry-run"
|
|
1536
|
+
}
|
|
1537
|
+
};
|
|
1538
|
+
}
|
|
1539
|
+
const collection = await this.collectionsClient.registerRoot({
|
|
1540
|
+
...metadata,
|
|
1541
|
+
rootPi: batchResult.rootPi
|
|
1542
|
+
});
|
|
1543
|
+
return {
|
|
1544
|
+
...batchResult,
|
|
1545
|
+
collection
|
|
1546
|
+
};
|
|
1547
|
+
}
|
|
1548
|
+
/**
|
|
1549
|
+
* Add files to an existing collection
|
|
1550
|
+
*
|
|
1551
|
+
* Requires owner or editor role on the collection containing the parent PI.
|
|
1552
|
+
* Use this to add a folder or files to an existing collection hierarchy.
|
|
1553
|
+
*
|
|
1554
|
+
* Note: Permission checks are enforced server-side by the ingest worker.
|
|
1555
|
+
* The server will return 403 if the user lacks edit access to the parent PI.
|
|
1556
|
+
*/
|
|
1557
|
+
async addToCollection(options) {
|
|
1558
|
+
const { files, parentPi, customPrompts, processing, onProgress, dryRun } = options;
|
|
1559
|
+
const uploader = new ArkeUploader({
|
|
1560
|
+
gatewayUrl: this.config.gatewayUrl,
|
|
1561
|
+
authToken: this.config.authToken,
|
|
1562
|
+
uploader: this.config.uploader,
|
|
1563
|
+
parentPi,
|
|
1564
|
+
customPrompts,
|
|
1565
|
+
processing
|
|
1566
|
+
});
|
|
1567
|
+
return uploader.uploadBatch(files, {
|
|
1568
|
+
onProgress,
|
|
1569
|
+
dryRun
|
|
1570
|
+
});
|
|
1571
|
+
}
|
|
1572
|
+
/**
|
|
1573
|
+
* Check if you can edit a specific PI (i.e., add files to its collection)
|
|
1574
|
+
*/
|
|
1575
|
+
async canEdit(pi) {
|
|
1576
|
+
return this.collectionsClient.getPiPermissions(pi);
|
|
1577
|
+
}
|
|
1578
|
+
/**
|
|
1579
|
+
* Get access to the underlying collections client for other operations
|
|
1580
|
+
*/
|
|
1581
|
+
get collections() {
|
|
1582
|
+
return this.collectionsClient;
|
|
1583
|
+
}
|
|
1584
|
+
};
|
|
1585
|
+
|
|
1586
|
+
// src/upload/index.ts
|
|
1587
|
+
init_errors();
|
|
1588
|
+
export {
|
|
1589
|
+
ArkeUploader,
|
|
1590
|
+
NetworkError,
|
|
1591
|
+
ScanError,
|
|
1592
|
+
UploadClient,
|
|
1593
|
+
UploadError,
|
|
1594
|
+
ValidationError,
|
|
1595
|
+
WorkerAPIError
|
|
1596
|
+
};
|
|
1597
|
+
//# sourceMappingURL=index.js.map
|