@arke-institute/sdk 0.1.0 → 0.1.1

package/dist/index.js

@@ -1,35 +1,630 @@
-"use strict";
 var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
 var __export = (target, all) => {
   for (var name in all)
     __defProp(target, name, { get: all[name], enumerable: true });
 };
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+
+// src/upload/utils/errors.ts
+function isRetryableError(error) {
+  if (error instanceof NetworkError) {
+    return true;
   }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+  if (error instanceof WorkerAPIError) {
+    return error.statusCode ? error.statusCode >= 500 : false;
+  }
+  if (error instanceof UploadError) {
+    if (error.statusCode) {
+      return error.statusCode >= 500 || error.statusCode === 429;
+    }
+    return false;
+  }
+  if (error.code === "ECONNRESET" || error.code === "ETIMEDOUT" || error.code === "ENOTFOUND" || error.code === "ECONNREFUSED") {
+    return true;
+  }
+  return false;
+}
+var WorkerAPIError, UploadError, ValidationError, NetworkError, ScanError;
+var init_errors = __esm({
+  "src/upload/utils/errors.ts"() {
+    "use strict";
+    WorkerAPIError = class extends Error {
+      constructor(message, statusCode, details) {
+        super(message);
+        this.statusCode = statusCode;
+        this.details = details;
+        this.name = "WorkerAPIError";
+        Error.captureStackTrace(this, this.constructor);
+      }
+    };
+    UploadError = class extends Error {
+      constructor(message, fileName, statusCode, cause) {
+        super(message);
+        this.fileName = fileName;
+        this.statusCode = statusCode;
+        this.cause = cause;
+        this.name = "UploadError";
+        Error.captureStackTrace(this, this.constructor);
+      }
+    };
+    ValidationError = class extends Error {
+      constructor(message, field) {
+        super(message);
+        this.field = field;
+        this.name = "ValidationError";
+        Error.captureStackTrace(this, this.constructor);
+      }
+    };
+    NetworkError = class extends Error {
+      constructor(message, cause) {
+        super(message);
+        this.cause = cause;
+        this.name = "NetworkError";
+        Error.captureStackTrace(this, this.constructor);
+      }
+    };
+    ScanError = class extends Error {
+      constructor(message, path2) {
+        super(message);
+        this.path = path2;
+        this.name = "ScanError";
+        Error.captureStackTrace(this, this.constructor);
+      }
+    };
+  }
+});
 
-// src/index.ts
-var src_exports = {};
-__export(src_exports, {
-  CollectionsClient: () => CollectionsClient,
-  CollectionsError: () => CollectionsError
+// src/upload/platforms/common.ts
+function detectPlatform() {
+  if (typeof process !== "undefined" && process.versions != null && process.versions.node != null) {
+    return "node";
+  }
+  if (typeof window !== "undefined" && typeof document !== "undefined") {
+    return "browser";
+  }
+  return "unknown";
+}
+function normalizePath(p) {
+  return p.replace(/\\/g, "/");
+}
+function getExtension(filename) {
+  const lastDot = filename.lastIndexOf(".");
+  return lastDot === -1 ? "" : filename.slice(lastDot + 1).toLowerCase();
+}
+function getMimeType(filename) {
+  const ext = getExtension(filename);
+  const mimeTypes = {
+    // Images
+    "jpg": "image/jpeg",
+    "jpeg": "image/jpeg",
+    "png": "image/png",
+    "gif": "image/gif",
+    "webp": "image/webp",
+    "tif": "image/tiff",
+    "tiff": "image/tiff",
+    "bmp": "image/bmp",
+    "svg": "image/svg+xml",
+    // Documents
+    "pdf": "application/pdf",
+    "txt": "text/plain",
+    "json": "application/json",
+    "xml": "application/xml",
+    "html": "text/html",
+    "htm": "text/html",
+    "css": "text/css",
+    "js": "application/javascript",
+    // Archives
+    "zip": "application/zip",
+    "tar": "application/x-tar",
+    "gz": "application/gzip",
+    // Audio
+    "mp3": "audio/mpeg",
+    "wav": "audio/wav",
+    "ogg": "audio/ogg",
+    // Video
+    "mp4": "video/mp4",
+    "webm": "video/webm",
+    "mov": "video/quicktime"
+  };
+  return mimeTypes[ext] || "application/octet-stream";
+}
+var init_common = __esm({
+  "src/upload/platforms/common.ts"() {
+    "use strict";
+  }
+});
+
+// src/upload/lib/validation.ts
+function validateFileSize(size) {
+  if (size <= 0) {
+    throw new ValidationError("File size must be greater than 0");
+  }
+  if (size > MAX_FILE_SIZE) {
+    throw new ValidationError(
+      `File size (${formatBytes(size)}) exceeds maximum allowed size (${formatBytes(MAX_FILE_SIZE)})`
+    );
+  }
+}
+function validateBatchSize(totalSize) {
+  if (totalSize > MAX_BATCH_SIZE) {
+    throw new ValidationError(
+      `Total batch size (${formatBytes(totalSize)}) exceeds maximum allowed size (${formatBytes(MAX_BATCH_SIZE)})`
+    );
+  }
+}
+function validateLogicalPath(path2) {
+  if (!path2.startsWith("/")) {
+    throw new ValidationError("Logical path must start with /", "path");
+  }
+  if (INVALID_PATH_CHARS.test(path2)) {
+    throw new ValidationError(
+      "Logical path contains invalid characters",
+      "path"
+    );
+  }
+  const segments = path2.split("/").filter((s) => s.length > 0);
+  if (segments.length === 0 && path2 !== "/") {
+    throw new ValidationError("Logical path cannot be empty", "path");
+  }
+  for (const segment of segments) {
+    if (segment === "." || segment === "..") {
+      throw new ValidationError(
+        "Logical path cannot contain . or .. segments",
+        "path"
+      );
+    }
+  }
+}
+function validateRefJson(content, fileName, logger) {
+  let parsed;
+  try {
+    parsed = JSON.parse(content);
+  } catch (error) {
+    throw new ValidationError(
+      `Invalid JSON in ${fileName}: ${error.message}`,
+      "ref"
+    );
+  }
+  if (typeof parsed !== "object" || Array.isArray(parsed) || parsed === null) {
+    throw new ValidationError(
+      `${fileName} must contain a JSON object`,
+      "ref"
+    );
+  }
+  if (!parsed.url || typeof parsed.url !== "string") {
+    throw new ValidationError(
+      `${fileName} must contain a 'url' field with a string value`,
+      "ref"
+    );
+  }
+  try {
+    const url = new URL(parsed.url);
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+      throw new Error("URL must use HTTP or HTTPS protocol");
+    }
+  } catch (error) {
+    throw new ValidationError(
+      `Invalid URL in ${fileName}: ${error.message}`,
+      "ref"
+    );
+  }
+  if (!parsed.type) {
+    if (logger) {
+      logger.warn(`${fileName}: Missing 'type' field (optional but recommended)`);
+    }
+  }
+  if (parsed.type && OCR_PROCESSABLE_TYPES.includes(parsed.type)) {
+    const typeToExt = {
+      "image/jpeg": ".jpg",
+      "image/png": ".png",
+      "image/webp": ".webp"
+    };
+    const expectedExt = typeToExt[parsed.type];
+    if (expectedExt && !fileName.includes(`${expectedExt}.ref.json`)) {
+      if (logger) {
+        logger.warn(
+          `${fileName}: Type is '${parsed.type}' but filename doesn't include '${expectedExt}.ref.json' pattern. This file may not be processed by OCR. Consider renaming to include the extension (e.g., 'photo${expectedExt}.ref.json').`
+        );
+      }
+    }
+  }
+}
+function formatBytes(bytes) {
+  if (bytes === 0) return "0 B";
+  const k = 1024;
+  const sizes = ["B", "KB", "MB", "GB", "TB"];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+  return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`;
+}
+function validateCustomPrompts(prompts) {
+  if (!prompts) return;
+  const MAX_LENGTH = 5e4;
+  const MAX_TOTAL_LENGTH = 75e3;
+  const fields = [
+    "general",
+    "reorganization",
+    "pinax",
+    "description",
+    "cheimarros"
+  ];
+  let totalLength = 0;
+  for (const field of fields) {
+    const value = prompts[field];
+    if (value) {
+      if (value.length > MAX_LENGTH) {
+        throw new ValidationError(
+          `Custom prompt '${field}' exceeds maximum length of ${MAX_LENGTH} characters (current: ${value.length})`,
+          "customPrompts"
+        );
+      }
+      totalLength += value.length;
+    }
+  }
+  if (totalLength > MAX_TOTAL_LENGTH) {
+    throw new ValidationError(
+      `Total custom prompts length (${totalLength}) exceeds maximum of ${MAX_TOTAL_LENGTH} characters`,
+      "customPrompts"
+    );
+  }
+}
+function validateCustomPromptsLocation(processingConfig) {
+  if (!processingConfig) return;
+  if ("customPrompts" in processingConfig) {
+    throw new ValidationError(
+      "customPrompts must be a top-level field in UploaderConfig, not inside the processing config. Use: new ArkeUploader({ customPrompts: {...}, processing: {...} }) NOT: new ArkeUploader({ processing: { customPrompts: {...} } })",
+      "processing"
+    );
+  }
+}
+var MAX_FILE_SIZE, MAX_BATCH_SIZE, INVALID_PATH_CHARS, OCR_PROCESSABLE_TYPES;
+var init_validation = __esm({
+  "src/upload/lib/validation.ts"() {
+    "use strict";
+    init_errors();
+    MAX_FILE_SIZE = 5 * 1024 * 1024 * 1024;
+    MAX_BATCH_SIZE = 100 * 1024 * 1024 * 1024;
+    INVALID_PATH_CHARS = /[<>:"|?*\x00-\x1f]/;
+    OCR_PROCESSABLE_TYPES = [
+      "image/jpeg",
+      "image/png",
+      "image/webp"
+    ];
+  }
+});
+
+// src/upload/utils/hash.ts
+import { CID } from "multiformats/cid";
+import * as raw from "multiformats/codecs/raw";
+import { sha256 } from "multiformats/hashes/sha2";
+async function computeFileCID(filePath) {
+  const fs2 = await import("fs/promises");
+  try {
+    const fileBuffer = await fs2.readFile(filePath);
+    const hash = await sha256.digest(fileBuffer);
+    const cid = CID.create(1, raw.code, hash);
+    return cid.toString();
+  } catch (error) {
+    throw new Error(`CID computation failed: ${error.message}`);
+  }
+}
+async function computeCIDFromBuffer(data) {
+  const hash = await sha256.digest(data);
+  const cid = CID.create(1, raw.code, hash);
+  return cid.toString();
+}
+var init_hash = __esm({
+  "src/upload/utils/hash.ts"() {
+    "use strict";
+  }
+});
+
+// src/upload/types/processing.ts
+var DEFAULT_PROCESSING_CONFIG;
+var init_processing = __esm({
+  "src/upload/types/processing.ts"() {
+    "use strict";
+    DEFAULT_PROCESSING_CONFIG = {
+      ocr: true,
+      describe: true,
+      pinax: true
+    };
+  }
+});
+
+// src/upload/platforms/node.ts
+var node_exports = {};
+__export(node_exports, {
+  NodeScanner: () => NodeScanner
+});
+import fs from "fs/promises";
+import path from "path";
+var NodeScanner;
+var init_node = __esm({
+  "src/upload/platforms/node.ts"() {
+    "use strict";
+    init_errors();
+    init_validation();
+    init_hash();
+    init_processing();
+    init_common();
+    NodeScanner = class {
+      /**
+       * Scan directory recursively and collect file metadata
+       */
+      async scanFiles(source, options) {
+        const dirPath = Array.isArray(source) ? source[0] : source;
+        if (!dirPath || typeof dirPath !== "string") {
+          throw new ScanError("Node.js scanner requires a directory path", "");
+        }
+        const files = [];
+        try {
+          const stats = await fs.stat(dirPath);
+          if (!stats.isDirectory()) {
+            throw new ScanError(`Path is not a directory: ${dirPath}`, dirPath);
+          }
+        } catch (error) {
+          if (error.code === "ENOENT") {
+            throw new ScanError(`Directory not found: ${dirPath}`, dirPath);
+          }
+          throw new ScanError(`Cannot access directory: ${error.message}`, dirPath);
+        }
+        validateLogicalPath(options.rootPath);
+        const globalProcessingConfig = options.defaultProcessingConfig || DEFAULT_PROCESSING_CONFIG;
+        async function loadDirectoryProcessingConfig(dirPath2) {
+          const configPath = path.join(dirPath2, ".arke-process.json");
+          try {
+            const content = await fs.readFile(configPath, "utf-8");
+            return JSON.parse(content);
+          } catch (error) {
+            if (error.code !== "ENOENT") {
+              console.warn(`Error reading processing config ${configPath}: ${error.message}`);
+            }
+            return null;
+          }
+        }
+        function mergeProcessingConfig(defaults, override) {
+          if (!override) return defaults;
+          return {
+            ocr: override.ocr ?? defaults.ocr,
+            describe: override.describe ?? defaults.describe,
+            pinax: override.pinax ?? defaults.pinax
+          };
+        }
+        async function walk(currentPath, relativePath = "") {
+          const dirConfigOverride = await loadDirectoryProcessingConfig(currentPath);
+          const currentProcessingConfig = mergeProcessingConfig(
+            globalProcessingConfig,
+            dirConfigOverride
+          );
+          let entries;
+          try {
+            entries = await fs.readdir(currentPath, { withFileTypes: true });
+          } catch (error) {
+            console.warn(`Cannot read directory: ${currentPath}`, error.message);
+            return;
+          }
+          for (const entry of entries) {
+            const fullPath = path.join(currentPath, entry.name);
+            const relPath = path.join(relativePath, entry.name);
+            try {
+              if (entry.isSymbolicLink()) {
+                if (!options.followSymlinks) {
+                  continue;
+                }
+                const stats = await fs.stat(fullPath);
+                if (stats.isDirectory()) {
+                  await walk(fullPath, relPath);
+                } else if (stats.isFile()) {
+                  await processFile(fullPath, relPath, stats.size, currentProcessingConfig);
+                }
+                continue;
+              }
+              if (entry.isDirectory()) {
+                await walk(fullPath, relPath);
+                continue;
+              }
+              if (entry.isFile()) {
+                const stats = await fs.stat(fullPath);
+                await processFile(fullPath, relPath, stats.size, currentProcessingConfig);
+              }
+            } catch (error) {
+              if (error instanceof ScanError && error.message.includes(".ref.json")) {
+                throw error;
+              }
+              console.warn(`Error processing ${fullPath}: ${error.message}`);
+              continue;
+            }
+          }
+        }
+        async function processFile(fullPath, relativePath, size, processingConfig) {
+          const fileName = path.basename(fullPath);
+          if (fileName === ".arke-process.json") {
+            return;
+          }
+          if (fileName.endsWith(".ref.json")) {
+            try {
+              const content = await fs.readFile(fullPath, "utf-8");
+              validateRefJson(content, fileName, console);
+            } catch (error) {
+              throw new ScanError(
+                `Invalid .ref.json file: ${fileName} - ${error.message}`,
+                fullPath
+              );
+            }
+          }
+          try {
+            validateFileSize(size);
+          } catch (error) {
+            console.warn(`Skipping file that exceeds size limit: ${fileName}`, error.message);
+            return;
+          }
+          const normalizedRelPath = normalizePath(relativePath);
+          const logicalPath = path.posix.join(options.rootPath, normalizedRelPath);
+          try {
+            validateLogicalPath(logicalPath);
+          } catch (error) {
+            console.warn(`Skipping file with invalid logical path: ${logicalPath}`, error.message);
+            return;
+          }
+          const contentType = getMimeType(fileName);
+          try {
+            await fs.access(fullPath, fs.constants.R_OK);
+          } catch (error) {
+            console.warn(`Skipping unreadable file: ${fullPath}`);
+            return;
+          }
+          let cid;
+          try {
+            cid = await computeFileCID(fullPath);
+          } catch (error) {
+            console.warn(`Warning: CID computation failed for ${fullPath}, continuing without CID:`, error.message);
+            cid = void 0;
+          }
+          files.push({
+            localPath: fullPath,
+            logicalPath,
+            fileName,
+            size,
+            contentType,
+            cid,
+            processingConfig
+          });
+        }
+        await walk(dirPath);
+        files.sort((a, b) => a.size - b.size);
+        return files;
+      }
+      /**
+       * Read file contents as ArrayBuffer
+       */
+      async readFile(file) {
+        const buffer = await fs.readFile(file.localPath);
+        return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+      }
+    };
+  }
+});
+
+// src/upload/platforms/browser.ts
+var browser_exports = {};
+__export(browser_exports, {
+  BrowserScanner: () => BrowserScanner
+});
+var BrowserScanner;
+var init_browser = __esm({
+  "src/upload/platforms/browser.ts"() {
+    "use strict";
+    init_errors();
+    init_validation();
+    init_hash();
+    init_processing();
+    init_common();
+    BrowserScanner = class {
+      /**
+       * Scan files from File or FileList
+       */
+      async scanFiles(source, options) {
+        const fileList = Array.isArray(source) ? source : [source];
+        if (fileList.length === 0) {
+          throw new ScanError("No files provided", "");
+        }
+        validateLogicalPath(options.rootPath);
+        const globalProcessingConfig = options.defaultProcessingConfig || DEFAULT_PROCESSING_CONFIG;
+        const files = [];
+        for (const file of fileList) {
+          try {
+            const fileInfo = await this.processFile(file, options.rootPath, globalProcessingConfig);
+            if (fileInfo) {
+              files.push(fileInfo);
+            }
+          } catch (error) {
+            console.warn(`Error processing ${file.name}: ${error.message}`);
+            continue;
+          }
+        }
+        files.sort((a, b) => a.size - b.size);
+        return files;
+      }
+      /**
+       * Process a single File object
+       */
+      async processFile(file, rootPath, processingConfig) {
+        const fileName = file.name;
+        const size = file.size;
+        if (fileName.startsWith(".")) {
+          return null;
+        }
+        const skipFiles = ["Thumbs.db", "desktop.ini", "__MACOSX"];
+        if (skipFiles.includes(fileName)) {
+          return null;
+        }
+        if (fileName === ".arke-process.json") {
+          return null;
+        }
+        try {
+          validateFileSize(size);
+        } catch (error) {
+          console.warn(`Skipping file that exceeds size limit: ${fileName}`, error.message);
+          return null;
+        }
+        let relativePath = "";
+        if ("webkitRelativePath" in file && file.webkitRelativePath) {
+          const parts = file.webkitRelativePath.split("/");
+          if (parts.length > 1) {
+            relativePath = parts.slice(1).join("/");
+          } else {
+            relativePath = fileName;
+          }
+        } else {
+          relativePath = fileName;
+        }
+        const normalizedRelPath = normalizePath(relativePath);
+        const logicalPath = `${rootPath}/${normalizedRelPath}`.replace(/\/+/g, "/");
+        try {
+          validateLogicalPath(logicalPath);
+        } catch (error) {
+          console.warn(`Skipping file with invalid logical path: ${logicalPath}`, error.message);
+          return null;
+        }
+        const contentType = file.type || getMimeType(fileName);
+        let cid;
+        try {
+          const buffer = await file.arrayBuffer();
+          cid = await computeCIDFromBuffer(new Uint8Array(buffer));
+        } catch (error) {
+          console.warn(`Warning: CID computation failed for ${fileName}, continuing without CID:`, error.message);
+          cid = void 0;
+        }
+        return {
+          localPath: `__browser_file__${fileName}`,
+          // Special marker for browser files
+          logicalPath,
+          fileName,
+          size,
+          contentType,
+          cid,
+          processingConfig
+        };
+      }
+      /**
+       * Read file contents as ArrayBuffer
+       * Note: In browser context, the File object should be passed directly
+       */
+      async readFile(file) {
+        throw new Error("Browser scanner requires File objects to be provided directly during upload");
+      }
+    };
+  }
 });
-module.exports = __toCommonJS(src_exports);
 
 // src/collections/errors.ts
 var CollectionsError = class extends Error {
-  constructor(message, code = "UNKNOWN_ERROR", details) {
+  constructor(message, code2 = "UNKNOWN_ERROR", details) {
     super(message);
-    this.code = code;
+    this.code = code2;
     this.details = details;
     this.name = "CollectionsError";
   }
@@ -48,8 +643,8 @@ var CollectionsClient = class {
   // ---------------------------------------------------------------------------
   // Request helpers
   // ---------------------------------------------------------------------------
-  buildUrl(path, query) {
-    const url = new URL(`${this.baseUrl}${path}`);
+  buildUrl(path2, query) {
+    const url = new URL(`${this.baseUrl}${path2}`);
     if (query) {
       Object.entries(query).forEach(([key, value]) => {
         if (value !== void 0 && value !== null) {
@@ -71,9 +666,9 @@ var CollectionsClient = class {
     }
     return headers;
   }
-  async request(path, options = {}) {
+  async request(path2, options = {}) {
     const authRequired = options.authRequired ?? false;
-    const url = this.buildUrl(path, options.query);
+    const url = this.buildUrl(path2, options.query);
     const headers = new Headers(this.getHeaders(authRequired));
     if (options.headers) {
       Object.entries(options.headers).forEach(([k, v]) => {
@@ -92,10 +687,11 @@ var CollectionsClient = class {
       return await response.text();
     }
     let body;
+    const text = await response.text();
     try {
-      body = await response.json();
+      body = JSON.parse(text);
     } catch {
-      body = await response.text();
+      body = text;
     }
     const message = body?.error && typeof body.error === "string" ? body.error : `Request failed with status ${response.status}`;
     throw new CollectionsError(message, "HTTP_ERROR", {
@@ -224,9 +820,791 @@ var CollectionsClient = class {
     return this.request(`/pi/${pi}/permissions`, { method: "GET" });
   }
 };
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
+
+// src/upload/lib/worker-client-fetch.ts
+init_errors();
+
+// src/upload/utils/retry.ts
+init_errors();
+var DEFAULT_OPTIONS = {
+  maxRetries: 3,
+  initialDelay: 1e3,
+  // 1 second
+  maxDelay: 3e4,
+  // 30 seconds
+  shouldRetry: isRetryableError,
+  jitter: true
+};
+async function retryWithBackoff(fn, options = {}) {
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  let lastError;
+  for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      if (attempt >= opts.maxRetries) {
+        throw error;
+      }
+      if (opts.shouldRetry && !opts.shouldRetry(error)) {
+        throw error;
+      }
+      let delay;
+      if (error.statusCode === 429 && error.retryAfter) {
+        delay = Math.min(error.retryAfter * 1e3, opts.maxDelay);
+      } else {
+        delay = Math.min(
+          opts.initialDelay * Math.pow(2, attempt),
+          opts.maxDelay
+        );
+      }
+      if (opts.jitter) {
+        const jitterAmount = delay * 0.25;
+        delay = delay + (Math.random() * jitterAmount * 2 - jitterAmount);
+      }
+      await sleep(Math.floor(delay));
+    }
+  }
+  throw lastError;
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// src/upload/lib/worker-client-fetch.ts
+var WorkerClient = class {
+  constructor(config) {
+    this.baseUrl = config.baseUrl.replace(/\/$/, "");
+    this.authToken = config.authToken;
+    this.timeout = config.timeout ?? 3e4;
+    this.maxRetries = config.maxRetries ?? 3;
+    this.retryInitialDelay = config.retryInitialDelay ?? 1e3;
+    this.retryMaxDelay = config.retryMaxDelay ?? 3e4;
+    this.retryJitter = config.retryJitter ?? true;
+    this.debug = config.debug ?? false;
+  }
+  setAuthToken(token) {
+    this.authToken = token;
+  }
+  /**
+   * Make HTTP request with fetch
+   */
+  async request(method, path2, body) {
+    const url = `${this.baseUrl}${path2}`;
+    if (this.debug) {
+      console.log(`HTTP Request: ${method} ${url}`, body);
+    }
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+      const headers = {
+        "Content-Type": "application/json"
+      };
+      if (this.authToken) {
+        headers["Authorization"] = `Bearer ${this.authToken}`;
+      }
+      const response = await fetch(url, {
+        method,
+        headers,
+        body: body ? JSON.stringify(body) : void 0,
+        signal: controller.signal
+      });
+      clearTimeout(timeoutId);
+      const data = await response.json();
+      if (this.debug) {
+        console.log(`HTTP Response: ${response.status}`, data);
+      }
+      if (!response.ok) {
+        const errorData = data;
+        throw new WorkerAPIError(
+          errorData.error || "Request failed",
+          response.status,
+          errorData.details
+        );
+      }
+      return data;
+    } catch (error) {
+      if (error instanceof WorkerAPIError) {
+        throw error;
+      }
+      if (error.name === "AbortError") {
+        throw new NetworkError(`Request timeout after ${this.timeout}ms`);
+      }
+      throw new NetworkError(`Network request failed: ${error.message}`);
+    }
+  }
+  /**
+   * Initialize a new batch upload
+   */
+  async initBatch(params) {
+    return retryWithBackoff(
+      () => this.request("POST", "/ingest/batches/init", params),
+      {
+        maxRetries: this.maxRetries,
+        initialDelay: this.retryInitialDelay,
+        maxDelay: this.retryMaxDelay,
+        jitter: this.retryJitter
+      }
+    );
+  }
+  /**
+   * Request presigned URLs for a file upload
+   */
+  async startFileUpload(batchId, params) {
+    return retryWithBackoff(
+      () => this.request(
+        "POST",
+        `/ingest/batches/${batchId}/files/start`,
+        params
+      ),
+      {
+        maxRetries: this.maxRetries,
+        initialDelay: this.retryInitialDelay,
+        maxDelay: this.retryMaxDelay,
+        jitter: this.retryJitter
+      }
+    );
+  }
+  /**
+   * Mark a file upload as complete
+   */
+  async completeFileUpload(batchId, params) {
+    return retryWithBackoff(
+      () => this.request(
+        "POST",
+        `/ingest/batches/${batchId}/files/complete`,
+        params
+      ),
+      {
+        maxRetries: this.maxRetries,
+        initialDelay: this.retryInitialDelay,
+        maxDelay: this.retryMaxDelay,
+        jitter: this.retryJitter
+      }
+    );
+  }
+  /**
+   * Finalize the batch after all files are uploaded
+   * Returns root_pi immediately for small batches, or status='discovery' for large batches
+   */
+  async finalizeBatch(batchId) {
+    return retryWithBackoff(
+      () => this.request(
+        "POST",
+        `/ingest/batches/${batchId}/finalize`,
+        {}
+      ),
+      {
+        maxRetries: this.maxRetries,
+        initialDelay: this.retryInitialDelay,
+        maxDelay: this.retryMaxDelay,
+        jitter: this.retryJitter
+      }
+    );
+  }
+  /**
+   * Get current batch status (used for polling during async discovery)
+   */
+  async getBatchStatus(batchId) {
+    return retryWithBackoff(
+      () => this.request(
+        "GET",
+        `/ingest/batches/${batchId}/status`
+      ),
+      {
+        maxRetries: this.maxRetries,
+        initialDelay: this.retryInitialDelay,
+        maxDelay: this.retryMaxDelay,
+        jitter: this.retryJitter
+      }
+    );
+  }
+};
+
+// src/upload/uploader.ts
+init_common();
+
+// src/upload/lib/simple-fetch.ts
+init_errors();
+async function uploadSimple(fileData, presignedUrl, contentType, options = {}) {
+  const { maxRetries = 3, retryInitialDelay, retryMaxDelay, retryJitter } = options;
+  await retryWithBackoff(
+    async () => {
+      let response;
+      try {
+        response = await fetch(presignedUrl, {
+          method: "PUT",
+          body: fileData,
+          headers: {
+            ...contentType ? { "Content-Type": contentType } : {}
+          }
+        });
+      } catch (error) {
+        throw new UploadError(`Upload failed: ${error.message}`, void 0, void 0, error);
+      }
+      if (!response.ok) {
+        const retryAfter = response.headers.get("retry-after");
+        const error = new UploadError(
+          `Upload failed with status ${response.status}: ${response.statusText}`,
+          void 0,
+          response.status
+        );
+        if (retryAfter && response.status === 429) {
+          error.retryAfter = parseInt(retryAfter, 10);
+        }
+        throw error;
+      }
+    },
+    {
+      maxRetries,
+      initialDelay: retryInitialDelay,
+      maxDelay: retryMaxDelay,
+      jitter: retryJitter
+    }
+  );
+}
+
+// src/upload/lib/multipart-fetch.ts
+init_errors();
+var DEFAULT_PART_SIZE = 10 * 1024 * 1024;
+async function uploadMultipart(fileData, presignedUrls, concurrency = 3, options = {}) {
+  const totalSize = fileData.byteLength;
+  const partSize = Math.ceil(totalSize / presignedUrls.length);
+  const parts = [];
+  const queue = [];
+  const { maxRetries = 3, retryInitialDelay, retryMaxDelay, retryJitter } = options;
+  for (let i = 0; i < presignedUrls.length; i++) {
+    const partNumber = i + 1;
+    const start = i * partSize;
+    const end = Math.min(start + partSize, totalSize);
+    const partData = fileData.slice(start, end);
+    const url = presignedUrls[i];
+    queue.push(async () => {
+      const etag = await uploadPart(partData, url, partNumber, maxRetries, {
+        initialDelay: retryInitialDelay,
+        maxDelay: retryMaxDelay,
+        jitter: retryJitter
+      });
+      parts.push({ part_number: partNumber, etag });
+    });
+  }
+  await executeWithConcurrency(queue, concurrency);
+  parts.sort((a, b) => a.part_number - b.part_number);
+  return parts;
+}
+async function uploadPart(partData, presignedUrl, partNumber, maxRetries = 3, retryOptions = {}) {
+  return retryWithBackoff(
+    async () => {
+      let response;
+      try {
+        response = await fetch(presignedUrl, {
+          method: "PUT",
+          body: partData
+        });
+      } catch (error) {
+        throw new UploadError(
+          `Part ${partNumber} upload failed: ${error.message}`,
+          void 0,
+          void 0,
+          error
+        );
+      }
+      if (!response.ok) {
+        const retryAfter = response.headers.get("retry-after");
+        const error = new UploadError(
+          `Part ${partNumber} upload failed with status ${response.status}: ${response.statusText}`,
+          void 0,
+          response.status
+        );
+        if (retryAfter && response.status === 429) {
+          error.retryAfter = parseInt(retryAfter, 10);
+        }
+        throw error;
+      }
+      const etag = response.headers.get("etag");
+      if (!etag) {
+        throw new UploadError(
+          `Part ${partNumber} upload succeeded but no ETag returned`,
+          void 0,
+          response.status
+        );
+      }
+      return etag.replace(/"/g, "");
+    },
+    {
+      maxRetries,
+      initialDelay: retryOptions.initialDelay,
+      maxDelay: retryOptions.maxDelay,
+      jitter: retryOptions.jitter
+    }
+  );
+}
+async function executeWithConcurrency(tasks, concurrency) {
+  const queue = [...tasks];
+  const workers = [];
+  const processNext = async () => {
+    while (queue.length > 0) {
+      const task = queue.shift();
+      await task();
+    }
+  };
+  for (let i = 0; i < Math.min(concurrency, tasks.length); i++) {
+    workers.push(processNext());
+  }
+  await Promise.all(workers);
+}
+
+// src/upload/uploader.ts
+init_errors();
+init_validation();
+var MULTIPART_THRESHOLD = 5 * 1024 * 1024;
+var ArkeUploader = class {
+  constructor(config) {
+    this.scanner = null;
+    validateCustomPromptsLocation(config.processing);
+    this.config = {
+      rootPath: "/uploads",
+      // Must have at least one segment (not just '/')
+      parallelUploads: 5,
+      parallelParts: 3,
+      ...config
+    };
+    this.workerClient = new WorkerClient({
+      baseUrl: config.gatewayUrl,
+      authToken: config.authToken,
+      timeout: config.timeout,
+      maxRetries: config.maxRetries,
+      retryInitialDelay: config.retryInitialDelay,
+      retryMaxDelay: config.retryMaxDelay,
+      retryJitter: config.retryJitter,
+      debug: false
+    });
+    this.platform = detectPlatform();
+  }
+  /**
+   * Get platform-specific scanner
+   */
+  async getScanner() {
+    if (this.scanner) {
+      return this.scanner;
+    }
+    if (this.platform === "node") {
+      const { NodeScanner: NodeScanner2 } = await Promise.resolve().then(() => (init_node(), node_exports));
+      this.scanner = new NodeScanner2();
+    } else if (this.platform === "browser") {
+      const { BrowserScanner: BrowserScanner2 } = await Promise.resolve().then(() => (init_browser(), browser_exports));
+      this.scanner = new BrowserScanner2();
+    } else {
+      throw new ValidationError("Unsupported platform");
+    }
+    return this.scanner;
+  }
+  /**
+   * Upload a batch of files
+   * @param source - Directory path (Node.js) or File[]/FileList (browser)
+   * @param options - Upload options
+   */
+  async uploadBatch(source, options = {}) {
+    const startTime = Date.now();
+    const { onProgress, dryRun = false } = options;
+    this.reportProgress(onProgress, {
+      phase: "scanning",
+      filesTotal: 0,
+      filesUploaded: 0,
+      bytesTotal: 0,
+      bytesUploaded: 0,
+      percentComplete: 0
+    });
+    const scanner = await this.getScanner();
+    const files = await scanner.scanFiles(source, {
+      rootPath: this.config.rootPath || "/",
+      followSymlinks: true,
+      defaultProcessingConfig: this.config.processing
+    });
+    if (files.length === 0) {
+      throw new ValidationError("No files found to upload");
+    }
+    const totalSize = files.reduce((sum, f) => sum + f.size, 0);
+    validateBatchSize(totalSize);
+    if (this.config.customPrompts) {
+      validateCustomPrompts(this.config.customPrompts);
+      const promptFields = Object.keys(this.config.customPrompts).filter(
+        (key) => this.config.customPrompts[key]
+      );
+      console.log(`[Arke Upload SDK] Custom prompts configured: ${promptFields.join(", ")}`);
+    }
+    if (dryRun) {
+      return {
+        batchId: "dry-run",
+        rootPi: "dry-run",
+        filesUploaded: files.length,
+        bytesUploaded: totalSize,
+        durationMs: Date.now() - startTime
+      };
+    }
+    const { batch_id } = await this.workerClient.initBatch({
+      uploader: this.config.uploader,
+      root_path: this.config.rootPath || "/",
+      parent_pi: this.config.parentPi || "",
+      metadata: this.config.metadata,
+      file_count: files.length,
+      total_size: totalSize,
+      custom_prompts: this.config.customPrompts
+    });
+    if (this.config.customPrompts) {
+      console.log(`[Arke Upload SDK] Custom prompts sent to worker for batch ${batch_id}`);
+    }
+    this.reportProgress(onProgress, {
+      phase: "uploading",
+      filesTotal: files.length,
+      filesUploaded: 0,
+      bytesTotal: totalSize,
+      bytesUploaded: 0,
+      percentComplete: 0
+    });
+    let filesUploaded = 0;
+    let bytesUploaded = 0;
+    const { failedFiles } = await this.uploadFilesWithConcurrency(
+      batch_id,
+      files,
+      source,
+      this.config.parallelUploads || 5,
+      (file, bytes) => {
+        filesUploaded++;
+        bytesUploaded += bytes;
+        this.reportProgress(onProgress, {
+          phase: "uploading",
+          filesTotal: files.length,
+          filesUploaded,
+          bytesTotal: totalSize,
+          bytesUploaded,
+          currentFile: file.fileName,
+          percentComplete: Math.round(bytesUploaded / totalSize * 100)
+        });
+      }
+    );
+    if (failedFiles.length === files.length) {
+      throw new ValidationError(
+        `All ${files.length} files failed to upload. First error: ${failedFiles[0]?.error || "Unknown"}`
+      );
+    }
+    if (failedFiles.length > 0) {
+      console.warn(
+        `Warning: ${failedFiles.length} of ${files.length} files failed to upload:`,
+        failedFiles.map((f) => `${f.file.fileName}: ${f.error}`).join(", ")
+      );
+    }
+    this.reportProgress(onProgress, {
+      phase: "finalizing",
+      filesTotal: files.length,
+      filesUploaded,
+      bytesTotal: totalSize,
+      bytesUploaded,
+      percentComplete: 95
+    });
+    const finalizeResult = await this.workerClient.finalizeBatch(batch_id);
+    let rootPi;
+    if (finalizeResult.root_pi) {
+      rootPi = finalizeResult.root_pi;
+    } else if (finalizeResult.status === "discovery") {
+      this.reportProgress(onProgress, {
+        phase: "discovery",
+        filesTotal: files.length,
+        filesUploaded,
+        bytesTotal: totalSize,
+        bytesUploaded,
+        percentComplete: 97
+      });
+      rootPi = await this.pollForRootPi(batch_id, onProgress, files.length, totalSize, bytesUploaded);
+    } else {
+      throw new ValidationError(
+        `Finalization returned unexpected status: ${finalizeResult.status} without root_pi`
+      );
+    }
+    this.reportProgress(onProgress, {
+      phase: "complete",
+      filesTotal: files.length,
+      filesUploaded,
+      bytesTotal: totalSize,
+      bytesUploaded,
+      percentComplete: 100
+    });
+    return {
+      batchId: batch_id,
+      rootPi,
+      filesUploaded,
+      bytesUploaded,
+      durationMs: Date.now() - startTime
+    };
+  }
+  /**
+   * Poll for root_pi during async discovery
+   */
+  async pollForRootPi(batchId, onProgress, filesTotal, bytesTotal, bytesUploaded) {
+    const POLL_INTERVAL_MS = 2e3;
+    const MAX_POLL_TIME_MS = 30 * 60 * 1e3;
+    const startTime = Date.now();
+    while (Date.now() - startTime < MAX_POLL_TIME_MS) {
+      const status = await this.workerClient.getBatchStatus(batchId);
+      if (status.root_pi) {
+        return status.root_pi;
+      }
+      if (status.status === "failed") {
+        throw new ValidationError(`Batch discovery failed`);
+      }
+      if (status.discovery_progress && onProgress) {
+        const { total, published } = status.discovery_progress;
+        const discoveryPercent = total > 0 ? Math.round(published / total * 100) : 0;
+        this.reportProgress(onProgress, {
+          phase: "discovery",
+          filesTotal,
+          filesUploaded: filesTotal,
+          bytesTotal,
+          bytesUploaded,
+          percentComplete: 95 + Math.round(discoveryPercent * 0.04)
+          // 95-99%
+        });
+      }
+      await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
+    }
+    throw new ValidationError(`Discovery timed out after ${MAX_POLL_TIME_MS / 1e3} seconds`);
+  }
+  /**
+   * Upload files with controlled concurrency
+   */
+  async uploadFilesWithConcurrency(batchId, files, source, concurrency, onFileComplete) {
+    const queue = [...files];
+    const workers = [];
+    const failedFiles = [];
+    const processNext = async () => {
+      while (queue.length > 0) {
+        const file = queue.shift();
+        try {
+          await this.uploadSingleFile(batchId, file, source);
+          onFileComplete(file, file.size);
+        } catch (error) {
+          const errorMessage = error.message || "Unknown error";
+          console.error(`Failed to upload ${file.fileName}: ${errorMessage}`);
+          failedFiles.push({ file, error: errorMessage });
+        }
+      }
+    };
+    for (let i = 0; i < Math.min(concurrency, files.length); i++) {
+      workers.push(processNext());
+    }
+    await Promise.all(workers);
+    return { failedFiles };
+  }
+  /**
+   * Upload a single file
+   */
+  async uploadSingleFile(batchId, file, source) {
+    const uploadInfo = await this.workerClient.startFileUpload(batchId, {
+      file_name: file.fileName,
+      file_size: file.size,
+      logical_path: file.logicalPath,
+      content_type: file.contentType,
+      cid: file.cid,
+      processing_config: file.processingConfig
+    });
+    const fileData = await this.getFileData(file, source);
+    const retryOptions = {
+      maxRetries: this.config.maxRetries,
+      retryInitialDelay: this.config.retryInitialDelay,
+      retryMaxDelay: this.config.retryMaxDelay,
+      retryJitter: this.config.retryJitter
+    };
+    if (uploadInfo.upload_type === "simple") {
+      await uploadSimple(fileData, uploadInfo.presigned_url, file.contentType, retryOptions);
+    } else {
+      const partUrls = uploadInfo.presigned_urls.map((p) => p.url);
+      const parts = await uploadMultipart(
+        fileData,
+        partUrls,
+        this.config.parallelParts || 3,
+        retryOptions
+      );
+      await this.workerClient.completeFileUpload(batchId, {
+        r2_key: uploadInfo.r2_key,
+        upload_id: uploadInfo.upload_id,
+        parts
+      });
+      return;
+    }
+    await this.workerClient.completeFileUpload(batchId, {
+      r2_key: uploadInfo.r2_key
+    });
+  }
+  /**
+   * Get file data based on platform
+   */
+  async getFileData(file, source) {
+    if (this.platform === "node") {
+      const fs2 = await import("fs/promises");
+      const buffer = await fs2.readFile(file.localPath);
+      return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+    } else if (this.platform === "browser") {
+      const files = Array.isArray(source) ? source : [source];
+      const browserFile = files.find(
+        (f) => f instanceof File && f.name === file.fileName
+      );
+      if (!browserFile) {
+        throw new Error(`Could not find browser File object for ${file.fileName}`);
+      }
+      return browserFile.arrayBuffer();
+    }
+    throw new Error("Unsupported platform for file reading");
+  }
+  /**
+   * Report progress to callback
+   */
+  reportProgress(callback, progress) {
+    if (callback) {
+      callback(progress);
+    }
+  }
+};
+
+// src/upload/client.ts
+init_errors();
+function getUserIdFromToken(token) {
+  try {
+    const parts = token.split(".");
+    if (parts.length !== 3) return null;
+    const payload = parts[1].replace(/-/g, "+").replace(/_/g, "/");
+    let decoded;
+    if (typeof atob === "function") {
+      decoded = atob(payload);
+    } else {
+      decoded = Buffer.from(payload, "base64").toString("utf-8");
+    }
+    const data = JSON.parse(decoded);
+    return data.sub || null;
+  } catch {
+    return null;
+  }
+}
+var UploadClient = class {
+  constructor(config) {
+    const uploader = config.uploader || getUserIdFromToken(config.authToken) || "unknown";
+    this.config = { ...config, uploader };
+    this.collectionsClient = new CollectionsClient({
+      gatewayUrl: config.gatewayUrl,
+      authToken: config.authToken,
+      fetchImpl: config.fetchImpl
+    });
+  }
+  /**
+   * Update the auth token (e.g., after token refresh)
+   */
+  setAuthToken(token) {
+    this.config = { ...this.config, authToken: token };
+    this.collectionsClient.setAuthToken(token);
+  }
+  /**
+   * Create a new collection and upload files to it
+   *
+   * Anyone authenticated can create a new collection.
+   * The root PI of the uploaded files becomes the collection's root.
+   */
+  async createCollection(options) {
+    const { files, collectionMetadata, customPrompts, processing, onProgress, dryRun } = options;
+    const metadata = {
+      ...collectionMetadata,
+      visibility: collectionMetadata.visibility || "public"
+    };
+    const uploader = new ArkeUploader({
+      gatewayUrl: this.config.gatewayUrl,
+      authToken: this.config.authToken,
+      uploader: this.config.uploader,
+      customPrompts,
+      processing
+    });
+    const batchResult = await uploader.uploadBatch(files, {
+      onProgress,
+      dryRun
+    });
+    if (dryRun) {
+      return {
+        ...batchResult,
+        collection: {
+          id: "dry-run",
+          title: metadata.title,
+          slug: metadata.slug,
+          description: metadata.description,
+          visibility: metadata.visibility,
+          rootPi: "dry-run"
+        }
+      };
+    }
+    const collection = await this.collectionsClient.registerRoot({
+      ...metadata,
+      rootPi: batchResult.rootPi
+    });
+    return {
+      ...batchResult,
+      collection
+    };
+  }
+  /**
+   * Add files to an existing collection
+   *
+   * Requires owner or editor role on the collection containing the parent PI.
+   * Use this to add a folder or files to an existing collection hierarchy.
+   */
+  async addToCollection(options) {
+    const { files, parentPi, customPrompts, processing, onProgress, dryRun } = options;
+    if (!dryRun) {
+      const permissions = await this.collectionsClient.getPiPermissions(parentPi);
+      if (!permissions.canEdit) {
+        if (!permissions.collection) {
+          throw new ValidationError(
+            `Cannot add files: PI "${parentPi}" is not part of any collection`
+          );
+        }
+        throw new ValidationError(
+          `Cannot add files to collection "${permissions.collection.title}": you need editor or owner role (current role: ${permissions.collection.role || "none"})`
+        );
+      }
+    }
+    const uploader = new ArkeUploader({
+      gatewayUrl: this.config.gatewayUrl,
+      authToken: this.config.authToken,
+      uploader: this.config.uploader,
+      parentPi,
+      customPrompts,
+      processing
+    });
+    return uploader.uploadBatch(files, {
+      onProgress,
+      dryRun
+    });
+  }
+  /**
+   * Check if you can edit a specific PI (i.e., add files to its collection)
+   */
+  async canEdit(pi) {
+    return this.collectionsClient.getPiPermissions(pi);
+  }
+  /**
+   * Get access to the underlying collections client for other operations
+   */
+  get collections() {
+    return this.collectionsClient;
+  }
+};
+
+// src/index.ts
+init_errors();
+export {
+  ArkeUploader,
   CollectionsClient,
-  CollectionsError
-});
+  CollectionsError,
+  NetworkError,
+  ScanError,
+  UploadClient,
+  UploadError,
+  ValidationError,
+  WorkerAPIError
+};
 //# sourceMappingURL=index.js.map