@qaecy/cue-cli 0.0.31 → 0.0.33

package/assets/wasm/dir_scanner_wasm.mjs

@@ -1,5 +1,27 @@
 /* @ts-self-types="./dir_scanner_wasm.d.ts" */
 
+/**
+ * Attempts to extract the unit count from any supported file and returns a plain JS
+ * object `{ units: number, error: string | null }`.
+ *
+ * `original_path` is used only for extension detection.
+ *
+ * ```js
+ * const buf = await file.arrayBuffer();
+ * const result = debug_file(file.name, new Uint8Array(buf));
+ * console.log(result); // { units: 12, error: null }
+ * ```
+ * @param {string} original_path
+ * @param {Uint8Array} data
+ * @returns {any}
+ */
+export function debug_file(original_path, data) {
+    const ptr0 = passStringToWasm0(original_path, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    const len0 = WASM_VECTOR_LEN;
+    const ret = wasm.debug_file(ptr0, len0, data);
+    return ret;
+}
+
 /**
  * Scans a list of in-memory files and returns per-extension metrics sorted by
  * file count descending.
@@ -101,6 +123,9 @@ function __wbg_get_imports() {
             const ret = arg0.length;
             return ret;
         },
+        __wbg_log_7e1aa9064a1dbdbd: function(arg0) {
+            console.log(arg0);
+        },
         __wbg_new_0c7403db6e782f19: function(arg0) {
             const ret = new Uint8Array(arg0);
             return ret;
@@ -126,6 +151,9 @@ function __wbg_get_imports() {
         __wbg_set_6be42768c690e380: function(arg0, arg1, arg2) {
             arg0[arg1] = arg2;
         },
+        __wbg_warn_3cc416af27dbdc02: function(arg0) {
+            console.warn(arg0);
+        },
         __wbindgen_cast_0000000000000001: function(arg0) {
             // Cast intrinsic for `F64 -> Externref`.
             const ret = arg0;

package/main.js

@@ -3827,19 +3827,6 @@ function generateFileUUID(filepath, providerId = "") {
   return contextBasedGuid(`${providerId}${filepath}`);
 }
 
-// libs/js/id-builders/src/lib/id-extractor.ts
-var extractIdsFromRawPath = (filePath) => {
-  if (filePath.startsWith("/")) {
-    filePath = filePath.slice(1);
-  }
-  const pathParts = filePath.split("/");
-  const projectId = pathParts[0];
-  const fileName = pathParts.pop() ?? "";
-  const suffix = `.${fileName.split(".").pop()?.toLowerCase()}`;
-  const documentUUID = fileName.replace(/\.[^.]+$/, "");
-  return { projectId, documentUUID, suffix };
-};
-
 // libs/js/id-builders/src/lib/md5-builder.ts
 var import_spark_md5 = __toESM(require("spark-md5"));
 async function fromReadStream(readStream) {
@@ -4154,6 +4141,7 @@ var CueAuth = class {
 // libs/js/cue-sdk/src/lib/api.ts
 var ENDPOINT_SEARCH = "/assistant/search";
 var ENDPOINT_SPARQL = "/triplestore/query";
+var ENDPOINT_CONSUMPTION = "/data-views/admin/consumption/units";
 var CueApi = class {
   constructor(_auth, _gatewayUrl, projects, sync) {
     this._auth = _auth;
@@ -4213,6 +4201,16 @@ var CueApi = class {
     }
     return response.json();
   }
+  async getConsumption(projectId) {
+    const headers = await this._authHeaders();
+    const response = await fetch(`${this._gatewayUrl}${ENDPOINT_CONSUMPTION}`, {
+      headers: { ...headers, "x-project-id": projectId }
+    });
+    if (!response.ok) {
+      throw new Error(`Failed to fetch consumption: ${response.status} ${response.statusText}`);
+    }
+    return response.json();
+  }
 };
 
 // libs/js/cue-sdk/src/lib/project.ts
@@ -4317,7 +4315,6 @@ var CueProjects = class {
 // libs/js/cue-sdk/src/lib/profile.ts
 var import_auth4 = require("firebase/auth");
 var import_firestore3 = require("firebase/firestore");
-var import_nanoid = require("nanoid");
 var COLLECTION_API_KEYS2 = "apiKeys";
 var CueProfile = class {
   constructor(_auth, app, useEmulator, emulatorHost, emulatorPort) {
@@ -4405,7 +4402,9 @@ var CueProfile = class {
     const uid = this._auth.currentUser?.uid;
     if (!uid)
       throw new Error("User not authenticated");
-    const data = { key: `cue-${(0, import_nanoid.nanoid)(150)}`, uid, expiration };
+    const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
+    const randomKey = Array.from(globalThis.crypto.getRandomValues(new Uint8Array(150)), (b) => alphabet[b & 63]).join("");
+    const data = { key: `cue-${randomKey}`, uid, expiration };
     const col = (0, import_firestore3.collection)(this._db, COLLECTION_API_KEYS2);
     this._apiKeyDocRef = await (0, import_firestore3.addDoc)(col, data);
     return data;
@@ -4598,9 +4597,18 @@ var Fuseki = class _Fuseki {
 };
 
 // libs/js/databases/src/lib/graph/qlever.ts
+var import_zlib = require("zlib");
+var import_n3 = require("n3");
+var QLeverLockedError = class extends Error {
+  constructor(body) {
+    super(`QLever is locked (rebuild in progress): ${body}`);
+    this.name = "QLeverLockedError";
+  }
+};
 var QLever = class _QLever {
   queryEndpoint;
   updateEndpoint;
+  dataEndpoint;
   baseHeaders;
   static RELEVANT_HEADER_KEYS = [
     "authorization",
@@ -4609,14 +4617,44 @@ var QLever = class _QLever {
     "x-user-roles"
     // add more if needed
   ];
+  /** Max retries on 423 Locked (rebuild in progress). */
+  static LOCKED_MAX_RETRIES = parseInt(process.env["QLEVER_LOCKED_MAX_RETRIES"] ?? "10", 10);
+  /** Base delay (ms) for exponential backoff on 423. */
+  static LOCKED_BASE_DELAY_MS = parseInt(process.env["QLEVER_LOCKED_BASE_DELAY_MS"] ?? "2000", 10);
+  /**
+   * Retry an async write operation on 423 Locked with exponential backoff + jitter.
+   * 423 means qlever accessor has an ongoing rebuild; we should wait and retry.
+   */
+  static async _retryOnLocked(fn) {
+    const maxRetries = _QLever.LOCKED_MAX_RETRIES;
+    const baseDelayMs = _QLever.LOCKED_BASE_DELAY_MS;
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        return await fn();
+      } catch (err) {
+        lastError = err;
+        if (err instanceof QLeverLockedError && attempt < maxRetries) {
+          const jitter = 0.5 + Math.random();
+          const delay = baseDelayMs * Math.pow(2, attempt) * jitter;
+          await new Promise((resolve2) => setTimeout(resolve2, delay));
+          continue;
+        }
+        throw err;
+      }
+    }
+    throw lastError;
+  }
   constructor(graphOptions) {
     this.queryEndpoint = graphOptions.queryEndpoint;
     this.updateEndpoint = graphOptions.updateEndpoint;
+    this.dataEndpoint = this.updateEndpoint.replace(/\/update$/, "/data");
     this.baseHeaders = Object.fromEntries(
       Object.entries(graphOptions.originalHeaders || {}).filter(
         ([key]) => _QLever.RELEVANT_HEADER_KEYS.includes(key)
       )
     );
+    this.baseHeaders["x-user-roles"] = "admin";
   }
   async ping() {
     const query3 = "ASK { }";
@@ -4626,7 +4664,6 @@ var QLever = class _QLever {
   async query(query3, accept = "application/sparql-results+json") {
     let res;
     try {
-      console.log(this.queryEndpoint);
       res = await fetch(this.queryEndpoint, {
         headers: {
           ...this.baseHeaders,
@@ -4660,19 +4697,65 @@ var QLever = class _QLever {
     return await res.text();
   }
   async update(update) {
-    const res = await fetch(this.updateEndpoint, {
-      headers: {
-        ...this.baseHeaders,
-        "Content-Type": "application/x-www-form-urlencoded"
-      },
-      method: "POST",
-      body: new URLSearchParams({ update })
+    return _QLever._retryOnLocked(async () => {
+      const res = await fetch(this.updateEndpoint, {
+        headers: {
+          ...this.baseHeaders,
+          "Content-Type": "application/x-www-form-urlencoded"
+        },
+        method: "POST",
+        body: new URLSearchParams({ update })
+      });
+      if (!res.ok) {
+        const body = await res.text();
+        if (res.status === 423)
+          throw new QLeverLockedError(body);
+        throw new Error(`SPARQL update failed (HTTP ${res.status}): ${body}`);
+      }
+      return await res.json();
+    });
+  }
+  /**
+   * Insert quads via the /data endpoint, grouped by named graph.
+   * This is preferred over SPARQL INSERT DATA for QLever because
+   * the /data endpoint correctly registers named graphs in the index.
+   */
+  async insertData(quads) {
+    await this._postToDataEndpoint(quads, this.dataEndpoint);
+  }
+  /**
+   * Delete quads via the /data/delete endpoint, grouped by named graph.
+   */
+  async deleteData(quads) {
+    await this._postToDataEndpoint(quads, `${this.dataEndpoint}/delete`);
+  }
+  async _postToDataEndpoint(quads, baseUrl) {
+    const nquads = await this._quadsToNQuads(quads);
+    const body = new Uint8Array((0, import_zlib.gzipSync)(Buffer.from(nquads, "utf-8")));
+    await _QLever._retryOnLocked(async () => {
+      const res = await fetch(baseUrl, {
+        method: "POST",
+        headers: {
+          ...this.baseHeaders,
+          "Content-Type": "application/n-quads",
+          "Content-Encoding": "gzip"
+        },
+        body
+      });
+      if (!res.ok) {
+        const text = await res.text();
+        if (res.status === 423)
+          throw new QLeverLockedError(text);
+        throw new Error(`QLever data POST failed (HTTP ${res.status}): ${text}`);
+      }
+    });
+  }
+  _quadsToNQuads(quads) {
+    return new Promise((resolve2, reject) => {
+      const writer = new import_n3.Writer({ format: "application/n-quads" });
+      writer.addQuads(quads);
+      writer.end((err, result) => err ? reject(err) : resolve2(result));
     });
-    if (!res.ok) {
-      const body = await res.text();
-      throw new Error(`SPARQL update failed (HTTP ${res.status}): ${body}`);
-    }
-    return await res.json();
   }
 };
 
@@ -4717,6 +4800,32 @@ var CueGraphDatabase = class {
   update(updateString) {
     return this._db.update(updateString);
   }
+  /** Returns true if this backend supports the /data bulk-insert endpoint (QLever only). */
+  supportsDataEndpoint() {
+    return this.options.graphType === "qlever";
+  }
+  /**
+   * Insert quads using the backend's preferred bulk-insert mechanism.
+   * For QLever: uses the /data endpoint (correctly registers named graphs).
+   * For Fuseki: falls back to SPARQL INSERT DATA.
+   */
+  insertData(quads) {
+    if (this.options.graphType === "qlever") {
+      return this._db.insertData(quads);
+    }
+    return Promise.reject(new Error("insertData not supported for Fuseki \u2014 use update() with SPARQL INSERT DATA"));
+  }
+  /**
+   * Delete quads using the backend's preferred bulk-delete mechanism.
+   * For QLever: uses the /data/delete endpoint.
+   * For Fuseki: falls back to SPARQL DELETE DATA.
+   */
+  deleteData(quads) {
+    if (this.options.graphType === "qlever") {
+      return this._db.deleteData(quads);
+    }
+    return Promise.reject(new Error("deleteData not supported for Fuseki \u2014 use update() with SPARQL DELETE DATA"));
+  }
 };
 
 // libs/js/databases/src/lib/blob/blob.ts
@@ -4777,6 +4886,7 @@ var CueBlobStorage = class {
 var import_promises3 = require("fs/promises");
 var import_path3 = require("path");
 var import_url = require("url");
+var import_os = require("os");
 
 // libs/js/models/src/lib/file-extensions.ts
 var fileExtensionsInfo = {
@@ -5677,106 +5787,20 @@ var import_uuid4 = require("uuid");
 var import_uuid5 = require("uuid");
 
 // libs/js/rdf-document-writers/src/lib/alternative-representation.ts
-var import_n32 = require("n3");
+var import_n33 = require("n3");
 
 // libs/js/rdf-document-writers/src/lib/file-location.ts
-var import_n3 = require("n3");
-var { namedNode, literal } = import_n3.DataFactory;
+var import_n32 = require("n3");
+var { namedNode, literal } = import_n32.DataFactory;
 var a = namedNode("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
 var prefixes = {
   qcy: qaecyPrefixes["qcy"],
   "qcy-e": qaecyPrefixes["qcy-e"],
   xsd: prefixCC["xsd"]
 };
-function fileLocationRaw(metadata, size, fileContentClasses = ["qcy:FileContent"], writer = new import_n3.Writer({ prefixes }), remoteProviderType = "GCP" /* GCP */) {
-  if (metadata.md5_hash === void 0)
-    throw new Error("md5_hash must be specified");
-  if (metadata.blob_name === void 0)
-    throw new Error("blob_name must be specified");
-  if (metadata.suffix === void 0)
-    throw new Error("suffix must be specified");
-  if (metadata.name === void 0)
-    throw new Error("name must be specified");
-  const ids = extractIdsFromRawPath(metadata.blob_name);
-  const fileInfo = getFileInfo(metadata.suffix);
-  const mimeCategory = fileTypeToMimeCategory[fileInfo.type];
-  const originalName = metadata.name.split("/").pop() ?? metadata.name;
-  const fileContent = namedNode(ids.documentUUID);
-  const fileLocationUUID = generateFileUUID(
-    metadata.name,
-    metadata.provider_id ?? ""
-  );
-  const fileLocation = namedNode(fileLocationUUID);
-  fileContentClasses.forEach((cl) => {
-    writer.addQuad(fileContent, a, namedNode(cl));
-  });
-  writer.addQuad(
-    fileContent,
-    namedNode("qcy:hasMimeCategory"),
-    namedNode(`qcy-e:${mimeCategory}`)
-  );
-  writer.addQuad(
-    fileContent,
-    namedNode("qcy:md5Hash"),
-    literal(metadata.md5_hash)
-  );
-  writer.addQuad(fileContent, namedNode("qcy:mime"), literal(fileInfo.mime));
-  writer.addQuad(
-    fileContent,
-    namedNode("qcy:openFormat"),
-    literal(fileInfo?.open)
-  );
-  writer.addQuad(fileContent, namedNode("qcy:sizeBytes"), literal(size));
-  writer.addQuad(fileContent, namedNode("qcy:hasFileLocation"), fileLocation);
-  writer.addQuad(fileLocation, a, namedNode("qcy:FileLocation"));
-  writer.addQuad(
-    fileLocation,
-    namedNode("qcy:filePath"),
-    literal(metadata.name)
-  );
-  writer.addQuad(
-    fileLocation,
-    namedNode("qcy:remoteProviderType"),
-    namedNode(`qcy-e:${remoteProviderType}`)
-  );
-  writer.addQuad(
-    fileLocation,
-    namedNode("qcy:remoteProviderId"),
-    literal(metadata.provider_id ?? "")
-  );
-  writer.addQuad(
-    fileLocation,
-    namedNode("qcy:remoteRelativePath"),
-    literal(metadata.blob_name)
-  );
-  writer.addQuad(
-    fileLocation,
-    namedNode("qcy:suffix"),
-    literal(metadata.suffix)
-  );
-  writer.addQuad(
-    fileLocation,
-    namedNode("qcy:dateCreated"),
-    literal((/* @__PURE__ */ new Date()).toISOString(), namedNode("xsd:dateTime"))
-  );
-  writer.addQuad(fileLocation, namedNode("qcy:value"), literal(originalName));
-  return writer;
-}
-function getFileInfo(suffix) {
-  let fileInfo = fileExtensionsInfo[suffix];
-  if (fileInfo === void 0) {
-    fileInfo = {
-      type: "unknown" /* UNKNOWN */,
-      open: false,
-      suffix,
-      mime: "application/octet-stream"
-    };
-  }
-  return fileInfo;
-}
 
 // libs/js/rdf-document-writers/src/lib/alternative-representation.ts
-var { namedNode: namedNode2 } = import_n32.DataFactory;
+var { namedNode: namedNode2 } = import_n33.DataFactory;
 var prefixes2 = {
   qcy: qaecyPrefixes["qcy"],
   "qcy-e": qaecyPrefixes["qcy-e"],
@@ -5784,12 +5808,12 @@ var prefixes2 = {
 };
 
 // libs/js/rdf-document-writers/src/lib/custom-template-parser.ts
-var import_n33 = require("n3");
-var { namedNode: namedNode3, literal: literal2 } = import_n33.DataFactory;
+var import_n34 = require("n3");
+var { namedNode: namedNode3, literal: literal2 } = import_n34.DataFactory;
 var a2 = namedNode3("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
 
 // libs/js/rdf-document-writers/src/lib/document-file.ts
-var import_n34 = require("n3");
+var import_n35 = require("n3");
 
 // libs/js/rdf-document-writers/src/lib/file-suffix.ts
 function getFileSuffix(filename) {
@@ -5801,28 +5825,13 @@ function getFileSuffix(filename) {
 }
 
 // libs/js/rdf-document-writers/src/lib/document-file.ts
-var { namedNode: namedNode4, literal: literal3 } = import_n34.DataFactory;
+var { namedNode: namedNode4, literal: literal3 } = import_n35.DataFactory;
 var a3 = namedNode4("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
 
 // libs/js/rdf-document-writers/src/lib/process-logs.ts
-var import_n35 = require("n3");
-
-// libs/js/rdf-document-writers/src/lib/serialize-rdf.ts
-async function serializeRDF(namespace, writer, format = "ttl") {
-  return new Promise((resolve2, reject) => {
-    writer.end((err, res) => {
-      if (err)
-        reject(err);
-      const addBase = format === "ttl" && namespace !== "";
-      resolve2(addBase ? `@base <${namespace}>.
-${res}` : res);
-    });
-  });
-}
-
-// libs/js/rdf-document-writers/src/lib/process-logs.ts
+var import_n36 = require("n3");
 var import_uuid6 = require("uuid");
-var { namedNode: namedNode5, literal: literal4 } = import_n35.DataFactory;
+var { namedNode: namedNode5, literal: literal4 } = import_n36.DataFactory;
 var a4 = namedNode5("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
 var prefixes3 = {
   qcy: qaecyPrefixes["qcy"],
@@ -5860,24 +5869,9 @@ function uploadedFileMetadata(originalName, projectId, userId, md5, providerId,
     })
   };
 }
-function turtleFileMetadata(sourceFile, processorId, locationUUID, stored = false) {
-  const ids = extractIdsFromRawPath(sourceFile);
-  const blob_name = locationUUID !== void 0 ? `${ids.projectId}/triples/${ids.documentUUID}_${locationUUID}_${processorId}.ttl` : `${ids.projectId}/triples/${ids.documentUUID}_${processorId}.ttl`;
-  const identifier = locationUUID !== void 0 ? `${ids.documentUUID}_${locationUUID}` : ids.documentUUID;
-  return {
-    blob_name,
-    processor: processorId,
-    space_id: ids.projectId,
-    stored: stored ? "True" : "False",
-    suffix: ".ttl",
-    identifier,
-    source: sourceFile
-  };
-}
 
 // libs/js/cue-sdk/src/lib/sync.ts
 var _scanFn = null;
-var _fileMapFn = null;
 var _wasmInitPromise = null;
 async function _initWasm() {
   const wasmDir = (0, import_path3.join)(__dirname, "assets", "wasm");
@@ -5889,25 +5883,91 @@ async function _initWasm() {
   );
   await mod.default({ module_or_path: wasmBinary });
   _scanFn = mod.scan;
-  _fileMapFn = mod.scan_file_map;
 }
 var DEFAULT_GRAPH_TYPE = "fuseki";
-var DEFAULT_SERVICE_ID = "cue-cli";
-var DEFAULT_RDF_BASE = "https://cue.qaecy.com/r/";
 var ENDPOINT_FUSEKI_QUERY = "/triplestore/query";
 var ENDPOINT_QLEVER_QUERY = "/sparql/query";
 var ENDPOINT_FUSEKI_UPDATE = "/triplestore/update";
 var ENDPOINT_QLEVER_UPDATE = "/sparql/update";
+var ENDPOINT_FSS_BATCH = "/commands/file-system-structure/batch";
+var FSS_BATCH_CHUNK_SIZE = 1e3;
+var PENDING_LS_PREFIX = "cue:pending:";
+function _pendingFilePath(spaceId) {
+  return (0, import_path3.join)((0, import_os.tmpdir)(), `cue-sync-pending-${spaceId}.json`);
+}
+async function _loadPending(spaceId) {
+  if (typeof window !== "undefined") {
+    const raw = window.localStorage.getItem(`${PENDING_LS_PREFIX}${spaceId}`);
+    return raw ? JSON.parse(raw) : null;
+  }
+  try {
+    const raw = await (0, import_promises3.readFile)(_pendingFilePath(spaceId), "utf-8");
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+async function _savePending(batch) {
+  const data = JSON.stringify(batch);
+  if (typeof window !== "undefined") {
+    window.localStorage.setItem(`${PENDING_LS_PREFIX}${batch.spaceId}`, data);
+    return;
+  }
+  await (0, import_promises3.writeFile)(_pendingFilePath(batch.spaceId), data, "utf-8");
+}
+async function _clearPending(spaceId) {
+  if (typeof window !== "undefined") {
+    window.localStorage.removeItem(`${PENDING_LS_PREFIX}${spaceId}`);
+    return;
+  }
+  try {
+    await (0, import_promises3.unlink)(_pendingFilePath(spaceId));
+  } catch {
+  }
+}
 var CueSyncApi = class {
-  constructor(_auth, _projects, _blob, _gatewayUrl, _serviceId = DEFAULT_SERVICE_ID, _rdfBase = DEFAULT_RDF_BASE) {
+  constructor(_auth, _projects, _blob, _gatewayUrl) {
     this._auth = _auth;
     this._projects = _projects;
     this._blob = _blob;
     this._gatewayUrl = _gatewayUrl;
-    this._serviceId = _serviceId;
-    this._rdfBase = _rdfBase;
   }
   _graphMap = /* @__PURE__ */ new Map();
+  _api;
+  _pendingItems = [];
+  _pendingSpaceId = null;
+  _flushTimer = null;
+  /** @internal Injected by CueApi after construction to avoid circular dependency. */
+  _bindApi(api) {
+    this._api = api;
+  }
+  /**
+   * Returns a preview of what would be synced: cost breakdown for new files only,
+   * units required, and units still available. Use this before calling {@link sync}
+   * to show the user an accurate cost estimate.
+   */
+  async previewSync(localFiles, options) {
+    const { spaceId, providerId, verbose } = options;
+    const token = await this._auth.getToken();
+    if (!token)
+      throw new Error("Not authenticated. Call cue.auth.signIn() first.");
+    const graph = await this._getOrCreateGraph(spaceId, token);
+    const [remoteFiles, consumption] = await Promise.all([
+      this._listRemoteFiles(graph, spaceId, providerId, verbose),
+      this._api?.getConsumption(spaceId) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
+    ]);
+    const report = await compareLocalRemote(localFiles, remoteFiles);
+    const toUpload = report.localNotOnRemote ?? [];
+    const costRecords = toUpload.length > 0 ? await this.scanCost(toUpload) : [];
+    const unitsToConsume = costRecords.reduce((sum, r) => sum + r.units, 0);
+    return {
+      costRecords,
+      unitsToConsume,
+      unitsAvailable: consumption.unitsAvailable,
+      filesToUpload: toUpload.length,
+      totalLocalFiles: localFiles.length
+    };
+  }
   async sync(localFiles, options) {
     const { spaceId, providerId, userId, verbose } = options;
     const token = await this._auth.getToken();
@@ -5916,7 +5976,11 @@ var CueSyncApi = class {
     const graph = await this._getOrCreateGraph(spaceId, token);
     if (verbose)
       console.info("Listing remote files \u23F3");
-    const remoteFiles = await this._listRemoteFiles(graph, spaceId, providerId, verbose);
+    const [remoteFiles, consumption] = await Promise.all([
+      this._listRemoteFiles(graph, spaceId, providerId, verbose),
+      this._api?.getConsumption(spaceId) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
+    ]);
+    const { unitsAvailable } = consumption;
     const report = await compareLocalRemote(localFiles, remoteFiles);
     if (verbose) {
       console.info(`Total local files: ${localFiles.length}`);
@@ -5929,27 +5993,20 @@ var CueSyncApi = class {
     let syncSize = report.syncSize;
     let failedUploads = 0;
     let rdfWritten = false;
-    let unitsConsumed = 0;
-    let fileUnits = {};
-    try {
-      if (!_wasmInitPromise)
-        _wasmInitPromise = _initWasm();
-      await _wasmInitPromise;
-      const entries = await Promise.all(
-        localFiles.map(async (f) => ({
-          originalPath: f.relativePath,
-          data: new Uint8Array(await (0, import_promises3.readFile)(f.fullPath))
-        }))
-      );
-      fileUnits = _fileMapFn(entries);
-    } catch (err) {
-      console.warn("[CueSyncApi] Unit map failed, proceeding without unit tracking:", err);
-    }
     const toUpload = report.localNotOnRemote ?? [];
+    if (toUpload.length > 0) {
+      const costRecords = await this.scanCost(toUpload);
+      const unitsToConsume = costRecords.reduce((sum, r) => sum + r.units, 0);
+      if (unitsToConsume > unitsAvailable) {
+        throw new Error(
+          `Insufficient units: ${unitsToConsume} units required but only ${unitsAvailable} available.`
+        );
+      }
+    }
+    await this._initPendingBatch(spaceId, token, verbose);
     if (verbose && toUpload.length)
       console.info("Syncing missing files \u23F3");
     for (const file of toUpload) {
-      let uploadSucceeded = false;
       try {
         const rawMeta = uploadedFileMetadata(
           file.relativePath,
@@ -5966,12 +6023,16 @@ var CueSyncApi = class {
           new Uint8Array(fileBuffer),
           rawMeta
         );
-        const uploaded = await this._uploadRdfMetadata(file, rawMeta, verbose);
-        if (uploaded)
-          rdfWritten = true;
+        await this._queueFileLocation({
+          relativePath: file.relativePath,
+          md5: file.md5,
+          size: file.size,
+          providerId,
+          fileContentExists: false
+        });
+        rdfWritten = true;
         syncCount += 1;
         syncSize += file.size || 0;
-        uploadSucceeded = true;
         this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, verbose);
       } catch (err) {
         failedUploads += 1;
@@ -5979,54 +6040,31 @@ var CueSyncApi = class {
         if (verbose)
           console.error("[CueSyncApi] Upload error details:", err);
       }
-      if (uploadSucceeded) {
-        const fileUnitsCount = fileUnits[file.relativePath] ?? 0;
-        if (fileUnitsCount > 0) {
-          unitsConsumed += fileUnitsCount;
-          try {
-            await this._projects.incrementUnitsConsumed(spaceId, fileUnitsCount, userId);
-          } catch (err) {
-            console.error(`[CueSyncApi] Failed to write clientSync entry for ${file.relativePath}:`, err);
-          }
-        }
-      }
     }
     if (verbose && report.localNotOnRemotePathOnly.length)
       console.info(`Syncing missing file locations (on provider "${providerId}") \u23F3`);
     for (const file of report.localNotOnRemotePathOnly) {
-      const rawMeta = uploadedFileMetadata(
-        file.relativePath,
-        spaceId,
-        userId,
-        file.md5,
-        providerId
-      );
-      if (!rawMeta.blob_name)
-        throw new Error(`blob_name missing for ${file.relativePath}`);
-      const uploaded = await this._uploadRdfMetadata(file, rawMeta, verbose);
-      if (uploaded)
-        rdfWritten = true;
+      await this._queueFileLocation({
+        relativePath: file.relativePath,
+        md5: file.md5,
+        size: file.size,
+        providerId,
+        fileContentExists: true
+      });
+      rdfWritten = true;
       syncCount += 1;
       syncSize += file.size || 0;
-      const fileUnitsCount = fileUnits[file.relativePath] ?? 0;
-      if (fileUnitsCount > 0) {
-        unitsConsumed += fileUnitsCount;
-        try {
-          await this._projects.incrementUnitsConsumed(spaceId, fileUnitsCount, userId);
-        } catch (err) {
-          console.error(`[CueSyncApi] Failed to write clientSync entry for ${file.relativePath}:`, err);
-        }
-      }
       this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, verbose);
     }
+    await this._drainPending(verbose);
+    this._stopFlushTimer();
     return {
       syncCount,
       syncSize,
       failedUploads,
       totalCount: report.totalCount,
       totalSize: report.totalSize,
-      rdfWritten,
-      unitsConsumed
+      rdfWritten
     };
   }
   async _getOrCreateGraph(spaceId, token) {
@@ -6105,30 +6143,77 @@ WHERE {
     }
     return map;
   }
-  async _uploadRdfMetadata(file, rawMeta, verbose) {
-    if (!rawMeta.blob_name)
-      throw new Error(`blob_name missing for ${file.relativePath}`);
-    const ttlMeta = turtleFileMetadata(
-      rawMeta.blob_name,
-      this._serviceId,
-      file.locationUUID
-    );
-    if (!ttlMeta.blob_name)
-      throw new Error(`ttl blob_name missing for ${rawMeta.blob_name}`);
-    const writer = fileLocationRaw(rawMeta, file.size);
-    const namespace = `${this._rdfBase}${ttlMeta.space_id}/`;
-    const triples = await serializeRDF(namespace, writer);
-    const uploaded = await this._blob.uploadProcessed(
-      ttlMeta.blob_name,
-      new Uint8Array(Buffer.from(triples, "utf-8")),
-      ttlMeta
-    );
-    if (!uploaded && verbose) {
-      console.info(
-        `Graph data for ${file.relativePath} already exists \u2014 skipping \u26A0\uFE0F`
+  async _initPendingBatch(spaceId, token, verbose) {
+    this._pendingSpaceId = spaceId;
+    this._pendingItems = [];
+    const existing = await _loadPending(spaceId);
+    if (existing && existing.items.length > 0) {
+      if (verbose)
+        console.info(`Flushing ${existing.items.length} pending file location(s) from previous sync \u23F3`);
+      await this._flushBatch(existing.items, spaceId, token, verbose);
+    }
+    const timer = setInterval(() => {
+      this._drainPending(verbose).catch(
+        (err) => console.error("[CueSyncApi] Periodic flush failed:", err)
       );
+    }, 6e4);
+    if (typeof timer === "object" && typeof timer.unref === "function") {
+      timer.unref();
+    }
+    this._flushTimer = timer;
+  }
+  async _queueFileLocation(item) {
+    this._pendingItems.push(item);
+    if (this._pendingSpaceId) {
+      await _savePending({ spaceId: this._pendingSpaceId, items: this._pendingItems });
+    }
+  }
+  async _drainPending(verbose) {
+    if (!this._pendingSpaceId || this._pendingItems.length === 0)
+      return;
+    const token = await this._auth.getToken();
+    if (!token)
+      return;
+    await this._flushBatch(this._pendingItems, this._pendingSpaceId, token, verbose);
+  }
+  async _flushBatch(items, spaceId, token, verbose) {
+    const snapshot = [...items];
+    if (this._pendingSpaceId === spaceId)
+      this._pendingItems = [];
+    await _clearPending(spaceId);
+    try {
+      for (let i = 0; i < snapshot.length; i += FSS_BATCH_CHUNK_SIZE) {
+        await this._postFssBatch(snapshot.slice(i, i + FSS_BATCH_CHUNK_SIZE), spaceId, token);
+      }
+      if (verbose)
+        console.info(`Wrote ${snapshot.length} file location(s) to commands API \u2705`);
+    } catch (err) {
+      const restored = [...snapshot, ...this._pendingItems];
+      this._pendingItems = restored;
+      await _savePending({ spaceId, items: restored });
+      throw err;
+    }
+  }
+  async _postFssBatch(items, spaceId, token) {
+    const response = await fetch(`${this._gatewayUrl}${ENDPOINT_FSS_BATCH}`, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${token}`,
+        "Content-Type": "application/json",
+        "x-project-id": spaceId
+      },
+      body: JSON.stringify({ items })
+    });
+    if (!response.ok) {
+      const body = await response.text().catch(() => "");
+      throw new Error(`File structure batch POST failed: ${response.status} ${response.statusText}${body ? ` \u2014 ${body}` : ""}`);
+    }
+  }
+  _stopFlushTimer() {
+    if (this._flushTimer !== null) {
+      clearInterval(this._flushTimer);
+      this._flushTimer = null;
     }
-    return uploaded;
   }
   /**
    * Scans `localFiles` and returns a per-extension cost breakdown.
@@ -6141,13 +6226,31 @@ WHERE {
     if (!_wasmInitPromise)
       _wasmInitPromise = _initWasm();
     await _wasmInitPromise;
-    const entries = await Promise.all(
-      localFiles.map(async (f) => ({
-        originalPath: f.relativePath,
-        data: new Uint8Array(await (0, import_promises3.readFile)(f.fullPath))
-      }))
-    );
-    return _scanFn(entries);
+    if (!_scanFn)
+      throw new Error("WASM scan function not initialised");
+    const BATCH_SIZE = 200;
+    const merged = /* @__PURE__ */ new Map();
+    for (let i = 0; i < localFiles.length; i += BATCH_SIZE) {
+      const batch = localFiles.slice(i, i + BATCH_SIZE);
+      const entries = await Promise.all(
+        batch.map(async (f) => ({
+          originalPath: f.relativePath,
+          data: new Uint8Array(await (0, import_promises3.readFile)(f.fullPath))
+        }))
+      );
+      const records = _scanFn(entries);
+      for (const r of records) {
+        const existing = merged.get(r.ext);
+        if (existing) {
+          existing.count += r.count;
+          existing.units += r.units;
+          existing.sizeMb += r.sizeMb;
+        } else {
+          merged.set(r.ext, { ...r });
+        }
+      }
+    }
+    return Array.from(merged.values());
   }
   _logProgress(syncCount, totalCount, syncSize, totalSize, verbose) {
     if (!verbose || totalCount === 0)
@@ -6179,7 +6282,9 @@ var CueNode = class extends Cue {
     }
     const blob = new CueBlobStorage({ storageRaw, storageProcessed });
     const syncApi = new CueSyncApi(this.auth, projects, blob, this._endpoints.gatewayUrl);
-    return new CueApi(this.auth, this._endpoints.gatewayUrl, projects, syncApi);
+    const api = new CueApi(this.auth, this._endpoints.gatewayUrl, projects, syncApi);
+    syncApi._bindApi(api);
+    return api;
   }
 };
 
@@ -6362,7 +6467,7 @@ async function dumpProcessedHandler(options) {
 var import_fs2 = require("fs");
 var import_stream = require("stream");
 var import_fs3 = require("fs");
-var import_zlib = require("zlib");
+var import_zlib2 = require("zlib");
 var import_promises5 = require("stream/promises");
 var import_promises6 = require("fs/promises");
 var import_auth9 = require("firebase/auth");
@@ -6542,7 +6647,7 @@ async function retryWithBackoff(fn, maxRetries, delayMs, label) {
 async function _doGzip(filePath) {
   const gzFilePath = `${filePath}.gz`;
   await new Promise((resolve2, reject) => {
-    const gzip = (0, import_zlib.createGzip)();
+    const gzip = (0, import_zlib2.createGzip)();
     const source = (0, import_fs3.createReadStream)(filePath);
     const dest = (0, import_fs2.createWriteStream)(gzFilePath);
     (0, import_promises5.pipeline)(source, gzip, dest).then(resolve2).catch(reject);
@@ -6898,24 +7003,46 @@ async function syncHandler(options) {
     }
     if (verbose)
       console.info("Built sync base \u2705\n");
-    const costRecords = await cue.api.sync.scanCost(localFiles);
-    const totalUnits = costRecords.reduce((sum, r) => sum + r.units, 0);
-    console.info("Cost estimate:");
-    for (const r of costRecords) {
-      console.info(`  .${r.ext}: ${r.count} file(s) \u2192 ${r.units} unit(s)`);
+    if (verbose)
+      console.info("Authenticating \u23F3");
+    const user = await cue.auth.signInWithApiKey(key, space);
+    if (verbose)
+      console.info("Authenticated \u2705\n");
+    if (verbose)
+      console.info("Checking sync preview \u23F3");
+    const preview = await cue.api.sync.previewSync(localFiles, {
+      spaceId: space,
+      providerId: provider,
+      userId: user.uid,
+      verbose
+    });
+    console.info("Cost estimate (new files only):");
+    if (preview.costRecords.length === 0) {
+      console.info("  Nothing new to sync.");
+    } else {
+      for (const r of preview.costRecords) {
+        console.info(`  .${r.ext}: ${r.count} file(s) \u2192 ${r.units} unit(s)`);
+      }
     }
-    console.info(`  Total: ${totalUnits} unit(s) across ${localFiles.length} file(s)
+    console.info(`  New files: ${preview.filesToUpload}/${preview.totalLocalFiles}`);
+    console.info(`  Units required: ${preview.unitsToConsume}`);
+    console.info(`  Units available: ${preview.unitsAvailable}
 `);
+    if (preview.filesToUpload === 0) {
+      console.info("Everything is already synced.");
+      process.exit(0);
+    }
+    if (preview.unitsToConsume > preview.unitsAvailable) {
+      console.error(
+        `Insufficient units: ${preview.unitsToConsume} required but only ${preview.unitsAvailable} available.`
+      );
+      process.exit(1);
+    }
     const confirmed = await askConfirm("Continue with sync? (y/n) ");
     if (!confirmed) {
       console.info("Sync cancelled.");
       process.exit(0);
     }
-    if (verbose)
-      console.info("Authenticating \u23F3");
-    const user = await cue.auth.signInWithApiKey(key, space);
-    if (verbose)
-      console.info("Authenticated \u2705\n");
     const result = await cue.api.sync.sync(localFiles, {
       spaceId: space,
       providerId: provider,
@@ -6933,7 +7060,7 @@ async function syncHandler(options) {
           console.info(`Threw RDF_WRITING_IDLE topic \u2705`);
       } catch (error) {
         if (verbose)
-          console.error(`Error throwing RDF_WRITING_IDLE topic: ${error.message}`);
+          console.error(`Error throwing RDF_WRITING_IDLE topic: ${error instanceof Error ? error.message : String(error)}`);
         if (verbose)
           console.warn("Continuing despite PubSub error in emulator mode...");
       }
@@ -6961,22 +7088,22 @@ async function syncHandler(options) {
 
 // apps/desktop/cue-cli/src/cue-cli-util-remove-rdf-star.ts
 var import_fs7 = require("fs");
-var import_zlib2 = require("zlib");
+var import_zlib3 = require("zlib");
 
 // libs/js/rdf-tools/src/lib/nq-to-nt.ts
-var import_n36 = require("n3");
-var { quad, defaultGraph } = import_n36.DataFactory;
+var import_n37 = require("n3");
+var { quad, defaultGraph } = import_n37.DataFactory;
 
 // libs/js/rdf-tools/src/lib/remove-rdf-star.ts
-var import_n37 = require("n3");
+var import_n38 = require("n3");
 var import_stream2 = require("stream");
 function isRDFStarTerm(term) {
   return term.termType === "Quad";
 }
 function removeRDFStar(inputStream, starCount) {
-  const parser = new import_n37.Parser({ format: "N-Quads*" });
+  const parser = new import_n38.Parser({ format: "N-Quads*" });
   const outputStream = new import_stream2.PassThrough();
-  const writer = new import_n37.StreamWriter({ format: "N-Quads" });
+  const writer = new import_n38.StreamWriter({ format: "N-Quads" });
   writer.pipe(outputStream);
   let count = 0;
   parser.parse(inputStream, (error, quad2, prefixes4) => {
@@ -7009,7 +7136,7 @@ async function utilRemoveRdfStarHandler(options) {
   if (verbose)
     console.info(`Output: ${output}`);
   const fileStream = (0, import_fs7.createReadStream)(input);
-  const inputStream = isGzipped ? fileStream.pipe((0, import_zlib2.createGunzip)()) : fileStream;
+  const inputStream = isGzipped ? fileStream.pipe((0, import_zlib3.createGunzip)()) : fileStream;
   let removed = 0;
   const cleanStream = removeRDFStar(inputStream, (count) => {
     removed = count;
@@ -7019,7 +7146,7 @@ async function utilRemoveRdfStarHandler(options) {
   });
   const writeStream = (0, import_fs7.createWriteStream)(output);
   if (isGzipped) {
-    const gzip = (0, import_zlib2.createGzip)();
+    const gzip = (0, import_zlib3.createGzip)();
     gzip.pipe(writeStream);
     for await (const chunk of cleanStream) {
       gzip.write(chunk);
@@ -7047,11 +7174,11 @@ async function utilRemoveRdfStarHandler(options) {
 var import_fs8 = require("fs");
 
 // libs/js/rdf-compare/src/lib/js-rdf-compare.ts
-var import_n38 = require("n3");
+var import_n39 = require("n3");
 var import_jsonld = require("jsonld");
 function parseTurtle(content) {
   return new Promise((resolve2, reject) => {
-    const parser = new import_n38.Parser({ format: "Turtle" });
+    const parser = new import_n39.Parser({ format: "Turtle" });
     const quads = [];
     parser.parse(content, (error, quad2) => {
       if (error)
@@ -7065,7 +7192,7 @@ function parseTurtle(content) {
 }
 function quadsToNQuads(quads) {
   return new Promise((resolve2, reject) => {
-    const writer = new import_n38.Writer({ format: "N-Quads" });
+    const writer = new import_n39.Writer({ format: "N-Quads" });
     writer.addQuads(quads);
     writer.end((error, result) => {
       if (error)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qaecy/cue-cli",
-  "version": "0.0.31",
+  "version": "0.0.33",
   "description": "Cue CLI for QAECY platform",
   "main": "main.js",
   "bin": {
@@ -21,7 +21,6 @@
     "jsonld": "8.3.3",
     "jszip": "3.10.1",
     "n3": "1.26.0",
-    "nanoid": "5.1.6",
     "oxigraph": "0.5.3",
     "protobufjs": "7.5.4",
     "spark-md5": "3.0.2",