@qaecy/cue-cli 0.0.33 → 0.0.34

package/main.js

@@ -107,7 +107,6 @@ var QLEVER_UPDATE_ENDPOINT_EMULATOR = process.env.CUE_QLEVER_ENDPOINT ? `${proce
 var SPARQL_ENDPOINT_EMULATOR = `${EMULATOR_API_GATEWAY_BASE}/triplestore/query`;
 var SPARQL_ENDPOINT = "https://accessors-api-gateway-ueyeemwf2a-oa.a.run.app/triplestore/query";
 var HASH_WORKER_PATH = (0, import_path.join)(__dirname, "hash-worker.js");
-var PROJECT_ID = "qaecy-mvp-406413.appspot.com";
 var FIREBASE_CONFIG = (useEmulator = false) => ({
   apiKey: "AIzaSyCLhz5Wa3ZCERQZVurSt9bqupPeREALFLk",
   appId: "1:151132927589:web:d11c64cc55bdc0f13ab88c",
@@ -4141,7 +4140,7 @@ var CueAuth = class {
 // libs/js/cue-sdk/src/lib/api.ts
 var ENDPOINT_SEARCH = "/assistant/search";
 var ENDPOINT_SPARQL = "/triplestore/query";
-var ENDPOINT_CONSUMPTION = "/data-views/admin/consumption/units";
+var ENDPOINT_CONSUMPTION = "/data-views/admin/consumption";
 var CueApi = class {
   constructor(_auth, _gatewayUrl, projects, sync) {
     this._auth = _auth;
@@ -4201,10 +4200,14 @@ var CueApi = class {
     }
     return response.json();
   }
-  async getConsumption(projectId) {
+  async getConsumption(projectId, tier) {
     const headers = await this._authHeaders();
     const response = await fetch(`${this._gatewayUrl}${ENDPOINT_CONSUMPTION}`, {
-      headers: { ...headers, "x-project-id": projectId }
+      headers: {
+        ...headers,
+        "x-project-id": projectId,
+        ...tier ? { "x-tier": tier } : {}
+      }
     });
     if (!response.ok) {
       throw new Error(`Failed to fetch consumption: ${response.status} ${response.statusText}`);
@@ -4880,6 +4883,30 @@ var CueBlobStorage = class {
     const result = await (0, import_storage3.listAll)(listRef);
     return result.items.map((item) => item.name);
   }
+  /**
+   * Download a file from the public bucket and return its contents as a string.
+   */
+  async downloadPublic(blobName) {
+    const fileRef = (0, import_storage3.ref)(this.options.storagePublic, blobName);
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 1e4);
+    try {
+      const [url, metadata] = await Promise.all([
+        (0, import_storage3.getDownloadURL)(fileRef),
+        (0, import_storage3.getMetadata)(fileRef)
+      ]);
+      const cacheBustedUrl = `${url}&t=${encodeURIComponent(metadata.updated)}`;
+      const res = await fetch(cacheBustedUrl, { signal: controller.signal });
+      if (!res.ok)
+        throw new Error(`HTTP ${res.status}`);
+      return res.text();
+    } catch (err) {
+      const isTimeout = err instanceof Error && err.name === "AbortError";
+      throw new Error(isTimeout ? `Download timed out: ${blobName}` : err instanceof Error ? err.message : String(err));
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
 };
 
 // libs/js/cue-sdk/src/lib/sync.ts
@@ -5941,6 +5968,29 @@ var CueSyncApi = class {
   _bindApi(api) {
     this._api = api;
   }
+  /**
+   * Flushes any pending metadata items from a previous interrupted sync.
+   * Safe to call even when there is nothing new to upload.
+   */
+  async flushPendingMetadata(spaceId, verbose) {
+    const token = await this._auth.getToken();
+    if (!token)
+      throw new Error("Not authenticated. Call cue.auth.signIn() first.");
+    const existing = await _loadPending(spaceId);
+    if (!existing || existing.items.length === 0)
+      return;
+    console.info(`Trying to upload metadata (${existing.items.length} item(s))...`);
+    if (verbose)
+      console.info(`Flushing ${existing.items.length} pending file location(s) from previous sync \u23F3`);
+    try {
+      this._pendingSpaceId = spaceId;
+      this._pendingItems = [];
+      await this._flushBatch(existing.items, spaceId, token, verbose);
+      console.info("Metadata uploaded \u2705");
+    } catch (err) {
+      throw new Error(`METADATA_SYNC_FAILED: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
   /**
    * Returns a preview of what would be synced: cost breakdown for new files only,
    * units required, and units still available. Use this before calling {@link sync}
@@ -5952,33 +6002,56 @@ var CueSyncApi = class {
     if (!token)
       throw new Error("Not authenticated. Call cue.auth.signIn() first.");
     const graph = await this._getOrCreateGraph(spaceId, token);
-    const [remoteFiles, consumption] = await Promise.all([
+    const project = await this._projects.getProject(spaceId);
+    const tier = project?.projectSettings?.tier ?? "l";
+    const [remoteFiles, consumption, creditMap, tierNames] = await Promise.all([
       this._listRemoteFiles(graph, spaceId, providerId, verbose),
-      this._api?.getConsumption(spaceId) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
+      this._api?.getConsumption(spaceId, tier) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance")),
+      this._fetchUnitCreditMap(verbose),
+      this._fetchTierNames()
     ]);
     const report = await compareLocalRemote(localFiles, remoteFiles);
     const toUpload = report.localNotOnRemote ?? [];
     const costRecords = toUpload.length > 0 ? await this.scanCost(toUpload) : [];
-    const unitsToConsume = costRecords.reduce((sum, r) => sum + r.units, 0);
+    let unitsToConsume = 0;
+    let creditsToConsume = 0;
+    for (const r of costRecords) {
+      unitsToConsume += r.units;
+      const tierMap = creditMap[tier];
+      const creditPerUnit = tierMap?.[r.ext] ?? 1;
+      if (verbose && tierMap && !(r.ext in tierMap)) {
+        console.info(`  Unknown format: .${r.ext} (using default rate of 1 credit/unit)`);
+      }
+      const credits = r.units * creditPerUnit;
+      creditsToConsume += credits;
+      r.credits = Math.round(credits);
+    }
+    const tierName = tierNames[tier] ?? tier;
     return {
       costRecords,
+      tier,
+      tierName,
       unitsToConsume,
+      creditsToConsume: Math.round(creditsToConsume),
+      creditsAvailable: consumption.creditsAvailable,
       unitsAvailable: consumption.unitsAvailable,
       filesToUpload: toUpload.length,
       totalLocalFiles: localFiles.length
     };
   }
   async sync(localFiles, options) {
-    const { spaceId, providerId, userId, verbose } = options;
+    const { spaceId, providerId, userId, verbose, onProgress } = options;
     const token = await this._auth.getToken();
     if (!token)
       throw new Error("Not authenticated. Call cue.auth.signIn() first.");
     const graph = await this._getOrCreateGraph(spaceId, token);
     if (verbose)
       console.info("Listing remote files \u23F3");
+    const project = await this._projects.getProject(spaceId);
+    const tier = project?.projectSettings?.tier ?? "l";
     const [remoteFiles, consumption] = await Promise.all([
       this._listRemoteFiles(graph, spaceId, providerId, verbose),
-      this._api?.getConsumption(spaceId) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
+      this._api?.getConsumption(spaceId, tier) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
     ]);
     const { unitsAvailable } = consumption;
     const report = await compareLocalRemote(localFiles, remoteFiles);
@@ -6033,7 +6106,7 @@ var CueSyncApi = class {
         rdfWritten = true;
         syncCount += 1;
         syncSize += file.size || 0;
-        this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, verbose);
+        this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, onProgress);
       } catch (err) {
         failedUploads += 1;
         console.error(`[CueSyncApi] Failed to upload file: ${file.fullPath}`);
@@ -6054,17 +6127,19 @@ var CueSyncApi = class {
       rdfWritten = true;
       syncCount += 1;
       syncSize += file.size || 0;
-      this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, verbose);
+      this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, onProgress);
     }
     await this._drainPending(verbose);
     this._stopFlushTimer();
+    const postSyncConsumption = await (this._api?.getConsumption(spaceId, tier) ?? Promise.resolve({ creditsAvailable: 0 }));
     return {
       syncCount,
       syncSize,
       failedUploads,
       totalCount: report.totalCount,
       totalSize: report.totalSize,
-      rdfWritten
+      rdfWritten,
+      creditsAvailable: postSyncConsumption.creditsAvailable
     };
   }
   async _getOrCreateGraph(spaceId, token) {
@@ -6090,9 +6165,15 @@ var CueSyncApi = class {
   async _listRemoteFiles(graph, spaceId, providerId, verbose) {
     if (verbose)
       console.info(`Listing files in raw space: ${spaceId}`);
+    const graphFilesWithTimeout = Promise.race([
+      this._getGraphFiles(graph, providerId),
+      new Promise(
+        (_, reject) => setTimeout(() => reject(new Error("GRAPH_TIMEOUT: Knowledge graph query timed out")), 15e3)
+      )
+    ]);
     const [blobNames, locationMap] = await Promise.all([
       this._blob.listRaw(spaceId),
-      this._getGraphFiles(graph, providerId)
+      graphFilesWithTimeout
     ]);
     if (verbose)
       console.info(`Found ${blobNames.length} files in raw store for space: ${spaceId}`);
@@ -6148,9 +6229,15 @@ WHERE {
     this._pendingItems = [];
     const existing = await _loadPending(spaceId);
     if (existing && existing.items.length > 0) {
+      console.info(`Trying to upload metadata from interrupted sync (${existing.items.length} item(s))...`);
       if (verbose)
         console.info(`Flushing ${existing.items.length} pending file location(s) from previous sync \u23F3`);
-      await this._flushBatch(existing.items, spaceId, token, verbose);
+      try {
+        await this._flushBatch(existing.items, spaceId, token, verbose);
+        console.info("Metadata uploaded \u2705");
+      } catch (err) {
+        throw new Error(`METADATA_SYNC_FAILED: ${err instanceof Error ? err.message : String(err)}`);
+      }
     }
     const timer = setInterval(() => {
       this._drainPending(verbose).catch(
@@ -6195,15 +6282,26 @@ WHERE {
     }
   }
   async _postFssBatch(items, spaceId, token) {
-    const response = await fetch(`${this._gatewayUrl}${ENDPOINT_FSS_BATCH}`, {
-      method: "POST",
-      headers: {
-        Authorization: `Bearer ${token}`,
-        "Content-Type": "application/json",
-        "x-project-id": spaceId
-      },
-      body: JSON.stringify({ items })
-    });
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 15e3);
+    let response;
+    try {
+      response = await fetch(`${this._gatewayUrl}${ENDPOINT_FSS_BATCH}`, {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${token}`,
+          "Content-Type": "application/json",
+          "x-project-id": spaceId
+        },
+        body: JSON.stringify({ items }),
+        signal: controller.signal
+      });
+    } catch (err) {
+      const isTimeout = err instanceof Error && err.name === "AbortError";
+      throw new Error(`File structure batch POST failed: ${isTimeout ? "request timed out" : err instanceof Error ? err.message : String(err)}`);
+    } finally {
+      clearTimeout(timeout);
+    }
     if (!response.ok) {
       const body = await response.text().catch(() => "");
       throw new Error(`File structure batch POST failed: ${response.status} ${response.statusText}${body ? ` \u2014 ${body}` : ""}`);
@@ -6252,21 +6350,41 @@ WHERE {
     }
     return Array.from(merged.values());
   }
-  _logProgress(syncCount, totalCount, syncSize, totalSize, verbose) {
-    if (!verbose || totalCount === 0)
-      return;
-    if (syncCount % Math.ceil(totalCount / 100) !== 0)
+  async _fetchTierNames() {
+    try {
+      const text = await this._blob.downloadPublic("tier-names.json");
+      return JSON.parse(text);
+    } catch {
+      return {};
+    }
+  }
+  async _fetchUnitCreditMap(verbose) {
+    let text;
+    try {
+      text = await this._blob.downloadPublic("unit-credit.json");
+    } catch (err) {
+      throw new Error(`Couldn't fetch credits table: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (verbose)
+      console.info(`Price file: ${text.length} bytes`);
+    try {
+      return JSON.parse(text);
+    } catch {
+      throw new Error(`Credits table is not valid JSON (${text.length} bytes)`);
+    }
+  }
+  _logProgress(syncCount, totalCount, syncSize, totalSize, onProgress) {
+    if (!onProgress || totalCount === 0)
       return;
     const pct = Math.floor(syncCount / totalCount * 100);
-    console.info(
-      `Progress: ${pct}% (${syncCount}/${totalCount} files, ${syncSize}/${totalSize} bytes)`
-    );
+    onProgress({ percent: pct, syncCount, totalCount, syncSize, totalSize });
   }
 };
 
 // libs/js/cue-sdk/src/lib/cue-node.ts
 var BUCKET_RAW2 = "spaces_raw_eu_west6";
 var BUCKET_PROCESSED2 = "spaces_processed_eu_west6";
+var BUCKET_PUBLIC2 = "cue_public_eu_west6";
 var CueNode = class extends Cue {
   constructor(config) {
     super(config);
@@ -6274,13 +6392,15 @@ var CueNode = class extends Cue {
   _buildApi(projects) {
     const storageRaw = (0, import_storage4.getStorage)(this._app, BUCKET_RAW2);
     const storageProcessed = (0, import_storage4.getStorage)(this._app, BUCKET_PROCESSED2);
+    const storagePublic = (0, import_storage4.getStorage)(this._app, BUCKET_PUBLIC2);
     if (this._isEmulator) {
       const storageHost = this._endpoints.storageEmulatorHost;
       const storagePort = this._endpoints.storageEmulatorPort;
       (0, import_storage4.connectStorageEmulator)(storageRaw, storageHost, storagePort);
       (0, import_storage4.connectStorageEmulator)(storageProcessed, storageHost, storagePort);
+      (0, import_storage4.connectStorageEmulator)(storagePublic, storageHost, storagePort);
     }
-    const blob = new CueBlobStorage({ storageRaw, storageProcessed });
+    const blob = new CueBlobStorage({ storageRaw, storageProcessed, storagePublic });
     const syncApi = new CueSyncApi(this.auth, projects, blob, this._endpoints.gatewayUrl);
     const api = new CueApi(this.auth, this._endpoints.gatewayUrl, projects, syncApi);
     syncApi._bindApi(api);
@@ -6896,46 +7016,6 @@ async function repairTtlHandler(options) {
   }
 }
 
-// apps/desktop/cue-cli/src/helpers/emit-idle.ts
-var import_pubsub = require("@google-cloud/pubsub");
-async function emitIdle(spaceId, hoursBack, useEmulator) {
-  const from = /* @__PURE__ */ new Date();
-  from.setHours(from.getHours() - hoursBack);
-  const message = {
-    space_id: spaceId,
-    first_write: from.toISOString(),
-    last_write: (/* @__PURE__ */ new Date()).toISOString()
-  };
-  await publishMessage("RDF_WRITING_IDLE", message, useEmulator);
-}
-async function publishMessage(topicName, data, useEmulator) {
-  if (useEmulator) {
-    process.env.PUBSUB_EMULATOR_HOST = process.env.PUBSUB_EMULATOR_HOST || "localhost:8085";
-  }
-  const projectId = useEmulator ? "demo-project" : PROJECT_ID;
-  const pubSubClient = new import_pubsub.PubSub({ projectId });
-  const topic = pubSubClient.topic(topicName);
-  const dataBuffer = Buffer.from(JSON.stringify(data));
-  try {
-    if (useEmulator) {
-      const publishPromise = topic.publishMessage({ data: dataBuffer });
-      const timeoutPromise = new Promise(
-        (_, reject) => setTimeout(() => reject(new Error("PubSub publish timeout")), 5e3)
-      );
-      await Promise.race([publishPromise, timeoutPromise]);
-    } else {
-      await topic.publishMessage({ data: dataBuffer });
-    }
-  } catch (error) {
-    console.error(`Error publishing message: ${error.message}`);
-    if (useEmulator) {
-      console.warn("Continuing despite PubSub error in emulator mode...");
-    } else {
-      throw error;
-    }
-  }
-}
-
 // apps/desktop/cue-cli/src/cue-cli-sync.ts
 var import_promises7 = require("fs/promises");
 var import_fs6 = require("fs");
@@ -7005,7 +7085,14 @@ async function syncHandler(options) {
       console.info("Built sync base \u2705\n");
     if (verbose)
       console.info("Authenticating \u23F3");
-    const user = await cue.auth.signInWithApiKey(key, space);
+    let user;
+    try {
+      user = await cue.auth.signInWithApiKey(key, space);
+    } catch (err) {
+      const cause = err instanceof Error ? err.message : String(err);
+      console.error(`Couldn't authenticate. ${cause}`);
+      process.exit(1);
+    }
     if (verbose)
       console.info("Authenticated \u2705\n");
     if (verbose)
@@ -7016,25 +7103,28 @@ async function syncHandler(options) {
       userId: user.uid,
       verbose
     });
+    console.info(`Project is configured at the ${preview.tierName} tier`);
     console.info("Cost estimate (new files only):");
     if (preview.costRecords.length === 0) {
       console.info("  Nothing new to sync.");
     } else {
       for (const r of preview.costRecords) {
-        console.info(`  .${r.ext}: ${r.count} file(s) \u2192 ${r.units} unit(s)`);
+        const creditsStr = r.credits !== void 0 ? ` (${Math.round(r.credits)} credits)` : "";
+        console.info(`  .${r.ext}: ${r.count} file(s)${creditsStr}`);
       }
     }
     console.info(`  New files: ${preview.filesToUpload}/${preview.totalLocalFiles}`);
-    console.info(`  Units required: ${preview.unitsToConsume}`);
-    console.info(`  Units available: ${preview.unitsAvailable}
+    console.info(`  Credits required: ${Math.round(preview.creditsToConsume)}`);
+    console.info(`  Credits available: ${Math.round(preview.creditsAvailable)}
 `);
+    await cue.api.sync.flushPendingMetadata(space, verbose);
     if (preview.filesToUpload === 0) {
       console.info("Everything is already synced.");
       process.exit(0);
     }
-    if (preview.unitsToConsume > preview.unitsAvailable) {
+    if (preview.creditsToConsume > preview.creditsAvailable) {
       console.error(
-        `Insufficient units: ${preview.unitsToConsume} required but only ${preview.unitsAvailable} available.`
+        `Insufficient credits: ${Math.round(preview.creditsToConsume)} required, ${Math.round(preview.creditsAvailable)} available.`
       );
       process.exit(1);
     }
@@ -7047,24 +7137,14 @@ async function syncHandler(options) {
       spaceId: space,
       providerId: provider,
       userId: user.uid,
-      verbose
-    });
-    if (result.rdfWritten && emulators) {
-      if (verbose) {
-        console.info("");
-        console.info(`Throwing RDF_WRITING_IDLE topic (only in emulators) \u23F3`);
+      verbose,
+      onProgress: ({ percent, syncCount, totalCount }) => {
+        const filled = Math.round(percent / 5);
+        const bar = "\u2588".repeat(filled) + "\u2591".repeat(20 - filled);
+        process.stdout.write(`\r  [${bar}] ${percent}% (${syncCount}/${totalCount} files) `);
       }
-      try {
-        await emitIdle(space, 1, emulators);
-        if (verbose)
-          console.info(`Threw RDF_WRITING_IDLE topic \u2705`);
-      } catch (error) {
-        if (verbose)
-          console.error(`Error throwing RDF_WRITING_IDLE topic: ${error instanceof Error ? error.message : String(error)}`);
-        if (verbose)
-          console.warn("Continuing despite PubSub error in emulator mode...");
-      }
-    }
+    });
+    process.stdout.write("\n");
     const zipDeletePromise = zip && !isFile ? deleteUnzipped(path) : Promise.resolve();
     await zipDeletePromise;
     if (zip && verbose)
@@ -7074,6 +7154,7 @@ async function syncHandler(options) {
       console.info(
         `Synced: ${result.syncCount}/${result.totalCount} files (${fileSizePretty(result.syncSize)}/${fileSizePretty(result.totalSize)})`
       );
+      console.info(`Credits remaining: ${Math.round(result.creditsAvailable)}`);
       console.info(`Sync finished \u{1F680}\u{1F680}\u{1F680}`);
       if (result.failedUploads > 0) {
         console.warn(`Total files failed to upload: ${result.failedUploads}`);
@@ -7081,7 +7162,18 @@ async function syncHandler(options) {
     }
     process.exit(0);
   } catch (err) {
-    console.error("[syncHandler] Unexpected error:", err);
+    const msg = err instanceof Error ? err.message : String(err);
+    if (msg.includes("GRAPH_TIMEOUT")) {
+      console.error("Could not reach the knowledge graph. Make sure all required services are running.");
+    } else if (msg.includes("METADATA_SYNC_FAILED")) {
+      console.error("Metadata sync failed. Try again.");
+    } else if (msg.includes("LEDGER_WRITE_FAILED") || msg.includes("File structure batch POST failed")) {
+      console.error(
+        "Failed communication with the knowledge graph. The files are up to date but metadata is not. Run the tool again to sync metadata. This is a swift job."
+      );
+    } else {
+      console.error("[syncHandler] Unexpected error:", err);
+    }
     process.exit(1);
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qaecy/cue-cli",
-  "version": "0.0.33",
+  "version": "0.0.34",
   "description": "Cue CLI for QAECY platform",
   "main": "main.js",
   "bin": {