@qaecy/cue-cli 0.0.15 → 0.0.19

package/main.js

@@ -67,20 +67,20 @@ var init_worker_pool = __esm({
       }
       _next() {
         if (this.queue.length > 0 && this.idleWorkers.length > 0) {
-          const { filePath, resolve, reject } = this.queue.shift();
+          const { filePath, resolve: resolve2, reject } = this.queue.shift();
           const worker = this.idleWorkers.pop();
           worker._currentCallback = (result) => {
             if (result.error)
               reject(new Error(result.error));
             else
-              resolve(result.hash);
+              resolve2(result.hash);
           };
           worker.postMessage(filePath);
         }
       }
       hashFile(filePath) {
-        return new Promise((resolve, reject) => {
-          this.queue.push({ filePath, resolve, reject });
+        return new Promise((resolve2, reject) => {
+          this.queue.push({ filePath, resolve: resolve2, reject });
           this._next();
         });
       }
@@ -93,8 +93,8 @@ var init_worker_pool = __esm({
 
 // apps/desktop/cue-cli/src/main.ts
 var import_commander = require("commander");
-var import_fs6 = require("fs");
-var import_path4 = require("path");
+var import_fs8 = require("fs");
+var import_path5 = require("path");
 
 // apps/desktop/cue-cli/src/variables.ts
 var import_path = require("path");
@@ -119,7 +119,7 @@ var IGNORED_LOCAL = {
 
 // libs/js/sync-tools/src/lib/compare-local-remote.ts
 function compareLocalRemote(localFiles, remoteFiles) {
-  return new Promise((resolve) => {
+  return new Promise((resolve2) => {
     const localByContentUUID = /* @__PURE__ */ new Map();
     const localBylocationUUID = /* @__PURE__ */ new Map();
     for (const lf of localFiles) {
@@ -180,7 +180,7 @@ function compareLocalRemote(localFiles, remoteFiles) {
       }
     }
     const synctPctSize = totalSize > 0 ? syncSize / totalSize : 1;
-    resolve({
+    resolve2({
       localNotOnRemote,
       localNotOnRemotePathOnly,
       remoteNotOnLocal,
@@ -212,6 +212,7 @@ var BUCKET_RAW = "spaces_raw_eu_west6";
 var BUCKET_PROCESSED = "spaces_processed_eu_west6";
 var BUCKET_LOGS = "spaces_logs_eu_west6";
 var BUCKET_PUBLIC = "cue_public_eu_west6";
+var BUCKET_PERSISTENCE = "db_persistence_eu_west6";
 var COLLECTION_CHAT_SESSIONS = "chatSessions";
 var COLLECTION_ORGANIZATIONS = "organizations";
 var COLLECTION_PROJECTS = "projects";
@@ -227,9 +228,10 @@ var import_firestore = require("firebase/firestore");
 var import_auth = require("firebase/auth");
 var import_app = require("firebase/app");
 var CueFirebase = class _CueFirebase {
+  static _instance;
+  _muted = true;
+  _emulator = false;
   constructor(config, auth, muted = false) {
-    this._muted = true;
-    this._emulator = false;
     this._muted = muted;
     this._init(config, auth);
   }
@@ -243,6 +245,28 @@ var CueFirebase = class _CueFirebase {
     }
     return _CueFirebase._instance;
   }
+  _functionAcceptTerms;
+  _functionEmitMessage;
+  _functionGetUserInfo;
+  _functionChangeUserRoleOnProject;
+  _functionInviteUserToProject;
+  _functionRemoveUserFromProject;
+  _storageProcessed;
+  _storageRaw;
+  _storageLogs;
+  _storageChatSessions;
+  _storagePublic;
+  _storagePersistence;
+  _collectionChatSessions;
+  _collectionOrganizations;
+  _collectionProjects;
+  _collectionRDFWriting;
+  _collectionAPIKeys;
+  _collectionUsers;
+  _collectionUserTermsAcceptance;
+  _collectionTiers;
+  _auth;
+  _app;
   get functionAcceptTerms() {
     return this._functionAcceptTerms;
   }
@@ -276,6 +300,9 @@ var CueFirebase = class _CueFirebase {
   get storagePublic() {
     return this._storagePublic;
   }
+  get storagePersistence() {
+    return this._storagePersistence;
+  }
   get collectionChatSessions() {
     return this._collectionChatSessions;
   }
@@ -347,6 +374,7 @@ var CueFirebase = class _CueFirebase {
     this._storageChatSessions = (0, import_storage.getStorage)(app, BUCKET_CHAT_SESSIONS);
     this._storageLogs = (0, import_storage.getStorage)(app, BUCKET_LOGS);
     this._storagePublic = (0, import_storage.getStorage)(app, BUCKET_PUBLIC);
+    this._storagePersistence = (0, import_storage.getStorage)(app, BUCKET_PERSISTENCE);
     this._collectionChatSessions = (0, import_firestore.collection)(
       (0, import_firestore.getFirestore)(app),
       COLLECTION_CHAT_SESSIONS
@@ -384,6 +412,8 @@ var CueFirebase = class _CueFirebase {
       throw new Error("Storage logs is not initialized");
     if (this._storagePublic === void 0)
       throw new Error("Storage public is not initialized");
+    if (this._storagePersistence === void 0)
+      throw new Error("Storage persistence is not initialized");
     if (this._app === void 0)
       throw new Error("App is not initialized");
     const functions = (0, import_functions.getFunctions)(this._app, GCP_REGION);
@@ -395,6 +425,7 @@ var CueFirebase = class _CueFirebase {
     (0, import_storage.connectStorageEmulator)(this._storageChatSessions, "localhost", 9199);
     (0, import_storage.connectStorageEmulator)(this._storageLogs, "localhost", 9199);
     (0, import_storage.connectStorageEmulator)(this._storagePublic, "localhost", 9199);
+    (0, import_storage.connectStorageEmulator)(this._storagePersistence, "localhost", 9199);
     if (!this._muted)
       console.info("Firebase emulators attached");
   }
@@ -3782,20 +3813,20 @@ var extractIdsFromRawPath = (filePath) => {
   const projectId = pathParts[0];
   const fileName = pathParts.pop() ?? "";
   const suffix = `.${fileName.split(".").pop()?.toLowerCase()}`;
-  const documentUUID = fileName.split(".")[0];
+  const documentUUID = fileName.replace(/\.[^.]+$/, "");
   return { projectId, documentUUID, suffix };
 };
 
 // libs/js/id-builders/src/lib/md5-builder.ts
 var import_spark_md5 = __toESM(require("spark-md5"));
 async function fromReadStream(readStream) {
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve2, reject) => {
     const spark = new import_spark_md5.default.ArrayBuffer();
     readStream.on("data", (chunk) => {
       spark.append(chunk);
     });
     readStream.on("end", () => {
-      resolve(spark.end());
+      resolve2(spark.end());
     });
     readStream.on("error", (err) => reject(err));
   });
@@ -4405,11 +4436,11 @@ async function retryWithBackoff(fn, maxRetries, delayMs, label) {
 }
 async function _doGzip(filePath) {
   const gzFilePath = `${filePath}.gz`;
-  await new Promise((resolve, reject) => {
+  await new Promise((resolve2, reject) => {
     const gzip = (0, import_zlib.createGzip)();
     const source = (0, import_fs3.createReadStream)(filePath);
     const dest = (0, import_fs2.createWriteStream)(gzFilePath);
-    (0, import_promises4.pipeline)(source, gzip, dest).then(resolve).catch(reject);
+    (0, import_promises4.pipeline)(source, gzip, dest).then(resolve2).catch(reject);
   });
   await (0, import_promises5.unlink)(filePath);
 }
@@ -5663,8 +5694,13 @@ var prefixes2 = {
   "xsd": prefixCC["xsd"]
 };
 
-// libs/js/rdf-document-writers/src/lib/document-file.ts
+// libs/js/rdf-document-writers/src/lib/custom-template-parser.ts
 var import_n33 = require("n3");
+var { namedNode: namedNode3, literal: literal2 } = import_n33.DataFactory;
+var a2 = namedNode3("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
+
+// libs/js/rdf-document-writers/src/lib/document-file.ts
+var import_n34 = require("n3");
 
 // libs/js/rdf-document-writers/src/lib/file-suffix.ts
 function getFileSuffix(filename) {
@@ -5676,17 +5712,17 @@ function getFileSuffix(filename) {
 }
 
 // libs/js/rdf-document-writers/src/lib/document-file.ts
-var { namedNode: namedNode3, literal: literal2 } = import_n33.DataFactory;
-var a2 = namedNode3("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
+var { namedNode: namedNode4, literal: literal3 } = import_n34.DataFactory;
+var a3 = namedNode4("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
 
 // libs/js/rdf-document-writers/src/lib/serialize-rdf.ts
 async function serializeRDF(namespace, writer) {
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve2, reject) => {
     writer.end((err, res) => {
       if (err)
         reject(err);
-      resolve(`@base <${namespace}>.
-${res}`);
+      resolve2(namespace !== "" ? `@base <${namespace}>.
+${res}` : res);
     });
   });
 }
@@ -5750,7 +5786,7 @@ async function uploadFile(file, spaceId, userId, providerId) {
   let lastError = null;
   while (attempt < maxRetries) {
     try {
-      await new Promise((resolve, reject) => {
+      await new Promise((resolve2, reject) => {
         const uploadTask = (0, import_storage5.uploadBytesResumable)(fileRef, fileBuffer, {
           customMetadata: rawFileMetadata
         });
@@ -5776,7 +5812,7 @@ async function uploadFile(file, spaceId, userId, providerId) {
             });
             reject(error);
           },
-          () => resolve()
+          () => resolve2()
         );
         if (!uploadTask) {
           console.error("[uploadFile] Upload task could not be created:", {
@@ -5874,6 +5910,9 @@ async function publishMessage(topicName, data, useEmulator) {
 }
 
 // apps/desktop/cue-cli/src/cue-cli-sync.ts
+var import_promises7 = require("fs/promises");
+var import_fs6 = require("fs");
+var import_path4 = require("path");
 async function syncHandler(options) {
   const { space, path, verbose, provider, emulators, zip } = options;
   try {
@@ -5881,8 +5920,28 @@ async function syncHandler(options) {
     if (verbose)
       console.info("Building sync base \u23F3");
     const qh = async (query) => queryHandler(query, space, emulators);
-    const [localFiles, remoteFiles] = await Promise.all([
-      listLocalFiles(
+    const resolvedPath = (0, import_path4.resolve)(path);
+    const pathStat = await (0, import_promises7.stat)(resolvedPath);
+    const isFile = pathStat.isFile();
+    let localFiles;
+    if (isFile) {
+      if (verbose)
+        console.info(`Path is a file, syncing single file: ${resolvedPath}`);
+      const md5 = await fromReadStream((0, import_fs6.createReadStream)(resolvedPath));
+      const relativePath = (0, import_path4.basename)(resolvedPath);
+      const contentUUID = contextBasedGuid(md5);
+      const locationUUID = generateFileUUID(relativePath, provider);
+      localFiles = [{
+        relativePath,
+        fullPath: resolvedPath,
+        md5,
+        contentUUID,
+        locationUUID,
+        mtimeMs: pathStat.mtimeMs,
+        size: pathStat.size
+      }];
+    } else {
+      localFiles = await listLocalFiles(
         path,
         provider,
         verbose,
@@ -5890,9 +5949,9 @@ async function syncHandler(options) {
         IGNORED_LOCAL,
         HASH_WORKER_PATH,
         zip
-      ),
-      listRemoteFiles(space, provider, qh, verbose)
-    ]);
+      );
+    }
+    const remoteFiles = await listRemoteFiles(space, provider, qh, verbose);
     const report = await compareLocalRemote(localFiles, remoteFiles);
     if (verbose) {
       console.info("Built sync base \u2705");
@@ -5948,7 +6007,7 @@ async function syncHandler(options) {
         continue;
       }
     }
-    const zipDeletePromise = zip ? deleteUnzipped(path) : Promise.resolve();
+    const zipDeletePromise = zip && !isFile ? deleteUnzipped(path) : Promise.resolve();
     if (verbose && report.localNotOnRemotePathOnly.length)
       console.info(
         `Starting sync of missing file locations (on provider "${provider}") \u23F3`
@@ -6006,13 +6065,97 @@ async function syncHandler(options) {
   }
 }
 
+// apps/desktop/cue-cli/src/cue-cli-util-remove-rdf-star.ts
+var import_fs7 = require("fs");
+var import_zlib2 = require("zlib");
+
+// libs/js/rdf-tools/src/lib/nq-to-nt.ts
+var import_n35 = require("n3");
+var { quad, defaultGraph } = import_n35.DataFactory;
+
+// libs/js/rdf-tools/src/lib/remove-rdf-star.ts
+var import_n36 = require("n3");
+var import_stream2 = require("stream");
+function isRDFStarTerm(term) {
+  return term.termType === "Quad";
+}
+function removeRDFStar(inputStream, starCount) {
+  const parser = new import_n36.Parser({ format: "N-Quads*" });
+  const outputStream = new import_stream2.PassThrough();
+  const writer = new import_n36.StreamWriter({ format: "N-Quads" });
+  writer.pipe(outputStream);
+  let count = 0;
+  parser.parse(inputStream, (error, quad2, prefixes3) => {
+    if (error) {
+      outputStream.emit("error", error);
+      writer.end();
+      return;
+    }
+    if (quad2) {
+      if (!isRDFStarTerm(quad2.subject) && !isRDFStarTerm(quad2.object)) {
+        writer.write(quad2);
+      } else if (starCount) {
+        count++;
+        starCount(count);
+      }
+    } else {
+      console.log("Finished processing input stream of RDF*, ending writer.");
+      writer.end();
+    }
+  });
+  return outputStream;
+}
+
+// apps/desktop/cue-cli/src/cue-cli-util-remove-rdf-star.ts
+async function utilRemoveRdfStarHandler(options) {
+  const { input, output, verbose } = options;
+  const isGzipped = input.endsWith(".gz");
+  if (verbose)
+    console.info(`Input: ${input} (${isGzipped ? "gzipped" : "plain"})`);
+  if (verbose)
+    console.info(`Output: ${output}`);
+  const fileStream = (0, import_fs7.createReadStream)(input);
+  const inputStream = isGzipped ? fileStream.pipe((0, import_zlib2.createGunzip)()) : fileStream;
+  let removed = 0;
+  const cleanStream = removeRDFStar(inputStream, (count) => {
+    removed = count;
+    if (verbose && count % 1e3 === 0) {
+      console.info(`Removed RDF-star triples so far: ${count}`);
+    }
+  });
+  const writeStream = (0, import_fs7.createWriteStream)(output);
+  if (isGzipped) {
+    const gzip = (0, import_zlib2.createGzip)();
+    gzip.pipe(writeStream);
+    for await (const chunk of cleanStream) {
+      gzip.write(chunk);
+    }
+    gzip.end();
+    await new Promise((resolve2, reject) => {
+      writeStream.on("finish", resolve2);
+      writeStream.on("error", reject);
+      gzip.on("error", reject);
+    });
+  } else {
+    for await (const chunk of cleanStream) {
+      writeStream.write(chunk);
+    }
+    writeStream.end();
+    await new Promise((resolve2, reject) => {
+      writeStream.on("finish", resolve2);
+      writeStream.on("error", reject);
+    });
+  }
+  console.info(`Done. Removed ${removed} RDF-star triple(s). Output written to: ${output}`);
+}
+
 // apps/desktop/cue-cli/src/main.ts
 var packageJson;
 try {
-  packageJson = JSON.parse((0, import_fs6.readFileSync)((0, import_path4.join)(__dirname, "package.json"), "utf8"));
+  packageJson = JSON.parse((0, import_fs8.readFileSync)((0, import_path5.join)(__dirname, "package.json"), "utf8"));
 } catch {
   try {
-    packageJson = JSON.parse((0, import_fs6.readFileSync)((0, import_path4.join)(__dirname, "../package.json"), "utf8"));
+    packageJson = JSON.parse((0, import_fs8.readFileSync)((0, import_path5.join)(__dirname, "../package.json"), "utf8"));
   } catch {
     packageJson = { version: "0.0.0" };
     console.warn("Could not find package.json, using fallback version");
@@ -6025,4 +6168,5 @@ program.command("dump").description("Dump Cue Knowledge Graph data to file\n
 program.command("compare").description("Compares folder content to files already updated to Cue").requiredOption("-s, --space <id>", "Specify the space ID (required)").requiredOption("-p, --path <id>", "Specify the folder path (required)").option("-k, --key <api-key>", "Specify the API key (or set CUE_API_KEY env variable)").option("--provider <provider ID>", "Specify the provider ID (eg. sharepoint, drive, dropbox) or leave empty for default provider", "").option("-v, --verbose", "Enable verbose output", false).option("-e, --emulators", "Uses emulators for sync", false).option("-z, --zip", "Include zipped content (will temporarily unzip files with same logic as when syncing and delete them again after the comparison)", false).action(compareHandler);
 program.command("dump-processed").description("Dump processed files to local folder").requiredOption("-s, --space <id>", "Specify the space ID (required)").requiredOption("-p, --processor <id>", "Id of the processor to dump processed files from (required) [eg. writers-blob, processors-cad-files]").option("-k, --key <api-key>", "Specify the API key (or set CUE_API_KEY env variable)").option("-v, --verbose", "Enable verbose output", false).option("-e, --emulators", "Uses emulators for sync", false).action(dumpProcessedHandler);
 program.command("repair-ttl").description("Repair TTL files in the specified space").requiredOption("-s, --space <id>", "Specify the space ID (required)").requiredOption("-p, --processor <id>", "Id of the processor to repair TTL files (required) [eg. writers-blob, processors-cad-files]").requiredOption("-f, --from <subString>", "Substring (Regex) to replace in the TTL file (required)").requiredOption("-t, --to <substituteString>", "Substring to replace in the TTL file (required)").option("-k, --key <api-key>", "Specify the API key (or set CUE_API_KEY env variable)").option("-v, --verbose", "Enable verbose output", false).option("-e, --emulators", "Uses emulators for sync", false).action(repairTtlHandler);
+program.command("util-remove-rdf-star").description("Remove RDF-star (quoted) triples from an NQuads file. Supports .nq and .nq.gz input/output.").requiredOption("-i, --input <path>", "Input file path (.nq or .nq.gz)").requiredOption("-o, --output <path>", "Output file path (.nq or .nq.gz)").option("-v, --verbose", "Enable verbose output", false).action(utilRemoveRdfStarHandler);
 program.parse(process.argv);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qaecy/cue-cli",
-  "version": "0.0.15",
+  "version": "0.0.19",
   "description": "Cue CLI for QAECY platform",
   "main": "main.js",
   "bin": {
@@ -20,13 +20,13 @@
     "jsonld": "8.3.3",
     "jszip": "3.10.1",
     "n3": "1.26.0",
-    "oxigraph": "0.5.0",
+    "oxigraph": "0.5.3",
     "protobufjs": "7.5.4",
     "spark-md5": "3.0.2",
     "sparqljs": "3.7.3",
     "tslib": "2.3.0",
     "uuid": "9.0.0",
-    "axios": "1.13.1"
+    "axios": "1.13.2"
   },
   "engines": {
     "node": ">=18"

package/readme.md

@@ -32,3 +32,17 @@ To sync the current dir to the space with id `<space-id>` under the provider id
 | `-v, --verbose`       | Enable verbose output                                                                                | `false`      |
 | `-e, --emulators`     | Use emulators for sync                                                                               | `false`      |
 | `-z, --zip`           | Include zipped content. Will be unzipped to `<zip_path>_unzipped`. Max uncompressed size: 500 MB, max recursion depth: 3. Cleans up unzipped files after sync. | `false`      |
+
+### util-remove-rdf-star
+
+Remove RDF-star (quoted triple) statements from an NQuads file. Supports both plain `.nq` and gzipped `.nq.gz` files. If the input is gzipped the output will also be gzipped.
+
+`npx @qaecy/cue-cli util-remove-rdf-star -i <input.nq.gz> -o <output.nq.gz> -v`
+
+#### Options
+
+| Option               | Description                             | Default |
+|----------------------|-----------------------------------------|---------|
+| `-i, --input <path>` | Input file path (`.nq` or `.nq.gz`) (required) | N/A |
+| `-o, --output <path>`| Output file path (`.nq` or `.nq.gz`) (required) | N/A |
+| `-v, --verbose`      | Enable verbose output                   | `false` |