@qaecy/cue-cli 0.0.32 → 0.0.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
File without changes
@@ -1,5 +1,27 @@
1
1
  /* @ts-self-types="./dir_scanner_wasm.d.ts" */
2
2
 
3
+ /**
4
+ * Attempts to extract the unit count from any supported file and returns a plain JS
5
+ * object `{ units: number, error: string | null }`.
6
+ *
7
+ * `original_path` is used only for extension detection.
8
+ *
9
+ * ```js
10
+ * const buf = await file.arrayBuffer();
11
+ * const result = debug_file(file.name, new Uint8Array(buf));
12
+ * console.log(result); // { units: 12, error: null }
13
+ * ```
14
+ * @param {string} original_path
15
+ * @param {Uint8Array} data
16
+ * @returns {any}
17
+ */
18
+ export function debug_file(original_path, data) {
19
+ const ptr0 = passStringToWasm0(original_path, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
20
+ const len0 = WASM_VECTOR_LEN;
21
+ const ret = wasm.debug_file(ptr0, len0, data);
22
+ return ret;
23
+ }
24
+
3
25
  /**
4
26
  * Scans a list of in-memory files and returns per-extension metrics sorted by
5
27
  * file count descending.
@@ -101,6 +123,9 @@ function __wbg_get_imports() {
101
123
  const ret = arg0.length;
102
124
  return ret;
103
125
  },
126
+ __wbg_log_7e1aa9064a1dbdbd: function(arg0) {
127
+ console.log(arg0);
128
+ },
104
129
  __wbg_new_0c7403db6e782f19: function(arg0) {
105
130
  const ret = new Uint8Array(arg0);
106
131
  return ret;
@@ -126,6 +151,9 @@ function __wbg_get_imports() {
126
151
  __wbg_set_6be42768c690e380: function(arg0, arg1, arg2) {
127
152
  arg0[arg1] = arg2;
128
153
  },
154
+ __wbg_warn_3cc416af27dbdc02: function(arg0) {
155
+ console.warn(arg0);
156
+ },
129
157
  __wbindgen_cast_0000000000000001: function(arg0) {
130
158
  // Cast intrinsic for `F64 -> Externref`.
131
159
  const ret = arg0;
package/main.js CHANGED
@@ -107,7 +107,6 @@ var QLEVER_UPDATE_ENDPOINT_EMULATOR = process.env.CUE_QLEVER_ENDPOINT ? `${proce
107
107
  var SPARQL_ENDPOINT_EMULATOR = `${EMULATOR_API_GATEWAY_BASE}/triplestore/query`;
108
108
  var SPARQL_ENDPOINT = "https://accessors-api-gateway-ueyeemwf2a-oa.a.run.app/triplestore/query";
109
109
  var HASH_WORKER_PATH = (0, import_path.join)(__dirname, "hash-worker.js");
110
- var PROJECT_ID = "qaecy-mvp-406413.appspot.com";
111
110
  var FIREBASE_CONFIG = (useEmulator = false) => ({
112
111
  apiKey: "AIzaSyCLhz5Wa3ZCERQZVurSt9bqupPeREALFLk",
113
112
  appId: "1:151132927589:web:d11c64cc55bdc0f13ab88c",
@@ -3827,19 +3826,6 @@ function generateFileUUID(filepath, providerId = "") {
3827
3826
  return contextBasedGuid(`${providerId}${filepath}`);
3828
3827
  }
3829
3828
 
3830
- // libs/js/id-builders/src/lib/id-extractor.ts
3831
- var extractIdsFromRawPath = (filePath) => {
3832
- if (filePath.startsWith("/")) {
3833
- filePath = filePath.slice(1);
3834
- }
3835
- const pathParts = filePath.split("/");
3836
- const projectId = pathParts[0];
3837
- const fileName = pathParts.pop() ?? "";
3838
- const suffix = `.${fileName.split(".").pop()?.toLowerCase()}`;
3839
- const documentUUID = fileName.replace(/\.[^.]+$/, "");
3840
- return { projectId, documentUUID, suffix };
3841
- };
3842
-
3843
3829
  // libs/js/id-builders/src/lib/md5-builder.ts
3844
3830
  var import_spark_md5 = __toESM(require("spark-md5"));
3845
3831
  async function fromReadStream(readStream) {
@@ -4154,6 +4140,7 @@ var CueAuth = class {
4154
4140
  // libs/js/cue-sdk/src/lib/api.ts
4155
4141
  var ENDPOINT_SEARCH = "/assistant/search";
4156
4142
  var ENDPOINT_SPARQL = "/triplestore/query";
4143
+ var ENDPOINT_CONSUMPTION = "/data-views/admin/consumption";
4157
4144
  var CueApi = class {
4158
4145
  constructor(_auth, _gatewayUrl, projects, sync) {
4159
4146
  this._auth = _auth;
@@ -4213,6 +4200,20 @@ var CueApi = class {
4213
4200
  }
4214
4201
  return response.json();
4215
4202
  }
4203
+ async getConsumption(projectId, tier) {
4204
+ const headers = await this._authHeaders();
4205
+ const response = await fetch(`${this._gatewayUrl}${ENDPOINT_CONSUMPTION}`, {
4206
+ headers: {
4207
+ ...headers,
4208
+ "x-project-id": projectId,
4209
+ ...tier ? { "x-tier": tier } : {}
4210
+ }
4211
+ });
4212
+ if (!response.ok) {
4213
+ throw new Error(`Failed to fetch consumption: ${response.status} ${response.statusText}`);
4214
+ }
4215
+ return response.json();
4216
+ }
4216
4217
  };
4217
4218
 
4218
4219
  // libs/js/cue-sdk/src/lib/project.ts
@@ -4599,9 +4600,18 @@ var Fuseki = class _Fuseki {
4599
4600
  };
4600
4601
 
4601
4602
  // libs/js/databases/src/lib/graph/qlever.ts
4603
+ var import_zlib = require("zlib");
4604
+ var import_n3 = require("n3");
4605
+ var QLeverLockedError = class extends Error {
4606
+ constructor(body) {
4607
+ super(`QLever is locked (rebuild in progress): ${body}`);
4608
+ this.name = "QLeverLockedError";
4609
+ }
4610
+ };
4602
4611
  var QLever = class _QLever {
4603
4612
  queryEndpoint;
4604
4613
  updateEndpoint;
4614
+ dataEndpoint;
4605
4615
  baseHeaders;
4606
4616
  static RELEVANT_HEADER_KEYS = [
4607
4617
  "authorization",
@@ -4610,14 +4620,44 @@ var QLever = class _QLever {
4610
4620
  "x-user-roles"
4611
4621
  // add more if needed
4612
4622
  ];
4623
+ /** Max retries on 423 Locked (rebuild in progress). */
4624
+ static LOCKED_MAX_RETRIES = parseInt(process.env["QLEVER_LOCKED_MAX_RETRIES"] ?? "10", 10);
4625
+ /** Base delay (ms) for exponential backoff on 423. */
4626
+ static LOCKED_BASE_DELAY_MS = parseInt(process.env["QLEVER_LOCKED_BASE_DELAY_MS"] ?? "2000", 10);
4627
+ /**
4628
+ * Retry an async write operation on 423 Locked with exponential backoff + jitter.
4629
+ * 423 means qlever accessor has an ongoing rebuild; we should wait and retry.
4630
+ */
4631
+ static async _retryOnLocked(fn) {
4632
+ const maxRetries = _QLever.LOCKED_MAX_RETRIES;
4633
+ const baseDelayMs = _QLever.LOCKED_BASE_DELAY_MS;
4634
+ let lastError;
4635
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
4636
+ try {
4637
+ return await fn();
4638
+ } catch (err) {
4639
+ lastError = err;
4640
+ if (err instanceof QLeverLockedError && attempt < maxRetries) {
4641
+ const jitter = 0.5 + Math.random();
4642
+ const delay = baseDelayMs * Math.pow(2, attempt) * jitter;
4643
+ await new Promise((resolve2) => setTimeout(resolve2, delay));
4644
+ continue;
4645
+ }
4646
+ throw err;
4647
+ }
4648
+ }
4649
+ throw lastError;
4650
+ }
4613
4651
  constructor(graphOptions) {
4614
4652
  this.queryEndpoint = graphOptions.queryEndpoint;
4615
4653
  this.updateEndpoint = graphOptions.updateEndpoint;
4654
+ this.dataEndpoint = this.updateEndpoint.replace(/\/update$/, "/data");
4616
4655
  this.baseHeaders = Object.fromEntries(
4617
4656
  Object.entries(graphOptions.originalHeaders || {}).filter(
4618
4657
  ([key]) => _QLever.RELEVANT_HEADER_KEYS.includes(key)
4619
4658
  )
4620
4659
  );
4660
+ this.baseHeaders["x-user-roles"] = "admin";
4621
4661
  }
4622
4662
  async ping() {
4623
4663
  const query3 = "ASK { }";
@@ -4627,7 +4667,6 @@ var QLever = class _QLever {
4627
4667
  async query(query3, accept = "application/sparql-results+json") {
4628
4668
  let res;
4629
4669
  try {
4630
- console.log(this.queryEndpoint);
4631
4670
  res = await fetch(this.queryEndpoint, {
4632
4671
  headers: {
4633
4672
  ...this.baseHeaders,
@@ -4661,19 +4700,65 @@ var QLever = class _QLever {
4661
4700
  return await res.text();
4662
4701
  }
4663
4702
  async update(update) {
4664
- const res = await fetch(this.updateEndpoint, {
4665
- headers: {
4666
- ...this.baseHeaders,
4667
- "Content-Type": "application/x-www-form-urlencoded"
4668
- },
4669
- method: "POST",
4670
- body: new URLSearchParams({ update })
4703
+ return _QLever._retryOnLocked(async () => {
4704
+ const res = await fetch(this.updateEndpoint, {
4705
+ headers: {
4706
+ ...this.baseHeaders,
4707
+ "Content-Type": "application/x-www-form-urlencoded"
4708
+ },
4709
+ method: "POST",
4710
+ body: new URLSearchParams({ update })
4711
+ });
4712
+ if (!res.ok) {
4713
+ const body = await res.text();
4714
+ if (res.status === 423)
4715
+ throw new QLeverLockedError(body);
4716
+ throw new Error(`SPARQL update failed (HTTP ${res.status}): ${body}`);
4717
+ }
4718
+ return await res.json();
4719
+ });
4720
+ }
4721
+ /**
4722
+ * Insert quads via the /data endpoint, grouped by named graph.
4723
+ * This is preferred over SPARQL INSERT DATA for QLever because
4724
+ * the /data endpoint correctly registers named graphs in the index.
4725
+ */
4726
+ async insertData(quads) {
4727
+ await this._postToDataEndpoint(quads, this.dataEndpoint);
4728
+ }
4729
+ /**
4730
+ * Delete quads via the /data/delete endpoint, grouped by named graph.
4731
+ */
4732
+ async deleteData(quads) {
4733
+ await this._postToDataEndpoint(quads, `${this.dataEndpoint}/delete`);
4734
+ }
4735
+ async _postToDataEndpoint(quads, baseUrl) {
4736
+ const nquads = await this._quadsToNQuads(quads);
4737
+ const body = new Uint8Array((0, import_zlib.gzipSync)(Buffer.from(nquads, "utf-8")));
4738
+ await _QLever._retryOnLocked(async () => {
4739
+ const res = await fetch(baseUrl, {
4740
+ method: "POST",
4741
+ headers: {
4742
+ ...this.baseHeaders,
4743
+ "Content-Type": "application/n-quads",
4744
+ "Content-Encoding": "gzip"
4745
+ },
4746
+ body
4747
+ });
4748
+ if (!res.ok) {
4749
+ const text = await res.text();
4750
+ if (res.status === 423)
4751
+ throw new QLeverLockedError(text);
4752
+ throw new Error(`QLever data POST failed (HTTP ${res.status}): ${text}`);
4753
+ }
4754
+ });
4755
+ }
4756
+ _quadsToNQuads(quads) {
4757
+ return new Promise((resolve2, reject) => {
4758
+ const writer = new import_n3.Writer({ format: "application/n-quads" });
4759
+ writer.addQuads(quads);
4760
+ writer.end((err, result) => err ? reject(err) : resolve2(result));
4671
4761
  });
4672
- if (!res.ok) {
4673
- const body = await res.text();
4674
- throw new Error(`SPARQL update failed (HTTP ${res.status}): ${body}`);
4675
- }
4676
- return await res.json();
4677
4762
  }
4678
4763
  };
4679
4764
 
@@ -4718,6 +4803,32 @@ var CueGraphDatabase = class {
4718
4803
  update(updateString) {
4719
4804
  return this._db.update(updateString);
4720
4805
  }
4806
+ /** Returns true if this backend supports the /data bulk-insert endpoint (QLever only). */
4807
+ supportsDataEndpoint() {
4808
+ return this.options.graphType === "qlever";
4809
+ }
4810
+ /**
4811
+ * Insert quads using the backend's preferred bulk-insert mechanism.
4812
+ * For QLever: uses the /data endpoint (correctly registers named graphs).
4813
+ * For Fuseki: falls back to SPARQL INSERT DATA.
4814
+ */
4815
+ insertData(quads) {
4816
+ if (this.options.graphType === "qlever") {
4817
+ return this._db.insertData(quads);
4818
+ }
4819
+ return Promise.reject(new Error("insertData not supported for Fuseki \u2014 use update() with SPARQL INSERT DATA"));
4820
+ }
4821
+ /**
4822
+ * Delete quads using the backend's preferred bulk-delete mechanism.
4823
+ * For QLever: uses the /data/delete endpoint.
4824
+ * For Fuseki: falls back to SPARQL DELETE DATA.
4825
+ */
4826
+ deleteData(quads) {
4827
+ if (this.options.graphType === "qlever") {
4828
+ return this._db.deleteData(quads);
4829
+ }
4830
+ return Promise.reject(new Error("deleteData not supported for Fuseki \u2014 use update() with SPARQL DELETE DATA"));
4831
+ }
4721
4832
  };
4722
4833
 
4723
4834
  // libs/js/databases/src/lib/blob/blob.ts
@@ -4772,12 +4883,37 @@ var CueBlobStorage = class {
4772
4883
  const result = await (0, import_storage3.listAll)(listRef);
4773
4884
  return result.items.map((item) => item.name);
4774
4885
  }
4886
+ /**
4887
+ * Download a file from the public bucket and return its contents as a string.
4888
+ */
4889
+ async downloadPublic(blobName) {
4890
+ const fileRef = (0, import_storage3.ref)(this.options.storagePublic, blobName);
4891
+ const controller = new AbortController();
4892
+ const timeout = setTimeout(() => controller.abort(), 1e4);
4893
+ try {
4894
+ const [url, metadata] = await Promise.all([
4895
+ (0, import_storage3.getDownloadURL)(fileRef),
4896
+ (0, import_storage3.getMetadata)(fileRef)
4897
+ ]);
4898
+ const cacheBustedUrl = `${url}&t=${encodeURIComponent(metadata.updated)}`;
4899
+ const res = await fetch(cacheBustedUrl, { signal: controller.signal });
4900
+ if (!res.ok)
4901
+ throw new Error(`HTTP ${res.status}`);
4902
+ return res.text();
4903
+ } catch (err) {
4904
+ const isTimeout = err instanceof Error && err.name === "AbortError";
4905
+ throw new Error(isTimeout ? `Download timed out: ${blobName}` : err instanceof Error ? err.message : String(err));
4906
+ } finally {
4907
+ clearTimeout(timeout);
4908
+ }
4909
+ }
4775
4910
  };
4776
4911
 
4777
4912
  // libs/js/cue-sdk/src/lib/sync.ts
4778
4913
  var import_promises3 = require("fs/promises");
4779
4914
  var import_path3 = require("path");
4780
4915
  var import_url = require("url");
4916
+ var import_os = require("os");
4781
4917
 
4782
4918
  // libs/js/models/src/lib/file-extensions.ts
4783
4919
  var fileExtensionsInfo = {
@@ -5678,106 +5814,20 @@ var import_uuid4 = require("uuid");
5678
5814
  var import_uuid5 = require("uuid");
5679
5815
 
5680
5816
  // libs/js/rdf-document-writers/src/lib/alternative-representation.ts
5681
- var import_n32 = require("n3");
5817
+ var import_n33 = require("n3");
5682
5818
 
5683
5819
  // libs/js/rdf-document-writers/src/lib/file-location.ts
5684
- var import_n3 = require("n3");
5685
- var { namedNode, literal } = import_n3.DataFactory;
5820
+ var import_n32 = require("n3");
5821
+ var { namedNode, literal } = import_n32.DataFactory;
5686
5822
  var a = namedNode("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
5687
5823
  var prefixes = {
5688
5824
  qcy: qaecyPrefixes["qcy"],
5689
5825
  "qcy-e": qaecyPrefixes["qcy-e"],
5690
5826
  xsd: prefixCC["xsd"]
5691
5827
  };
5692
- function fileLocationRaw(metadata, size, fileContentClasses = ["qcy:FileContent"], writer = new import_n3.Writer({ prefixes }), remoteProviderType = "GCP" /* GCP */) {
5693
- if (metadata.md5_hash === void 0)
5694
- throw new Error("md5_hash must be specified");
5695
- if (metadata.blob_name === void 0)
5696
- throw new Error("blob_name must be specified");
5697
- if (metadata.suffix === void 0)
5698
- throw new Error("suffix must be specified");
5699
- if (metadata.name === void 0)
5700
- throw new Error("name must be specified");
5701
- const ids = extractIdsFromRawPath(metadata.blob_name);
5702
- const fileInfo = getFileInfo(metadata.suffix);
5703
- const mimeCategory = fileTypeToMimeCategory[fileInfo.type];
5704
- const originalName = metadata.name.split("/").pop() ?? metadata.name;
5705
- const fileContent = namedNode(ids.documentUUID);
5706
- const fileLocationUUID = generateFileUUID(
5707
- metadata.name,
5708
- metadata.provider_id ?? ""
5709
- );
5710
- const fileLocation = namedNode(fileLocationUUID);
5711
- fileContentClasses.forEach((cl) => {
5712
- writer.addQuad(fileContent, a, namedNode(cl));
5713
- });
5714
- writer.addQuad(
5715
- fileContent,
5716
- namedNode("qcy:hasMimeCategory"),
5717
- namedNode(`qcy-e:${mimeCategory}`)
5718
- );
5719
- writer.addQuad(
5720
- fileContent,
5721
- namedNode("qcy:md5Hash"),
5722
- literal(metadata.md5_hash)
5723
- );
5724
- writer.addQuad(fileContent, namedNode("qcy:mime"), literal(fileInfo.mime));
5725
- writer.addQuad(
5726
- fileContent,
5727
- namedNode("qcy:openFormat"),
5728
- literal(fileInfo?.open)
5729
- );
5730
- writer.addQuad(fileContent, namedNode("qcy:sizeBytes"), literal(size));
5731
- writer.addQuad(fileContent, namedNode("qcy:hasFileLocation"), fileLocation);
5732
- writer.addQuad(fileLocation, a, namedNode("qcy:FileLocation"));
5733
- writer.addQuad(
5734
- fileLocation,
5735
- namedNode("qcy:filePath"),
5736
- literal(metadata.name)
5737
- );
5738
- writer.addQuad(
5739
- fileLocation,
5740
- namedNode("qcy:remoteProviderType"),
5741
- namedNode(`qcy-e:${remoteProviderType}`)
5742
- );
5743
- writer.addQuad(
5744
- fileLocation,
5745
- namedNode("qcy:remoteProviderId"),
5746
- literal(metadata.provider_id ?? "")
5747
- );
5748
- writer.addQuad(
5749
- fileLocation,
5750
- namedNode("qcy:remoteRelativePath"),
5751
- literal(metadata.blob_name)
5752
- );
5753
- writer.addQuad(
5754
- fileLocation,
5755
- namedNode("qcy:suffix"),
5756
- literal(metadata.suffix)
5757
- );
5758
- writer.addQuad(
5759
- fileLocation,
5760
- namedNode("qcy:dateCreated"),
5761
- literal((/* @__PURE__ */ new Date()).toISOString(), namedNode("xsd:dateTime"))
5762
- );
5763
- writer.addQuad(fileLocation, namedNode("qcy:value"), literal(originalName));
5764
- return writer;
5765
- }
5766
- function getFileInfo(suffix) {
5767
- let fileInfo = fileExtensionsInfo[suffix];
5768
- if (fileInfo === void 0) {
5769
- fileInfo = {
5770
- type: "unknown" /* UNKNOWN */,
5771
- open: false,
5772
- suffix,
5773
- mime: "application/octet-stream"
5774
- };
5775
- }
5776
- return fileInfo;
5777
- }
5778
5828
 
5779
5829
  // libs/js/rdf-document-writers/src/lib/alternative-representation.ts
5780
- var { namedNode: namedNode2 } = import_n32.DataFactory;
5830
+ var { namedNode: namedNode2 } = import_n33.DataFactory;
5781
5831
  var prefixes2 = {
5782
5832
  qcy: qaecyPrefixes["qcy"],
5783
5833
  "qcy-e": qaecyPrefixes["qcy-e"],
@@ -5785,12 +5835,12 @@ var prefixes2 = {
5785
5835
  };
5786
5836
 
5787
5837
  // libs/js/rdf-document-writers/src/lib/custom-template-parser.ts
5788
- var import_n33 = require("n3");
5789
- var { namedNode: namedNode3, literal: literal2 } = import_n33.DataFactory;
5838
+ var import_n34 = require("n3");
5839
+ var { namedNode: namedNode3, literal: literal2 } = import_n34.DataFactory;
5790
5840
  var a2 = namedNode3("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
5791
5841
 
5792
5842
  // libs/js/rdf-document-writers/src/lib/document-file.ts
5793
- var import_n34 = require("n3");
5843
+ var import_n35 = require("n3");
5794
5844
 
5795
5845
  // libs/js/rdf-document-writers/src/lib/file-suffix.ts
5796
5846
  function getFileSuffix(filename) {
@@ -5802,28 +5852,13 @@ function getFileSuffix(filename) {
5802
5852
  }
5803
5853
 
5804
5854
  // libs/js/rdf-document-writers/src/lib/document-file.ts
5805
- var { namedNode: namedNode4, literal: literal3 } = import_n34.DataFactory;
5855
+ var { namedNode: namedNode4, literal: literal3 } = import_n35.DataFactory;
5806
5856
  var a3 = namedNode4("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
5807
5857
 
5808
5858
  // libs/js/rdf-document-writers/src/lib/process-logs.ts
5809
- var import_n35 = require("n3");
5810
-
5811
- // libs/js/rdf-document-writers/src/lib/serialize-rdf.ts
5812
- async function serializeRDF(namespace, writer, format = "ttl") {
5813
- return new Promise((resolve2, reject) => {
5814
- writer.end((err, res) => {
5815
- if (err)
5816
- reject(err);
5817
- const addBase = format === "ttl" && namespace !== "";
5818
- resolve2(addBase ? `@base <${namespace}>.
5819
- ${res}` : res);
5820
- });
5821
- });
5822
- }
5823
-
5824
- // libs/js/rdf-document-writers/src/lib/process-logs.ts
5859
+ var import_n36 = require("n3");
5825
5860
  var import_uuid6 = require("uuid");
5826
- var { namedNode: namedNode5, literal: literal4 } = import_n35.DataFactory;
5861
+ var { namedNode: namedNode5, literal: literal4 } = import_n36.DataFactory;
5827
5862
  var a4 = namedNode5("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
5828
5863
  var prefixes3 = {
5829
5864
  qcy: qaecyPrefixes["qcy"],
@@ -5861,20 +5896,6 @@ function uploadedFileMetadata(originalName, projectId, userId, md5, providerId,
5861
5896
  })
5862
5897
  };
5863
5898
  }
5864
- function turtleFileMetadata(sourceFile, processorId, locationUUID, stored = false) {
5865
- const ids = extractIdsFromRawPath(sourceFile);
5866
- const blob_name = locationUUID !== void 0 ? `${ids.projectId}/triples/${ids.documentUUID}_${locationUUID}_${processorId}.ttl` : `${ids.projectId}/triples/${ids.documentUUID}_${processorId}.ttl`;
5867
- const identifier = locationUUID !== void 0 ? `${ids.documentUUID}_${locationUUID}` : ids.documentUUID;
5868
- return {
5869
- blob_name,
5870
- processor: processorId,
5871
- space_id: ids.projectId,
5872
- stored: stored ? "True" : "False",
5873
- suffix: ".ttl",
5874
- identifier,
5875
- source: sourceFile
5876
- };
5877
- }
5878
5899
 
5879
5900
  // libs/js/cue-sdk/src/lib/sync.ts
5880
5901
  var _scanFn = null;
@@ -5891,31 +5912,148 @@ async function _initWasm() {
5891
5912
  _scanFn = mod.scan;
5892
5913
  }
5893
5914
  var DEFAULT_GRAPH_TYPE = "fuseki";
5894
- var DEFAULT_SERVICE_ID = "cue-cli";
5895
- var DEFAULT_RDF_BASE = "https://cue.qaecy.com/r/";
5896
5915
  var ENDPOINT_FUSEKI_QUERY = "/triplestore/query";
5897
5916
  var ENDPOINT_QLEVER_QUERY = "/sparql/query";
5898
5917
  var ENDPOINT_FUSEKI_UPDATE = "/triplestore/update";
5899
5918
  var ENDPOINT_QLEVER_UPDATE = "/sparql/update";
5919
+ var ENDPOINT_FSS_BATCH = "/commands/file-system-structure/batch";
5920
+ var FSS_BATCH_CHUNK_SIZE = 1e3;
5921
+ var PENDING_LS_PREFIX = "cue:pending:";
5922
+ function _pendingFilePath(spaceId) {
5923
+ return (0, import_path3.join)((0, import_os.tmpdir)(), `cue-sync-pending-${spaceId}.json`);
5924
+ }
5925
+ async function _loadPending(spaceId) {
5926
+ if (typeof window !== "undefined") {
5927
+ const raw = window.localStorage.getItem(`${PENDING_LS_PREFIX}${spaceId}`);
5928
+ return raw ? JSON.parse(raw) : null;
5929
+ }
5930
+ try {
5931
+ const raw = await (0, import_promises3.readFile)(_pendingFilePath(spaceId), "utf-8");
5932
+ return JSON.parse(raw);
5933
+ } catch {
5934
+ return null;
5935
+ }
5936
+ }
5937
+ async function _savePending(batch) {
5938
+ const data = JSON.stringify(batch);
5939
+ if (typeof window !== "undefined") {
5940
+ window.localStorage.setItem(`${PENDING_LS_PREFIX}${batch.spaceId}`, data);
5941
+ return;
5942
+ }
5943
+ await (0, import_promises3.writeFile)(_pendingFilePath(batch.spaceId), data, "utf-8");
5944
+ }
5945
+ async function _clearPending(spaceId) {
5946
+ if (typeof window !== "undefined") {
5947
+ window.localStorage.removeItem(`${PENDING_LS_PREFIX}${spaceId}`);
5948
+ return;
5949
+ }
5950
+ try {
5951
+ await (0, import_promises3.unlink)(_pendingFilePath(spaceId));
5952
+ } catch {
5953
+ }
5954
+ }
5900
5955
  var CueSyncApi = class {
5901
- constructor(_auth, _projects, _blob, _gatewayUrl, _serviceId = DEFAULT_SERVICE_ID, _rdfBase = DEFAULT_RDF_BASE) {
5956
+ constructor(_auth, _projects, _blob, _gatewayUrl) {
5902
5957
  this._auth = _auth;
5903
5958
  this._projects = _projects;
5904
5959
  this._blob = _blob;
5905
5960
  this._gatewayUrl = _gatewayUrl;
5906
- this._serviceId = _serviceId;
5907
- this._rdfBase = _rdfBase;
5908
5961
  }
5909
5962
  _graphMap = /* @__PURE__ */ new Map();
5963
+ _api;
5964
+ _pendingItems = [];
5965
+ _pendingSpaceId = null;
5966
+ _flushTimer = null;
5967
+ /** @internal Injected by CueApi after construction to avoid circular dependency. */
5968
+ _bindApi(api) {
5969
+ this._api = api;
5970
+ }
5971
+ /**
5972
+ * Flushes any pending metadata items from a previous interrupted sync.
5973
+ * Safe to call even when there is nothing new to upload.
5974
+ */
5975
+ async flushPendingMetadata(spaceId, verbose) {
5976
+ const token = await this._auth.getToken();
5977
+ if (!token)
5978
+ throw new Error("Not authenticated. Call cue.auth.signIn() first.");
5979
+ const existing = await _loadPending(spaceId);
5980
+ if (!existing || existing.items.length === 0)
5981
+ return;
5982
+ console.info(`Trying to upload metadata (${existing.items.length} item(s))...`);
5983
+ if (verbose)
5984
+ console.info(`Flushing ${existing.items.length} pending file location(s) from previous sync \u23F3`);
5985
+ try {
5986
+ this._pendingSpaceId = spaceId;
5987
+ this._pendingItems = [];
5988
+ await this._flushBatch(existing.items, spaceId, token, verbose);
5989
+ console.info("Metadata uploaded \u2705");
5990
+ } catch (err) {
5991
+ throw new Error(`METADATA_SYNC_FAILED: ${err instanceof Error ? err.message : String(err)}`);
5992
+ }
5993
+ }
5994
+ /**
5995
+ * Returns a preview of what would be synced: cost breakdown for new files only,
5996
+ * units required, and units still available. Use this before calling {@link sync}
5997
+ * to show the user an accurate cost estimate.
5998
+ */
5999
+ async previewSync(localFiles, options) {
6000
+ const { spaceId, providerId, verbose } = options;
6001
+ const token = await this._auth.getToken();
6002
+ if (!token)
6003
+ throw new Error("Not authenticated. Call cue.auth.signIn() first.");
6004
+ const graph = await this._getOrCreateGraph(spaceId, token);
6005
+ const project = await this._projects.getProject(spaceId);
6006
+ const tier = project?.projectSettings?.tier ?? "l";
6007
+ const [remoteFiles, consumption, creditMap, tierNames] = await Promise.all([
6008
+ this._listRemoteFiles(graph, spaceId, providerId, verbose),
6009
+ this._api?.getConsumption(spaceId, tier) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance")),
6010
+ this._fetchUnitCreditMap(verbose),
6011
+ this._fetchTierNames()
6012
+ ]);
6013
+ const report = await compareLocalRemote(localFiles, remoteFiles);
6014
+ const toUpload = report.localNotOnRemote ?? [];
6015
+ const costRecords = toUpload.length > 0 ? await this.scanCost(toUpload) : [];
6016
+ let unitsToConsume = 0;
6017
+ let creditsToConsume = 0;
6018
+ for (const r of costRecords) {
6019
+ unitsToConsume += r.units;
6020
+ const tierMap = creditMap[tier];
6021
+ const creditPerUnit = tierMap?.[r.ext] ?? 1;
6022
+ if (verbose && tierMap && !(r.ext in tierMap)) {
6023
+ console.info(` Unknown format: .${r.ext} (using default rate of 1 credit/unit)`);
6024
+ }
6025
+ const credits = r.units * creditPerUnit;
6026
+ creditsToConsume += credits;
6027
+ r.credits = Math.round(credits);
6028
+ }
6029
+ const tierName = tierNames[tier] ?? tier;
6030
+ return {
6031
+ costRecords,
6032
+ tier,
6033
+ tierName,
6034
+ unitsToConsume,
6035
+ creditsToConsume: Math.round(creditsToConsume),
6036
+ creditsAvailable: consumption.creditsAvailable,
6037
+ unitsAvailable: consumption.unitsAvailable,
6038
+ filesToUpload: toUpload.length,
6039
+ totalLocalFiles: localFiles.length
6040
+ };
6041
+ }
5910
6042
  async sync(localFiles, options) {
5911
- const { spaceId, providerId, userId, verbose } = options;
6043
+ const { spaceId, providerId, userId, verbose, onProgress } = options;
5912
6044
  const token = await this._auth.getToken();
5913
6045
  if (!token)
5914
6046
  throw new Error("Not authenticated. Call cue.auth.signIn() first.");
5915
6047
  const graph = await this._getOrCreateGraph(spaceId, token);
5916
6048
  if (verbose)
5917
6049
  console.info("Listing remote files \u23F3");
5918
- const remoteFiles = await this._listRemoteFiles(graph, spaceId, providerId, verbose);
6050
+ const project = await this._projects.getProject(spaceId);
6051
+ const tier = project?.projectSettings?.tier ?? "l";
6052
+ const [remoteFiles, consumption] = await Promise.all([
6053
+ this._listRemoteFiles(graph, spaceId, providerId, verbose),
6054
+ this._api?.getConsumption(spaceId, tier) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
6055
+ ]);
6056
+ const { unitsAvailable } = consumption;
5919
6057
  const report = await compareLocalRemote(localFiles, remoteFiles);
5920
6058
  if (verbose) {
5921
6059
  console.info(`Total local files: ${localFiles.length}`);
@@ -5929,6 +6067,16 @@ var CueSyncApi = class {
5929
6067
  let failedUploads = 0;
5930
6068
  let rdfWritten = false;
5931
6069
  const toUpload = report.localNotOnRemote ?? [];
6070
+ if (toUpload.length > 0) {
6071
+ const costRecords = await this.scanCost(toUpload);
6072
+ const unitsToConsume = costRecords.reduce((sum, r) => sum + r.units, 0);
6073
+ if (unitsToConsume > unitsAvailable) {
6074
+ throw new Error(
6075
+ `Insufficient units: ${unitsToConsume} units required but only ${unitsAvailable} available.`
6076
+ );
6077
+ }
6078
+ }
6079
+ await this._initPendingBatch(spaceId, token, verbose);
5932
6080
  if (verbose && toUpload.length)
5933
6081
  console.info("Syncing missing files \u23F3");
5934
6082
  for (const file of toUpload) {
@@ -5948,12 +6096,17 @@ var CueSyncApi = class {
5948
6096
  new Uint8Array(fileBuffer),
5949
6097
  rawMeta
5950
6098
  );
5951
- const uploaded = await this._uploadRdfMetadata(file, rawMeta, verbose);
5952
- if (uploaded)
5953
- rdfWritten = true;
6099
+ await this._queueFileLocation({
6100
+ relativePath: file.relativePath,
6101
+ md5: file.md5,
6102
+ size: file.size,
6103
+ providerId,
6104
+ fileContentExists: false
6105
+ });
6106
+ rdfWritten = true;
5954
6107
  syncCount += 1;
5955
6108
  syncSize += file.size || 0;
5956
- this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, verbose);
6109
+ this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, onProgress);
5957
6110
  } catch (err) {
5958
6111
  failedUploads += 1;
5959
6112
  console.error(`[CueSyncApi] Failed to upload file: ${file.fullPath}`);
@@ -5964,29 +6117,29 @@ var CueSyncApi = class {
5964
6117
  if (verbose && report.localNotOnRemotePathOnly.length)
5965
6118
  console.info(`Syncing missing file locations (on provider "${providerId}") \u23F3`);
5966
6119
  for (const file of report.localNotOnRemotePathOnly) {
5967
- const rawMeta = uploadedFileMetadata(
5968
- file.relativePath,
5969
- spaceId,
5970
- userId,
5971
- file.md5,
5972
- providerId
5973
- );
5974
- if (!rawMeta.blob_name)
5975
- throw new Error(`blob_name missing for ${file.relativePath}`);
5976
- const uploaded = await this._uploadRdfMetadata(file, rawMeta, verbose);
5977
- if (uploaded)
5978
- rdfWritten = true;
6120
+ await this._queueFileLocation({
6121
+ relativePath: file.relativePath,
6122
+ md5: file.md5,
6123
+ size: file.size,
6124
+ providerId,
6125
+ fileContentExists: true
6126
+ });
6127
+ rdfWritten = true;
5979
6128
  syncCount += 1;
5980
6129
  syncSize += file.size || 0;
5981
- this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, verbose);
6130
+ this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, onProgress);
5982
6131
  }
6132
+ await this._drainPending(verbose);
6133
+ this._stopFlushTimer();
6134
+ const postSyncConsumption = await (this._api?.getConsumption(spaceId, tier) ?? Promise.resolve({ creditsAvailable: 0 }));
5983
6135
  return {
5984
6136
  syncCount,
5985
6137
  syncSize,
5986
6138
  failedUploads,
5987
6139
  totalCount: report.totalCount,
5988
6140
  totalSize: report.totalSize,
5989
- rdfWritten
6141
+ rdfWritten,
6142
+ creditsAvailable: postSyncConsumption.creditsAvailable
5990
6143
  };
5991
6144
  }
5992
6145
  async _getOrCreateGraph(spaceId, token) {
@@ -6012,9 +6165,15 @@ var CueSyncApi = class {
6012
6165
  async _listRemoteFiles(graph, spaceId, providerId, verbose) {
6013
6166
  if (verbose)
6014
6167
  console.info(`Listing files in raw space: ${spaceId}`);
6168
+ const graphFilesWithTimeout = Promise.race([
6169
+ this._getGraphFiles(graph, providerId),
6170
+ new Promise(
6171
+ (_, reject) => setTimeout(() => reject(new Error("GRAPH_TIMEOUT: Knowledge graph query timed out")), 15e3)
6172
+ )
6173
+ ]);
6015
6174
  const [blobNames, locationMap] = await Promise.all([
6016
6175
  this._blob.listRaw(spaceId),
6017
- this._getGraphFiles(graph, providerId)
6176
+ graphFilesWithTimeout
6018
6177
  ]);
6019
6178
  if (verbose)
6020
6179
  console.info(`Found ${blobNames.length} files in raw store for space: ${spaceId}`);
@@ -6065,30 +6224,94 @@ WHERE {
6065
6224
  }
6066
6225
  return map;
6067
6226
  }
6068
- async _uploadRdfMetadata(file, rawMeta, verbose) {
6069
- if (!rawMeta.blob_name)
6070
- throw new Error(`blob_name missing for ${file.relativePath}`);
6071
- const ttlMeta = turtleFileMetadata(
6072
- rawMeta.blob_name,
6073
- this._serviceId,
6074
- file.locationUUID
6075
- );
6076
- if (!ttlMeta.blob_name)
6077
- throw new Error(`ttl blob_name missing for ${rawMeta.blob_name}`);
6078
- const writer = fileLocationRaw(rawMeta, file.size);
6079
- const namespace = `${this._rdfBase}${ttlMeta.space_id}/`;
6080
- const triples = await serializeRDF(namespace, writer);
6081
- const uploaded = await this._blob.uploadProcessed(
6082
- ttlMeta.blob_name,
6083
- new Uint8Array(Buffer.from(triples, "utf-8")),
6084
- ttlMeta
6085
- );
6086
- if (!uploaded && verbose) {
6087
- console.info(
6088
- `Graph data for ${file.relativePath} already exists \u2014 skipping \u26A0\uFE0F`
6227
+ async _initPendingBatch(spaceId, token, verbose) {
6228
+ this._pendingSpaceId = spaceId;
6229
+ this._pendingItems = [];
6230
+ const existing = await _loadPending(spaceId);
6231
+ if (existing && existing.items.length > 0) {
6232
+ console.info(`Trying to upload metadata from interrupted sync (${existing.items.length} item(s))...`);
6233
+ if (verbose)
6234
+ console.info(`Flushing ${existing.items.length} pending file location(s) from previous sync \u23F3`);
6235
+ try {
6236
+ await this._flushBatch(existing.items, spaceId, token, verbose);
6237
+ console.info("Metadata uploaded \u2705");
6238
+ } catch (err) {
6239
+ throw new Error(`METADATA_SYNC_FAILED: ${err instanceof Error ? err.message : String(err)}`);
6240
+ }
6241
+ }
6242
+ const timer = setInterval(() => {
6243
+ this._drainPending(verbose).catch(
6244
+ (err) => console.error("[CueSyncApi] Periodic flush failed:", err)
6089
6245
  );
6246
+ }, 6e4);
6247
+ if (typeof timer === "object" && typeof timer.unref === "function") {
6248
+ timer.unref();
6249
+ }
6250
+ this._flushTimer = timer;
6251
+ }
6252
+ async _queueFileLocation(item) {
6253
+ this._pendingItems.push(item);
6254
+ if (this._pendingSpaceId) {
6255
+ await _savePending({ spaceId: this._pendingSpaceId, items: this._pendingItems });
6256
+ }
6257
+ }
6258
+ async _drainPending(verbose) {
6259
+ if (!this._pendingSpaceId || this._pendingItems.length === 0)
6260
+ return;
6261
+ const token = await this._auth.getToken();
6262
+ if (!token)
6263
+ return;
6264
+ await this._flushBatch(this._pendingItems, this._pendingSpaceId, token, verbose);
6265
+ }
6266
+ async _flushBatch(items, spaceId, token, verbose) {
6267
+ const snapshot = [...items];
6268
+ if (this._pendingSpaceId === spaceId)
6269
+ this._pendingItems = [];
6270
+ await _clearPending(spaceId);
6271
+ try {
6272
+ for (let i = 0; i < snapshot.length; i += FSS_BATCH_CHUNK_SIZE) {
6273
+ await this._postFssBatch(snapshot.slice(i, i + FSS_BATCH_CHUNK_SIZE), spaceId, token);
6274
+ }
6275
+ if (verbose)
6276
+ console.info(`Wrote ${snapshot.length} file location(s) to commands API \u2705`);
6277
+ } catch (err) {
6278
+ const restored = [...snapshot, ...this._pendingItems];
6279
+ this._pendingItems = restored;
6280
+ await _savePending({ spaceId, items: restored });
6281
+ throw err;
6282
+ }
6283
+ }
6284
+ async _postFssBatch(items, spaceId, token) {
6285
+ const controller = new AbortController();
6286
+ const timeout = setTimeout(() => controller.abort(), 15e3);
6287
+ let response;
6288
+ try {
6289
+ response = await fetch(`${this._gatewayUrl}${ENDPOINT_FSS_BATCH}`, {
6290
+ method: "POST",
6291
+ headers: {
6292
+ Authorization: `Bearer ${token}`,
6293
+ "Content-Type": "application/json",
6294
+ "x-project-id": spaceId
6295
+ },
6296
+ body: JSON.stringify({ items }),
6297
+ signal: controller.signal
6298
+ });
6299
+ } catch (err) {
6300
+ const isTimeout = err instanceof Error && err.name === "AbortError";
6301
+ throw new Error(`File structure batch POST failed: ${isTimeout ? "request timed out" : err instanceof Error ? err.message : String(err)}`);
6302
+ } finally {
6303
+ clearTimeout(timeout);
6304
+ }
6305
+ if (!response.ok) {
6306
+ const body = await response.text().catch(() => "");
6307
+ throw new Error(`File structure batch POST failed: ${response.status} ${response.statusText}${body ? ` \u2014 ${body}` : ""}`);
6308
+ }
6309
+ }
6310
+ _stopFlushTimer() {
6311
+ if (this._flushTimer !== null) {
6312
+ clearInterval(this._flushTimer);
6313
+ this._flushTimer = null;
6090
6314
  }
6091
- return uploaded;
6092
6315
  }
6093
6316
  /**
6094
6317
  * Scans `localFiles` and returns a per-extension cost breakdown.
@@ -6101,31 +6324,67 @@ WHERE {
6101
6324
  if (!_wasmInitPromise)
6102
6325
  _wasmInitPromise = _initWasm();
6103
6326
  await _wasmInitPromise;
6104
- const entries = await Promise.all(
6105
- localFiles.map(async (f) => ({
6106
- originalPath: f.relativePath,
6107
- data: new Uint8Array(await (0, import_promises3.readFile)(f.fullPath))
6108
- }))
6109
- );
6110
6327
  if (!_scanFn)
6111
6328
  throw new Error("WASM scan function not initialised");
6112
- return _scanFn(entries);
6329
+ const BATCH_SIZE = 200;
6330
+ const merged = /* @__PURE__ */ new Map();
6331
+ for (let i = 0; i < localFiles.length; i += BATCH_SIZE) {
6332
+ const batch = localFiles.slice(i, i + BATCH_SIZE);
6333
+ const entries = await Promise.all(
6334
+ batch.map(async (f) => ({
6335
+ originalPath: f.relativePath,
6336
+ data: new Uint8Array(await (0, import_promises3.readFile)(f.fullPath))
6337
+ }))
6338
+ );
6339
+ const records = _scanFn(entries);
6340
+ for (const r of records) {
6341
+ const existing = merged.get(r.ext);
6342
+ if (existing) {
6343
+ existing.count += r.count;
6344
+ existing.units += r.units;
6345
+ existing.sizeMb += r.sizeMb;
6346
+ } else {
6347
+ merged.set(r.ext, { ...r });
6348
+ }
6349
+ }
6350
+ }
6351
+ return Array.from(merged.values());
6113
6352
  }
6114
- _logProgress(syncCount, totalCount, syncSize, totalSize, verbose) {
6115
- if (!verbose || totalCount === 0)
6116
- return;
6117
- if (syncCount % Math.ceil(totalCount / 100) !== 0)
6353
+ async _fetchTierNames() {
6354
+ try {
6355
+ const text = await this._blob.downloadPublic("tier-names.json");
6356
+ return JSON.parse(text);
6357
+ } catch {
6358
+ return {};
6359
+ }
6360
+ }
6361
+ async _fetchUnitCreditMap(verbose) {
6362
+ let text;
6363
+ try {
6364
+ text = await this._blob.downloadPublic("unit-credit.json");
6365
+ } catch (err) {
6366
+ throw new Error(`Couldn't fetch credits table: ${err instanceof Error ? err.message : String(err)}`);
6367
+ }
6368
+ if (verbose)
6369
+ console.info(`Price file: ${text.length} bytes`);
6370
+ try {
6371
+ return JSON.parse(text);
6372
+ } catch {
6373
+ throw new Error(`Credits table is not valid JSON (${text.length} bytes)`);
6374
+ }
6375
+ }
6376
+ _logProgress(syncCount, totalCount, syncSize, totalSize, onProgress) {
6377
+ if (!onProgress || totalCount === 0)
6118
6378
  return;
6119
6379
  const pct = Math.floor(syncCount / totalCount * 100);
6120
- console.info(
6121
- `Progress: ${pct}% (${syncCount}/${totalCount} files, ${syncSize}/${totalSize} bytes)`
6122
- );
6380
+ onProgress({ percent: pct, syncCount, totalCount, syncSize, totalSize });
6123
6381
  }
6124
6382
  };
6125
6383
 
6126
6384
  // libs/js/cue-sdk/src/lib/cue-node.ts
6127
6385
  var BUCKET_RAW2 = "spaces_raw_eu_west6";
6128
6386
  var BUCKET_PROCESSED2 = "spaces_processed_eu_west6";
6387
+ var BUCKET_PUBLIC2 = "cue_public_eu_west6";
6129
6388
  var CueNode = class extends Cue {
6130
6389
  constructor(config) {
6131
6390
  super(config);
@@ -6133,15 +6392,19 @@ var CueNode = class extends Cue {
6133
6392
  _buildApi(projects) {
6134
6393
  const storageRaw = (0, import_storage4.getStorage)(this._app, BUCKET_RAW2);
6135
6394
  const storageProcessed = (0, import_storage4.getStorage)(this._app, BUCKET_PROCESSED2);
6395
+ const storagePublic = (0, import_storage4.getStorage)(this._app, BUCKET_PUBLIC2);
6136
6396
  if (this._isEmulator) {
6137
6397
  const storageHost = this._endpoints.storageEmulatorHost;
6138
6398
  const storagePort = this._endpoints.storageEmulatorPort;
6139
6399
  (0, import_storage4.connectStorageEmulator)(storageRaw, storageHost, storagePort);
6140
6400
  (0, import_storage4.connectStorageEmulator)(storageProcessed, storageHost, storagePort);
6401
+ (0, import_storage4.connectStorageEmulator)(storagePublic, storageHost, storagePort);
6141
6402
  }
6142
- const blob = new CueBlobStorage({ storageRaw, storageProcessed });
6403
+ const blob = new CueBlobStorage({ storageRaw, storageProcessed, storagePublic });
6143
6404
  const syncApi = new CueSyncApi(this.auth, projects, blob, this._endpoints.gatewayUrl);
6144
- return new CueApi(this.auth, this._endpoints.gatewayUrl, projects, syncApi);
6405
+ const api = new CueApi(this.auth, this._endpoints.gatewayUrl, projects, syncApi);
6406
+ syncApi._bindApi(api);
6407
+ return api;
6145
6408
  }
6146
6409
  };
6147
6410
 
@@ -6324,7 +6587,7 @@ async function dumpProcessedHandler(options) {
6324
6587
  var import_fs2 = require("fs");
6325
6588
  var import_stream = require("stream");
6326
6589
  var import_fs3 = require("fs");
6327
- var import_zlib = require("zlib");
6590
+ var import_zlib2 = require("zlib");
6328
6591
  var import_promises5 = require("stream/promises");
6329
6592
  var import_promises6 = require("fs/promises");
6330
6593
  var import_auth9 = require("firebase/auth");
@@ -6504,7 +6767,7 @@ async function retryWithBackoff(fn, maxRetries, delayMs, label) {
6504
6767
  async function _doGzip(filePath) {
6505
6768
  const gzFilePath = `${filePath}.gz`;
6506
6769
  await new Promise((resolve2, reject) => {
6507
- const gzip = (0, import_zlib.createGzip)();
6770
+ const gzip = (0, import_zlib2.createGzip)();
6508
6771
  const source = (0, import_fs3.createReadStream)(filePath);
6509
6772
  const dest = (0, import_fs2.createWriteStream)(gzFilePath);
6510
6773
  (0, import_promises5.pipeline)(source, gzip, dest).then(resolve2).catch(reject);
@@ -6753,46 +7016,6 @@ async function repairTtlHandler(options) {
6753
7016
  }
6754
7017
  }
6755
7018
 
6756
- // apps/desktop/cue-cli/src/helpers/emit-idle.ts
6757
- var import_pubsub = require("@google-cloud/pubsub");
6758
- async function emitIdle(spaceId, hoursBack, useEmulator) {
6759
- const from = /* @__PURE__ */ new Date();
6760
- from.setHours(from.getHours() - hoursBack);
6761
- const message = {
6762
- space_id: spaceId,
6763
- first_write: from.toISOString(),
6764
- last_write: (/* @__PURE__ */ new Date()).toISOString()
6765
- };
6766
- await publishMessage("RDF_WRITING_IDLE", message, useEmulator);
6767
- }
6768
- async function publishMessage(topicName, data, useEmulator) {
6769
- if (useEmulator) {
6770
- process.env.PUBSUB_EMULATOR_HOST = process.env.PUBSUB_EMULATOR_HOST || "localhost:8085";
6771
- }
6772
- const projectId = useEmulator ? "demo-project" : PROJECT_ID;
6773
- const pubSubClient = new import_pubsub.PubSub({ projectId });
6774
- const topic = pubSubClient.topic(topicName);
6775
- const dataBuffer = Buffer.from(JSON.stringify(data));
6776
- try {
6777
- if (useEmulator) {
6778
- const publishPromise = topic.publishMessage({ data: dataBuffer });
6779
- const timeoutPromise = new Promise(
6780
- (_, reject) => setTimeout(() => reject(new Error("PubSub publish timeout")), 5e3)
6781
- );
6782
- await Promise.race([publishPromise, timeoutPromise]);
6783
- } else {
6784
- await topic.publishMessage({ data: dataBuffer });
6785
- }
6786
- } catch (error) {
6787
- console.error(`Error publishing message: ${error.message}`);
6788
- if (useEmulator) {
6789
- console.warn("Continuing despite PubSub error in emulator mode...");
6790
- } else {
6791
- throw error;
6792
- }
6793
- }
6794
- }
6795
-
6796
7019
  // apps/desktop/cue-cli/src/cue-cli-sync.ts
6797
7020
  var import_promises7 = require("fs/promises");
6798
7021
  var import_fs6 = require("fs");
@@ -6860,46 +7083,68 @@ async function syncHandler(options) {
6860
7083
  }
6861
7084
  if (verbose)
6862
7085
  console.info("Built sync base \u2705\n");
6863
- const costRecords = await cue.api.sync.scanCost(localFiles);
6864
- const totalUnits = costRecords.reduce((sum, r) => sum + r.units, 0);
6865
- console.info("Cost estimate:");
6866
- for (const r of costRecords) {
6867
- console.info(` .${r.ext}: ${r.count} file(s) \u2192 ${r.units} unit(s)`);
7086
+ if (verbose)
7087
+ console.info("Authenticating \u23F3");
7088
+ let user;
7089
+ try {
7090
+ user = await cue.auth.signInWithApiKey(key, space);
7091
+ } catch (err) {
7092
+ const cause = err instanceof Error ? err.message : String(err);
7093
+ console.error(`Couldn't authenticate. ${cause}`);
7094
+ process.exit(1);
7095
+ }
7096
+ if (verbose)
7097
+ console.info("Authenticated \u2705\n");
7098
+ if (verbose)
7099
+ console.info("Checking sync preview \u23F3");
7100
+ const preview = await cue.api.sync.previewSync(localFiles, {
7101
+ spaceId: space,
7102
+ providerId: provider,
7103
+ userId: user.uid,
7104
+ verbose
7105
+ });
7106
+ console.info(`Project is configured at the ${preview.tierName} tier`);
7107
+ console.info("Cost estimate (new files only):");
7108
+ if (preview.costRecords.length === 0) {
7109
+ console.info(" Nothing new to sync.");
7110
+ } else {
7111
+ for (const r of preview.costRecords) {
7112
+ const creditsStr = r.credits !== void 0 ? ` (${Math.round(r.credits)} credits)` : "";
7113
+ console.info(` .${r.ext}: ${r.count} file(s)${creditsStr}`);
7114
+ }
6868
7115
  }
6869
- console.info(` Total: ${totalUnits} unit(s) across ${localFiles.length} file(s)
7116
+ console.info(` New files: ${preview.filesToUpload}/${preview.totalLocalFiles}`);
7117
+ console.info(` Credits required: ${Math.round(preview.creditsToConsume)}`);
7118
+ console.info(` Credits available: ${Math.round(preview.creditsAvailable)}
6870
7119
  `);
7120
+ await cue.api.sync.flushPendingMetadata(space, verbose);
7121
+ if (preview.filesToUpload === 0) {
7122
+ console.info("Everything is already synced.");
7123
+ process.exit(0);
7124
+ }
7125
+ if (preview.creditsToConsume > preview.creditsAvailable) {
7126
+ console.error(
7127
+ `Insufficient credits: ${Math.round(preview.creditsToConsume)} required, ${Math.round(preview.creditsAvailable)} available.`
7128
+ );
7129
+ process.exit(1);
7130
+ }
6871
7131
  const confirmed = await askConfirm("Continue with sync? (y/n) ");
6872
7132
  if (!confirmed) {
6873
7133
  console.info("Sync cancelled.");
6874
7134
  process.exit(0);
6875
7135
  }
6876
- if (verbose)
6877
- console.info("Authenticating \u23F3");
6878
- const user = await cue.auth.signInWithApiKey(key, space);
6879
- if (verbose)
6880
- console.info("Authenticated \u2705\n");
6881
7136
  const result = await cue.api.sync.sync(localFiles, {
6882
7137
  spaceId: space,
6883
7138
  providerId: provider,
6884
7139
  userId: user.uid,
6885
- verbose
6886
- });
6887
- if (result.rdfWritten && emulators) {
6888
- if (verbose) {
6889
- console.info("");
6890
- console.info(`Throwing RDF_WRITING_IDLE topic (only in emulators) \u23F3`);
7140
+ verbose,
7141
+ onProgress: ({ percent, syncCount, totalCount }) => {
7142
+ const filled = Math.round(percent / 5);
7143
+ const bar = "\u2588".repeat(filled) + "\u2591".repeat(20 - filled);
7144
+ process.stdout.write(`\r [${bar}] ${percent}% (${syncCount}/${totalCount} files) `);
6891
7145
  }
6892
- try {
6893
- await emitIdle(space, 1, emulators);
6894
- if (verbose)
6895
- console.info(`Threw RDF_WRITING_IDLE topic \u2705`);
6896
- } catch (error) {
6897
- if (verbose)
6898
- console.error(`Error throwing RDF_WRITING_IDLE topic: ${error instanceof Error ? error.message : String(error)}`);
6899
- if (verbose)
6900
- console.warn("Continuing despite PubSub error in emulator mode...");
6901
- }
6902
- }
7146
+ });
7147
+ process.stdout.write("\n");
6903
7148
  const zipDeletePromise = zip && !isFile ? deleteUnzipped(path) : Promise.resolve();
6904
7149
  await zipDeletePromise;
6905
7150
  if (zip && verbose)
@@ -6909,6 +7154,7 @@ async function syncHandler(options) {
6909
7154
  console.info(
6910
7155
  `Synced: ${result.syncCount}/${result.totalCount} files (${fileSizePretty(result.syncSize)}/${fileSizePretty(result.totalSize)})`
6911
7156
  );
7157
+ console.info(`Credits remaining: ${Math.round(result.creditsAvailable)}`);
6912
7158
  console.info(`Sync finished \u{1F680}\u{1F680}\u{1F680}`);
6913
7159
  if (result.failedUploads > 0) {
6914
7160
  console.warn(`Total files failed to upload: ${result.failedUploads}`);
@@ -6916,29 +7162,40 @@ async function syncHandler(options) {
6916
7162
  }
6917
7163
  process.exit(0);
6918
7164
  } catch (err) {
6919
- console.error("[syncHandler] Unexpected error:", err);
7165
+ const msg = err instanceof Error ? err.message : String(err);
7166
+ if (msg.includes("GRAPH_TIMEOUT")) {
7167
+ console.error("Could not reach the knowledge graph. Make sure all required services are running.");
7168
+ } else if (msg.includes("METADATA_SYNC_FAILED")) {
7169
+ console.error("Metadata sync failed. Try again.");
7170
+ } else if (msg.includes("LEDGER_WRITE_FAILED") || msg.includes("File structure batch POST failed")) {
7171
+ console.error(
7172
+ "Failed communication with the knowledge graph. The files are up to date but metadata is not. Run the tool again to sync metadata. This is a swift job."
7173
+ );
7174
+ } else {
7175
+ console.error("[syncHandler] Unexpected error:", err);
7176
+ }
6920
7177
  process.exit(1);
6921
7178
  }
6922
7179
  }
6923
7180
 
6924
7181
  // apps/desktop/cue-cli/src/cue-cli-util-remove-rdf-star.ts
6925
7182
  var import_fs7 = require("fs");
6926
- var import_zlib2 = require("zlib");
7183
+ var import_zlib3 = require("zlib");
6927
7184
 
6928
7185
  // libs/js/rdf-tools/src/lib/nq-to-nt.ts
6929
- var import_n36 = require("n3");
6930
- var { quad, defaultGraph } = import_n36.DataFactory;
7186
+ var import_n37 = require("n3");
7187
+ var { quad, defaultGraph } = import_n37.DataFactory;
6931
7188
 
6932
7189
  // libs/js/rdf-tools/src/lib/remove-rdf-star.ts
6933
- var import_n37 = require("n3");
7190
+ var import_n38 = require("n3");
6934
7191
  var import_stream2 = require("stream");
6935
7192
  function isRDFStarTerm(term) {
6936
7193
  return term.termType === "Quad";
6937
7194
  }
6938
7195
  function removeRDFStar(inputStream, starCount) {
6939
- const parser = new import_n37.Parser({ format: "N-Quads*" });
7196
+ const parser = new import_n38.Parser({ format: "N-Quads*" });
6940
7197
  const outputStream = new import_stream2.PassThrough();
6941
- const writer = new import_n37.StreamWriter({ format: "N-Quads" });
7198
+ const writer = new import_n38.StreamWriter({ format: "N-Quads" });
6942
7199
  writer.pipe(outputStream);
6943
7200
  let count = 0;
6944
7201
  parser.parse(inputStream, (error, quad2, prefixes4) => {
@@ -6971,7 +7228,7 @@ async function utilRemoveRdfStarHandler(options) {
6971
7228
  if (verbose)
6972
7229
  console.info(`Output: ${output}`);
6973
7230
  const fileStream = (0, import_fs7.createReadStream)(input);
6974
- const inputStream = isGzipped ? fileStream.pipe((0, import_zlib2.createGunzip)()) : fileStream;
7231
+ const inputStream = isGzipped ? fileStream.pipe((0, import_zlib3.createGunzip)()) : fileStream;
6975
7232
  let removed = 0;
6976
7233
  const cleanStream = removeRDFStar(inputStream, (count) => {
6977
7234
  removed = count;
@@ -6981,7 +7238,7 @@ async function utilRemoveRdfStarHandler(options) {
6981
7238
  });
6982
7239
  const writeStream = (0, import_fs7.createWriteStream)(output);
6983
7240
  if (isGzipped) {
6984
- const gzip = (0, import_zlib2.createGzip)();
7241
+ const gzip = (0, import_zlib3.createGzip)();
6985
7242
  gzip.pipe(writeStream);
6986
7243
  for await (const chunk of cleanStream) {
6987
7244
  gzip.write(chunk);
@@ -7009,11 +7266,11 @@ async function utilRemoveRdfStarHandler(options) {
7009
7266
  var import_fs8 = require("fs");
7010
7267
 
7011
7268
  // libs/js/rdf-compare/src/lib/js-rdf-compare.ts
7012
- var import_n38 = require("n3");
7269
+ var import_n39 = require("n3");
7013
7270
  var import_jsonld = require("jsonld");
7014
7271
  function parseTurtle(content) {
7015
7272
  return new Promise((resolve2, reject) => {
7016
- const parser = new import_n38.Parser({ format: "Turtle" });
7273
+ const parser = new import_n39.Parser({ format: "Turtle" });
7017
7274
  const quads = [];
7018
7275
  parser.parse(content, (error, quad2) => {
7019
7276
  if (error)
@@ -7027,7 +7284,7 @@ function parseTurtle(content) {
7027
7284
  }
7028
7285
  function quadsToNQuads(quads) {
7029
7286
  return new Promise((resolve2, reject) => {
7030
- const writer = new import_n38.Writer({ format: "N-Quads" });
7287
+ const writer = new import_n39.Writer({ format: "N-Quads" });
7031
7288
  writer.addQuads(quads);
7032
7289
  writer.end((error, result) => {
7033
7290
  if (error)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@qaecy/cue-cli",
3
- "version": "0.0.32",
3
+ "version": "0.0.34",
4
4
  "description": "Cue CLI for QAECY platform",
5
5
  "main": "main.js",
6
6
  "bin": {
@@ -21,7 +21,6 @@
21
21
  "jsonld": "8.3.3",
22
22
  "jszip": "3.10.1",
23
23
  "n3": "1.26.0",
24
- "nanoid": "5.1.6",
25
24
  "oxigraph": "0.5.3",
26
25
  "protobufjs": "7.5.4",
27
26
  "spark-md5": "3.0.2",