@qaecy/cue-cli 0.0.32 → 0.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/.gitkeep +0 -0
- package/assets/wasm/dir_scanner_wasm.mjs +28 -0
- package/assets/wasm/dir_scanner_wasm_bg.wasm +0 -0
- package/main.js +555 -298
- package/package.json +1 -2
package/assets/.gitkeep
ADDED
|
File without changes
|
|
@@ -1,5 +1,27 @@
|
|
|
1
1
|
/* @ts-self-types="./dir_scanner_wasm.d.ts" */
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Attempts to extract the unit count from any supported file and returns a plain JS
|
|
5
|
+
* object `{ units: number, error: string | null }`.
|
|
6
|
+
*
|
|
7
|
+
* `original_path` is used only for extension detection.
|
|
8
|
+
*
|
|
9
|
+
* ```js
|
|
10
|
+
* const buf = await file.arrayBuffer();
|
|
11
|
+
* const result = debug_file(file.name, new Uint8Array(buf));
|
|
12
|
+
* console.log(result); // { units: 12, error: null }
|
|
13
|
+
* ```
|
|
14
|
+
* @param {string} original_path
|
|
15
|
+
* @param {Uint8Array} data
|
|
16
|
+
* @returns {any}
|
|
17
|
+
*/
|
|
18
|
+
export function debug_file(original_path, data) {
|
|
19
|
+
const ptr0 = passStringToWasm0(original_path, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
20
|
+
const len0 = WASM_VECTOR_LEN;
|
|
21
|
+
const ret = wasm.debug_file(ptr0, len0, data);
|
|
22
|
+
return ret;
|
|
23
|
+
}
|
|
24
|
+
|
|
3
25
|
/**
|
|
4
26
|
* Scans a list of in-memory files and returns per-extension metrics sorted by
|
|
5
27
|
* file count descending.
|
|
@@ -101,6 +123,9 @@ function __wbg_get_imports() {
|
|
|
101
123
|
const ret = arg0.length;
|
|
102
124
|
return ret;
|
|
103
125
|
},
|
|
126
|
+
__wbg_log_7e1aa9064a1dbdbd: function(arg0) {
|
|
127
|
+
console.log(arg0);
|
|
128
|
+
},
|
|
104
129
|
__wbg_new_0c7403db6e782f19: function(arg0) {
|
|
105
130
|
const ret = new Uint8Array(arg0);
|
|
106
131
|
return ret;
|
|
@@ -126,6 +151,9 @@ function __wbg_get_imports() {
|
|
|
126
151
|
__wbg_set_6be42768c690e380: function(arg0, arg1, arg2) {
|
|
127
152
|
arg0[arg1] = arg2;
|
|
128
153
|
},
|
|
154
|
+
__wbg_warn_3cc416af27dbdc02: function(arg0) {
|
|
155
|
+
console.warn(arg0);
|
|
156
|
+
},
|
|
129
157
|
__wbindgen_cast_0000000000000001: function(arg0) {
|
|
130
158
|
// Cast intrinsic for `F64 -> Externref`.
|
|
131
159
|
const ret = arg0;
|
|
Binary file
|
package/main.js
CHANGED
|
@@ -107,7 +107,6 @@ var QLEVER_UPDATE_ENDPOINT_EMULATOR = process.env.CUE_QLEVER_ENDPOINT ? `${proce
|
|
|
107
107
|
var SPARQL_ENDPOINT_EMULATOR = `${EMULATOR_API_GATEWAY_BASE}/triplestore/query`;
|
|
108
108
|
var SPARQL_ENDPOINT = "https://accessors-api-gateway-ueyeemwf2a-oa.a.run.app/triplestore/query";
|
|
109
109
|
var HASH_WORKER_PATH = (0, import_path.join)(__dirname, "hash-worker.js");
|
|
110
|
-
var PROJECT_ID = "qaecy-mvp-406413.appspot.com";
|
|
111
110
|
var FIREBASE_CONFIG = (useEmulator = false) => ({
|
|
112
111
|
apiKey: "AIzaSyCLhz5Wa3ZCERQZVurSt9bqupPeREALFLk",
|
|
113
112
|
appId: "1:151132927589:web:d11c64cc55bdc0f13ab88c",
|
|
@@ -3827,19 +3826,6 @@ function generateFileUUID(filepath, providerId = "") {
|
|
|
3827
3826
|
return contextBasedGuid(`${providerId}${filepath}`);
|
|
3828
3827
|
}
|
|
3829
3828
|
|
|
3830
|
-
// libs/js/id-builders/src/lib/id-extractor.ts
|
|
3831
|
-
var extractIdsFromRawPath = (filePath) => {
|
|
3832
|
-
if (filePath.startsWith("/")) {
|
|
3833
|
-
filePath = filePath.slice(1);
|
|
3834
|
-
}
|
|
3835
|
-
const pathParts = filePath.split("/");
|
|
3836
|
-
const projectId = pathParts[0];
|
|
3837
|
-
const fileName = pathParts.pop() ?? "";
|
|
3838
|
-
const suffix = `.${fileName.split(".").pop()?.toLowerCase()}`;
|
|
3839
|
-
const documentUUID = fileName.replace(/\.[^.]+$/, "");
|
|
3840
|
-
return { projectId, documentUUID, suffix };
|
|
3841
|
-
};
|
|
3842
|
-
|
|
3843
3829
|
// libs/js/id-builders/src/lib/md5-builder.ts
|
|
3844
3830
|
var import_spark_md5 = __toESM(require("spark-md5"));
|
|
3845
3831
|
async function fromReadStream(readStream) {
|
|
@@ -4154,6 +4140,7 @@ var CueAuth = class {
|
|
|
4154
4140
|
// libs/js/cue-sdk/src/lib/api.ts
|
|
4155
4141
|
var ENDPOINT_SEARCH = "/assistant/search";
|
|
4156
4142
|
var ENDPOINT_SPARQL = "/triplestore/query";
|
|
4143
|
+
var ENDPOINT_CONSUMPTION = "/data-views/admin/consumption";
|
|
4157
4144
|
var CueApi = class {
|
|
4158
4145
|
constructor(_auth, _gatewayUrl, projects, sync) {
|
|
4159
4146
|
this._auth = _auth;
|
|
@@ -4213,6 +4200,20 @@ var CueApi = class {
|
|
|
4213
4200
|
}
|
|
4214
4201
|
return response.json();
|
|
4215
4202
|
}
|
|
4203
|
+
async getConsumption(projectId, tier) {
|
|
4204
|
+
const headers = await this._authHeaders();
|
|
4205
|
+
const response = await fetch(`${this._gatewayUrl}${ENDPOINT_CONSUMPTION}`, {
|
|
4206
|
+
headers: {
|
|
4207
|
+
...headers,
|
|
4208
|
+
"x-project-id": projectId,
|
|
4209
|
+
...tier ? { "x-tier": tier } : {}
|
|
4210
|
+
}
|
|
4211
|
+
});
|
|
4212
|
+
if (!response.ok) {
|
|
4213
|
+
throw new Error(`Failed to fetch consumption: ${response.status} ${response.statusText}`);
|
|
4214
|
+
}
|
|
4215
|
+
return response.json();
|
|
4216
|
+
}
|
|
4216
4217
|
};
|
|
4217
4218
|
|
|
4218
4219
|
// libs/js/cue-sdk/src/lib/project.ts
|
|
@@ -4599,9 +4600,18 @@ var Fuseki = class _Fuseki {
|
|
|
4599
4600
|
};
|
|
4600
4601
|
|
|
4601
4602
|
// libs/js/databases/src/lib/graph/qlever.ts
|
|
4603
|
+
var import_zlib = require("zlib");
|
|
4604
|
+
var import_n3 = require("n3");
|
|
4605
|
+
var QLeverLockedError = class extends Error {
|
|
4606
|
+
constructor(body) {
|
|
4607
|
+
super(`QLever is locked (rebuild in progress): ${body}`);
|
|
4608
|
+
this.name = "QLeverLockedError";
|
|
4609
|
+
}
|
|
4610
|
+
};
|
|
4602
4611
|
var QLever = class _QLever {
|
|
4603
4612
|
queryEndpoint;
|
|
4604
4613
|
updateEndpoint;
|
|
4614
|
+
dataEndpoint;
|
|
4605
4615
|
baseHeaders;
|
|
4606
4616
|
static RELEVANT_HEADER_KEYS = [
|
|
4607
4617
|
"authorization",
|
|
@@ -4610,14 +4620,44 @@ var QLever = class _QLever {
|
|
|
4610
4620
|
"x-user-roles"
|
|
4611
4621
|
// add more if needed
|
|
4612
4622
|
];
|
|
4623
|
+
/** Max retries on 423 Locked (rebuild in progress). */
|
|
4624
|
+
static LOCKED_MAX_RETRIES = parseInt(process.env["QLEVER_LOCKED_MAX_RETRIES"] ?? "10", 10);
|
|
4625
|
+
/** Base delay (ms) for exponential backoff on 423. */
|
|
4626
|
+
static LOCKED_BASE_DELAY_MS = parseInt(process.env["QLEVER_LOCKED_BASE_DELAY_MS"] ?? "2000", 10);
|
|
4627
|
+
/**
|
|
4628
|
+
* Retry an async write operation on 423 Locked with exponential backoff + jitter.
|
|
4629
|
+
* 423 means qlever accessor has an ongoing rebuild; we should wait and retry.
|
|
4630
|
+
*/
|
|
4631
|
+
static async _retryOnLocked(fn) {
|
|
4632
|
+
const maxRetries = _QLever.LOCKED_MAX_RETRIES;
|
|
4633
|
+
const baseDelayMs = _QLever.LOCKED_BASE_DELAY_MS;
|
|
4634
|
+
let lastError;
|
|
4635
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
4636
|
+
try {
|
|
4637
|
+
return await fn();
|
|
4638
|
+
} catch (err) {
|
|
4639
|
+
lastError = err;
|
|
4640
|
+
if (err instanceof QLeverLockedError && attempt < maxRetries) {
|
|
4641
|
+
const jitter = 0.5 + Math.random();
|
|
4642
|
+
const delay = baseDelayMs * Math.pow(2, attempt) * jitter;
|
|
4643
|
+
await new Promise((resolve2) => setTimeout(resolve2, delay));
|
|
4644
|
+
continue;
|
|
4645
|
+
}
|
|
4646
|
+
throw err;
|
|
4647
|
+
}
|
|
4648
|
+
}
|
|
4649
|
+
throw lastError;
|
|
4650
|
+
}
|
|
4613
4651
|
constructor(graphOptions) {
|
|
4614
4652
|
this.queryEndpoint = graphOptions.queryEndpoint;
|
|
4615
4653
|
this.updateEndpoint = graphOptions.updateEndpoint;
|
|
4654
|
+
this.dataEndpoint = this.updateEndpoint.replace(/\/update$/, "/data");
|
|
4616
4655
|
this.baseHeaders = Object.fromEntries(
|
|
4617
4656
|
Object.entries(graphOptions.originalHeaders || {}).filter(
|
|
4618
4657
|
([key]) => _QLever.RELEVANT_HEADER_KEYS.includes(key)
|
|
4619
4658
|
)
|
|
4620
4659
|
);
|
|
4660
|
+
this.baseHeaders["x-user-roles"] = "admin";
|
|
4621
4661
|
}
|
|
4622
4662
|
async ping() {
|
|
4623
4663
|
const query3 = "ASK { }";
|
|
@@ -4627,7 +4667,6 @@ var QLever = class _QLever {
|
|
|
4627
4667
|
async query(query3, accept = "application/sparql-results+json") {
|
|
4628
4668
|
let res;
|
|
4629
4669
|
try {
|
|
4630
|
-
console.log(this.queryEndpoint);
|
|
4631
4670
|
res = await fetch(this.queryEndpoint, {
|
|
4632
4671
|
headers: {
|
|
4633
4672
|
...this.baseHeaders,
|
|
@@ -4661,19 +4700,65 @@ var QLever = class _QLever {
|
|
|
4661
4700
|
return await res.text();
|
|
4662
4701
|
}
|
|
4663
4702
|
async update(update) {
|
|
4664
|
-
|
|
4665
|
-
|
|
4666
|
-
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
|
|
4703
|
+
return _QLever._retryOnLocked(async () => {
|
|
4704
|
+
const res = await fetch(this.updateEndpoint, {
|
|
4705
|
+
headers: {
|
|
4706
|
+
...this.baseHeaders,
|
|
4707
|
+
"Content-Type": "application/x-www-form-urlencoded"
|
|
4708
|
+
},
|
|
4709
|
+
method: "POST",
|
|
4710
|
+
body: new URLSearchParams({ update })
|
|
4711
|
+
});
|
|
4712
|
+
if (!res.ok) {
|
|
4713
|
+
const body = await res.text();
|
|
4714
|
+
if (res.status === 423)
|
|
4715
|
+
throw new QLeverLockedError(body);
|
|
4716
|
+
throw new Error(`SPARQL update failed (HTTP ${res.status}): ${body}`);
|
|
4717
|
+
}
|
|
4718
|
+
return await res.json();
|
|
4719
|
+
});
|
|
4720
|
+
}
|
|
4721
|
+
/**
|
|
4722
|
+
* Insert quads via the /data endpoint, grouped by named graph.
|
|
4723
|
+
* This is preferred over SPARQL INSERT DATA for QLever because
|
|
4724
|
+
* the /data endpoint correctly registers named graphs in the index.
|
|
4725
|
+
*/
|
|
4726
|
+
async insertData(quads) {
|
|
4727
|
+
await this._postToDataEndpoint(quads, this.dataEndpoint);
|
|
4728
|
+
}
|
|
4729
|
+
/**
|
|
4730
|
+
* Delete quads via the /data/delete endpoint, grouped by named graph.
|
|
4731
|
+
*/
|
|
4732
|
+
async deleteData(quads) {
|
|
4733
|
+
await this._postToDataEndpoint(quads, `${this.dataEndpoint}/delete`);
|
|
4734
|
+
}
|
|
4735
|
+
async _postToDataEndpoint(quads, baseUrl) {
|
|
4736
|
+
const nquads = await this._quadsToNQuads(quads);
|
|
4737
|
+
const body = new Uint8Array((0, import_zlib.gzipSync)(Buffer.from(nquads, "utf-8")));
|
|
4738
|
+
await _QLever._retryOnLocked(async () => {
|
|
4739
|
+
const res = await fetch(baseUrl, {
|
|
4740
|
+
method: "POST",
|
|
4741
|
+
headers: {
|
|
4742
|
+
...this.baseHeaders,
|
|
4743
|
+
"Content-Type": "application/n-quads",
|
|
4744
|
+
"Content-Encoding": "gzip"
|
|
4745
|
+
},
|
|
4746
|
+
body
|
|
4747
|
+
});
|
|
4748
|
+
if (!res.ok) {
|
|
4749
|
+
const text = await res.text();
|
|
4750
|
+
if (res.status === 423)
|
|
4751
|
+
throw new QLeverLockedError(text);
|
|
4752
|
+
throw new Error(`QLever data POST failed (HTTP ${res.status}): ${text}`);
|
|
4753
|
+
}
|
|
4754
|
+
});
|
|
4755
|
+
}
|
|
4756
|
+
_quadsToNQuads(quads) {
|
|
4757
|
+
return new Promise((resolve2, reject) => {
|
|
4758
|
+
const writer = new import_n3.Writer({ format: "application/n-quads" });
|
|
4759
|
+
writer.addQuads(quads);
|
|
4760
|
+
writer.end((err, result) => err ? reject(err) : resolve2(result));
|
|
4671
4761
|
});
|
|
4672
|
-
if (!res.ok) {
|
|
4673
|
-
const body = await res.text();
|
|
4674
|
-
throw new Error(`SPARQL update failed (HTTP ${res.status}): ${body}`);
|
|
4675
|
-
}
|
|
4676
|
-
return await res.json();
|
|
4677
4762
|
}
|
|
4678
4763
|
};
|
|
4679
4764
|
|
|
@@ -4718,6 +4803,32 @@ var CueGraphDatabase = class {
|
|
|
4718
4803
|
update(updateString) {
|
|
4719
4804
|
return this._db.update(updateString);
|
|
4720
4805
|
}
|
|
4806
|
+
/** Returns true if this backend supports the /data bulk-insert endpoint (QLever only). */
|
|
4807
|
+
supportsDataEndpoint() {
|
|
4808
|
+
return this.options.graphType === "qlever";
|
|
4809
|
+
}
|
|
4810
|
+
/**
|
|
4811
|
+
* Insert quads using the backend's preferred bulk-insert mechanism.
|
|
4812
|
+
* For QLever: uses the /data endpoint (correctly registers named graphs).
|
|
4813
|
+
* For Fuseki: falls back to SPARQL INSERT DATA.
|
|
4814
|
+
*/
|
|
4815
|
+
insertData(quads) {
|
|
4816
|
+
if (this.options.graphType === "qlever") {
|
|
4817
|
+
return this._db.insertData(quads);
|
|
4818
|
+
}
|
|
4819
|
+
return Promise.reject(new Error("insertData not supported for Fuseki \u2014 use update() with SPARQL INSERT DATA"));
|
|
4820
|
+
}
|
|
4821
|
+
/**
|
|
4822
|
+
* Delete quads using the backend's preferred bulk-delete mechanism.
|
|
4823
|
+
* For QLever: uses the /data/delete endpoint.
|
|
4824
|
+
* For Fuseki: falls back to SPARQL DELETE DATA.
|
|
4825
|
+
*/
|
|
4826
|
+
deleteData(quads) {
|
|
4827
|
+
if (this.options.graphType === "qlever") {
|
|
4828
|
+
return this._db.deleteData(quads);
|
|
4829
|
+
}
|
|
4830
|
+
return Promise.reject(new Error("deleteData not supported for Fuseki \u2014 use update() with SPARQL DELETE DATA"));
|
|
4831
|
+
}
|
|
4721
4832
|
};
|
|
4722
4833
|
|
|
4723
4834
|
// libs/js/databases/src/lib/blob/blob.ts
|
|
@@ -4772,12 +4883,37 @@ var CueBlobStorage = class {
|
|
|
4772
4883
|
const result = await (0, import_storage3.listAll)(listRef);
|
|
4773
4884
|
return result.items.map((item) => item.name);
|
|
4774
4885
|
}
|
|
4886
|
+
/**
|
|
4887
|
+
* Download a file from the public bucket and return its contents as a string.
|
|
4888
|
+
*/
|
|
4889
|
+
async downloadPublic(blobName) {
|
|
4890
|
+
const fileRef = (0, import_storage3.ref)(this.options.storagePublic, blobName);
|
|
4891
|
+
const controller = new AbortController();
|
|
4892
|
+
const timeout = setTimeout(() => controller.abort(), 1e4);
|
|
4893
|
+
try {
|
|
4894
|
+
const [url, metadata] = await Promise.all([
|
|
4895
|
+
(0, import_storage3.getDownloadURL)(fileRef),
|
|
4896
|
+
(0, import_storage3.getMetadata)(fileRef)
|
|
4897
|
+
]);
|
|
4898
|
+
const cacheBustedUrl = `${url}&t=${encodeURIComponent(metadata.updated)}`;
|
|
4899
|
+
const res = await fetch(cacheBustedUrl, { signal: controller.signal });
|
|
4900
|
+
if (!res.ok)
|
|
4901
|
+
throw new Error(`HTTP ${res.status}`);
|
|
4902
|
+
return res.text();
|
|
4903
|
+
} catch (err) {
|
|
4904
|
+
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
4905
|
+
throw new Error(isTimeout ? `Download timed out: ${blobName}` : err instanceof Error ? err.message : String(err));
|
|
4906
|
+
} finally {
|
|
4907
|
+
clearTimeout(timeout);
|
|
4908
|
+
}
|
|
4909
|
+
}
|
|
4775
4910
|
};
|
|
4776
4911
|
|
|
4777
4912
|
// libs/js/cue-sdk/src/lib/sync.ts
|
|
4778
4913
|
var import_promises3 = require("fs/promises");
|
|
4779
4914
|
var import_path3 = require("path");
|
|
4780
4915
|
var import_url = require("url");
|
|
4916
|
+
var import_os = require("os");
|
|
4781
4917
|
|
|
4782
4918
|
// libs/js/models/src/lib/file-extensions.ts
|
|
4783
4919
|
var fileExtensionsInfo = {
|
|
@@ -5678,106 +5814,20 @@ var import_uuid4 = require("uuid");
|
|
|
5678
5814
|
var import_uuid5 = require("uuid");
|
|
5679
5815
|
|
|
5680
5816
|
// libs/js/rdf-document-writers/src/lib/alternative-representation.ts
|
|
5681
|
-
var
|
|
5817
|
+
var import_n33 = require("n3");
|
|
5682
5818
|
|
|
5683
5819
|
// libs/js/rdf-document-writers/src/lib/file-location.ts
|
|
5684
|
-
var
|
|
5685
|
-
var { namedNode, literal } =
|
|
5820
|
+
var import_n32 = require("n3");
|
|
5821
|
+
var { namedNode, literal } = import_n32.DataFactory;
|
|
5686
5822
|
var a = namedNode("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
|
|
5687
5823
|
var prefixes = {
|
|
5688
5824
|
qcy: qaecyPrefixes["qcy"],
|
|
5689
5825
|
"qcy-e": qaecyPrefixes["qcy-e"],
|
|
5690
5826
|
xsd: prefixCC["xsd"]
|
|
5691
5827
|
};
|
|
5692
|
-
function fileLocationRaw(metadata, size, fileContentClasses = ["qcy:FileContent"], writer = new import_n3.Writer({ prefixes }), remoteProviderType = "GCP" /* GCP */) {
|
|
5693
|
-
if (metadata.md5_hash === void 0)
|
|
5694
|
-
throw new Error("md5_hash must be specified");
|
|
5695
|
-
if (metadata.blob_name === void 0)
|
|
5696
|
-
throw new Error("blob_name must be specified");
|
|
5697
|
-
if (metadata.suffix === void 0)
|
|
5698
|
-
throw new Error("suffix must be specified");
|
|
5699
|
-
if (metadata.name === void 0)
|
|
5700
|
-
throw new Error("name must be specified");
|
|
5701
|
-
const ids = extractIdsFromRawPath(metadata.blob_name);
|
|
5702
|
-
const fileInfo = getFileInfo(metadata.suffix);
|
|
5703
|
-
const mimeCategory = fileTypeToMimeCategory[fileInfo.type];
|
|
5704
|
-
const originalName = metadata.name.split("/").pop() ?? metadata.name;
|
|
5705
|
-
const fileContent = namedNode(ids.documentUUID);
|
|
5706
|
-
const fileLocationUUID = generateFileUUID(
|
|
5707
|
-
metadata.name,
|
|
5708
|
-
metadata.provider_id ?? ""
|
|
5709
|
-
);
|
|
5710
|
-
const fileLocation = namedNode(fileLocationUUID);
|
|
5711
|
-
fileContentClasses.forEach((cl) => {
|
|
5712
|
-
writer.addQuad(fileContent, a, namedNode(cl));
|
|
5713
|
-
});
|
|
5714
|
-
writer.addQuad(
|
|
5715
|
-
fileContent,
|
|
5716
|
-
namedNode("qcy:hasMimeCategory"),
|
|
5717
|
-
namedNode(`qcy-e:${mimeCategory}`)
|
|
5718
|
-
);
|
|
5719
|
-
writer.addQuad(
|
|
5720
|
-
fileContent,
|
|
5721
|
-
namedNode("qcy:md5Hash"),
|
|
5722
|
-
literal(metadata.md5_hash)
|
|
5723
|
-
);
|
|
5724
|
-
writer.addQuad(fileContent, namedNode("qcy:mime"), literal(fileInfo.mime));
|
|
5725
|
-
writer.addQuad(
|
|
5726
|
-
fileContent,
|
|
5727
|
-
namedNode("qcy:openFormat"),
|
|
5728
|
-
literal(fileInfo?.open)
|
|
5729
|
-
);
|
|
5730
|
-
writer.addQuad(fileContent, namedNode("qcy:sizeBytes"), literal(size));
|
|
5731
|
-
writer.addQuad(fileContent, namedNode("qcy:hasFileLocation"), fileLocation);
|
|
5732
|
-
writer.addQuad(fileLocation, a, namedNode("qcy:FileLocation"));
|
|
5733
|
-
writer.addQuad(
|
|
5734
|
-
fileLocation,
|
|
5735
|
-
namedNode("qcy:filePath"),
|
|
5736
|
-
literal(metadata.name)
|
|
5737
|
-
);
|
|
5738
|
-
writer.addQuad(
|
|
5739
|
-
fileLocation,
|
|
5740
|
-
namedNode("qcy:remoteProviderType"),
|
|
5741
|
-
namedNode(`qcy-e:${remoteProviderType}`)
|
|
5742
|
-
);
|
|
5743
|
-
writer.addQuad(
|
|
5744
|
-
fileLocation,
|
|
5745
|
-
namedNode("qcy:remoteProviderId"),
|
|
5746
|
-
literal(metadata.provider_id ?? "")
|
|
5747
|
-
);
|
|
5748
|
-
writer.addQuad(
|
|
5749
|
-
fileLocation,
|
|
5750
|
-
namedNode("qcy:remoteRelativePath"),
|
|
5751
|
-
literal(metadata.blob_name)
|
|
5752
|
-
);
|
|
5753
|
-
writer.addQuad(
|
|
5754
|
-
fileLocation,
|
|
5755
|
-
namedNode("qcy:suffix"),
|
|
5756
|
-
literal(metadata.suffix)
|
|
5757
|
-
);
|
|
5758
|
-
writer.addQuad(
|
|
5759
|
-
fileLocation,
|
|
5760
|
-
namedNode("qcy:dateCreated"),
|
|
5761
|
-
literal((/* @__PURE__ */ new Date()).toISOString(), namedNode("xsd:dateTime"))
|
|
5762
|
-
);
|
|
5763
|
-
writer.addQuad(fileLocation, namedNode("qcy:value"), literal(originalName));
|
|
5764
|
-
return writer;
|
|
5765
|
-
}
|
|
5766
|
-
function getFileInfo(suffix) {
|
|
5767
|
-
let fileInfo = fileExtensionsInfo[suffix];
|
|
5768
|
-
if (fileInfo === void 0) {
|
|
5769
|
-
fileInfo = {
|
|
5770
|
-
type: "unknown" /* UNKNOWN */,
|
|
5771
|
-
open: false,
|
|
5772
|
-
suffix,
|
|
5773
|
-
mime: "application/octet-stream"
|
|
5774
|
-
};
|
|
5775
|
-
}
|
|
5776
|
-
return fileInfo;
|
|
5777
|
-
}
|
|
5778
5828
|
|
|
5779
5829
|
// libs/js/rdf-document-writers/src/lib/alternative-representation.ts
|
|
5780
|
-
var { namedNode: namedNode2 } =
|
|
5830
|
+
var { namedNode: namedNode2 } = import_n33.DataFactory;
|
|
5781
5831
|
var prefixes2 = {
|
|
5782
5832
|
qcy: qaecyPrefixes["qcy"],
|
|
5783
5833
|
"qcy-e": qaecyPrefixes["qcy-e"],
|
|
@@ -5785,12 +5835,12 @@ var prefixes2 = {
|
|
|
5785
5835
|
};
|
|
5786
5836
|
|
|
5787
5837
|
// libs/js/rdf-document-writers/src/lib/custom-template-parser.ts
|
|
5788
|
-
var
|
|
5789
|
-
var { namedNode: namedNode3, literal: literal2 } =
|
|
5838
|
+
var import_n34 = require("n3");
|
|
5839
|
+
var { namedNode: namedNode3, literal: literal2 } = import_n34.DataFactory;
|
|
5790
5840
|
var a2 = namedNode3("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
|
|
5791
5841
|
|
|
5792
5842
|
// libs/js/rdf-document-writers/src/lib/document-file.ts
|
|
5793
|
-
var
|
|
5843
|
+
var import_n35 = require("n3");
|
|
5794
5844
|
|
|
5795
5845
|
// libs/js/rdf-document-writers/src/lib/file-suffix.ts
|
|
5796
5846
|
function getFileSuffix(filename) {
|
|
@@ -5802,28 +5852,13 @@ function getFileSuffix(filename) {
|
|
|
5802
5852
|
}
|
|
5803
5853
|
|
|
5804
5854
|
// libs/js/rdf-document-writers/src/lib/document-file.ts
|
|
5805
|
-
var { namedNode: namedNode4, literal: literal3 } =
|
|
5855
|
+
var { namedNode: namedNode4, literal: literal3 } = import_n35.DataFactory;
|
|
5806
5856
|
var a3 = namedNode4("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
|
|
5807
5857
|
|
|
5808
5858
|
// libs/js/rdf-document-writers/src/lib/process-logs.ts
|
|
5809
|
-
var
|
|
5810
|
-
|
|
5811
|
-
// libs/js/rdf-document-writers/src/lib/serialize-rdf.ts
|
|
5812
|
-
async function serializeRDF(namespace, writer, format = "ttl") {
|
|
5813
|
-
return new Promise((resolve2, reject) => {
|
|
5814
|
-
writer.end((err, res) => {
|
|
5815
|
-
if (err)
|
|
5816
|
-
reject(err);
|
|
5817
|
-
const addBase = format === "ttl" && namespace !== "";
|
|
5818
|
-
resolve2(addBase ? `@base <${namespace}>.
|
|
5819
|
-
${res}` : res);
|
|
5820
|
-
});
|
|
5821
|
-
});
|
|
5822
|
-
}
|
|
5823
|
-
|
|
5824
|
-
// libs/js/rdf-document-writers/src/lib/process-logs.ts
|
|
5859
|
+
var import_n36 = require("n3");
|
|
5825
5860
|
var import_uuid6 = require("uuid");
|
|
5826
|
-
var { namedNode: namedNode5, literal: literal4 } =
|
|
5861
|
+
var { namedNode: namedNode5, literal: literal4 } = import_n36.DataFactory;
|
|
5827
5862
|
var a4 = namedNode5("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
|
|
5828
5863
|
var prefixes3 = {
|
|
5829
5864
|
qcy: qaecyPrefixes["qcy"],
|
|
@@ -5861,20 +5896,6 @@ function uploadedFileMetadata(originalName, projectId, userId, md5, providerId,
|
|
|
5861
5896
|
})
|
|
5862
5897
|
};
|
|
5863
5898
|
}
|
|
5864
|
-
function turtleFileMetadata(sourceFile, processorId, locationUUID, stored = false) {
|
|
5865
|
-
const ids = extractIdsFromRawPath(sourceFile);
|
|
5866
|
-
const blob_name = locationUUID !== void 0 ? `${ids.projectId}/triples/${ids.documentUUID}_${locationUUID}_${processorId}.ttl` : `${ids.projectId}/triples/${ids.documentUUID}_${processorId}.ttl`;
|
|
5867
|
-
const identifier = locationUUID !== void 0 ? `${ids.documentUUID}_${locationUUID}` : ids.documentUUID;
|
|
5868
|
-
return {
|
|
5869
|
-
blob_name,
|
|
5870
|
-
processor: processorId,
|
|
5871
|
-
space_id: ids.projectId,
|
|
5872
|
-
stored: stored ? "True" : "False",
|
|
5873
|
-
suffix: ".ttl",
|
|
5874
|
-
identifier,
|
|
5875
|
-
source: sourceFile
|
|
5876
|
-
};
|
|
5877
|
-
}
|
|
5878
5899
|
|
|
5879
5900
|
// libs/js/cue-sdk/src/lib/sync.ts
|
|
5880
5901
|
var _scanFn = null;
|
|
@@ -5891,31 +5912,148 @@ async function _initWasm() {
|
|
|
5891
5912
|
_scanFn = mod.scan;
|
|
5892
5913
|
}
|
|
5893
5914
|
var DEFAULT_GRAPH_TYPE = "fuseki";
|
|
5894
|
-
var DEFAULT_SERVICE_ID = "cue-cli";
|
|
5895
|
-
var DEFAULT_RDF_BASE = "https://cue.qaecy.com/r/";
|
|
5896
5915
|
var ENDPOINT_FUSEKI_QUERY = "/triplestore/query";
|
|
5897
5916
|
var ENDPOINT_QLEVER_QUERY = "/sparql/query";
|
|
5898
5917
|
var ENDPOINT_FUSEKI_UPDATE = "/triplestore/update";
|
|
5899
5918
|
var ENDPOINT_QLEVER_UPDATE = "/sparql/update";
|
|
5919
|
+
var ENDPOINT_FSS_BATCH = "/commands/file-system-structure/batch";
|
|
5920
|
+
var FSS_BATCH_CHUNK_SIZE = 1e3;
|
|
5921
|
+
var PENDING_LS_PREFIX = "cue:pending:";
|
|
5922
|
+
function _pendingFilePath(spaceId) {
|
|
5923
|
+
return (0, import_path3.join)((0, import_os.tmpdir)(), `cue-sync-pending-${spaceId}.json`);
|
|
5924
|
+
}
|
|
5925
|
+
async function _loadPending(spaceId) {
|
|
5926
|
+
if (typeof window !== "undefined") {
|
|
5927
|
+
const raw = window.localStorage.getItem(`${PENDING_LS_PREFIX}${spaceId}`);
|
|
5928
|
+
return raw ? JSON.parse(raw) : null;
|
|
5929
|
+
}
|
|
5930
|
+
try {
|
|
5931
|
+
const raw = await (0, import_promises3.readFile)(_pendingFilePath(spaceId), "utf-8");
|
|
5932
|
+
return JSON.parse(raw);
|
|
5933
|
+
} catch {
|
|
5934
|
+
return null;
|
|
5935
|
+
}
|
|
5936
|
+
}
|
|
5937
|
+
async function _savePending(batch) {
|
|
5938
|
+
const data = JSON.stringify(batch);
|
|
5939
|
+
if (typeof window !== "undefined") {
|
|
5940
|
+
window.localStorage.setItem(`${PENDING_LS_PREFIX}${batch.spaceId}`, data);
|
|
5941
|
+
return;
|
|
5942
|
+
}
|
|
5943
|
+
await (0, import_promises3.writeFile)(_pendingFilePath(batch.spaceId), data, "utf-8");
|
|
5944
|
+
}
|
|
5945
|
+
async function _clearPending(spaceId) {
|
|
5946
|
+
if (typeof window !== "undefined") {
|
|
5947
|
+
window.localStorage.removeItem(`${PENDING_LS_PREFIX}${spaceId}`);
|
|
5948
|
+
return;
|
|
5949
|
+
}
|
|
5950
|
+
try {
|
|
5951
|
+
await (0, import_promises3.unlink)(_pendingFilePath(spaceId));
|
|
5952
|
+
} catch {
|
|
5953
|
+
}
|
|
5954
|
+
}
|
|
5900
5955
|
var CueSyncApi = class {
|
|
5901
|
-
constructor(_auth, _projects, _blob, _gatewayUrl
|
|
5956
|
+
constructor(_auth, _projects, _blob, _gatewayUrl) {
|
|
5902
5957
|
this._auth = _auth;
|
|
5903
5958
|
this._projects = _projects;
|
|
5904
5959
|
this._blob = _blob;
|
|
5905
5960
|
this._gatewayUrl = _gatewayUrl;
|
|
5906
|
-
this._serviceId = _serviceId;
|
|
5907
|
-
this._rdfBase = _rdfBase;
|
|
5908
5961
|
}
|
|
5909
5962
|
_graphMap = /* @__PURE__ */ new Map();
|
|
5963
|
+
_api;
|
|
5964
|
+
_pendingItems = [];
|
|
5965
|
+
_pendingSpaceId = null;
|
|
5966
|
+
_flushTimer = null;
|
|
5967
|
+
/** @internal Injected by CueApi after construction to avoid circular dependency. */
|
|
5968
|
+
_bindApi(api) {
|
|
5969
|
+
this._api = api;
|
|
5970
|
+
}
|
|
5971
|
+
/**
|
|
5972
|
+
* Flushes any pending metadata items from a previous interrupted sync.
|
|
5973
|
+
* Safe to call even when there is nothing new to upload.
|
|
5974
|
+
*/
|
|
5975
|
+
async flushPendingMetadata(spaceId, verbose) {
|
|
5976
|
+
const token = await this._auth.getToken();
|
|
5977
|
+
if (!token)
|
|
5978
|
+
throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
5979
|
+
const existing = await _loadPending(spaceId);
|
|
5980
|
+
if (!existing || existing.items.length === 0)
|
|
5981
|
+
return;
|
|
5982
|
+
console.info(`Trying to upload metadata (${existing.items.length} item(s))...`);
|
|
5983
|
+
if (verbose)
|
|
5984
|
+
console.info(`Flushing ${existing.items.length} pending file location(s) from previous sync \u23F3`);
|
|
5985
|
+
try {
|
|
5986
|
+
this._pendingSpaceId = spaceId;
|
|
5987
|
+
this._pendingItems = [];
|
|
5988
|
+
await this._flushBatch(existing.items, spaceId, token, verbose);
|
|
5989
|
+
console.info("Metadata uploaded \u2705");
|
|
5990
|
+
} catch (err) {
|
|
5991
|
+
throw new Error(`METADATA_SYNC_FAILED: ${err instanceof Error ? err.message : String(err)}`);
|
|
5992
|
+
}
|
|
5993
|
+
}
|
|
5994
|
+
/**
|
|
5995
|
+
* Returns a preview of what would be synced: cost breakdown for new files only,
|
|
5996
|
+
* units required, and units still available. Use this before calling {@link sync}
|
|
5997
|
+
* to show the user an accurate cost estimate.
|
|
5998
|
+
*/
|
|
5999
|
+
async previewSync(localFiles, options) {
|
|
6000
|
+
const { spaceId, providerId, verbose } = options;
|
|
6001
|
+
const token = await this._auth.getToken();
|
|
6002
|
+
if (!token)
|
|
6003
|
+
throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
6004
|
+
const graph = await this._getOrCreateGraph(spaceId, token);
|
|
6005
|
+
const project = await this._projects.getProject(spaceId);
|
|
6006
|
+
const tier = project?.projectSettings?.tier ?? "l";
|
|
6007
|
+
const [remoteFiles, consumption, creditMap, tierNames] = await Promise.all([
|
|
6008
|
+
this._listRemoteFiles(graph, spaceId, providerId, verbose),
|
|
6009
|
+
this._api?.getConsumption(spaceId, tier) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance")),
|
|
6010
|
+
this._fetchUnitCreditMap(verbose),
|
|
6011
|
+
this._fetchTierNames()
|
|
6012
|
+
]);
|
|
6013
|
+
const report = await compareLocalRemote(localFiles, remoteFiles);
|
|
6014
|
+
const toUpload = report.localNotOnRemote ?? [];
|
|
6015
|
+
const costRecords = toUpload.length > 0 ? await this.scanCost(toUpload) : [];
|
|
6016
|
+
let unitsToConsume = 0;
|
|
6017
|
+
let creditsToConsume = 0;
|
|
6018
|
+
for (const r of costRecords) {
|
|
6019
|
+
unitsToConsume += r.units;
|
|
6020
|
+
const tierMap = creditMap[tier];
|
|
6021
|
+
const creditPerUnit = tierMap?.[r.ext] ?? 1;
|
|
6022
|
+
if (verbose && tierMap && !(r.ext in tierMap)) {
|
|
6023
|
+
console.info(` Unknown format: .${r.ext} (using default rate of 1 credit/unit)`);
|
|
6024
|
+
}
|
|
6025
|
+
const credits = r.units * creditPerUnit;
|
|
6026
|
+
creditsToConsume += credits;
|
|
6027
|
+
r.credits = Math.round(credits);
|
|
6028
|
+
}
|
|
6029
|
+
const tierName = tierNames[tier] ?? tier;
|
|
6030
|
+
return {
|
|
6031
|
+
costRecords,
|
|
6032
|
+
tier,
|
|
6033
|
+
tierName,
|
|
6034
|
+
unitsToConsume,
|
|
6035
|
+
creditsToConsume: Math.round(creditsToConsume),
|
|
6036
|
+
creditsAvailable: consumption.creditsAvailable,
|
|
6037
|
+
unitsAvailable: consumption.unitsAvailable,
|
|
6038
|
+
filesToUpload: toUpload.length,
|
|
6039
|
+
totalLocalFiles: localFiles.length
|
|
6040
|
+
};
|
|
6041
|
+
}
|
|
5910
6042
|
async sync(localFiles, options) {
|
|
5911
|
-
const { spaceId, providerId, userId, verbose } = options;
|
|
6043
|
+
const { spaceId, providerId, userId, verbose, onProgress } = options;
|
|
5912
6044
|
const token = await this._auth.getToken();
|
|
5913
6045
|
if (!token)
|
|
5914
6046
|
throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
5915
6047
|
const graph = await this._getOrCreateGraph(spaceId, token);
|
|
5916
6048
|
if (verbose)
|
|
5917
6049
|
console.info("Listing remote files \u23F3");
|
|
5918
|
-
const
|
|
6050
|
+
const project = await this._projects.getProject(spaceId);
|
|
6051
|
+
const tier = project?.projectSettings?.tier ?? "l";
|
|
6052
|
+
const [remoteFiles, consumption] = await Promise.all([
|
|
6053
|
+
this._listRemoteFiles(graph, spaceId, providerId, verbose),
|
|
6054
|
+
this._api?.getConsumption(spaceId, tier) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
|
|
6055
|
+
]);
|
|
6056
|
+
const { unitsAvailable } = consumption;
|
|
5919
6057
|
const report = await compareLocalRemote(localFiles, remoteFiles);
|
|
5920
6058
|
if (verbose) {
|
|
5921
6059
|
console.info(`Total local files: ${localFiles.length}`);
|
|
@@ -5929,6 +6067,16 @@ var CueSyncApi = class {
|
|
|
5929
6067
|
let failedUploads = 0;
|
|
5930
6068
|
let rdfWritten = false;
|
|
5931
6069
|
const toUpload = report.localNotOnRemote ?? [];
|
|
6070
|
+
if (toUpload.length > 0) {
|
|
6071
|
+
const costRecords = await this.scanCost(toUpload);
|
|
6072
|
+
const unitsToConsume = costRecords.reduce((sum, r) => sum + r.units, 0);
|
|
6073
|
+
if (unitsToConsume > unitsAvailable) {
|
|
6074
|
+
throw new Error(
|
|
6075
|
+
`Insufficient units: ${unitsToConsume} units required but only ${unitsAvailable} available.`
|
|
6076
|
+
);
|
|
6077
|
+
}
|
|
6078
|
+
}
|
|
6079
|
+
await this._initPendingBatch(spaceId, token, verbose);
|
|
5932
6080
|
if (verbose && toUpload.length)
|
|
5933
6081
|
console.info("Syncing missing files \u23F3");
|
|
5934
6082
|
for (const file of toUpload) {
|
|
@@ -5948,12 +6096,17 @@ var CueSyncApi = class {
|
|
|
5948
6096
|
new Uint8Array(fileBuffer),
|
|
5949
6097
|
rawMeta
|
|
5950
6098
|
);
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
6099
|
+
await this._queueFileLocation({
|
|
6100
|
+
relativePath: file.relativePath,
|
|
6101
|
+
md5: file.md5,
|
|
6102
|
+
size: file.size,
|
|
6103
|
+
providerId,
|
|
6104
|
+
fileContentExists: false
|
|
6105
|
+
});
|
|
6106
|
+
rdfWritten = true;
|
|
5954
6107
|
syncCount += 1;
|
|
5955
6108
|
syncSize += file.size || 0;
|
|
5956
|
-
this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize,
|
|
6109
|
+
this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, onProgress);
|
|
5957
6110
|
} catch (err) {
|
|
5958
6111
|
failedUploads += 1;
|
|
5959
6112
|
console.error(`[CueSyncApi] Failed to upload file: ${file.fullPath}`);
|
|
@@ -5964,29 +6117,29 @@ var CueSyncApi = class {
|
|
|
5964
6117
|
if (verbose && report.localNotOnRemotePathOnly.length)
|
|
5965
6118
|
console.info(`Syncing missing file locations (on provider "${providerId}") \u23F3`);
|
|
5966
6119
|
for (const file of report.localNotOnRemotePathOnly) {
|
|
5967
|
-
|
|
5968
|
-
file.relativePath,
|
|
5969
|
-
|
|
5970
|
-
|
|
5971
|
-
|
|
5972
|
-
|
|
5973
|
-
);
|
|
5974
|
-
|
|
5975
|
-
throw new Error(`blob_name missing for ${file.relativePath}`);
|
|
5976
|
-
const uploaded = await this._uploadRdfMetadata(file, rawMeta, verbose);
|
|
5977
|
-
if (uploaded)
|
|
5978
|
-
rdfWritten = true;
|
|
6120
|
+
await this._queueFileLocation({
|
|
6121
|
+
relativePath: file.relativePath,
|
|
6122
|
+
md5: file.md5,
|
|
6123
|
+
size: file.size,
|
|
6124
|
+
providerId,
|
|
6125
|
+
fileContentExists: true
|
|
6126
|
+
});
|
|
6127
|
+
rdfWritten = true;
|
|
5979
6128
|
syncCount += 1;
|
|
5980
6129
|
syncSize += file.size || 0;
|
|
5981
|
-
this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize,
|
|
6130
|
+
this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, onProgress);
|
|
5982
6131
|
}
|
|
6132
|
+
await this._drainPending(verbose);
|
|
6133
|
+
this._stopFlushTimer();
|
|
6134
|
+
const postSyncConsumption = await (this._api?.getConsumption(spaceId, tier) ?? Promise.resolve({ creditsAvailable: 0 }));
|
|
5983
6135
|
return {
|
|
5984
6136
|
syncCount,
|
|
5985
6137
|
syncSize,
|
|
5986
6138
|
failedUploads,
|
|
5987
6139
|
totalCount: report.totalCount,
|
|
5988
6140
|
totalSize: report.totalSize,
|
|
5989
|
-
rdfWritten
|
|
6141
|
+
rdfWritten,
|
|
6142
|
+
creditsAvailable: postSyncConsumption.creditsAvailable
|
|
5990
6143
|
};
|
|
5991
6144
|
}
|
|
5992
6145
|
async _getOrCreateGraph(spaceId, token) {
|
|
@@ -6012,9 +6165,15 @@ var CueSyncApi = class {
|
|
|
6012
6165
|
async _listRemoteFiles(graph, spaceId, providerId, verbose) {
|
|
6013
6166
|
if (verbose)
|
|
6014
6167
|
console.info(`Listing files in raw space: ${spaceId}`);
|
|
6168
|
+
const graphFilesWithTimeout = Promise.race([
|
|
6169
|
+
this._getGraphFiles(graph, providerId),
|
|
6170
|
+
new Promise(
|
|
6171
|
+
(_, reject) => setTimeout(() => reject(new Error("GRAPH_TIMEOUT: Knowledge graph query timed out")), 15e3)
|
|
6172
|
+
)
|
|
6173
|
+
]);
|
|
6015
6174
|
const [blobNames, locationMap] = await Promise.all([
|
|
6016
6175
|
this._blob.listRaw(spaceId),
|
|
6017
|
-
|
|
6176
|
+
graphFilesWithTimeout
|
|
6018
6177
|
]);
|
|
6019
6178
|
if (verbose)
|
|
6020
6179
|
console.info(`Found ${blobNames.length} files in raw store for space: ${spaceId}`);
|
|
@@ -6065,30 +6224,94 @@ WHERE {
|
|
|
6065
6224
|
}
|
|
6066
6225
|
return map;
|
|
6067
6226
|
}
|
|
6068
|
-
async
|
|
6069
|
-
|
|
6070
|
-
|
|
6071
|
-
const
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
6076
|
-
|
|
6077
|
-
|
|
6078
|
-
|
|
6079
|
-
|
|
6080
|
-
|
|
6081
|
-
|
|
6082
|
-
|
|
6083
|
-
|
|
6084
|
-
|
|
6085
|
-
|
|
6086
|
-
if (!uploaded && verbose) {
|
|
6087
|
-
console.info(
|
|
6088
|
-
`Graph data for ${file.relativePath} already exists \u2014 skipping \u26A0\uFE0F`
|
|
6227
|
+
async _initPendingBatch(spaceId, token, verbose) {
|
|
6228
|
+
this._pendingSpaceId = spaceId;
|
|
6229
|
+
this._pendingItems = [];
|
|
6230
|
+
const existing = await _loadPending(spaceId);
|
|
6231
|
+
if (existing && existing.items.length > 0) {
|
|
6232
|
+
console.info(`Trying to upload metadata from interrupted sync (${existing.items.length} item(s))...`);
|
|
6233
|
+
if (verbose)
|
|
6234
|
+
console.info(`Flushing ${existing.items.length} pending file location(s) from previous sync \u23F3`);
|
|
6235
|
+
try {
|
|
6236
|
+
await this._flushBatch(existing.items, spaceId, token, verbose);
|
|
6237
|
+
console.info("Metadata uploaded \u2705");
|
|
6238
|
+
} catch (err) {
|
|
6239
|
+
throw new Error(`METADATA_SYNC_FAILED: ${err instanceof Error ? err.message : String(err)}`);
|
|
6240
|
+
}
|
|
6241
|
+
}
|
|
6242
|
+
const timer = setInterval(() => {
|
|
6243
|
+
this._drainPending(verbose).catch(
|
|
6244
|
+
(err) => console.error("[CueSyncApi] Periodic flush failed:", err)
|
|
6089
6245
|
);
|
|
6246
|
+
}, 6e4);
|
|
6247
|
+
if (typeof timer === "object" && typeof timer.unref === "function") {
|
|
6248
|
+
timer.unref();
|
|
6249
|
+
}
|
|
6250
|
+
this._flushTimer = timer;
|
|
6251
|
+
}
|
|
6252
|
+
async _queueFileLocation(item) {
|
|
6253
|
+
this._pendingItems.push(item);
|
|
6254
|
+
if (this._pendingSpaceId) {
|
|
6255
|
+
await _savePending({ spaceId: this._pendingSpaceId, items: this._pendingItems });
|
|
6256
|
+
}
|
|
6257
|
+
}
|
|
6258
|
+
async _drainPending(verbose) {
|
|
6259
|
+
if (!this._pendingSpaceId || this._pendingItems.length === 0)
|
|
6260
|
+
return;
|
|
6261
|
+
const token = await this._auth.getToken();
|
|
6262
|
+
if (!token)
|
|
6263
|
+
return;
|
|
6264
|
+
await this._flushBatch(this._pendingItems, this._pendingSpaceId, token, verbose);
|
|
6265
|
+
}
|
|
6266
|
+
async _flushBatch(items, spaceId, token, verbose) {
|
|
6267
|
+
const snapshot = [...items];
|
|
6268
|
+
if (this._pendingSpaceId === spaceId)
|
|
6269
|
+
this._pendingItems = [];
|
|
6270
|
+
await _clearPending(spaceId);
|
|
6271
|
+
try {
|
|
6272
|
+
for (let i = 0; i < snapshot.length; i += FSS_BATCH_CHUNK_SIZE) {
|
|
6273
|
+
await this._postFssBatch(snapshot.slice(i, i + FSS_BATCH_CHUNK_SIZE), spaceId, token);
|
|
6274
|
+
}
|
|
6275
|
+
if (verbose)
|
|
6276
|
+
console.info(`Wrote ${snapshot.length} file location(s) to commands API \u2705`);
|
|
6277
|
+
} catch (err) {
|
|
6278
|
+
const restored = [...snapshot, ...this._pendingItems];
|
|
6279
|
+
this._pendingItems = restored;
|
|
6280
|
+
await _savePending({ spaceId, items: restored });
|
|
6281
|
+
throw err;
|
|
6282
|
+
}
|
|
6283
|
+
}
|
|
6284
|
+
async _postFssBatch(items, spaceId, token) {
|
|
6285
|
+
const controller = new AbortController();
|
|
6286
|
+
const timeout = setTimeout(() => controller.abort(), 15e3);
|
|
6287
|
+
let response;
|
|
6288
|
+
try {
|
|
6289
|
+
response = await fetch(`${this._gatewayUrl}${ENDPOINT_FSS_BATCH}`, {
|
|
6290
|
+
method: "POST",
|
|
6291
|
+
headers: {
|
|
6292
|
+
Authorization: `Bearer ${token}`,
|
|
6293
|
+
"Content-Type": "application/json",
|
|
6294
|
+
"x-project-id": spaceId
|
|
6295
|
+
},
|
|
6296
|
+
body: JSON.stringify({ items }),
|
|
6297
|
+
signal: controller.signal
|
|
6298
|
+
});
|
|
6299
|
+
} catch (err) {
|
|
6300
|
+
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
6301
|
+
throw new Error(`File structure batch POST failed: ${isTimeout ? "request timed out" : err instanceof Error ? err.message : String(err)}`);
|
|
6302
|
+
} finally {
|
|
6303
|
+
clearTimeout(timeout);
|
|
6304
|
+
}
|
|
6305
|
+
if (!response.ok) {
|
|
6306
|
+
const body = await response.text().catch(() => "");
|
|
6307
|
+
throw new Error(`File structure batch POST failed: ${response.status} ${response.statusText}${body ? ` \u2014 ${body}` : ""}`);
|
|
6308
|
+
}
|
|
6309
|
+
}
|
|
6310
|
+
_stopFlushTimer() {
|
|
6311
|
+
if (this._flushTimer !== null) {
|
|
6312
|
+
clearInterval(this._flushTimer);
|
|
6313
|
+
this._flushTimer = null;
|
|
6090
6314
|
}
|
|
6091
|
-
return uploaded;
|
|
6092
6315
|
}
|
|
6093
6316
|
/**
|
|
6094
6317
|
* Scans `localFiles` and returns a per-extension cost breakdown.
|
|
@@ -6101,31 +6324,67 @@ WHERE {
|
|
|
6101
6324
|
if (!_wasmInitPromise)
|
|
6102
6325
|
_wasmInitPromise = _initWasm();
|
|
6103
6326
|
await _wasmInitPromise;
|
|
6104
|
-
const entries = await Promise.all(
|
|
6105
|
-
localFiles.map(async (f) => ({
|
|
6106
|
-
originalPath: f.relativePath,
|
|
6107
|
-
data: new Uint8Array(await (0, import_promises3.readFile)(f.fullPath))
|
|
6108
|
-
}))
|
|
6109
|
-
);
|
|
6110
6327
|
if (!_scanFn)
|
|
6111
6328
|
throw new Error("WASM scan function not initialised");
|
|
6112
|
-
|
|
6329
|
+
const BATCH_SIZE = 200;
|
|
6330
|
+
const merged = /* @__PURE__ */ new Map();
|
|
6331
|
+
for (let i = 0; i < localFiles.length; i += BATCH_SIZE) {
|
|
6332
|
+
const batch = localFiles.slice(i, i + BATCH_SIZE);
|
|
6333
|
+
const entries = await Promise.all(
|
|
6334
|
+
batch.map(async (f) => ({
|
|
6335
|
+
originalPath: f.relativePath,
|
|
6336
|
+
data: new Uint8Array(await (0, import_promises3.readFile)(f.fullPath))
|
|
6337
|
+
}))
|
|
6338
|
+
);
|
|
6339
|
+
const records = _scanFn(entries);
|
|
6340
|
+
for (const r of records) {
|
|
6341
|
+
const existing = merged.get(r.ext);
|
|
6342
|
+
if (existing) {
|
|
6343
|
+
existing.count += r.count;
|
|
6344
|
+
existing.units += r.units;
|
|
6345
|
+
existing.sizeMb += r.sizeMb;
|
|
6346
|
+
} else {
|
|
6347
|
+
merged.set(r.ext, { ...r });
|
|
6348
|
+
}
|
|
6349
|
+
}
|
|
6350
|
+
}
|
|
6351
|
+
return Array.from(merged.values());
|
|
6113
6352
|
}
|
|
6114
|
-
|
|
6115
|
-
|
|
6116
|
-
|
|
6117
|
-
|
|
6353
|
+
async _fetchTierNames() {
|
|
6354
|
+
try {
|
|
6355
|
+
const text = await this._blob.downloadPublic("tier-names.json");
|
|
6356
|
+
return JSON.parse(text);
|
|
6357
|
+
} catch {
|
|
6358
|
+
return {};
|
|
6359
|
+
}
|
|
6360
|
+
}
|
|
6361
|
+
async _fetchUnitCreditMap(verbose) {
|
|
6362
|
+
let text;
|
|
6363
|
+
try {
|
|
6364
|
+
text = await this._blob.downloadPublic("unit-credit.json");
|
|
6365
|
+
} catch (err) {
|
|
6366
|
+
throw new Error(`Couldn't fetch credits table: ${err instanceof Error ? err.message : String(err)}`);
|
|
6367
|
+
}
|
|
6368
|
+
if (verbose)
|
|
6369
|
+
console.info(`Price file: ${text.length} bytes`);
|
|
6370
|
+
try {
|
|
6371
|
+
return JSON.parse(text);
|
|
6372
|
+
} catch {
|
|
6373
|
+
throw new Error(`Credits table is not valid JSON (${text.length} bytes)`);
|
|
6374
|
+
}
|
|
6375
|
+
}
|
|
6376
|
+
_logProgress(syncCount, totalCount, syncSize, totalSize, onProgress) {
|
|
6377
|
+
if (!onProgress || totalCount === 0)
|
|
6118
6378
|
return;
|
|
6119
6379
|
const pct = Math.floor(syncCount / totalCount * 100);
|
|
6120
|
-
|
|
6121
|
-
`Progress: ${pct}% (${syncCount}/${totalCount} files, ${syncSize}/${totalSize} bytes)`
|
|
6122
|
-
);
|
|
6380
|
+
onProgress({ percent: pct, syncCount, totalCount, syncSize, totalSize });
|
|
6123
6381
|
}
|
|
6124
6382
|
};
|
|
6125
6383
|
|
|
6126
6384
|
// libs/js/cue-sdk/src/lib/cue-node.ts
|
|
6127
6385
|
var BUCKET_RAW2 = "spaces_raw_eu_west6";
|
|
6128
6386
|
var BUCKET_PROCESSED2 = "spaces_processed_eu_west6";
|
|
6387
|
+
var BUCKET_PUBLIC2 = "cue_public_eu_west6";
|
|
6129
6388
|
var CueNode = class extends Cue {
|
|
6130
6389
|
constructor(config) {
|
|
6131
6390
|
super(config);
|
|
@@ -6133,15 +6392,19 @@ var CueNode = class extends Cue {
|
|
|
6133
6392
|
_buildApi(projects) {
|
|
6134
6393
|
const storageRaw = (0, import_storage4.getStorage)(this._app, BUCKET_RAW2);
|
|
6135
6394
|
const storageProcessed = (0, import_storage4.getStorage)(this._app, BUCKET_PROCESSED2);
|
|
6395
|
+
const storagePublic = (0, import_storage4.getStorage)(this._app, BUCKET_PUBLIC2);
|
|
6136
6396
|
if (this._isEmulator) {
|
|
6137
6397
|
const storageHost = this._endpoints.storageEmulatorHost;
|
|
6138
6398
|
const storagePort = this._endpoints.storageEmulatorPort;
|
|
6139
6399
|
(0, import_storage4.connectStorageEmulator)(storageRaw, storageHost, storagePort);
|
|
6140
6400
|
(0, import_storage4.connectStorageEmulator)(storageProcessed, storageHost, storagePort);
|
|
6401
|
+
(0, import_storage4.connectStorageEmulator)(storagePublic, storageHost, storagePort);
|
|
6141
6402
|
}
|
|
6142
|
-
const blob = new CueBlobStorage({ storageRaw, storageProcessed });
|
|
6403
|
+
const blob = new CueBlobStorage({ storageRaw, storageProcessed, storagePublic });
|
|
6143
6404
|
const syncApi = new CueSyncApi(this.auth, projects, blob, this._endpoints.gatewayUrl);
|
|
6144
|
-
|
|
6405
|
+
const api = new CueApi(this.auth, this._endpoints.gatewayUrl, projects, syncApi);
|
|
6406
|
+
syncApi._bindApi(api);
|
|
6407
|
+
return api;
|
|
6145
6408
|
}
|
|
6146
6409
|
};
|
|
6147
6410
|
|
|
@@ -6324,7 +6587,7 @@ async function dumpProcessedHandler(options) {
|
|
|
6324
6587
|
var import_fs2 = require("fs");
|
|
6325
6588
|
var import_stream = require("stream");
|
|
6326
6589
|
var import_fs3 = require("fs");
|
|
6327
|
-
var
|
|
6590
|
+
var import_zlib2 = require("zlib");
|
|
6328
6591
|
var import_promises5 = require("stream/promises");
|
|
6329
6592
|
var import_promises6 = require("fs/promises");
|
|
6330
6593
|
var import_auth9 = require("firebase/auth");
|
|
@@ -6504,7 +6767,7 @@ async function retryWithBackoff(fn, maxRetries, delayMs, label) {
|
|
|
6504
6767
|
async function _doGzip(filePath) {
|
|
6505
6768
|
const gzFilePath = `${filePath}.gz`;
|
|
6506
6769
|
await new Promise((resolve2, reject) => {
|
|
6507
|
-
const gzip = (0,
|
|
6770
|
+
const gzip = (0, import_zlib2.createGzip)();
|
|
6508
6771
|
const source = (0, import_fs3.createReadStream)(filePath);
|
|
6509
6772
|
const dest = (0, import_fs2.createWriteStream)(gzFilePath);
|
|
6510
6773
|
(0, import_promises5.pipeline)(source, gzip, dest).then(resolve2).catch(reject);
|
|
@@ -6753,46 +7016,6 @@ async function repairTtlHandler(options) {
|
|
|
6753
7016
|
}
|
|
6754
7017
|
}
|
|
6755
7018
|
|
|
6756
|
-
// apps/desktop/cue-cli/src/helpers/emit-idle.ts
|
|
6757
|
-
var import_pubsub = require("@google-cloud/pubsub");
|
|
6758
|
-
async function emitIdle(spaceId, hoursBack, useEmulator) {
|
|
6759
|
-
const from = /* @__PURE__ */ new Date();
|
|
6760
|
-
from.setHours(from.getHours() - hoursBack);
|
|
6761
|
-
const message = {
|
|
6762
|
-
space_id: spaceId,
|
|
6763
|
-
first_write: from.toISOString(),
|
|
6764
|
-
last_write: (/* @__PURE__ */ new Date()).toISOString()
|
|
6765
|
-
};
|
|
6766
|
-
await publishMessage("RDF_WRITING_IDLE", message, useEmulator);
|
|
6767
|
-
}
|
|
6768
|
-
async function publishMessage(topicName, data, useEmulator) {
|
|
6769
|
-
if (useEmulator) {
|
|
6770
|
-
process.env.PUBSUB_EMULATOR_HOST = process.env.PUBSUB_EMULATOR_HOST || "localhost:8085";
|
|
6771
|
-
}
|
|
6772
|
-
const projectId = useEmulator ? "demo-project" : PROJECT_ID;
|
|
6773
|
-
const pubSubClient = new import_pubsub.PubSub({ projectId });
|
|
6774
|
-
const topic = pubSubClient.topic(topicName);
|
|
6775
|
-
const dataBuffer = Buffer.from(JSON.stringify(data));
|
|
6776
|
-
try {
|
|
6777
|
-
if (useEmulator) {
|
|
6778
|
-
const publishPromise = topic.publishMessage({ data: dataBuffer });
|
|
6779
|
-
const timeoutPromise = new Promise(
|
|
6780
|
-
(_, reject) => setTimeout(() => reject(new Error("PubSub publish timeout")), 5e3)
|
|
6781
|
-
);
|
|
6782
|
-
await Promise.race([publishPromise, timeoutPromise]);
|
|
6783
|
-
} else {
|
|
6784
|
-
await topic.publishMessage({ data: dataBuffer });
|
|
6785
|
-
}
|
|
6786
|
-
} catch (error) {
|
|
6787
|
-
console.error(`Error publishing message: ${error.message}`);
|
|
6788
|
-
if (useEmulator) {
|
|
6789
|
-
console.warn("Continuing despite PubSub error in emulator mode...");
|
|
6790
|
-
} else {
|
|
6791
|
-
throw error;
|
|
6792
|
-
}
|
|
6793
|
-
}
|
|
6794
|
-
}
|
|
6795
|
-
|
|
6796
7019
|
// apps/desktop/cue-cli/src/cue-cli-sync.ts
|
|
6797
7020
|
var import_promises7 = require("fs/promises");
|
|
6798
7021
|
var import_fs6 = require("fs");
|
|
@@ -6860,46 +7083,68 @@ async function syncHandler(options) {
|
|
|
6860
7083
|
}
|
|
6861
7084
|
if (verbose)
|
|
6862
7085
|
console.info("Built sync base \u2705\n");
|
|
6863
|
-
|
|
6864
|
-
|
|
6865
|
-
|
|
6866
|
-
|
|
6867
|
-
|
|
7086
|
+
if (verbose)
|
|
7087
|
+
console.info("Authenticating \u23F3");
|
|
7088
|
+
let user;
|
|
7089
|
+
try {
|
|
7090
|
+
user = await cue.auth.signInWithApiKey(key, space);
|
|
7091
|
+
} catch (err) {
|
|
7092
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
7093
|
+
console.error(`Couldn't authenticate. ${cause}`);
|
|
7094
|
+
process.exit(1);
|
|
7095
|
+
}
|
|
7096
|
+
if (verbose)
|
|
7097
|
+
console.info("Authenticated \u2705\n");
|
|
7098
|
+
if (verbose)
|
|
7099
|
+
console.info("Checking sync preview \u23F3");
|
|
7100
|
+
const preview = await cue.api.sync.previewSync(localFiles, {
|
|
7101
|
+
spaceId: space,
|
|
7102
|
+
providerId: provider,
|
|
7103
|
+
userId: user.uid,
|
|
7104
|
+
verbose
|
|
7105
|
+
});
|
|
7106
|
+
console.info(`Project is configured at the ${preview.tierName} tier`);
|
|
7107
|
+
console.info("Cost estimate (new files only):");
|
|
7108
|
+
if (preview.costRecords.length === 0) {
|
|
7109
|
+
console.info(" Nothing new to sync.");
|
|
7110
|
+
} else {
|
|
7111
|
+
for (const r of preview.costRecords) {
|
|
7112
|
+
const creditsStr = r.credits !== void 0 ? ` (${Math.round(r.credits)} credits)` : "";
|
|
7113
|
+
console.info(` .${r.ext}: ${r.count} file(s)${creditsStr}`);
|
|
7114
|
+
}
|
|
6868
7115
|
}
|
|
6869
|
-
console.info(`
|
|
7116
|
+
console.info(` New files: ${preview.filesToUpload}/${preview.totalLocalFiles}`);
|
|
7117
|
+
console.info(` Credits required: ${Math.round(preview.creditsToConsume)}`);
|
|
7118
|
+
console.info(` Credits available: ${Math.round(preview.creditsAvailable)}
|
|
6870
7119
|
`);
|
|
7120
|
+
await cue.api.sync.flushPendingMetadata(space, verbose);
|
|
7121
|
+
if (preview.filesToUpload === 0) {
|
|
7122
|
+
console.info("Everything is already synced.");
|
|
7123
|
+
process.exit(0);
|
|
7124
|
+
}
|
|
7125
|
+
if (preview.creditsToConsume > preview.creditsAvailable) {
|
|
7126
|
+
console.error(
|
|
7127
|
+
`Insufficient credits: ${Math.round(preview.creditsToConsume)} required, ${Math.round(preview.creditsAvailable)} available.`
|
|
7128
|
+
);
|
|
7129
|
+
process.exit(1);
|
|
7130
|
+
}
|
|
6871
7131
|
const confirmed = await askConfirm("Continue with sync? (y/n) ");
|
|
6872
7132
|
if (!confirmed) {
|
|
6873
7133
|
console.info("Sync cancelled.");
|
|
6874
7134
|
process.exit(0);
|
|
6875
7135
|
}
|
|
6876
|
-
if (verbose)
|
|
6877
|
-
console.info("Authenticating \u23F3");
|
|
6878
|
-
const user = await cue.auth.signInWithApiKey(key, space);
|
|
6879
|
-
if (verbose)
|
|
6880
|
-
console.info("Authenticated \u2705\n");
|
|
6881
7136
|
const result = await cue.api.sync.sync(localFiles, {
|
|
6882
7137
|
spaceId: space,
|
|
6883
7138
|
providerId: provider,
|
|
6884
7139
|
userId: user.uid,
|
|
6885
|
-
verbose
|
|
6886
|
-
|
|
6887
|
-
|
|
6888
|
-
|
|
6889
|
-
|
|
6890
|
-
console.info(`Throwing RDF_WRITING_IDLE topic (only in emulators) \u23F3`);
|
|
7140
|
+
verbose,
|
|
7141
|
+
onProgress: ({ percent, syncCount, totalCount }) => {
|
|
7142
|
+
const filled = Math.round(percent / 5);
|
|
7143
|
+
const bar = "\u2588".repeat(filled) + "\u2591".repeat(20 - filled);
|
|
7144
|
+
process.stdout.write(`\r [${bar}] ${percent}% (${syncCount}/${totalCount} files) `);
|
|
6891
7145
|
}
|
|
6892
|
-
|
|
6893
|
-
|
|
6894
|
-
if (verbose)
|
|
6895
|
-
console.info(`Threw RDF_WRITING_IDLE topic \u2705`);
|
|
6896
|
-
} catch (error) {
|
|
6897
|
-
if (verbose)
|
|
6898
|
-
console.error(`Error throwing RDF_WRITING_IDLE topic: ${error instanceof Error ? error.message : String(error)}`);
|
|
6899
|
-
if (verbose)
|
|
6900
|
-
console.warn("Continuing despite PubSub error in emulator mode...");
|
|
6901
|
-
}
|
|
6902
|
-
}
|
|
7146
|
+
});
|
|
7147
|
+
process.stdout.write("\n");
|
|
6903
7148
|
const zipDeletePromise = zip && !isFile ? deleteUnzipped(path) : Promise.resolve();
|
|
6904
7149
|
await zipDeletePromise;
|
|
6905
7150
|
if (zip && verbose)
|
|
@@ -6909,6 +7154,7 @@ async function syncHandler(options) {
|
|
|
6909
7154
|
console.info(
|
|
6910
7155
|
`Synced: ${result.syncCount}/${result.totalCount} files (${fileSizePretty(result.syncSize)}/${fileSizePretty(result.totalSize)})`
|
|
6911
7156
|
);
|
|
7157
|
+
console.info(`Credits remaining: ${Math.round(result.creditsAvailable)}`);
|
|
6912
7158
|
console.info(`Sync finished \u{1F680}\u{1F680}\u{1F680}`);
|
|
6913
7159
|
if (result.failedUploads > 0) {
|
|
6914
7160
|
console.warn(`Total files failed to upload: ${result.failedUploads}`);
|
|
@@ -6916,29 +7162,40 @@ async function syncHandler(options) {
|
|
|
6916
7162
|
}
|
|
6917
7163
|
process.exit(0);
|
|
6918
7164
|
} catch (err) {
|
|
6919
|
-
|
|
7165
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
7166
|
+
if (msg.includes("GRAPH_TIMEOUT")) {
|
|
7167
|
+
console.error("Could not reach the knowledge graph. Make sure all required services are running.");
|
|
7168
|
+
} else if (msg.includes("METADATA_SYNC_FAILED")) {
|
|
7169
|
+
console.error("Metadata sync failed. Try again.");
|
|
7170
|
+
} else if (msg.includes("LEDGER_WRITE_FAILED") || msg.includes("File structure batch POST failed")) {
|
|
7171
|
+
console.error(
|
|
7172
|
+
"Failed communication with the knowledge graph. The files are up to date but metadata is not. Run the tool again to sync metadata. This is a swift job."
|
|
7173
|
+
);
|
|
7174
|
+
} else {
|
|
7175
|
+
console.error("[syncHandler] Unexpected error:", err);
|
|
7176
|
+
}
|
|
6920
7177
|
process.exit(1);
|
|
6921
7178
|
}
|
|
6922
7179
|
}
|
|
6923
7180
|
|
|
6924
7181
|
// apps/desktop/cue-cli/src/cue-cli-util-remove-rdf-star.ts
|
|
6925
7182
|
var import_fs7 = require("fs");
|
|
6926
|
-
var
|
|
7183
|
+
var import_zlib3 = require("zlib");
|
|
6927
7184
|
|
|
6928
7185
|
// libs/js/rdf-tools/src/lib/nq-to-nt.ts
|
|
6929
|
-
var
|
|
6930
|
-
var { quad, defaultGraph } =
|
|
7186
|
+
var import_n37 = require("n3");
|
|
7187
|
+
var { quad, defaultGraph } = import_n37.DataFactory;
|
|
6931
7188
|
|
|
6932
7189
|
// libs/js/rdf-tools/src/lib/remove-rdf-star.ts
|
|
6933
|
-
var
|
|
7190
|
+
var import_n38 = require("n3");
|
|
6934
7191
|
var import_stream2 = require("stream");
|
|
6935
7192
|
function isRDFStarTerm(term) {
|
|
6936
7193
|
return term.termType === "Quad";
|
|
6937
7194
|
}
|
|
6938
7195
|
function removeRDFStar(inputStream, starCount) {
|
|
6939
|
-
const parser = new
|
|
7196
|
+
const parser = new import_n38.Parser({ format: "N-Quads*" });
|
|
6940
7197
|
const outputStream = new import_stream2.PassThrough();
|
|
6941
|
-
const writer = new
|
|
7198
|
+
const writer = new import_n38.StreamWriter({ format: "N-Quads" });
|
|
6942
7199
|
writer.pipe(outputStream);
|
|
6943
7200
|
let count = 0;
|
|
6944
7201
|
parser.parse(inputStream, (error, quad2, prefixes4) => {
|
|
@@ -6971,7 +7228,7 @@ async function utilRemoveRdfStarHandler(options) {
|
|
|
6971
7228
|
if (verbose)
|
|
6972
7229
|
console.info(`Output: ${output}`);
|
|
6973
7230
|
const fileStream = (0, import_fs7.createReadStream)(input);
|
|
6974
|
-
const inputStream = isGzipped ? fileStream.pipe((0,
|
|
7231
|
+
const inputStream = isGzipped ? fileStream.pipe((0, import_zlib3.createGunzip)()) : fileStream;
|
|
6975
7232
|
let removed = 0;
|
|
6976
7233
|
const cleanStream = removeRDFStar(inputStream, (count) => {
|
|
6977
7234
|
removed = count;
|
|
@@ -6981,7 +7238,7 @@ async function utilRemoveRdfStarHandler(options) {
|
|
|
6981
7238
|
});
|
|
6982
7239
|
const writeStream = (0, import_fs7.createWriteStream)(output);
|
|
6983
7240
|
if (isGzipped) {
|
|
6984
|
-
const gzip = (0,
|
|
7241
|
+
const gzip = (0, import_zlib3.createGzip)();
|
|
6985
7242
|
gzip.pipe(writeStream);
|
|
6986
7243
|
for await (const chunk of cleanStream) {
|
|
6987
7244
|
gzip.write(chunk);
|
|
@@ -7009,11 +7266,11 @@ async function utilRemoveRdfStarHandler(options) {
|
|
|
7009
7266
|
var import_fs8 = require("fs");
|
|
7010
7267
|
|
|
7011
7268
|
// libs/js/rdf-compare/src/lib/js-rdf-compare.ts
|
|
7012
|
-
var
|
|
7269
|
+
var import_n39 = require("n3");
|
|
7013
7270
|
var import_jsonld = require("jsonld");
|
|
7014
7271
|
function parseTurtle(content) {
|
|
7015
7272
|
return new Promise((resolve2, reject) => {
|
|
7016
|
-
const parser = new
|
|
7273
|
+
const parser = new import_n39.Parser({ format: "Turtle" });
|
|
7017
7274
|
const quads = [];
|
|
7018
7275
|
parser.parse(content, (error, quad2) => {
|
|
7019
7276
|
if (error)
|
|
@@ -7027,7 +7284,7 @@ function parseTurtle(content) {
|
|
|
7027
7284
|
}
|
|
7028
7285
|
function quadsToNQuads(quads) {
|
|
7029
7286
|
return new Promise((resolve2, reject) => {
|
|
7030
|
-
const writer = new
|
|
7287
|
+
const writer = new import_n39.Writer({ format: "N-Quads" });
|
|
7031
7288
|
writer.addQuads(quads);
|
|
7032
7289
|
writer.end((error, result) => {
|
|
7033
7290
|
if (error)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qaecy/cue-cli",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.34",
|
|
4
4
|
"description": "Cue CLI for QAECY platform",
|
|
5
5
|
"main": "main.js",
|
|
6
6
|
"bin": {
|
|
@@ -21,7 +21,6 @@
|
|
|
21
21
|
"jsonld": "8.3.3",
|
|
22
22
|
"jszip": "3.10.1",
|
|
23
23
|
"n3": "1.26.0",
|
|
24
|
-
"nanoid": "5.1.6",
|
|
25
24
|
"oxigraph": "0.5.3",
|
|
26
25
|
"protobufjs": "7.5.4",
|
|
27
26
|
"spark-md5": "3.0.2",
|