@qaecy/cue-cli 0.0.22 → 0.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/main.js +1710 -816
- package/package.json +6 -3
package/main.js
CHANGED
|
@@ -94,17 +94,19 @@ var init_worker_pool = __esm({
|
|
|
94
94
|
// apps/desktop/cue-cli/src/main.ts
|
|
95
95
|
var import_commander = require("commander");
|
|
96
96
|
var import_fs9 = require("fs");
|
|
97
|
-
var
|
|
97
|
+
var import_path6 = require("path");
|
|
98
98
|
|
|
99
99
|
// apps/desktop/cue-cli/src/variables.ts
|
|
100
100
|
var import_path = require("path");
|
|
101
|
-
var
|
|
102
|
-
var
|
|
103
|
-
var
|
|
101
|
+
var EMULATOR_API_GATEWAY_PORT = process.env.CUE_EMULATOR_API_GATEWAY_PORT ?? "8093";
|
|
102
|
+
var EMULATOR_API_GATEWAY_BASE = `http://localhost:${EMULATOR_API_GATEWAY_PORT}`;
|
|
103
|
+
var TOKEN_ENDPOINT_EMULATOR = `${EMULATOR_API_GATEWAY_BASE}/token`;
|
|
104
|
+
var EMULATOR_QLEVER_PORT = process.env.CUE_QLEVER_PORT ?? "8102";
|
|
105
|
+
var QLEVER_QUERY_ENDPOINT_EMULATOR = process.env.CUE_QLEVER_ENDPOINT ? `${process.env.CUE_QLEVER_ENDPOINT}/query` : `http://localhost:${EMULATOR_QLEVER_PORT}/query`;
|
|
106
|
+
var QLEVER_UPDATE_ENDPOINT_EMULATOR = process.env.CUE_QLEVER_ENDPOINT ? `${process.env.CUE_QLEVER_ENDPOINT}/update` : `http://localhost:${EMULATOR_QLEVER_PORT}/update`;
|
|
107
|
+
var SPARQL_ENDPOINT_EMULATOR = `${EMULATOR_API_GATEWAY_BASE}/triplestore/query`;
|
|
104
108
|
var SPARQL_ENDPOINT = "https://accessors-api-gateway-ueyeemwf2a-oa.a.run.app/triplestore/query";
|
|
105
109
|
var HASH_WORKER_PATH = (0, import_path.join)(__dirname, "hash-worker.js");
|
|
106
|
-
var SERVICE_ID = "cue-cli";
|
|
107
|
-
var RDF_BASE = "https://cue.qaecy.com/r/";
|
|
108
110
|
var PROJECT_ID = "qaecy-mvp-406413.appspot.com";
|
|
109
111
|
var FIREBASE_CONFIG = (useEmulator = false) => ({
|
|
110
112
|
apiKey: "AIzaSyCLhz5Wa3ZCERQZVurSt9bqupPeREALFLk",
|
|
@@ -416,16 +418,36 @@ var CueFirebase = class _CueFirebase {
|
|
|
416
418
|
throw new Error("Storage persistence is not initialized");
|
|
417
419
|
if (this._app === void 0)
|
|
418
420
|
throw new Error("App is not initialized");
|
|
421
|
+
const _env = typeof process !== "undefined" ? process.env : {};
|
|
422
|
+
const authEmulatorEnv = _env["FIREBASE_AUTH_EMULATOR_HOST"];
|
|
423
|
+
const authEmulatorUrl = authEmulatorEnv ? `http://${authEmulatorEnv}` : "http://localhost:9099";
|
|
424
|
+
const firestoreEnv = _env["FIRESTORE_EMULATOR_HOST"];
|
|
425
|
+
const [firestoreHost, firestorePortStr] = firestoreEnv ? firestoreEnv.split(":") : ["localhost", "8080"];
|
|
426
|
+
const firestorePort = parseInt(firestorePortStr ?? "8080", 10);
|
|
427
|
+
const storageEnv = _env["STORAGE_EMULATOR_HOST"];
|
|
428
|
+
let storageHost = "localhost";
|
|
429
|
+
let storagePort = 9199;
|
|
430
|
+
if (storageEnv) {
|
|
431
|
+
try {
|
|
432
|
+
const u = new URL(storageEnv.startsWith("http") ? storageEnv : `http://${storageEnv}`);
|
|
433
|
+
storageHost = u.hostname;
|
|
434
|
+
storagePort = parseInt(u.port || "9199", 10);
|
|
435
|
+
} catch {
|
|
436
|
+
const parts = storageEnv.split(":");
|
|
437
|
+
storageHost = parts[0];
|
|
438
|
+
storagePort = parseInt(parts[1] ?? "9199", 10);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
419
441
|
const functions = (0, import_functions.getFunctions)(this._app, GCP_REGION);
|
|
420
|
-
(0, import_auth.connectAuthEmulator)(this._auth,
|
|
421
|
-
(0, import_firestore.connectFirestoreEmulator)((0, import_firestore.getFirestore)(this._app),
|
|
442
|
+
(0, import_auth.connectAuthEmulator)(this._auth, authEmulatorUrl);
|
|
443
|
+
(0, import_firestore.connectFirestoreEmulator)((0, import_firestore.getFirestore)(this._app), firestoreHost, firestorePort);
|
|
422
444
|
(0, import_functions.connectFunctionsEmulator)(functions, "localhost", 5001);
|
|
423
|
-
(0, import_storage.connectStorageEmulator)(this._storageProcessed,
|
|
424
|
-
(0, import_storage.connectStorageEmulator)(this._storageRaw,
|
|
425
|
-
(0, import_storage.connectStorageEmulator)(this._storageChatSessions,
|
|
426
|
-
(0, import_storage.connectStorageEmulator)(this._storageLogs,
|
|
427
|
-
(0, import_storage.connectStorageEmulator)(this._storagePublic,
|
|
428
|
-
(0, import_storage.connectStorageEmulator)(this._storagePersistence,
|
|
445
|
+
(0, import_storage.connectStorageEmulator)(this._storageProcessed, storageHost, storagePort);
|
|
446
|
+
(0, import_storage.connectStorageEmulator)(this._storageRaw, storageHost, storagePort);
|
|
447
|
+
(0, import_storage.connectStorageEmulator)(this._storageChatSessions, storageHost, storagePort);
|
|
448
|
+
(0, import_storage.connectStorageEmulator)(this._storageLogs, storageHost, storagePort);
|
|
449
|
+
(0, import_storage.connectStorageEmulator)(this._storagePublic, storageHost, storagePort);
|
|
450
|
+
(0, import_storage.connectStorageEmulator)(this._storagePersistence, storageHost, storagePort);
|
|
429
451
|
if (!this._muted)
|
|
430
452
|
console.info("Firebase emulators attached");
|
|
431
453
|
}
|
|
@@ -441,6 +463,7 @@ var import_storage2 = require("firebase/storage");
|
|
|
441
463
|
var qaecyPrefixes = {
|
|
442
464
|
qcy: "https://dev.qaecy.com/ont#",
|
|
443
465
|
"qcy-e": "https://dev.qaecy.com/enum#",
|
|
466
|
+
"qcy-l": "https://dev.qaecy.com/logs#",
|
|
444
467
|
"qcy-f": "https://dev.qaecy.com/functions#",
|
|
445
468
|
obc: "https://w3id.org/obc#",
|
|
446
469
|
// OpenBIM Components
|
|
@@ -4024,7 +4047,7 @@ async function deleteUnzipped(dir) {
|
|
|
4024
4047
|
|
|
4025
4048
|
// apps/desktop/cue-cli/src/helpers/query-handler.ts
|
|
4026
4049
|
var import_auth2 = require("firebase/auth");
|
|
4027
|
-
async function queryHandler(
|
|
4050
|
+
async function queryHandler(query3, spaceId, useEmulator) {
|
|
4028
4051
|
const endpoint = useEmulator ? SPARQL_ENDPOINT_EMULATOR : SPARQL_ENDPOINT;
|
|
4029
4052
|
const token = await (0, import_auth2.getAuth)().currentUser?.getIdToken();
|
|
4030
4053
|
const res = await fetch(endpoint, {
|
|
@@ -4035,7 +4058,7 @@ async function queryHandler(query, spaceId, useEmulator) {
|
|
|
4035
4058
|
"Content-Type": "application/sparql-query",
|
|
4036
4059
|
Accept: "application/sparql-results+json"
|
|
4037
4060
|
},
|
|
4038
|
-
body:
|
|
4061
|
+
body: query3
|
|
4039
4062
|
});
|
|
4040
4063
|
if (!res.ok) {
|
|
4041
4064
|
console.error(
|
|
@@ -4055,639 +4078,705 @@ function fileSizePretty(size) {
|
|
|
4055
4078
|
return (size / Math.pow(1024, i)).toFixed(2) + " " + sizes[i];
|
|
4056
4079
|
}
|
|
4057
4080
|
|
|
4058
|
-
//
|
|
4059
|
-
|
|
4060
|
-
const url = useEmulator ? TOKEN_ENDPOINT_EMULATOR : TOKEN_ENDPOINT;
|
|
4061
|
-
const data = await fetch(url, {
|
|
4062
|
-
method: "GET",
|
|
4063
|
-
headers: {
|
|
4064
|
-
"x-project-id": pid,
|
|
4065
|
-
"cue-api-key": apiKey
|
|
4066
|
-
}
|
|
4067
|
-
});
|
|
4068
|
-
if (!data.ok) {
|
|
4069
|
-
console.error("Failed to fetch token:", data.statusText);
|
|
4070
|
-
process.exit(1);
|
|
4071
|
-
}
|
|
4072
|
-
const result = await data.json();
|
|
4073
|
-
return result.token;
|
|
4074
|
-
}
|
|
4081
|
+
// libs/js/cue-sdk/src/lib/cue.ts
|
|
4082
|
+
var import_app2 = require("firebase/app");
|
|
4075
4083
|
|
|
4076
|
-
//
|
|
4084
|
+
// libs/js/cue-sdk/src/lib/auth.ts
|
|
4077
4085
|
var import_auth3 = require("firebase/auth");
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
|
|
4081
|
-
|
|
4082
|
-
|
|
4083
|
-
|
|
4086
|
+
var MICROSOFT_PROVIDER_ID = "microsoft.com";
|
|
4087
|
+
var CueAuth = class {
|
|
4088
|
+
_auth;
|
|
4089
|
+
_endpoints;
|
|
4090
|
+
constructor(app, useEmulator = false, endpoints) {
|
|
4091
|
+
this._auth = (0, import_auth3.getAuth)(app);
|
|
4092
|
+
this._endpoints = endpoints;
|
|
4093
|
+
if (useEmulator) {
|
|
4094
|
+
(0, import_auth3.connectAuthEmulator)(this._auth, endpoints.authEmulatorUrl, {
|
|
4095
|
+
disableWarnings: true
|
|
4096
|
+
});
|
|
4097
|
+
}
|
|
4098
|
+
}
|
|
4099
|
+
async signIn(provider, credentials) {
|
|
4100
|
+
if (provider === "password") {
|
|
4101
|
+
if (!credentials)
|
|
4102
|
+
throw new Error("credentials are required for password sign-in");
|
|
4103
|
+
const result2 = await (0, import_auth3.signInWithEmailAndPassword)(
|
|
4104
|
+
this._auth,
|
|
4105
|
+
credentials.email,
|
|
4106
|
+
credentials.password
|
|
4084
4107
|
);
|
|
4085
|
-
|
|
4108
|
+
return result2.user;
|
|
4086
4109
|
}
|
|
4110
|
+
const firebaseProvider = provider === "google" ? new import_auth3.GoogleAuthProvider() : new import_auth3.OAuthProvider(MICROSOFT_PROVIDER_ID);
|
|
4111
|
+
const result = await (0, import_auth3.signInWithPopup)(this._auth, firebaseProvider);
|
|
4112
|
+
return result.user;
|
|
4087
4113
|
}
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4105
|
-
|
|
4106
|
-
|
|
4107
|
-
|
|
4114
|
+
/** Sign in with a Cue API key */
|
|
4115
|
+
async signInWithApiKey(cueApiKey, projectId) {
|
|
4116
|
+
const response = await fetch(this._endpoints.tokenUrl, {
|
|
4117
|
+
method: "GET",
|
|
4118
|
+
headers: {
|
|
4119
|
+
"x-project-id": projectId,
|
|
4120
|
+
"cue-api-key": cueApiKey
|
|
4121
|
+
}
|
|
4122
|
+
});
|
|
4123
|
+
if (!response.ok)
|
|
4124
|
+
throw new Error(`Failed to fetch custom token: ${response.statusText}`);
|
|
4125
|
+
const { token } = await response.json();
|
|
4126
|
+
const result = await (0, import_auth3.signInWithCustomToken)(this._auth, token);
|
|
4127
|
+
return result.user;
|
|
4128
|
+
}
|
|
4129
|
+
/** Sign out the current user */
|
|
4130
|
+
async signOut() {
|
|
4131
|
+
await (0, import_auth3.signOut)(this._auth);
|
|
4132
|
+
}
|
|
4133
|
+
/** Currently signed-in user, or null if not authenticated */
|
|
4134
|
+
get currentUser() {
|
|
4135
|
+
return this._auth.currentUser;
|
|
4136
|
+
}
|
|
4137
|
+
/** Subscribe to authentication state changes. Returns an unsubscribe function. */
|
|
4138
|
+
onAuthStateChanged(listener) {
|
|
4139
|
+
return (0, import_auth3.onAuthStateChanged)(this._auth, listener);
|
|
4140
|
+
}
|
|
4141
|
+
/** Get the Firebase ID token for the current user, or null if not authenticated */
|
|
4142
|
+
async getToken(forceRefresh = false) {
|
|
4143
|
+
const user = this._auth.currentUser;
|
|
4144
|
+
if (!user)
|
|
4145
|
+
return null;
|
|
4146
|
+
return user.getIdToken(forceRefresh);
|
|
4147
|
+
}
|
|
4148
|
+
/** Raw Firebase Auth instance, for advanced use cases */
|
|
4149
|
+
get firebaseAuth() {
|
|
4150
|
+
return this._auth;
|
|
4151
|
+
}
|
|
4152
|
+
};
|
|
4108
4153
|
|
|
4109
|
-
//
|
|
4110
|
-
|
|
4111
|
-
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
|
|
4154
|
+
// libs/js/cue-sdk/src/lib/api.ts
|
|
4155
|
+
var ENDPOINT_SEARCH = "/assistant/search";
|
|
4156
|
+
var ENDPOINT_SPARQL = "/triplestore/query";
|
|
4157
|
+
var CueApi = class {
|
|
4158
|
+
constructor(_auth, _gatewayUrl, projects, sync) {
|
|
4159
|
+
this._auth = _auth;
|
|
4160
|
+
this._gatewayUrl = _gatewayUrl;
|
|
4161
|
+
this.projects = projects;
|
|
4162
|
+
this.sync = sync;
|
|
4163
|
+
}
|
|
4164
|
+
async _authHeaders() {
|
|
4165
|
+
return this.getAuthHeaders();
|
|
4166
|
+
}
|
|
4167
|
+
/**
|
|
4168
|
+
* Returns standard authentication headers for the current user.
|
|
4169
|
+
* Useful when calling Cue-backed services directly (e.g. the GIS proxy).
|
|
4170
|
+
*/
|
|
4171
|
+
async getAuthHeaders() {
|
|
4172
|
+
const token = await this._auth.getToken();
|
|
4173
|
+
if (!token)
|
|
4174
|
+
throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
4175
|
+
return {
|
|
4176
|
+
Authorization: `Bearer ${token}`,
|
|
4177
|
+
"Content-Type": "application/json"
|
|
4178
|
+
};
|
|
4179
|
+
}
|
|
4180
|
+
/**
|
|
4181
|
+
* Search project documents using natural language.
|
|
4182
|
+
* The user must be authenticated before calling this.
|
|
4183
|
+
*/
|
|
4184
|
+
async search(request) {
|
|
4185
|
+
const headers = await this._authHeaders();
|
|
4186
|
+
const response = await fetch(`${this._gatewayUrl}${ENDPOINT_SEARCH}`, {
|
|
4187
|
+
method: "POST",
|
|
4188
|
+
headers,
|
|
4189
|
+
body: JSON.stringify({
|
|
4190
|
+
term: request.term,
|
|
4191
|
+
projectId: request.projectId,
|
|
4192
|
+
categories: request.categories ?? []
|
|
4193
|
+
})
|
|
4194
|
+
});
|
|
4195
|
+
if (!response.ok) {
|
|
4196
|
+
throw new Error(`Search request failed: ${response.status} ${response.statusText}`);
|
|
4133
4197
|
}
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4142
|
-
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
console.log(
|
|
4150
|
-
`Synchronized size: ${fileSizePretty(report.syncSize || 0)} (${(report.synctPctSize * 100).toFixed(2)}%)`
|
|
4151
|
-
);
|
|
4152
|
-
console.log("");
|
|
4153
|
-
if (report.localNotOnRemote) {
|
|
4154
|
-
console.log(
|
|
4155
|
-
`${report.localNotOnRemote.length} files do not exist on remote`
|
|
4156
|
-
);
|
|
4157
|
-
if (verbose && report.localNotOnRemote.length > 0) {
|
|
4158
|
-
for (const f of report.localNotOnRemote) {
|
|
4159
|
-
console.log(
|
|
4160
|
-
" - " + f.relativePath + " (" + fileSizePretty(f.size || 0) + ")"
|
|
4161
|
-
);
|
|
4162
|
-
}
|
|
4163
|
-
}
|
|
4164
|
-
console.log("");
|
|
4198
|
+
return response.json();
|
|
4199
|
+
}
|
|
4200
|
+
/**
|
|
4201
|
+
* Execute a SPARQL query against the project's triplestore.
|
|
4202
|
+
* The user must be authenticated before calling this.
|
|
4203
|
+
*/
|
|
4204
|
+
async sparql(query3, projectId) {
|
|
4205
|
+
const headers = await this._authHeaders();
|
|
4206
|
+
const response = await fetch(`${this._gatewayUrl}${ENDPOINT_SPARQL}`, {
|
|
4207
|
+
method: "POST",
|
|
4208
|
+
headers,
|
|
4209
|
+
body: JSON.stringify({ query: query3, projectId })
|
|
4210
|
+
});
|
|
4211
|
+
if (!response.ok) {
|
|
4212
|
+
throw new Error(`SPARQL query failed: ${response.status} ${response.statusText}`);
|
|
4165
4213
|
}
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4214
|
+
return response.json();
|
|
4215
|
+
}
|
|
4216
|
+
};
|
|
4217
|
+
|
|
4218
|
+
// libs/js/cue-sdk/src/lib/project.ts
|
|
4219
|
+
var import_firestore2 = require("firebase/firestore");
|
|
4220
|
+
var COLLECTION_PROJECTS2 = "projects";
|
|
4221
|
+
var CueProjects = class {
|
|
4222
|
+
constructor(_auth, app, useEmulator = false, endpoints) {
|
|
4223
|
+
this._auth = _auth;
|
|
4224
|
+
this._db = (0, import_firestore2.getFirestore)(app);
|
|
4225
|
+
if (useEmulator) {
|
|
4226
|
+
const host = endpoints?.firestoreEmulatorHost ?? "localhost";
|
|
4227
|
+
const port = endpoints?.firestoreEmulatorPort ?? 8080;
|
|
4228
|
+
(0, import_firestore2.connectFirestoreEmulator)(this._db, host, port);
|
|
4178
4229
|
}
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4230
|
+
}
|
|
4231
|
+
_db;
|
|
4232
|
+
_requireUser() {
|
|
4233
|
+
const user = this._auth.currentUser;
|
|
4234
|
+
if (!user)
|
|
4235
|
+
throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
4236
|
+
return user.uid;
|
|
4237
|
+
}
|
|
4238
|
+
/**
|
|
4239
|
+
* Create a new project. The authenticated user is automatically set as admin, syncer, and member.
|
|
4240
|
+
* Throws if a project with the given ID already exists.
|
|
4241
|
+
*/
|
|
4242
|
+
async createProject(options) {
|
|
4243
|
+
const userId = this._requireUser();
|
|
4244
|
+
const projectId = options.id ?? crypto.randomUUID();
|
|
4245
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4246
|
+
const projectSettings = { views: [], chatDisabled: false };
|
|
4247
|
+
const ref5 = (0, import_firestore2.doc)((0, import_firestore2.collection)(this._db, COLLECTION_PROJECTS2), projectId);
|
|
4248
|
+
const existing = await (0, import_firestore2.getDoc)(ref5);
|
|
4249
|
+
if (existing.exists()) {
|
|
4250
|
+
throw new Error(`Project with ID "${projectId}" already exists.`);
|
|
4192
4251
|
}
|
|
4193
|
-
|
|
4194
|
-
|
|
4195
|
-
|
|
4196
|
-
|
|
4197
|
-
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
|
|
4201
|
-
|
|
4202
|
-
|
|
4203
|
-
|
|
4204
|
-
|
|
4252
|
+
const data = {
|
|
4253
|
+
id: projectId,
|
|
4254
|
+
name: options.name,
|
|
4255
|
+
organizationID: options.organizationID,
|
|
4256
|
+
created: now,
|
|
4257
|
+
lastSync: null,
|
|
4258
|
+
isPublic: false,
|
|
4259
|
+
members: [userId],
|
|
4260
|
+
syncers: [userId],
|
|
4261
|
+
admins: [userId],
|
|
4262
|
+
alternativeIDs: [projectId],
|
|
4263
|
+
projectSettings
|
|
4264
|
+
};
|
|
4265
|
+
await (0, import_firestore2.setDoc)(ref5, data);
|
|
4266
|
+
return data;
|
|
4267
|
+
}
|
|
4268
|
+
/**
|
|
4269
|
+
* List all projects where the authenticated user appears in the members, syncers, or admins array.
|
|
4270
|
+
* Runs three parallel Firestore queries and deduplicates by project ID.
|
|
4271
|
+
*/
|
|
4272
|
+
async listProjects() {
|
|
4273
|
+
const userId = this._requireUser();
|
|
4274
|
+
const col = (0, import_firestore2.collection)(this._db, COLLECTION_PROJECTS2);
|
|
4275
|
+
const [memberSnap, syncerSnap, adminSnap] = await Promise.all([
|
|
4276
|
+
(0, import_firestore2.getDocs)((0, import_firestore2.query)(col, (0, import_firestore2.where)("members", "array-contains", userId))),
|
|
4277
|
+
(0, import_firestore2.getDocs)((0, import_firestore2.query)(col, (0, import_firestore2.where)("syncers", "array-contains", userId))),
|
|
4278
|
+
(0, import_firestore2.getDocs)((0, import_firestore2.query)(col, (0, import_firestore2.where)("admins", "array-contains", userId)))
|
|
4279
|
+
]);
|
|
4280
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4281
|
+
const results = [];
|
|
4282
|
+
for (const snap of [memberSnap, syncerSnap, adminSnap]) {
|
|
4283
|
+
for (const d of snap.docs) {
|
|
4284
|
+
const project = d.data();
|
|
4285
|
+
if (!seen.has(project.id)) {
|
|
4286
|
+
seen.add(project.id);
|
|
4287
|
+
results.push(project);
|
|
4205
4288
|
}
|
|
4206
4289
|
}
|
|
4207
|
-
console.log("");
|
|
4208
4290
|
}
|
|
4209
|
-
|
|
4210
|
-
console.error("Error:", err);
|
|
4211
|
-
process.exit(1);
|
|
4291
|
+
return results;
|
|
4212
4292
|
}
|
|
4213
|
-
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
const firebase = CueFirebase.getInstance();
|
|
4221
|
-
const storage = bucket === "raw" ? firebase.storageRaw : firebase.storageProcessed;
|
|
4222
|
-
console.log("Fetching files from storage bucket:", bucket, "in subdir:", subDir, "containing substring:", subString);
|
|
4223
|
-
const listResult = await (0, import_storage3.listAll)((0, import_storage3.ref)(storage, subDir));
|
|
4224
|
-
const outputDir = (0, import_path3.join)(process.cwd(), "downloaded_blobs", bucket, subDir);
|
|
4225
|
-
await (0, import_promises3.mkdir)(outputDir, { recursive: true });
|
|
4226
|
-
for (const fileRef of listResult.items) {
|
|
4227
|
-
console.log(fileRef.name);
|
|
4228
|
-
if (subDir && !fileRef.name.includes(subString))
|
|
4229
|
-
continue;
|
|
4230
|
-
const bytes = await (0, import_storage3.getBytes)(fileRef);
|
|
4231
|
-
const outputPath = (0, import_path3.join)(outputDir, fileRef.name);
|
|
4232
|
-
await (0, import_promises3.writeFile)(outputPath, Buffer.from(bytes));
|
|
4233
|
-
console.log(`Downloaded ${fileRef.name} to ${outputPath} \u2705`);
|
|
4293
|
+
/** Fetch a single project by ID. Returns null if not found. */
|
|
4294
|
+
async getProject(projectId) {
|
|
4295
|
+
const ref5 = (0, import_firestore2.doc)((0, import_firestore2.collection)(this._db, COLLECTION_PROJECTS2), projectId);
|
|
4296
|
+
const snap = await (0, import_firestore2.getDoc)(ref5);
|
|
4297
|
+
if (!snap.exists())
|
|
4298
|
+
return null;
|
|
4299
|
+
return snap.data();
|
|
4234
4300
|
}
|
|
4235
|
-
|
|
4301
|
+
/**
|
|
4302
|
+
* Atomically increments `unitsConsumed` on the top-level `clientSync/{projectId}`
|
|
4303
|
+
* document, creating it if it doesn't exist. Intended for pre-flight checks.
|
|
4304
|
+
*/
|
|
4305
|
+
async incrementUnitsConsumed(projectId, units, userId) {
|
|
4306
|
+
if (units <= 0)
|
|
4307
|
+
return;
|
|
4308
|
+
const ref5 = (0, import_firestore2.doc)(this._db, "clientSync", projectId);
|
|
4309
|
+
await (0, import_firestore2.setDoc)(ref5, {
|
|
4310
|
+
unitsConsumed: (0, import_firestore2.increment)(units),
|
|
4311
|
+
lastUpdated: (0, import_firestore2.serverTimestamp)(),
|
|
4312
|
+
lastUserId: userId
|
|
4313
|
+
}, { merge: true });
|
|
4314
|
+
}
|
|
4315
|
+
};
|
|
4236
4316
|
|
|
4237
|
-
//
|
|
4238
|
-
|
|
4239
|
-
|
|
4240
|
-
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4244
|
-
|
|
4317
|
+
// libs/js/cue-sdk/src/lib/profile.ts
|
|
4318
|
+
var import_auth4 = require("firebase/auth");
|
|
4319
|
+
var import_firestore3 = require("firebase/firestore");
|
|
4320
|
+
var import_nanoid = require("nanoid");
|
|
4321
|
+
var COLLECTION_API_KEYS2 = "apiKeys";
|
|
4322
|
+
var CueProfile = class {
|
|
4323
|
+
constructor(_auth, app, useEmulator, emulatorHost, emulatorPort) {
|
|
4324
|
+
this._auth = _auth;
|
|
4325
|
+
this._db = (0, import_firestore3.getFirestore)(app);
|
|
4326
|
+
if (useEmulator) {
|
|
4327
|
+
(0, import_firestore3.connectFirestoreEmulator)(this._db, emulatorHost, emulatorPort);
|
|
4245
4328
|
}
|
|
4246
|
-
if (verbose)
|
|
4247
|
-
console.info(`Dumping processed files for processor ${processor} \u23F3`);
|
|
4248
|
-
const files = await getFilesContainingSubstring("processed", `${space}/triples`, processor);
|
|
4249
|
-
console.log(files);
|
|
4250
|
-
} catch (err) {
|
|
4251
|
-
console.error("Error:", err);
|
|
4252
|
-
process.exit(1);
|
|
4253
4329
|
}
|
|
4254
|
-
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
var import_promises5 = require("fs/promises");
|
|
4263
|
-
var import_auth6 = require("firebase/auth");
|
|
4264
|
-
|
|
4265
|
-
// libs/js/size-tools/src/lib/js-size-tools.ts
|
|
4266
|
-
function humanFileSize(bytes, si = false, dp = 1) {
|
|
4267
|
-
const thresh = si ? 1e3 : 1024;
|
|
4268
|
-
if (Math.abs(bytes) < thresh) {
|
|
4269
|
-
return bytes + " B";
|
|
4330
|
+
_db;
|
|
4331
|
+
_apiKeyDocRef;
|
|
4332
|
+
/** Whether the current user has an active API key. */
|
|
4333
|
+
async hasAPIKey() {
|
|
4334
|
+
const uid = this._auth.currentUser?.uid;
|
|
4335
|
+
if (!uid)
|
|
4336
|
+
return false;
|
|
4337
|
+
return this._checkIfUserHasAPIKey(uid);
|
|
4270
4338
|
}
|
|
4271
|
-
|
|
4272
|
-
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
} while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1);
|
|
4278
|
-
return bytes.toFixed(dp) + " " + units[u];
|
|
4279
|
-
}
|
|
4280
|
-
|
|
4281
|
-
// apps/desktop/cue-cli/src/helpers/graph-dump.ts
|
|
4282
|
-
var import_fs4 = require("fs");
|
|
4283
|
-
async function dumpRdfGraphToFileJelly(spaceId, useEmulator = false, verbose = false) {
|
|
4284
|
-
return dumpRdfGraphToFile(
|
|
4285
|
-
spaceId,
|
|
4286
|
-
useEmulator,
|
|
4287
|
-
verbose,
|
|
4288
|
-
`${spaceId}.jelly`,
|
|
4289
|
-
"application/x-jelly-rdf"
|
|
4290
|
-
);
|
|
4291
|
-
}
|
|
4292
|
-
async function dumpRdfGraphToFile(spaceId, useEmulator = false, verbose = false, outFile, mimeType = "application/n-quads") {
|
|
4293
|
-
const endpoint = useEmulator ? SPARQL_ENDPOINT_EMULATOR : SPARQL_ENDPOINT;
|
|
4294
|
-
const dataUrl = endpoint.replace("/query", "/data");
|
|
4295
|
-
if (verbose)
|
|
4296
|
-
console.info(`Streaming RDF graph \u23F3`);
|
|
4297
|
-
const filePath = outFile || `${spaceId}.nq`;
|
|
4298
|
-
if ((0, import_fs4.existsSync)(filePath) || (0, import_fs4.existsSync)(`${filePath}.gz`)) {
|
|
4299
|
-
if (verbose)
|
|
4300
|
-
console.info(`File ${filePath} already exists, skipping download.`);
|
|
4301
|
-
return mimeType === "application/n-quads" ? `${filePath}.gz` : filePath;
|
|
4339
|
+
/** Returns the sign-in methods registered for the current user's email. */
|
|
4340
|
+
async getSignInMethods() {
|
|
4341
|
+
const user = this._auth.currentUser;
|
|
4342
|
+
if (!user?.email)
|
|
4343
|
+
throw new Error("User has no e-mail");
|
|
4344
|
+
return (0, import_auth4.fetchSignInMethodsForEmail)(this._auth.firebaseAuth, user.email);
|
|
4302
4345
|
}
|
|
4303
|
-
|
|
4304
|
-
|
|
4305
|
-
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
authorization: `Bearer ${token}`,
|
|
4309
|
-
Accept: mimeType
|
|
4310
|
-
}
|
|
4311
|
-
});
|
|
4312
|
-
if (!res.ok || !res.body) {
|
|
4313
|
-
throw new Error(`Failed to stream data: ${res.status} ${res.statusText}`);
|
|
4346
|
+
/** Builds a human-readable label from a Firebase UserInfo provider entry. */
|
|
4347
|
+
buildProviderLabel(userInfo) {
|
|
4348
|
+
if (userInfo.displayName && userInfo.email)
|
|
4349
|
+
return `${userInfo.displayName} (${userInfo.email})`;
|
|
4350
|
+
return userInfo.displayName ?? userInfo.email ?? "-";
|
|
4314
4351
|
}
|
|
4315
|
-
|
|
4316
|
-
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4352
|
+
/** Returns SSO accounts linked to the current user (excludes password). */
|
|
4353
|
+
getSSOAccounts() {
|
|
4354
|
+
const user = this._auth.currentUser;
|
|
4355
|
+
if (!user)
|
|
4356
|
+
return [];
|
|
4357
|
+
return user.providerData.filter((p) => p.providerId !== "password").map((p) => ({ id: p.providerId, label: this.buildProviderLabel(p) }));
|
|
4358
|
+
}
|
|
4359
|
+
/** Links a Google or Microsoft provider to the current account via popup. Returns the new provider label. */
|
|
4360
|
+
async linkProvider(ssoProvider) {
|
|
4361
|
+
const user = this._requireUser();
|
|
4362
|
+
const provider = ssoProvider === "google.com" ? new import_auth4.GoogleAuthProvider() : new import_auth4.OAuthProvider(ssoProvider);
|
|
4363
|
+
const credential = await (0, import_auth4.linkWithPopup)(user, provider);
|
|
4364
|
+
const userInfo = credential.user.providerData.find(
|
|
4365
|
+
(pd) => pd.providerId === ssoProvider
|
|
4366
|
+
);
|
|
4367
|
+
return userInfo ? this.buildProviderLabel(userInfo) : "-";
|
|
4368
|
+
}
|
|
4369
|
+
/** Unlinks a provider from the current account. */
|
|
4370
|
+
async unlinkProvider(providerId) {
|
|
4371
|
+
await (0, import_auth4.unlink)(this._requireUser(), providerId);
|
|
4372
|
+
}
|
|
4373
|
+
/** Changes the password. Reauthenticates first. */
|
|
4374
|
+
async updatePassword(currentPassword, newPassword) {
|
|
4375
|
+
const user = this._requireUser();
|
|
4376
|
+
if (!user.email)
|
|
4377
|
+
throw new Error("User has no e-mail");
|
|
4378
|
+
await (0, import_auth4.reauthenticateWithCredential)(
|
|
4379
|
+
user,
|
|
4380
|
+
import_auth4.EmailAuthProvider.credential(user.email, currentPassword)
|
|
4381
|
+
);
|
|
4382
|
+
await (0, import_auth4.updatePassword)(user, newPassword);
|
|
4383
|
+
}
|
|
4384
|
+
/** Adds (sets) a password for an account that currently only uses SSO. */
|
|
4385
|
+
async addPassword(password) {
|
|
4386
|
+
const user = this._requireUser();
|
|
4387
|
+
if (!user.email)
|
|
4388
|
+
throw new Error("User has no e-mail");
|
|
4389
|
+
await (0, import_auth4.linkWithCredential)(user, import_auth4.EmailAuthProvider.credential(user.email, password));
|
|
4390
|
+
}
|
|
4391
|
+
/** Requests an e-mail change. Sends a verification e-mail to the new address. */
|
|
4392
|
+
async updateEmail(newEmail, password) {
|
|
4393
|
+
const user = this._requireUser();
|
|
4394
|
+
if (!user.email)
|
|
4395
|
+
throw new Error("User e-mail not available");
|
|
4396
|
+
await (0, import_auth4.reauthenticateWithCredential)(
|
|
4397
|
+
user,
|
|
4398
|
+
import_auth4.EmailAuthProvider.credential(user.email, password)
|
|
4399
|
+
);
|
|
4400
|
+
await (0, import_auth4.verifyBeforeUpdateEmail)(user, newEmail);
|
|
4401
|
+
await (0, import_auth4.sendEmailVerification)(user);
|
|
4402
|
+
}
|
|
4403
|
+
/** Creates a new API key for the current user. */
|
|
4404
|
+
async createAPIKey(expiration) {
|
|
4405
|
+
const uid = this._auth.currentUser?.uid;
|
|
4406
|
+
if (!uid)
|
|
4407
|
+
throw new Error("User not authenticated");
|
|
4408
|
+
const data = { key: `cue-${(0, import_nanoid.nanoid)(150)}`, uid, expiration };
|
|
4409
|
+
const col = (0, import_firestore3.collection)(this._db, COLLECTION_API_KEYS2);
|
|
4410
|
+
this._apiKeyDocRef = await (0, import_firestore3.addDoc)(col, data);
|
|
4411
|
+
return data;
|
|
4412
|
+
}
|
|
4413
|
+
/** Revokes the current user's API key. */
|
|
4414
|
+
async revokeAPIKey() {
|
|
4415
|
+
if (!this._apiKeyDocRef)
|
|
4416
|
+
throw new Error("No API key document reference");
|
|
4417
|
+
await (0, import_firestore3.deleteDoc)(this._apiKeyDocRef);
|
|
4418
|
+
this._apiKeyDocRef = void 0;
|
|
4419
|
+
}
|
|
4420
|
+
/** Fetches the current user's existing API key. */
|
|
4421
|
+
async requestAPIKey() {
|
|
4422
|
+
if (!this._apiKeyDocRef)
|
|
4423
|
+
throw new Error("No API key document reference");
|
|
4424
|
+
const doc2 = (await (0, import_firestore3.getDoc)(this._apiKeyDocRef)).data();
|
|
4425
|
+
return { key: doc2.key, expiration: doc2.expiration };
|
|
4426
|
+
}
|
|
4427
|
+
async _checkIfUserHasAPIKey(uid) {
|
|
4428
|
+
const col = (0, import_firestore3.collection)(this._db, COLLECTION_API_KEYS2);
|
|
4429
|
+
const q = (0, import_firestore3.query)(col, (0, import_firestore3.where)("uid", "==", uid), (0, import_firestore3.limit)(1));
|
|
4430
|
+
const snapshot = await (0, import_firestore3.getDocs)(q);
|
|
4431
|
+
if (!snapshot.empty)
|
|
4432
|
+
this._apiKeyDocRef = snapshot.docs[0].ref;
|
|
4433
|
+
return !snapshot.empty;
|
|
4434
|
+
}
|
|
4435
|
+
_requireUser() {
|
|
4436
|
+
const user = this._auth.currentUser;
|
|
4437
|
+
if (!user)
|
|
4438
|
+
throw new Error("Not authenticated");
|
|
4439
|
+
return user;
|
|
4440
|
+
}
|
|
4441
|
+
};
|
|
4442
|
+
|
|
4443
|
+
// libs/js/cue-sdk/src/lib/cue.ts
|
|
4444
|
+
var FIREBASE_PROJECT_ID = "qaecy-mvp-406413";
|
|
4445
|
+
var FIREBASE_SENDER_ID = "734737865998";
|
|
4446
|
+
var ENDPOINTS = {
|
|
4447
|
+
production: {
|
|
4448
|
+
gatewayUrl: "https://accessors-api-gateway-ueyeemwf2a-oa.a.run.app",
|
|
4449
|
+
tokenUrl: "https://accessors-api-gateway-ueyeemwf2a-oa.a.run.app/token",
|
|
4450
|
+
authEmulatorUrl: "http://localhost:9099",
|
|
4451
|
+
storageEmulatorHost: "localhost",
|
|
4452
|
+
storageEmulatorPort: 9199,
|
|
4453
|
+
firestoreEmulatorHost: "localhost",
|
|
4454
|
+
firestoreEmulatorPort: 8080
|
|
4455
|
+
},
|
|
4456
|
+
emulator: {
|
|
4457
|
+
gatewayUrl: "http://localhost:8093",
|
|
4458
|
+
tokenUrl: "http://localhost:8093/token",
|
|
4459
|
+
authEmulatorUrl: "http://localhost:9099",
|
|
4460
|
+
storageEmulatorHost: "localhost",
|
|
4461
|
+
storageEmulatorPort: 9199,
|
|
4462
|
+
firestoreEmulatorHost: "localhost",
|
|
4463
|
+
firestoreEmulatorPort: 8080
|
|
4464
|
+
}
|
|
4465
|
+
};
|
|
4466
|
+
var Cue = class {
|
|
4467
|
+
auth;
|
|
4468
|
+
api;
|
|
4469
|
+
profile;
|
|
4470
|
+
_app;
|
|
4471
|
+
_endpoints;
|
|
4472
|
+
_isEmulator;
|
|
4473
|
+
constructor(config) {
|
|
4474
|
+
const env = config.environment ?? "production";
|
|
4475
|
+
this._endpoints = { ...ENDPOINTS[env], ...config.endpoints };
|
|
4476
|
+
this._isEmulator = env === "emulator";
|
|
4477
|
+
this._app = (0, import_app2.initializeApp)({
|
|
4478
|
+
apiKey: config.apiKey,
|
|
4479
|
+
appId: config.appId,
|
|
4480
|
+
measurementId: config.measurementId,
|
|
4481
|
+
authDomain: `${FIREBASE_PROJECT_ID}.firebaseapp.com`,
|
|
4482
|
+
projectId: FIREBASE_PROJECT_ID,
|
|
4483
|
+
messagingSenderId: FIREBASE_SENDER_ID
|
|
4484
|
+
});
|
|
4485
|
+
this.auth = new CueAuth(this._app, this._isEmulator, this._endpoints);
|
|
4486
|
+
const projects = new CueProjects(this.auth, this._app, this._isEmulator, this._endpoints);
|
|
4487
|
+
this.api = this._buildApi(projects);
|
|
4488
|
+
this.profile = new CueProfile(
|
|
4489
|
+
this.auth,
|
|
4490
|
+
this._app,
|
|
4491
|
+
this._isEmulator,
|
|
4492
|
+
this._endpoints.firestoreEmulatorHost,
|
|
4493
|
+
this._endpoints.firestoreEmulatorPort
|
|
4494
|
+
);
|
|
4495
|
+
}
|
|
4496
|
+
/** Override in subclasses to provide a custom CueApi (e.g. with sync). */
|
|
4497
|
+
_buildApi(projects) {
|
|
4498
|
+
return new CueApi(this.auth, this._endpoints.gatewayUrl, projects);
|
|
4499
|
+
}
|
|
4500
|
+
/** Convenience: get the current user's Firebase ID token */
|
|
4501
|
+
getToken(forceRefresh = false) {
|
|
4502
|
+
return this.auth.getToken(forceRefresh);
|
|
4503
|
+
}
|
|
4504
|
+
};
|
|
4505
|
+
|
|
4506
|
+
// libs/js/cue-sdk/src/lib/cue-node.ts
|
|
4507
|
+
var import_storage4 = require("firebase/storage");
|
|
4508
|
+
|
|
4509
|
+
// libs/js/databases/src/lib/graph/fuseki.ts
|
|
4510
|
+
var Fuseki = class _Fuseki {
|
|
4511
|
+
queryEndpoint;
|
|
4512
|
+
updateEndpoint;
|
|
4513
|
+
baseHeaders;
|
|
4514
|
+
static RELEVANT_HEADER_KEYS = [
|
|
4515
|
+
"authorization",
|
|
4516
|
+
"Authorization",
|
|
4517
|
+
"x-project-id"
|
|
4518
|
+
];
|
|
4519
|
+
constructor(graphOptions) {
|
|
4520
|
+
this.queryEndpoint = graphOptions.queryEndpoint;
|
|
4521
|
+
this.updateEndpoint = graphOptions.updateEndpoint;
|
|
4522
|
+
this.baseHeaders = Object.fromEntries(
|
|
4523
|
+
Object.entries(graphOptions.originalHeaders || {}).filter(
|
|
4524
|
+
([key]) => _Fuseki.RELEVANT_HEADER_KEYS.includes(key)
|
|
4525
|
+
)
|
|
4526
|
+
);
|
|
4527
|
+
if (graphOptions.authHeader !== void 0) {
|
|
4528
|
+
this.baseHeaders["Authorization"] = graphOptions.authHeader;
|
|
4327
4529
|
}
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4530
|
+
}
|
|
4531
|
+
async ping() {
|
|
4532
|
+
const query3 = "ASK { }";
|
|
4533
|
+
const res = await this.query(query3);
|
|
4534
|
+
return res.boolean;
|
|
4535
|
+
}
|
|
4536
|
+
async query(query3, accept = "application/sparql-results+json") {
|
|
4537
|
+
let res;
|
|
4538
|
+
try {
|
|
4539
|
+
res = await fetch(this.queryEndpoint, {
|
|
4540
|
+
headers: {
|
|
4541
|
+
...this.baseHeaders,
|
|
4542
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
4543
|
+
Accept: accept
|
|
4544
|
+
},
|
|
4545
|
+
method: "POST",
|
|
4546
|
+
body: new URLSearchParams({ query: query3 })
|
|
4547
|
+
});
|
|
4548
|
+
} catch (err) {
|
|
4549
|
+
throw new Error(
|
|
4550
|
+
`Fuseki is not accessible at ${this.queryEndpoint}: ${err instanceof Error ? err.message : String(err)}`
|
|
4331
4551
|
);
|
|
4332
|
-
});
|
|
4333
|
-
const streamTimeout = 3e5;
|
|
4334
|
-
hasDataTimeout = setTimeout(() => {
|
|
4335
|
-
const timeSinceLastChunk = Date.now() - lastChunkTime;
|
|
4336
|
-
if (timeSinceLastChunk > streamTimeout) {
|
|
4337
|
-
console.error(`Stream appears stalled - no data received for ${timeSinceLastChunk}ms`);
|
|
4338
|
-
nodeStream.destroy(new Error("Stream timeout"));
|
|
4339
4552
|
}
|
|
4340
|
-
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
} finally {
|
|
4344
|
-
if (hasDataTimeout) {
|
|
4345
|
-
clearTimeout(hasDataTimeout);
|
|
4553
|
+
if (!res.ok) {
|
|
4554
|
+
const body = await res.text();
|
|
4555
|
+
throw new Error(`Fuseki query failed (HTTP ${res.status}): ${body}`);
|
|
4346
4556
|
}
|
|
4557
|
+
return await res.json();
|
|
4347
4558
|
}
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4559
|
+
async subset(query3, accept = "text/turtle") {
|
|
4560
|
+
const res = await fetch(this.queryEndpoint, {
|
|
4561
|
+
headers: {
|
|
4562
|
+
...this.baseHeaders,
|
|
4563
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
4564
|
+
Authorization: this.baseHeaders["Authorization"] || "",
|
|
4565
|
+
Accept: accept
|
|
4566
|
+
},
|
|
4567
|
+
method: "POST",
|
|
4568
|
+
body: new URLSearchParams({ query: query3 })
|
|
4569
|
+
});
|
|
4570
|
+
if (accept === "application/ld+json") {
|
|
4571
|
+
return await res.json();
|
|
4572
|
+
}
|
|
4573
|
+
return await res.text();
|
|
4352
4574
|
}
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4575
|
+
async update(update) {
|
|
4576
|
+
const res = await fetch(this.updateEndpoint, {
|
|
4577
|
+
headers: {
|
|
4578
|
+
...this.baseHeaders,
|
|
4579
|
+
"Content-Type": "application/x-www-form-urlencoded"
|
|
4580
|
+
},
|
|
4581
|
+
method: "POST",
|
|
4582
|
+
body: new URLSearchParams({ update })
|
|
4583
|
+
});
|
|
4584
|
+
if (!res.ok) {
|
|
4585
|
+
const body = await res.text();
|
|
4586
|
+
throw new Error(`SPARQL update failed (HTTP ${res.status}): ${body}`);
|
|
4587
|
+
}
|
|
4588
|
+
const contentType = res.headers.get("content-type") ?? "";
|
|
4589
|
+
if (contentType.includes("application/json")) {
|
|
4590
|
+
return await res.json();
|
|
4591
|
+
}
|
|
4592
|
+
return {
|
|
4593
|
+
ok: res.ok,
|
|
4594
|
+
status: res.status,
|
|
4595
|
+
message: await res.text()
|
|
4596
|
+
};
|
|
4363
4597
|
}
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
4370
|
-
|
|
4371
|
-
|
|
4372
|
-
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
queries.map(
|
|
4385
|
-
(q, idx) => retryWithBackoff(
|
|
4386
|
-
() => _doQuery(q, spaceId, endpoint),
|
|
4387
|
-
maxRetries,
|
|
4388
|
-
retryDelayMs,
|
|
4389
|
-
verbose ? `Batch ${offset - batchSize * concurrency + idx * batchSize}` : void 0
|
|
4390
|
-
)
|
|
4598
|
+
};
|
|
4599
|
+
|
|
4600
|
+
// libs/js/databases/src/lib/graph/qlever.ts
|
|
4601
|
+
var QLever = class _QLever {
|
|
4602
|
+
queryEndpoint;
|
|
4603
|
+
updateEndpoint;
|
|
4604
|
+
baseHeaders;
|
|
4605
|
+
static RELEVANT_HEADER_KEYS = [
|
|
4606
|
+
"authorization",
|
|
4607
|
+
"Authorization",
|
|
4608
|
+
"x-project-id",
|
|
4609
|
+
"x-user-roles"
|
|
4610
|
+
// add more if needed
|
|
4611
|
+
];
|
|
4612
|
+
constructor(graphOptions) {
|
|
4613
|
+
this.queryEndpoint = graphOptions.queryEndpoint;
|
|
4614
|
+
this.updateEndpoint = graphOptions.updateEndpoint;
|
|
4615
|
+
this.baseHeaders = Object.fromEntries(
|
|
4616
|
+
Object.entries(graphOptions.originalHeaders || {}).filter(
|
|
4617
|
+
([key]) => _QLever.RELEVANT_HEADER_KEYS.includes(key)
|
|
4391
4618
|
)
|
|
4392
4619
|
);
|
|
4393
|
-
const batches = results.filter(Boolean);
|
|
4394
|
-
if (batches.length === 0) {
|
|
4395
|
-
hasMore = false;
|
|
4396
|
-
break;
|
|
4397
|
-
}
|
|
4398
|
-
for (const batch of batches) {
|
|
4399
|
-
fileStream.write(batch.trim() + "\n");
|
|
4400
|
-
}
|
|
4401
|
-
if (verbose)
|
|
4402
|
-
process.stdout.write(`Current offset: ${offset.toLocaleString()}\r`);
|
|
4403
|
-
if (delayBetweenBatchesMs > 0)
|
|
4404
|
-
await new Promise((res) => setTimeout(res, delayBetweenBatchesMs));
|
|
4405
4620
|
}
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
return `${filePath}.gz`;
|
|
4414
|
-
}
|
|
4415
|
-
async function retryWithBackoff(fn, maxRetries, delayMs, label) {
|
|
4416
|
-
let attempt = 0;
|
|
4417
|
-
let lastErr = null;
|
|
4418
|
-
while (attempt < maxRetries) {
|
|
4621
|
+
async ping() {
|
|
4622
|
+
const query3 = "ASK { }";
|
|
4623
|
+
const res = await this.query(query3);
|
|
4624
|
+
return res.boolean;
|
|
4625
|
+
}
|
|
4626
|
+
async query(query3, accept = "application/sparql-results+json") {
|
|
4627
|
+
let res;
|
|
4419
4628
|
try {
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4629
|
+
console.log(this.queryEndpoint);
|
|
4630
|
+
res = await fetch(this.queryEndpoint, {
|
|
4631
|
+
headers: {
|
|
4632
|
+
...this.baseHeaders,
|
|
4633
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
4634
|
+
Accept: accept
|
|
4635
|
+
},
|
|
4636
|
+
method: "POST",
|
|
4637
|
+
body: new URLSearchParams({ query: query3 })
|
|
4638
|
+
});
|
|
4423
4639
|
} catch (err) {
|
|
4424
|
-
|
|
4640
|
+
throw new Error(
|
|
4641
|
+
`QLever is not accessible at ${this.queryEndpoint}: ${err instanceof Error ? err.message : String(err)}`
|
|
4642
|
+
);
|
|
4425
4643
|
}
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
);
|
|
4644
|
+
if (!res.ok) {
|
|
4645
|
+
const body = await res.text();
|
|
4646
|
+
throw new Error(`QLever query failed (HTTP ${res.status}): ${body}`);
|
|
4647
|
+
}
|
|
4648
|
+
return await res.json();
|
|
4432
4649
|
}
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
const gzip = (0, import_zlib.createGzip)();
|
|
4441
|
-
const source = (0, import_fs3.createReadStream)(filePath);
|
|
4442
|
-
const dest = (0, import_fs2.createWriteStream)(gzFilePath);
|
|
4443
|
-
(0, import_promises4.pipeline)(source, gzip, dest).then(resolve2).catch(reject);
|
|
4444
|
-
});
|
|
4445
|
-
await (0, import_promises5.unlink)(filePath);
|
|
4446
|
-
}
|
|
4447
|
-
async function _doQuery(query, spaceId, url) {
|
|
4448
|
-
const token = await (0, import_auth6.getAuth)().currentUser?.getIdToken();
|
|
4449
|
-
try {
|
|
4450
|
-
const res = await fetch(url, {
|
|
4650
|
+
async subset(query3, accept = "text/turtle") {
|
|
4651
|
+
const res = await fetch(this.queryEndpoint, {
|
|
4652
|
+
headers: {
|
|
4653
|
+
...this.baseHeaders,
|
|
4654
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
4655
|
+
Accept: accept
|
|
4656
|
+
},
|
|
4451
4657
|
method: "POST",
|
|
4658
|
+
body: new URLSearchParams({ query: query3 })
|
|
4659
|
+
});
|
|
4660
|
+
return await res.text();
|
|
4661
|
+
}
|
|
4662
|
+
async update(update) {
|
|
4663
|
+
const res = await fetch(this.updateEndpoint, {
|
|
4452
4664
|
headers: {
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
"Content-Type": "application/sparql-query",
|
|
4456
|
-
Accept: "application/n-quads"
|
|
4665
|
+
...this.baseHeaders,
|
|
4666
|
+
"Content-Type": "application/x-www-form-urlencoded"
|
|
4457
4667
|
},
|
|
4458
|
-
|
|
4668
|
+
method: "POST",
|
|
4669
|
+
body: new URLSearchParams({ update })
|
|
4459
4670
|
});
|
|
4460
4671
|
if (!res.ok) {
|
|
4461
|
-
|
|
4462
|
-
|
|
4672
|
+
const body = await res.text();
|
|
4673
|
+
throw new Error(`SPARQL update failed (HTTP ${res.status}): ${body}`);
|
|
4463
4674
|
}
|
|
4464
|
-
return await res.
|
|
4465
|
-
} catch (e) {
|
|
4466
|
-
console.error(e);
|
|
4467
|
-
return null;
|
|
4675
|
+
return await res.json();
|
|
4468
4676
|
}
|
|
4469
|
-
}
|
|
4677
|
+
};
|
|
4470
4678
|
|
|
4471
|
-
//
|
|
4472
|
-
var
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
"
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
if (!res.ok) {
|
|
4486
|
-
throw new Error(`Failed to create graph: ${res.statusText}`);
|
|
4679
|
+
// libs/js/databases/src/lib/graph/main.ts
|
|
4680
|
+
var CueGraphDatabase = class {
|
|
4681
|
+
constructor(options) {
|
|
4682
|
+
this.options = options;
|
|
4683
|
+
switch (this.options.graphType) {
|
|
4684
|
+
case "qlever":
|
|
4685
|
+
this._db = new QLever(this.options);
|
|
4686
|
+
break;
|
|
4687
|
+
case "fuseki":
|
|
4688
|
+
this._db = new Fuseki(this.options);
|
|
4689
|
+
break;
|
|
4690
|
+
default:
|
|
4691
|
+
throw new Error(`Unsupported graph type: ${this.options.graphType}`);
|
|
4692
|
+
}
|
|
4487
4693
|
}
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4498
|
-
|
|
4694
|
+
_db;
|
|
4695
|
+
ping() {
|
|
4696
|
+
return this._db.ping();
|
|
4697
|
+
}
|
|
4698
|
+
query(queryString, accept) {
|
|
4699
|
+
return this._db.query(queryString, accept);
|
|
4700
|
+
}
|
|
4701
|
+
subset(queryString, accept) {
|
|
4702
|
+
if (this.options.graphType === "qlever") {
|
|
4703
|
+
if (accept && accept !== "text/turtle") {
|
|
4704
|
+
return Promise.reject(
|
|
4705
|
+
new Error(
|
|
4706
|
+
`QLever only supports 'text/turtle' for CONSTRUCT/DESCRIBE queries.`
|
|
4707
|
+
)
|
|
4708
|
+
);
|
|
4709
|
+
}
|
|
4710
|
+
return this._db.subset(
|
|
4711
|
+
queryString,
|
|
4712
|
+
accept
|
|
4713
|
+
);
|
|
4714
|
+
}
|
|
4715
|
+
return this._db.subset(queryString, accept);
|
|
4716
|
+
}
|
|
4717
|
+
update(updateString) {
|
|
4718
|
+
return this._db.update(updateString);
|
|
4719
|
+
}
|
|
4720
|
+
};
|
|
4499
4721
|
|
|
4722
|
+
// libs/js/databases/src/lib/blob/blob.ts
|
|
4723
|
+
var import_storage3 = require("firebase/storage");
|
|
4724
|
+
var CueBlobStorage = class {
|
|
4725
|
+
constructor(options) {
|
|
4726
|
+
this.options = options;
|
|
4727
|
+
}
|
|
4728
|
+
/**
|
|
4729
|
+
* Upload binary data to the raw storage bucket with retry logic.
|
|
4730
|
+
*/
|
|
4731
|
+
async uploadRaw(blobName, data, metadata, maxRetries = 3) {
|
|
4732
|
+
const fileRef = (0, import_storage3.ref)(this.options.storageRaw, blobName);
|
|
4733
|
+
let attempt = 0;
|
|
4734
|
+
let lastError;
|
|
4735
|
+
while (attempt < maxRetries) {
|
|
4736
|
+
try {
|
|
4737
|
+
await new Promise((resolve2, reject) => {
|
|
4738
|
+
const task = (0, import_storage3.uploadBytesResumable)(fileRef, data, {
|
|
4739
|
+
customMetadata: metadata
|
|
4740
|
+
});
|
|
4741
|
+
task.on("state_changed", null, reject, resolve2);
|
|
4742
|
+
});
|
|
4743
|
+
return;
|
|
4744
|
+
} catch (err) {
|
|
4745
|
+
lastError = err;
|
|
4746
|
+
attempt++;
|
|
4747
|
+
if (attempt < maxRetries) {
|
|
4748
|
+
await new Promise((res) => setTimeout(res, 1e3 * attempt));
|
|
4749
|
+
}
|
|
4750
|
+
}
|
|
4751
|
+
}
|
|
4752
|
+
throw lastError;
|
|
4753
|
+
}
|
|
4754
|
+
/**
|
|
4755
|
+
* Upload data to the processed storage bucket.
|
|
4756
|
+
* Skips upload and returns `false` if the blob already exists.
|
|
4757
|
+
*/
|
|
4758
|
+
async uploadProcessed(blobName, data, metadata) {
|
|
4759
|
+
const fileRef = (0, import_storage3.ref)(this.options.storageProcessed, blobName);
|
|
4760
|
+
const existing = await (0, import_storage3.getMetadata)(fileRef).catch(() => null);
|
|
4761
|
+
if (existing)
|
|
4762
|
+
return false;
|
|
4763
|
+
await (0, import_storage3.uploadBytes)(fileRef, data, { customMetadata: metadata });
|
|
4764
|
+
return true;
|
|
4765
|
+
}
|
|
4766
|
+
/**
|
|
4767
|
+
* List all blob names directly under `prefix` in the raw storage bucket.
|
|
4768
|
+
*/
|
|
4769
|
+
async listRaw(prefix) {
|
|
4770
|
+
const listRef = (0, import_storage3.ref)(this.options.storageRaw, prefix);
|
|
4771
|
+
const result = await (0, import_storage3.listAll)(listRef);
|
|
4772
|
+
return result.items.map((item) => item.name);
|
|
4773
|
+
}
|
|
4774
|
+
};
|
|
4500
4775
|
|
|
4501
|
-
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
tdb2:transactionLogType "journal" ;
|
|
4506
|
-
tdb2:blockFileSize "32M" ;
|
|
4507
|
-
tdb2:blockCacheSize "1G" ;
|
|
4508
|
-
tdb2:fileMode "direct" .
|
|
4509
|
-
|
|
4510
|
-
:service_tdb_all a fuseki:Service;
|
|
4511
|
-
fuseki:name "${partition}" ;
|
|
4512
|
-
rdfs:label "TDB2 ${partition}";
|
|
4513
|
-
fuseki:dataset :tdb_dataset_readwrite;
|
|
4514
|
-
fuseki:allowedUsers "admin";
|
|
4515
|
-
fuseki:endpoint [ fuseki:name "update";
|
|
4516
|
-
fuseki:operation fuseki:update
|
|
4517
|
-
];
|
|
4518
|
-
fuseki:endpoint [ fuseki:name "query";
|
|
4519
|
-
fuseki:operation fuseki:query
|
|
4520
|
-
];
|
|
4521
|
-
fuseki:endpoint [ fuseki:name "get";
|
|
4522
|
-
fuseki:operation fuseki:gsp-r
|
|
4523
|
-
];
|
|
4524
|
-
fuseki:endpoint [ fuseki:name "shacl";
|
|
4525
|
-
fuseki:operation fuseki:shacl
|
|
4526
|
-
];
|
|
4527
|
-
fuseki:endpoint [ fuseki:name "data";
|
|
4528
|
-
fuseki:operation fuseki:gsp-rw
|
|
4529
|
-
];
|
|
4530
|
-
fuseki:endpoint [ fuseki:name "sparql";
|
|
4531
|
-
fuseki:operation fuseki:query
|
|
4532
|
-
].`;
|
|
4533
|
-
}
|
|
4534
|
-
|
|
4535
|
-
// apps/desktop/cue-cli/src/helpers/graph-upload.ts
|
|
4536
|
-
var import_auth8 = require("firebase/auth");
|
|
4537
|
-
var import_fs5 = require("fs");
|
|
4538
|
-
async function uploadToLocalGraph(id, filePath, mimeType = "application/n-quads", zipped = false) {
|
|
4539
|
-
const stream = (0, import_fs5.createReadStream)(filePath);
|
|
4540
|
-
const token = await (0, import_auth8.getAuth)().currentUser?.getIdToken();
|
|
4541
|
-
const dataUrl = SPARQL_ENDPOINT_EMULATOR.replace("/query", "/data");
|
|
4542
|
-
const headers = {
|
|
4543
|
-
"x-project-id": id,
|
|
4544
|
-
authorization: `Bearer ${token}`,
|
|
4545
|
-
"Content-Type": mimeType
|
|
4546
|
-
};
|
|
4547
|
-
if (zipped) {
|
|
4548
|
-
headers["Content-Encoding"] = "gzip";
|
|
4549
|
-
}
|
|
4550
|
-
const res = await fetch(dataUrl, {
|
|
4551
|
-
method: "POST",
|
|
4552
|
-
headers,
|
|
4553
|
-
body: stream,
|
|
4554
|
-
duplex: "half"
|
|
4555
|
-
});
|
|
4556
|
-
const text = await res.text();
|
|
4557
|
-
console.log(text);
|
|
4558
|
-
if (!res.ok) {
|
|
4559
|
-
console.error(`Failed to upload graph: ${res.statusText}`);
|
|
4560
|
-
return;
|
|
4561
|
-
}
|
|
4562
|
-
}
|
|
4563
|
-
|
|
4564
|
-
// apps/desktop/cue-cli/src/cue-cli-dump.ts
|
|
4565
|
-
async function dumpHandler(options) {
|
|
4566
|
-
const { space, verbose, emulators, jelly, query, load } = options;
|
|
4567
|
-
try {
|
|
4568
|
-
const { isSuperAdmin } = await authenticate(emulators, options.key, verbose);
|
|
4569
|
-
if (!isSuperAdmin) {
|
|
4570
|
-
console.error("Sorry - this tool is only for super admin users");
|
|
4571
|
-
process.exit(1);
|
|
4572
|
-
}
|
|
4573
|
-
if (verbose)
|
|
4574
|
-
console.info("Dumping graph \u23F3");
|
|
4575
|
-
let filePath = "";
|
|
4576
|
-
if (jelly) {
|
|
4577
|
-
if (verbose)
|
|
4578
|
-
console.info("Setting: Jelly format");
|
|
4579
|
-
if (verbose)
|
|
4580
|
-
console.time("Downloaded graph \u2705");
|
|
4581
|
-
filePath = await dumpRdfGraphToFileJelly(space, emulators, verbose);
|
|
4582
|
-
if (verbose)
|
|
4583
|
-
console.timeEnd("Downloaded graph \u2705");
|
|
4584
|
-
} else {
|
|
4585
|
-
if (verbose)
|
|
4586
|
-
console.time("Downloaded and zipped graph \u2705");
|
|
4587
|
-
if (query) {
|
|
4588
|
-
if (verbose)
|
|
4589
|
-
console.info("Setting: Construct query");
|
|
4590
|
-
filePath = await dumpRdfGraphToFileConstruct(space, emulators, verbose);
|
|
4591
|
-
} else {
|
|
4592
|
-
if (verbose)
|
|
4593
|
-
console.info("Setting: /data endpoint");
|
|
4594
|
-
filePath = await dumpRdfGraphToFile(space, emulators, verbose);
|
|
4595
|
-
}
|
|
4596
|
-
if (verbose)
|
|
4597
|
-
console.timeEnd("Downloaded and zipped graph \u2705");
|
|
4598
|
-
}
|
|
4599
|
-
console.info(`File written: ${filePath}`);
|
|
4600
|
-
if (load && !emulators) {
|
|
4601
|
-
if (verbose)
|
|
4602
|
-
console.info(`Creating local RDF graph with id: ${space} \u23F3`);
|
|
4603
|
-
await createGraph(space);
|
|
4604
|
-
if (verbose)
|
|
4605
|
-
console.info(`Created local RDF graph \u2705`);
|
|
4606
|
-
if (verbose)
|
|
4607
|
-
console.info(`Uploading file to local RDF graph \u23F3`);
|
|
4608
|
-
await uploadToLocalGraph(
|
|
4609
|
-
space,
|
|
4610
|
-
filePath,
|
|
4611
|
-
jelly ? "application/jelly+json" : "application/n-quads",
|
|
4612
|
-
jelly ? false : true
|
|
4613
|
-
);
|
|
4614
|
-
if (verbose)
|
|
4615
|
-
console.info(`Uploaded file to local RDF graph \u2705`);
|
|
4616
|
-
}
|
|
4617
|
-
} catch (err) {
|
|
4618
|
-
console.error("Error:", err);
|
|
4619
|
-
process.exit(1);
|
|
4620
|
-
}
|
|
4621
|
-
}
|
|
4622
|
-
|
|
4623
|
-
// apps/desktop/cue-cli/src/helpers/repair-remote-ttl.ts
|
|
4624
|
-
var import_storage4 = require("firebase/storage");
|
|
4625
|
-
async function repairRemoteTTL(space, subString, regex, substituteString) {
|
|
4626
|
-
const firebase = CueFirebase.getInstance();
|
|
4627
|
-
const storage = firebase.storageProcessed;
|
|
4628
|
-
console.log("Fetching files from storage bucket 'triples' containing substring:", subString);
|
|
4629
|
-
const listResult = await (0, import_storage4.listAll)((0, import_storage4.ref)(storage, `${space}/triples`));
|
|
4630
|
-
for (const fileRef of listResult.items) {
|
|
4631
|
-
if (subString && !fileRef.name.match(subString))
|
|
4632
|
-
continue;
|
|
4633
|
-
const stream = await (0, import_storage4.getStream)(fileRef);
|
|
4634
|
-
const reader = stream.getReader();
|
|
4635
|
-
const chunks = [];
|
|
4636
|
-
let done = false;
|
|
4637
|
-
while (!done) {
|
|
4638
|
-
const { value, done: streamDone } = await reader.read();
|
|
4639
|
-
if (value) {
|
|
4640
|
-
chunks.push(value);
|
|
4641
|
-
}
|
|
4642
|
-
done = streamDone;
|
|
4643
|
-
}
|
|
4644
|
-
let fileContent = Buffer.concat(chunks).toString("utf8");
|
|
4645
|
-
let modified = false;
|
|
4646
|
-
if (substituteString && regex) {
|
|
4647
|
-
const newContent = fileContent.replace(regex, substituteString);
|
|
4648
|
-
if (newContent !== fileContent) {
|
|
4649
|
-
fileContent = newContent;
|
|
4650
|
-
modified = true;
|
|
4651
|
-
}
|
|
4652
|
-
}
|
|
4653
|
-
if (modified) {
|
|
4654
|
-
const buffer = Buffer.from(fileContent, "utf8");
|
|
4655
|
-
let existingMetadata = {};
|
|
4656
|
-
try {
|
|
4657
|
-
existingMetadata = await (0, import_storage4.getMetadata)(fileRef);
|
|
4658
|
-
} catch (err) {
|
|
4659
|
-
console.warn(`Could not fetch metadata for ${fileRef.name}, proceeding with default.`);
|
|
4660
|
-
}
|
|
4661
|
-
const customMetadata = { ...existingMetadata.customMetadata || {}, stored: "False" };
|
|
4662
|
-
const metadata = { customMetadata };
|
|
4663
|
-
await (0, import_storage4.uploadBytesResumable)((0, import_storage4.ref)(storage, `${space}/triples/${fileRef.name}`), buffer, metadata);
|
|
4664
|
-
console.log(`Fixed ${fileRef.name} \u2705`);
|
|
4665
|
-
} else {
|
|
4666
|
-
console.log(`No changes for ${fileRef.name}`);
|
|
4667
|
-
}
|
|
4668
|
-
}
|
|
4669
|
-
}
|
|
4670
|
-
|
|
4671
|
-
// apps/desktop/cue-cli/src/cue-cli-repair-ttl.ts
|
|
4672
|
-
async function repairTtlHandler(options) {
|
|
4673
|
-
const { space, verbose, emulators, processor, from, to } = options;
|
|
4674
|
-
try {
|
|
4675
|
-
const { isSuperAdmin } = await authenticate(emulators, options.key, verbose);
|
|
4676
|
-
if (!isSuperAdmin) {
|
|
4677
|
-
console.error("Sorry - this tool is only for super admin users");
|
|
4678
|
-
process.exit(1);
|
|
4679
|
-
}
|
|
4680
|
-
if (verbose)
|
|
4681
|
-
console.info(`Repairing TTL files for processor ${processor} \u23F3`);
|
|
4682
|
-
await repairRemoteTTL(space, processor, new RegExp(from, "g"), to);
|
|
4683
|
-
} catch (err) {
|
|
4684
|
-
console.error("Error:", err);
|
|
4685
|
-
process.exit(1);
|
|
4686
|
-
}
|
|
4687
|
-
}
|
|
4688
|
-
|
|
4689
|
-
// apps/desktop/cue-cli/src/helpers/upload-file.ts
|
|
4690
|
-
var import_storage5 = require("firebase/storage");
|
|
4776
|
+
// libs/js/cue-sdk/src/lib/sync.ts
|
|
4777
|
+
var import_promises3 = require("fs/promises");
|
|
4778
|
+
var import_path3 = require("path");
|
|
4779
|
+
var import_module = require("module");
|
|
4691
4780
|
|
|
4692
4781
|
// libs/js/models/src/lib/file-extensions.ts
|
|
4693
4782
|
var fileExtensionsInfo = {
|
|
@@ -5715,18 +5804,33 @@ function getFileSuffix(filename) {
|
|
|
5715
5804
|
var { namedNode: namedNode4, literal: literal3 } = import_n34.DataFactory;
|
|
5716
5805
|
var a3 = namedNode4("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
|
|
5717
5806
|
|
|
5807
|
+
// libs/js/rdf-document-writers/src/lib/process-logs.ts
|
|
5808
|
+
var import_n35 = require("n3");
|
|
5809
|
+
|
|
5718
5810
|
// libs/js/rdf-document-writers/src/lib/serialize-rdf.ts
|
|
5719
|
-
async function serializeRDF(namespace, writer) {
|
|
5811
|
+
async function serializeRDF(namespace, writer, format = "ttl") {
|
|
5720
5812
|
return new Promise((resolve2, reject) => {
|
|
5721
5813
|
writer.end((err, res) => {
|
|
5722
5814
|
if (err)
|
|
5723
5815
|
reject(err);
|
|
5724
|
-
|
|
5816
|
+
const addBase = format === "ttl" && namespace !== "";
|
|
5817
|
+
resolve2(addBase ? `@base <${namespace}>.
|
|
5725
5818
|
${res}` : res);
|
|
5726
5819
|
});
|
|
5727
5820
|
});
|
|
5728
5821
|
}
|
|
5729
5822
|
|
|
5823
|
+
// libs/js/rdf-document-writers/src/lib/process-logs.ts
|
|
5824
|
+
var import_uuid6 = require("uuid");
|
|
5825
|
+
var { namedNode: namedNode5, literal: literal4 } = import_n35.DataFactory;
|
|
5826
|
+
var a4 = namedNode5("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
|
|
5827
|
+
var prefixes3 = {
|
|
5828
|
+
qcy: qaecyPrefixes["qcy"],
|
|
5829
|
+
"qcy-e": qaecyPrefixes["qcy-e"],
|
|
5830
|
+
"qcy-l": qaecyPrefixes["qcy-l"],
|
|
5831
|
+
xsd: prefixCC["xsd"]
|
|
5832
|
+
};
|
|
5833
|
+
|
|
5730
5834
|
// libs/js/file-metadata-helpers/src/lib/js-file-metadata-helpers.ts
|
|
5731
5835
|
function uploadedFileMetadata(originalName, projectId, userId, md5, providerId, clientMetadata, mime, skipLoading = false, skipProcessing = false) {
|
|
5732
5836
|
const suffix = getFileSuffix(originalName);
|
|
@@ -5750,123 +5854,941 @@ function uploadedFileMetadata(originalName, projectId, userId, md5, providerId,
|
|
|
5750
5854
|
provider_id: providerId ?? "",
|
|
5751
5855
|
mime,
|
|
5752
5856
|
additional_metadata: clientMetadata ?? "",
|
|
5753
|
-
last_seen_as: JSON.stringify({
|
|
5857
|
+
last_seen_as: JSON.stringify({
|
|
5858
|
+
hostId: providerId ?? "",
|
|
5859
|
+
fileUUID
|
|
5860
|
+
})
|
|
5861
|
+
};
|
|
5862
|
+
}
|
|
5863
|
+
function turtleFileMetadata(sourceFile, processorId, locationUUID, stored = false) {
|
|
5864
|
+
const ids = extractIdsFromRawPath(sourceFile);
|
|
5865
|
+
const blob_name = locationUUID !== void 0 ? `${ids.projectId}/triples/${ids.documentUUID}_${locationUUID}_${processorId}.ttl` : `${ids.projectId}/triples/${ids.documentUUID}_${processorId}.ttl`;
|
|
5866
|
+
const identifier = locationUUID !== void 0 ? `${ids.documentUUID}_${locationUUID}` : ids.documentUUID;
|
|
5867
|
+
return {
|
|
5868
|
+
blob_name,
|
|
5869
|
+
processor: processorId,
|
|
5870
|
+
space_id: ids.projectId,
|
|
5871
|
+
stored: stored ? "True" : "False",
|
|
5872
|
+
suffix: ".ttl",
|
|
5873
|
+
identifier,
|
|
5874
|
+
source: sourceFile
|
|
5754
5875
|
};
|
|
5755
5876
|
}
|
|
5756
|
-
|
|
5757
|
-
|
|
5758
|
-
|
|
5759
|
-
|
|
5760
|
-
|
|
5761
|
-
|
|
5762
|
-
|
|
5763
|
-
|
|
5764
|
-
|
|
5765
|
-
|
|
5766
|
-
|
|
5877
|
+
|
|
5878
|
+
// libs/js/cue-sdk/src/lib/sync.ts
|
|
5879
|
+
var _scanFn = null;
|
|
5880
|
+
var _fileMapFn = null;
|
|
5881
|
+
var _wasmInitPromise = null;
|
|
5882
|
+
async function _initWasm() {
|
|
5883
|
+
const _require = (0, import_module.createRequire)((0, import_path3.resolve)(process.cwd(), "_anchor.js"));
|
|
5884
|
+
const pkgMain = _require.resolve("dir-scanner-wasm");
|
|
5885
|
+
const wasmBinary = await (0, import_promises3.readFile)((0, import_path3.join)((0, import_path3.dirname)(pkgMain), "dir_scanner_wasm_bg.wasm"));
|
|
5886
|
+
const mod = await import(
|
|
5887
|
+
/* @vite-ignore */
|
|
5888
|
+
pkgMain
|
|
5889
|
+
);
|
|
5890
|
+
await mod.default({ module_or_path: wasmBinary });
|
|
5891
|
+
_scanFn = mod.scan;
|
|
5892
|
+
_fileMapFn = mod.scan_file_map;
|
|
5893
|
+
}
|
|
5894
|
+
var DEFAULT_GRAPH_TYPE = "fuseki";
|
|
5895
|
+
var DEFAULT_SERVICE_ID = "cue-cli";
|
|
5896
|
+
var DEFAULT_RDF_BASE = "https://cue.qaecy.com/r/";
|
|
5897
|
+
var ENDPOINT_FUSEKI_QUERY = "/triplestore/query";
|
|
5898
|
+
var ENDPOINT_QLEVER_QUERY = "/sparql/query";
|
|
5899
|
+
var ENDPOINT_FUSEKI_UPDATE = "/triplestore/update";
|
|
5900
|
+
var ENDPOINT_QLEVER_UPDATE = "/sparql/update";
|
|
5901
|
+
var CueSyncApi = class {
|
|
5902
|
+
constructor(_auth, _projects, _blob, _gatewayUrl, _serviceId = DEFAULT_SERVICE_ID, _rdfBase = DEFAULT_RDF_BASE) {
|
|
5903
|
+
this._auth = _auth;
|
|
5904
|
+
this._projects = _projects;
|
|
5905
|
+
this._blob = _blob;
|
|
5906
|
+
this._gatewayUrl = _gatewayUrl;
|
|
5907
|
+
this._serviceId = _serviceId;
|
|
5908
|
+
this._rdfBase = _rdfBase;
|
|
5909
|
+
}
|
|
5910
|
+
_graphMap = /* @__PURE__ */ new Map();
|
|
5911
|
+
async sync(localFiles, options) {
|
|
5912
|
+
const { spaceId, providerId, userId, verbose } = options;
|
|
5913
|
+
const token = await this._auth.getToken();
|
|
5914
|
+
if (!token)
|
|
5915
|
+
throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
5916
|
+
const graph = await this._getOrCreateGraph(spaceId, token);
|
|
5917
|
+
if (verbose)
|
|
5918
|
+
console.info("Listing remote files \u23F3");
|
|
5919
|
+
const remoteFiles = await this._listRemoteFiles(graph, spaceId, providerId, verbose);
|
|
5920
|
+
const report = await compareLocalRemote(localFiles, remoteFiles);
|
|
5921
|
+
if (verbose) {
|
|
5922
|
+
console.info(`Total local files: ${localFiles.length}`);
|
|
5923
|
+
console.info(`Total remote files: ${remoteFiles.length}`);
|
|
5924
|
+
console.info(
|
|
5925
|
+
`Total files to sync: ${(report.localNotOnRemote?.length ?? 0) + report.localNotOnRemotePathOnly.length}`
|
|
5926
|
+
);
|
|
5927
|
+
}
|
|
5928
|
+
let syncCount = report.syncCount;
|
|
5929
|
+
let syncSize = report.syncSize;
|
|
5930
|
+
let failedUploads = 0;
|
|
5931
|
+
let rdfWritten = false;
|
|
5932
|
+
let unitsConsumed = 0;
|
|
5933
|
+
let fileUnits = {};
|
|
5934
|
+
try {
|
|
5935
|
+
if (!_wasmInitPromise)
|
|
5936
|
+
_wasmInitPromise = _initWasm();
|
|
5937
|
+
await _wasmInitPromise;
|
|
5938
|
+
const entries = await Promise.all(
|
|
5939
|
+
localFiles.map(async (f) => ({
|
|
5940
|
+
originalPath: f.relativePath,
|
|
5941
|
+
data: new Uint8Array(await (0, import_promises3.readFile)(f.fullPath))
|
|
5942
|
+
}))
|
|
5943
|
+
);
|
|
5944
|
+
fileUnits = _fileMapFn(entries);
|
|
5945
|
+
} catch (err) {
|
|
5946
|
+
console.warn("[CueSyncApi] Unit map failed, proceeding without unit tracking:", err);
|
|
5947
|
+
}
|
|
5948
|
+
const toUpload = report.localNotOnRemote ?? [];
|
|
5949
|
+
if (verbose && toUpload.length)
|
|
5950
|
+
console.info("Syncing missing files \u23F3");
|
|
5951
|
+
for (const file of toUpload) {
|
|
5952
|
+
let uploadSucceeded = false;
|
|
5953
|
+
try {
|
|
5954
|
+
const rawMeta = uploadedFileMetadata(
|
|
5955
|
+
file.relativePath,
|
|
5956
|
+
spaceId,
|
|
5957
|
+
userId,
|
|
5958
|
+
file.md5,
|
|
5959
|
+
providerId
|
|
5960
|
+
);
|
|
5961
|
+
if (!rawMeta.blob_name)
|
|
5962
|
+
throw new Error(`blob_name missing for ${file.relativePath}`);
|
|
5963
|
+
const fileBuffer = await (0, import_promises3.readFile)(file.fullPath);
|
|
5964
|
+
await this._blob.uploadRaw(
|
|
5965
|
+
rawMeta.blob_name,
|
|
5966
|
+
new Uint8Array(fileBuffer),
|
|
5967
|
+
rawMeta
|
|
5968
|
+
);
|
|
5969
|
+
const uploaded = await this._uploadRdfMetadata(file, rawMeta, verbose);
|
|
5970
|
+
if (uploaded)
|
|
5971
|
+
rdfWritten = true;
|
|
5972
|
+
syncCount += 1;
|
|
5973
|
+
syncSize += file.size || 0;
|
|
5974
|
+
uploadSucceeded = true;
|
|
5975
|
+
this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, verbose);
|
|
5976
|
+
} catch (err) {
|
|
5977
|
+
failedUploads += 1;
|
|
5978
|
+
console.error(`[CueSyncApi] Failed to upload file: ${file.fullPath}`);
|
|
5979
|
+
if (verbose)
|
|
5980
|
+
console.error("[CueSyncApi] Upload error details:", err);
|
|
5981
|
+
}
|
|
5982
|
+
if (uploadSucceeded) {
|
|
5983
|
+
const fileUnitsCount = fileUnits[file.relativePath] ?? 0;
|
|
5984
|
+
if (fileUnitsCount > 0) {
|
|
5985
|
+
unitsConsumed += fileUnitsCount;
|
|
5986
|
+
try {
|
|
5987
|
+
await this._projects.incrementUnitsConsumed(spaceId, fileUnitsCount, userId);
|
|
5988
|
+
} catch (err) {
|
|
5989
|
+
console.error(`[CueSyncApi] Failed to write clientSync entry for ${file.relativePath}:`, err);
|
|
5990
|
+
}
|
|
5991
|
+
}
|
|
5992
|
+
}
|
|
5993
|
+
}
|
|
5994
|
+
if (verbose && report.localNotOnRemotePathOnly.length)
|
|
5995
|
+
console.info(`Syncing missing file locations (on provider "${providerId}") \u23F3`);
|
|
5996
|
+
for (const file of report.localNotOnRemotePathOnly) {
|
|
5997
|
+
const rawMeta = uploadedFileMetadata(
|
|
5998
|
+
file.relativePath,
|
|
5999
|
+
spaceId,
|
|
6000
|
+
userId,
|
|
6001
|
+
file.md5,
|
|
6002
|
+
providerId
|
|
6003
|
+
);
|
|
6004
|
+
if (!rawMeta.blob_name)
|
|
6005
|
+
throw new Error(`blob_name missing for ${file.relativePath}`);
|
|
6006
|
+
const uploaded = await this._uploadRdfMetadata(file, rawMeta, verbose);
|
|
6007
|
+
if (uploaded)
|
|
6008
|
+
rdfWritten = true;
|
|
6009
|
+
syncCount += 1;
|
|
6010
|
+
syncSize += file.size || 0;
|
|
6011
|
+
const fileUnitsCount = fileUnits[file.relativePath] ?? 0;
|
|
6012
|
+
if (fileUnitsCount > 0) {
|
|
6013
|
+
unitsConsumed += fileUnitsCount;
|
|
6014
|
+
try {
|
|
6015
|
+
await this._projects.incrementUnitsConsumed(spaceId, fileUnitsCount, userId);
|
|
6016
|
+
} catch (err) {
|
|
6017
|
+
console.error(`[CueSyncApi] Failed to write clientSync entry for ${file.relativePath}:`, err);
|
|
6018
|
+
}
|
|
6019
|
+
}
|
|
6020
|
+
this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, verbose);
|
|
6021
|
+
}
|
|
6022
|
+
return {
|
|
6023
|
+
syncCount,
|
|
6024
|
+
syncSize,
|
|
6025
|
+
failedUploads,
|
|
6026
|
+
totalCount: report.totalCount,
|
|
6027
|
+
totalSize: report.totalSize,
|
|
6028
|
+
rdfWritten,
|
|
6029
|
+
unitsConsumed
|
|
6030
|
+
};
|
|
6031
|
+
}
|
|
6032
|
+
async _getOrCreateGraph(spaceId, token) {
|
|
6033
|
+
const cached = this._graphMap.get(spaceId);
|
|
6034
|
+
if (cached)
|
|
6035
|
+
return cached;
|
|
6036
|
+
const project = await this._projects.getProject(spaceId);
|
|
6037
|
+
const graphType = project?.projectSettings?.graph?.type ?? DEFAULT_GRAPH_TYPE;
|
|
6038
|
+
const queryEndpoint = graphType === "qlever" ? `${this._gatewayUrl}${ENDPOINT_QLEVER_QUERY}` : `${this._gatewayUrl}${ENDPOINT_FUSEKI_QUERY}`;
|
|
6039
|
+
const updateEndpoint = graphType === "qlever" ? `${this._gatewayUrl}${ENDPOINT_QLEVER_UPDATE}` : `${this._gatewayUrl}${ENDPOINT_FUSEKI_UPDATE}`;
|
|
6040
|
+
const graph = new CueGraphDatabase({
|
|
6041
|
+
graphType,
|
|
6042
|
+
queryEndpoint,
|
|
6043
|
+
updateEndpoint,
|
|
6044
|
+
originalHeaders: {
|
|
6045
|
+
"x-project-id": spaceId,
|
|
6046
|
+
authorization: `Bearer ${token}`
|
|
6047
|
+
}
|
|
6048
|
+
});
|
|
6049
|
+
this._graphMap.set(spaceId, graph);
|
|
6050
|
+
return graph;
|
|
6051
|
+
}
|
|
6052
|
+
async _listRemoteFiles(graph, spaceId, providerId, verbose) {
|
|
6053
|
+
if (verbose)
|
|
6054
|
+
console.info(`Listing files in raw space: ${spaceId}`);
|
|
6055
|
+
const [blobNames, locationMap] = await Promise.all([
|
|
6056
|
+
this._blob.listRaw(spaceId),
|
|
6057
|
+
this._getGraphFiles(graph, providerId)
|
|
6058
|
+
]);
|
|
6059
|
+
if (verbose)
|
|
6060
|
+
console.info(`Found ${blobNames.length} files in raw store for space: ${spaceId}`);
|
|
6061
|
+
const files = [];
|
|
6062
|
+
for (const name of blobNames) {
|
|
6063
|
+
const contentUUID = name.substring(0, 36);
|
|
6064
|
+
const locations = locationMap[contentUUID] ?? [];
|
|
6065
|
+
if (locations.length === 0) {
|
|
6066
|
+
if (verbose)
|
|
6067
|
+
console.warn(`No location data found for contentUUID: ${contentUUID}`);
|
|
6068
|
+
files.push({ contentUUID });
|
|
6069
|
+
} else {
|
|
6070
|
+
for (const loc of locations) {
|
|
6071
|
+
files.push({
|
|
6072
|
+
contentUUID,
|
|
6073
|
+
locationUUID: loc.locationUUID,
|
|
6074
|
+
created: loc.created,
|
|
6075
|
+
size: loc.size
|
|
6076
|
+
});
|
|
6077
|
+
}
|
|
6078
|
+
}
|
|
6079
|
+
}
|
|
6080
|
+
return files;
|
|
6081
|
+
}
|
|
6082
|
+
async _getGraphFiles(graph, providerId) {
|
|
6083
|
+
const q = `PREFIX qcy: <${qaecyPrefixes["qcy"]}>
|
|
6084
|
+
SELECT ?fc ?loc ?created ?fp ?size
|
|
6085
|
+
WHERE {
|
|
6086
|
+
?fc a qcy:FileContent ;
|
|
6087
|
+
qcy:sizeBytes ?size ;
|
|
6088
|
+
qcy:hasFileLocation ?loc .
|
|
6089
|
+
?loc qcy:remoteProviderId "${providerId}" ;
|
|
6090
|
+
qcy:dateCreated ?created ;
|
|
6091
|
+
qcy:filePath ?fp .
|
|
6092
|
+
}`;
|
|
6093
|
+
const res = await graph.query(q, "application/sparql-results+json");
|
|
6094
|
+
const map = {};
|
|
6095
|
+
if (typeof res === "string")
|
|
6096
|
+
return map;
|
|
6097
|
+
for (const b of res.results.bindings) {
|
|
6098
|
+
const locationUUID = b["loc"].value.split("/").at(-1) ?? "";
|
|
6099
|
+
const contentUUID = b["fc"].value.split("/").at(-1) ?? "";
|
|
6100
|
+
const created = b["created"].value;
|
|
6101
|
+
const size = b["size"].value;
|
|
6102
|
+
if (!map[contentUUID])
|
|
6103
|
+
map[contentUUID] = [];
|
|
6104
|
+
map[contentUUID].push({ locationUUID, created, size });
|
|
6105
|
+
}
|
|
6106
|
+
return map;
|
|
6107
|
+
}
|
|
6108
|
+
async _uploadRdfMetadata(file, rawMeta, verbose) {
|
|
6109
|
+
if (!rawMeta.blob_name)
|
|
6110
|
+
throw new Error(`blob_name missing for ${file.relativePath}`);
|
|
6111
|
+
const ttlMeta = turtleFileMetadata(
|
|
6112
|
+
rawMeta.blob_name,
|
|
6113
|
+
this._serviceId,
|
|
6114
|
+
file.locationUUID
|
|
6115
|
+
);
|
|
6116
|
+
if (!ttlMeta.blob_name)
|
|
6117
|
+
throw new Error(`ttl blob_name missing for ${rawMeta.blob_name}`);
|
|
6118
|
+
const writer = fileLocationRaw(rawMeta, file.size);
|
|
6119
|
+
const namespace = `${this._rdfBase}${ttlMeta.space_id}/`;
|
|
6120
|
+
const triples = await serializeRDF(namespace, writer);
|
|
6121
|
+
const uploaded = await this._blob.uploadProcessed(
|
|
6122
|
+
ttlMeta.blob_name,
|
|
6123
|
+
new Uint8Array(Buffer.from(triples, "utf-8")),
|
|
6124
|
+
ttlMeta
|
|
6125
|
+
);
|
|
6126
|
+
if (!uploaded && verbose) {
|
|
6127
|
+
console.info(
|
|
6128
|
+
`Graph data for ${file.relativePath} already exists \u2014 skipping \u26A0\uFE0F`
|
|
6129
|
+
);
|
|
6130
|
+
}
|
|
6131
|
+
return uploaded;
|
|
6132
|
+
}
|
|
6133
|
+
/**
|
|
6134
|
+
* Scans `localFiles` and returns a per-extension cost breakdown.
|
|
6135
|
+
*
|
|
6136
|
+
* Each {@link ScanOutputRecord} contains `units` — the billable metric for
|
|
6137
|
+
* that extension (e.g. pages for PDFs, rows for spreadsheets) — which can be
|
|
6138
|
+
* shown to the user before or after calling {@link sync}.
|
|
6139
|
+
*/
|
|
6140
|
+
async scanCost(localFiles) {
|
|
6141
|
+
if (!_wasmInitPromise)
|
|
6142
|
+
_wasmInitPromise = _initWasm();
|
|
6143
|
+
await _wasmInitPromise;
|
|
6144
|
+
const entries = await Promise.all(
|
|
6145
|
+
localFiles.map(async (f) => ({
|
|
6146
|
+
originalPath: f.relativePath,
|
|
6147
|
+
data: new Uint8Array(await (0, import_promises3.readFile)(f.fullPath))
|
|
6148
|
+
}))
|
|
6149
|
+
);
|
|
6150
|
+
return _scanFn(entries);
|
|
6151
|
+
}
|
|
6152
|
+
_logProgress(syncCount, totalCount, syncSize, totalSize, verbose) {
|
|
6153
|
+
if (!verbose || totalCount === 0)
|
|
6154
|
+
return;
|
|
6155
|
+
if (syncCount % Math.ceil(totalCount / 100) !== 0)
|
|
6156
|
+
return;
|
|
6157
|
+
const pct = Math.floor(syncCount / totalCount * 100);
|
|
6158
|
+
console.info(
|
|
6159
|
+
`Progress: ${pct}% (${syncCount}/${totalCount} files, ${syncSize}/${totalSize} bytes)`
|
|
6160
|
+
);
|
|
6161
|
+
}
|
|
6162
|
+
};
|
|
6163
|
+
|
|
6164
|
+
// libs/js/cue-sdk/src/lib/cue-node.ts
|
|
6165
|
+
var BUCKET_RAW2 = "spaces_raw_eu_west6";
|
|
6166
|
+
var BUCKET_PROCESSED2 = "spaces_processed_eu_west6";
|
|
6167
|
+
var CueNode = class extends Cue {
|
|
6168
|
+
constructor(config) {
|
|
6169
|
+
super(config);
|
|
6170
|
+
}
|
|
6171
|
+
_buildApi(projects) {
|
|
6172
|
+
const storageRaw = (0, import_storage4.getStorage)(this._app, BUCKET_RAW2);
|
|
6173
|
+
const storageProcessed = (0, import_storage4.getStorage)(this._app, BUCKET_PROCESSED2);
|
|
6174
|
+
if (this._isEmulator) {
|
|
6175
|
+
const storageHost = this._endpoints.storageEmulatorHost;
|
|
6176
|
+
const storagePort = this._endpoints.storageEmulatorPort;
|
|
6177
|
+
(0, import_storage4.connectStorageEmulator)(storageRaw, storageHost, storagePort);
|
|
6178
|
+
(0, import_storage4.connectStorageEmulator)(storageProcessed, storageHost, storagePort);
|
|
6179
|
+
}
|
|
6180
|
+
const blob = new CueBlobStorage({ storageRaw, storageProcessed });
|
|
6181
|
+
const syncApi = new CueSyncApi(this.auth, projects, blob, this._endpoints.gatewayUrl);
|
|
6182
|
+
return new CueApi(this.auth, this._endpoints.gatewayUrl, projects, syncApi);
|
|
6183
|
+
}
|
|
6184
|
+
};
|
|
6185
|
+
|
|
6186
|
+
// apps/desktop/cue-cli/src/helpers/auth.ts
|
|
6187
|
+
async function authenticate(emulators, space, key, verbose = false) {
|
|
6188
|
+
if (!key) {
|
|
6189
|
+
key = process.env.CUE_API_KEY;
|
|
6190
|
+
if (!key) {
|
|
6191
|
+
console.error(
|
|
6192
|
+
"API key is required. Provide it via --key option or CUE_API_KEY environment variable."
|
|
6193
|
+
);
|
|
6194
|
+
process.exit(1);
|
|
6195
|
+
}
|
|
6196
|
+
}
|
|
6197
|
+
const cue = new CueNode({
|
|
6198
|
+
apiKey: FIREBASE_CONFIG().apiKey,
|
|
6199
|
+
appId: FIREBASE_CONFIG().appId,
|
|
6200
|
+
measurementId: FIREBASE_CONFIG().measurementId,
|
|
6201
|
+
environment: emulators ? "emulator" : "production"
|
|
6202
|
+
});
|
|
6203
|
+
if (verbose)
|
|
6204
|
+
console.info("Authenticating \u23F3");
|
|
6205
|
+
const user = await cue.auth.signInWithApiKey(key, space);
|
|
6206
|
+
if (verbose)
|
|
6207
|
+
console.info("Authenticated \u2705");
|
|
6208
|
+
const idTokenResult = await user.getIdTokenResult();
|
|
6209
|
+
const role = idTokenResult.claims["role"];
|
|
6210
|
+
const isSuperAdmin = role === "superadmin";
|
|
6211
|
+
return { isSuperAdmin, userId: user.uid };
|
|
6212
|
+
}
|
|
6213
|
+
|
|
6214
|
+
// apps/desktop/cue-cli/src/cue-cli-compare.ts
|
|
6215
|
+
async function compareHandler(options) {
|
|
6216
|
+
const { space, path, verbose, provider, emulators, zip } = options;
|
|
6217
|
+
try {
|
|
6218
|
+
await authenticate(emulators, space, options.key, verbose);
|
|
6219
|
+
if (verbose)
|
|
6220
|
+
console.info("Building compare base \u23F3");
|
|
6221
|
+
const qh = async (query3) => queryHandler(query3, space, emulators);
|
|
6222
|
+
const [localFiles, remoteFiles] = await Promise.all([
|
|
6223
|
+
listLocalFiles(
|
|
6224
|
+
path,
|
|
6225
|
+
provider,
|
|
6226
|
+
verbose,
|
|
6227
|
+
5,
|
|
6228
|
+
IGNORED_LOCAL,
|
|
6229
|
+
HASH_WORKER_PATH,
|
|
6230
|
+
zip
|
|
6231
|
+
),
|
|
6232
|
+
listRemoteFiles(space, provider, qh, verbose)
|
|
6233
|
+
]);
|
|
6234
|
+
const unzipPromise = zip ? deleteUnzipped(path) : Promise.resolve();
|
|
6235
|
+
if (zip) {
|
|
6236
|
+
if (verbose)
|
|
6237
|
+
console.info("Started deletion of temp unzipped dirs \u23F3");
|
|
6238
|
+
}
|
|
6239
|
+
const report = await compareLocalRemote(localFiles, remoteFiles);
|
|
6240
|
+
await unzipPromise;
|
|
6241
|
+
if (zip && verbose)
|
|
6242
|
+
console.info("Cleaned up unzipped files \u2705");
|
|
6243
|
+
if (verbose)
|
|
6244
|
+
console.info("Built compare base \u2705");
|
|
6245
|
+
console.log("");
|
|
6246
|
+
console.log("--- Compare Report ---");
|
|
6247
|
+
console.log("");
|
|
6248
|
+
console.log(`Total files: ${report.totalCount}`);
|
|
6249
|
+
console.log(`Total size: ${fileSizePretty(report.totalSize || 0)}`);
|
|
6250
|
+
console.log("");
|
|
6251
|
+
console.log(
|
|
6252
|
+
`Files synchronized: ${report.syncCount} (${(report.synctPctCount * 100).toFixed(2)}%)`
|
|
6253
|
+
);
|
|
6254
|
+
console.log(
|
|
6255
|
+
`Synchronized size: ${fileSizePretty(report.syncSize || 0)} (${(report.synctPctSize * 100).toFixed(2)}%)`
|
|
6256
|
+
);
|
|
6257
|
+
console.log("");
|
|
6258
|
+
if (report.localNotOnRemote) {
|
|
6259
|
+
console.log(
|
|
6260
|
+
`${report.localNotOnRemote.length} files do not exist on remote`
|
|
6261
|
+
);
|
|
6262
|
+
if (verbose && report.localNotOnRemote.length > 0) {
|
|
6263
|
+
for (const f of report.localNotOnRemote) {
|
|
6264
|
+
console.log(
|
|
6265
|
+
" - " + f.relativePath + " (" + fileSizePretty(f.size || 0) + ")"
|
|
6266
|
+
);
|
|
6267
|
+
}
|
|
6268
|
+
}
|
|
6269
|
+
console.log("");
|
|
6270
|
+
}
|
|
6271
|
+
if (report.localNotOnRemotePathOnly) {
|
|
6272
|
+
console.log(
|
|
6273
|
+
`${report.localNotOnRemotePathOnly.length} file paths do not exist on remote on providerId "${provider}" (file duplicates)`
|
|
6274
|
+
);
|
|
6275
|
+
if (verbose && report.localNotOnRemotePathOnly.length > 0) {
|
|
6276
|
+
for (const f of report.localNotOnRemotePathOnly) {
|
|
6277
|
+
console.log(
|
|
6278
|
+
" - " + f.relativePath + " (" + fileSizePretty(f.size || 0) + ")"
|
|
6279
|
+
);
|
|
6280
|
+
}
|
|
6281
|
+
}
|
|
6282
|
+
console.log("");
|
|
6283
|
+
}
|
|
6284
|
+
if (report.remoteNotOnLocal) {
|
|
6285
|
+
console.log(`${report.remoteNotOnLocal.length} files do not exist locally`);
|
|
6286
|
+
console.log(
|
|
6287
|
+
"This might expected if the files belong to another provider or have been deleted locally"
|
|
6288
|
+
);
|
|
6289
|
+
if (verbose && report.remoteNotOnLocal.length > 0) {
|
|
6290
|
+
for (const f of report.remoteNotOnLocal) {
|
|
6291
|
+
console.log(
|
|
6292
|
+
" - " + f.contentUUID + " (" + fileSizePretty(f.size || 0) + ")"
|
|
6293
|
+
);
|
|
6294
|
+
}
|
|
6295
|
+
}
|
|
6296
|
+
console.log("");
|
|
6297
|
+
}
|
|
6298
|
+
if (report.remoteNotOnLocalPathOnly) {
|
|
6299
|
+
console.log(
|
|
6300
|
+
`${report.remoteNotOnLocalPathOnly.length} file paths on providerId "${provider}" do not exist locally`
|
|
6301
|
+
);
|
|
6302
|
+
console.log(
|
|
6303
|
+
"This might expected if the files belong to another provider or have been deleted locally"
|
|
6304
|
+
);
|
|
6305
|
+
if (verbose && report.remoteNotOnLocalPathOnly.length > 0) {
|
|
6306
|
+
for (const f of report.remoteNotOnLocalPathOnly) {
|
|
6307
|
+
console.log(
|
|
6308
|
+
" - " + f.contentUUID + " (" + fileSizePretty(f.size || 0) + ")"
|
|
6309
|
+
);
|
|
6310
|
+
}
|
|
6311
|
+
}
|
|
6312
|
+
console.log("");
|
|
6313
|
+
}
|
|
6314
|
+
} catch (err) {
|
|
6315
|
+
console.error("Error:", err);
|
|
6316
|
+
process.exit(1);
|
|
6317
|
+
}
|
|
6318
|
+
}
|
|
6319
|
+
|
|
6320
|
+
// apps/desktop/cue-cli/src/helpers/get-files-containing-substring.ts
|
|
6321
|
+
var import_storage5 = require("firebase/storage");
|
|
6322
|
+
var import_path4 = require("path");
|
|
6323
|
+
var import_promises4 = require("fs/promises");
|
|
6324
|
+
async function getFilesContainingSubstring(bucket, subDir = "", subString = "") {
|
|
6325
|
+
const firebase = CueFirebase.getInstance();
|
|
6326
|
+
const storage = bucket === "raw" ? firebase.storageRaw : firebase.storageProcessed;
|
|
6327
|
+
console.log("Fetching files from storage bucket:", bucket, "in subdir:", subDir, "containing substring:", subString);
|
|
6328
|
+
const listResult = await (0, import_storage5.listAll)((0, import_storage5.ref)(storage, subDir));
|
|
6329
|
+
const outputDir = (0, import_path4.join)(process.cwd(), "downloaded_blobs", bucket, subDir);
|
|
6330
|
+
await (0, import_promises4.mkdir)(outputDir, { recursive: true });
|
|
6331
|
+
for (const fileRef of listResult.items) {
|
|
6332
|
+
console.log(fileRef.name);
|
|
6333
|
+
if (subDir && !fileRef.name.includes(subString))
|
|
6334
|
+
continue;
|
|
6335
|
+
const bytes = await (0, import_storage5.getBytes)(fileRef);
|
|
6336
|
+
const outputPath = (0, import_path4.join)(outputDir, fileRef.name);
|
|
6337
|
+
await (0, import_promises4.writeFile)(outputPath, Buffer.from(bytes));
|
|
6338
|
+
console.log(`Downloaded ${fileRef.name} to ${outputPath} \u2705`);
|
|
6339
|
+
}
|
|
6340
|
+
}
|
|
6341
|
+
|
|
6342
|
+
// apps/desktop/cue-cli/src/cue-cli-dump-processed.ts
|
|
6343
|
+
async function dumpProcessedHandler(options) {
|
|
6344
|
+
const { space, verbose, emulators, processor } = options;
|
|
6345
|
+
try {
|
|
6346
|
+
const { isSuperAdmin } = await authenticate(emulators, space, options.key, verbose);
|
|
6347
|
+
if (!isSuperAdmin) {
|
|
6348
|
+
console.error("Sorry - this tool is only for super admin users");
|
|
6349
|
+
process.exit(1);
|
|
6350
|
+
}
|
|
6351
|
+
if (verbose)
|
|
6352
|
+
console.info(`Dumping processed files for processor ${processor} \u23F3`);
|
|
6353
|
+
const files = await getFilesContainingSubstring("processed", `${space}/triples`, processor);
|
|
6354
|
+
console.log(files);
|
|
6355
|
+
} catch (err) {
|
|
6356
|
+
console.error("Error:", err);
|
|
6357
|
+
process.exit(1);
|
|
6358
|
+
}
|
|
6359
|
+
}
|
|
6360
|
+
|
|
6361
|
+
// apps/desktop/cue-cli/src/helpers/graph-dump.ts
|
|
6362
|
+
var import_fs2 = require("fs");
|
|
6363
|
+
var import_stream = require("stream");
|
|
6364
|
+
var import_fs3 = require("fs");
|
|
6365
|
+
var import_zlib = require("zlib");
|
|
6366
|
+
var import_promises5 = require("stream/promises");
|
|
6367
|
+
var import_promises6 = require("fs/promises");
|
|
6368
|
+
var import_auth9 = require("firebase/auth");
|
|
6369
|
+
|
|
6370
|
+
// libs/js/size-tools/src/lib/js-size-tools.ts
|
|
6371
|
+
function humanFileSize(bytes, si = false, dp = 1) {
|
|
6372
|
+
const thresh = si ? 1e3 : 1024;
|
|
6373
|
+
if (Math.abs(bytes) < thresh) {
|
|
6374
|
+
return bytes + " B";
|
|
6375
|
+
}
|
|
6376
|
+
const units = si ? ["kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"] : ["KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"];
|
|
6377
|
+
let u = -1;
|
|
6378
|
+
const r = 10 ** dp;
|
|
6379
|
+
do {
|
|
6380
|
+
bytes /= thresh;
|
|
6381
|
+
++u;
|
|
6382
|
+
} while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1);
|
|
6383
|
+
return bytes.toFixed(dp) + " " + units[u];
|
|
6384
|
+
}
|
|
6385
|
+
|
|
6386
|
+
// apps/desktop/cue-cli/src/helpers/graph-dump.ts
|
|
6387
|
+
var import_fs4 = require("fs");
|
|
6388
|
+
async function dumpRdfGraphToFileJelly(spaceId, useEmulator = false, verbose = false) {
|
|
6389
|
+
return dumpRdfGraphToFile(
|
|
6390
|
+
spaceId,
|
|
6391
|
+
useEmulator,
|
|
6392
|
+
verbose,
|
|
6393
|
+
`${spaceId}.jelly`,
|
|
6394
|
+
"application/x-jelly-rdf"
|
|
6395
|
+
);
|
|
6396
|
+
}
|
|
6397
|
+
async function dumpRdfGraphToFile(spaceId, useEmulator = false, verbose = false, outFile, mimeType = "application/n-quads") {
|
|
6398
|
+
const endpoint = useEmulator ? SPARQL_ENDPOINT_EMULATOR : SPARQL_ENDPOINT;
|
|
6399
|
+
const dataUrl = endpoint.replace("/query", "/data");
|
|
6400
|
+
if (verbose)
|
|
6401
|
+
console.info(`Streaming RDF graph \u23F3`);
|
|
6402
|
+
const filePath = outFile || `${spaceId}.nq`;
|
|
6403
|
+
if ((0, import_fs4.existsSync)(filePath) || (0, import_fs4.existsSync)(`${filePath}.gz`)) {
|
|
6404
|
+
if (verbose)
|
|
6405
|
+
console.info(`File ${filePath} already exists, skipping download.`);
|
|
6406
|
+
return mimeType === "application/n-quads" ? `${filePath}.gz` : filePath;
|
|
6407
|
+
}
|
|
6408
|
+
const token = await (0, import_auth9.getAuth)().currentUser?.getIdToken();
|
|
6409
|
+
const res = await fetch(dataUrl, {
|
|
6410
|
+
method: "GET",
|
|
6411
|
+
headers: {
|
|
6412
|
+
"x-project-id": spaceId,
|
|
6413
|
+
authorization: `Bearer ${token}`,
|
|
6414
|
+
Accept: mimeType
|
|
6415
|
+
}
|
|
6416
|
+
});
|
|
6417
|
+
if (!res.ok || !res.body) {
|
|
6418
|
+
throw new Error(`Failed to stream data: ${res.status} ${res.statusText}`);
|
|
6419
|
+
}
|
|
6420
|
+
const fileStream = (0, import_fs2.createWriteStream)(filePath, { flags: "w" });
|
|
6421
|
+
const nodeStream = import_stream.Readable.fromWeb(res.body);
|
|
6422
|
+
let chunkCount = 0;
|
|
6423
|
+
let totalSize = 0;
|
|
6424
|
+
let hasDataTimeout = null;
|
|
6425
|
+
let lastChunkTime = Date.now();
|
|
6426
|
+
nodeStream.on("data", (chunk) => {
|
|
6427
|
+
lastChunkTime = Date.now();
|
|
6428
|
+
chunkCount++;
|
|
6429
|
+
totalSize += chunk.length;
|
|
6430
|
+
if (hasDataTimeout) {
|
|
6431
|
+
clearTimeout(hasDataTimeout);
|
|
6432
|
+
}
|
|
6433
|
+
if (verbose && totalSize >= 10 * 1024 * 1024 && totalSize % (10 * 1024 * 1024) < chunk.length)
|
|
6434
|
+
console.info(
|
|
6435
|
+
`Writing chunk #${chunkCount}, size: ${humanFileSize(totalSize)}`
|
|
6436
|
+
);
|
|
6437
|
+
});
|
|
6438
|
+
const streamTimeout = 3e5;
|
|
6439
|
+
hasDataTimeout = setTimeout(() => {
|
|
6440
|
+
const timeSinceLastChunk = Date.now() - lastChunkTime;
|
|
6441
|
+
if (timeSinceLastChunk > streamTimeout) {
|
|
6442
|
+
console.error(`Stream appears stalled - no data received for ${timeSinceLastChunk}ms`);
|
|
6443
|
+
nodeStream.destroy(new Error("Stream timeout"));
|
|
6444
|
+
}
|
|
6445
|
+
}, streamTimeout);
|
|
6446
|
+
try {
|
|
6447
|
+
await (0, import_promises5.pipeline)(nodeStream, fileStream);
|
|
6448
|
+
} finally {
|
|
6449
|
+
if (hasDataTimeout) {
|
|
6450
|
+
clearTimeout(hasDataTimeout);
|
|
6451
|
+
}
|
|
6452
|
+
}
|
|
6453
|
+
if (verbose)
|
|
6454
|
+
console.info(`Total chunks written: ${chunkCount}, total size: ${humanFileSize(totalSize)}`);
|
|
6455
|
+
if (totalSize === 0) {
|
|
6456
|
+
throw new Error("No data received from server - possible authentication or permission issue");
|
|
6457
|
+
}
|
|
6458
|
+
if (verbose)
|
|
6459
|
+
console.info(`Streamed RDF graph to ${filePath} \u2705`);
|
|
6460
|
+
if (mimeType === "application/n-quads") {
|
|
6461
|
+
if (verbose)
|
|
6462
|
+
console.info(`
|
|
6463
|
+
Gzipping to ${filePath}.gz \u23F3`);
|
|
6464
|
+
await _doGzip(filePath);
|
|
6465
|
+
if (verbose)
|
|
6466
|
+
console.info(`Gzipped to ${filePath}.gz \u2705`);
|
|
6467
|
+
return `${filePath}.gz`;
|
|
6468
|
+
}
|
|
6469
|
+
return filePath;
|
|
6470
|
+
}
|
|
6471
|
+
async function dumpRdfGraphToFileConstruct(spaceId, useEmulator = false, verbose = false, graphIRI, batchSize = 5e4, concurrency = 8, maxRetries = 5, retryDelayMs = 1e3, delayBetweenBatchesMs = 500) {
|
|
6472
|
+
const endpoint = useEmulator ? SPARQL_ENDPOINT_EMULATOR : SPARQL_ENDPOINT;
|
|
6473
|
+
const filePath = `${spaceId}.nq`;
|
|
6474
|
+
const graph = graphIRI !== void 0 ? `<${graphIRI}>` : "?g";
|
|
6475
|
+
const constructQuery = `
|
|
6476
|
+
CONSTRUCT { GRAPH ${graph} { ?s ?p ?o } }
|
|
6477
|
+
WHERE { GRAPH ${graph} { ?s ?p ?o } }
|
|
6478
|
+
`;
|
|
6479
|
+
let offset = 0;
|
|
6480
|
+
const fileStream = (0, import_fs2.createWriteStream)(filePath, { flags: "w" });
|
|
6481
|
+
let hasMore = true;
|
|
6482
|
+
while (hasMore) {
|
|
6483
|
+
const queries = Array.from(
|
|
6484
|
+
{ length: concurrency },
|
|
6485
|
+
(_, i) => `${constructQuery} LIMIT ${batchSize} OFFSET ${offset + i * batchSize}`
|
|
6486
|
+
);
|
|
6487
|
+
offset += batchSize * concurrency;
|
|
6488
|
+
const results = await Promise.all(
|
|
6489
|
+
queries.map(
|
|
6490
|
+
(q, idx) => retryWithBackoff(
|
|
6491
|
+
() => _doQuery(q, spaceId, endpoint),
|
|
6492
|
+
maxRetries,
|
|
6493
|
+
retryDelayMs,
|
|
6494
|
+
verbose ? `Batch ${offset - batchSize * concurrency + idx * batchSize}` : void 0
|
|
6495
|
+
)
|
|
6496
|
+
)
|
|
6497
|
+
);
|
|
6498
|
+
const batches = results.filter(Boolean);
|
|
6499
|
+
if (batches.length === 0) {
|
|
6500
|
+
hasMore = false;
|
|
6501
|
+
break;
|
|
6502
|
+
}
|
|
6503
|
+
for (const batch of batches) {
|
|
6504
|
+
fileStream.write(batch.trim() + "\n");
|
|
6505
|
+
}
|
|
6506
|
+
if (verbose)
|
|
6507
|
+
process.stdout.write(`Current offset: ${offset.toLocaleString()}\r`);
|
|
6508
|
+
if (delayBetweenBatchesMs > 0)
|
|
6509
|
+
await new Promise((res) => setTimeout(res, delayBetweenBatchesMs));
|
|
6510
|
+
}
|
|
6511
|
+
fileStream.end();
|
|
6512
|
+
if (verbose)
|
|
6513
|
+
console.info(`
|
|
6514
|
+
Gzipping to ${filePath}.gz \u23F3`);
|
|
6515
|
+
await _doGzip(filePath);
|
|
6516
|
+
if (verbose)
|
|
6517
|
+
console.info(`Gzipped to ${filePath}.gz \u2705`);
|
|
6518
|
+
return `${filePath}.gz`;
|
|
6519
|
+
}
|
|
6520
|
+
async function retryWithBackoff(fn, maxRetries, delayMs, label) {
|
|
6521
|
+
let attempt = 0;
|
|
6522
|
+
let lastErr = null;
|
|
6523
|
+
while (attempt < maxRetries) {
|
|
6524
|
+
try {
|
|
6525
|
+
const result = await fn();
|
|
6526
|
+
if (result !== null && result !== void 0)
|
|
6527
|
+
return result;
|
|
6528
|
+
} catch (err) {
|
|
6529
|
+
lastErr = err;
|
|
6530
|
+
}
|
|
6531
|
+
attempt++;
|
|
6532
|
+
if (label)
|
|
6533
|
+
console.warn(`${label}: Retry ${attempt}/${maxRetries}`);
|
|
6534
|
+
await new Promise(
|
|
6535
|
+
(res) => setTimeout(res, delayMs * Math.pow(2, attempt - 1))
|
|
6536
|
+
);
|
|
6537
|
+
}
|
|
6538
|
+
if (label)
|
|
6539
|
+
console.error(`${label}: Failed after ${maxRetries} retries`, lastErr);
|
|
6540
|
+
return null;
|
|
6541
|
+
}
|
|
6542
|
+
async function _doGzip(filePath) {
|
|
6543
|
+
const gzFilePath = `${filePath}.gz`;
|
|
6544
|
+
await new Promise((resolve2, reject) => {
|
|
6545
|
+
const gzip = (0, import_zlib.createGzip)();
|
|
6546
|
+
const source = (0, import_fs3.createReadStream)(filePath);
|
|
6547
|
+
const dest = (0, import_fs2.createWriteStream)(gzFilePath);
|
|
6548
|
+
(0, import_promises5.pipeline)(source, gzip, dest).then(resolve2).catch(reject);
|
|
6549
|
+
});
|
|
6550
|
+
await (0, import_promises6.unlink)(filePath);
|
|
6551
|
+
}
|
|
6552
|
+
async function _doQuery(query3, spaceId, url) {
|
|
6553
|
+
const token = await (0, import_auth9.getAuth)().currentUser?.getIdToken();
|
|
6554
|
+
try {
|
|
6555
|
+
const res = await fetch(url, {
|
|
6556
|
+
method: "POST",
|
|
6557
|
+
headers: {
|
|
6558
|
+
"x-project-id": spaceId,
|
|
6559
|
+
authorization: `Bearer ${token}`,
|
|
6560
|
+
"Content-Type": "application/sparql-query",
|
|
6561
|
+
Accept: "application/n-quads"
|
|
6562
|
+
},
|
|
6563
|
+
body: query3
|
|
6564
|
+
});
|
|
6565
|
+
if (!res.ok) {
|
|
6566
|
+
console.error(`Error: ${res.status} ${res.statusText}`);
|
|
6567
|
+
return null;
|
|
6568
|
+
}
|
|
6569
|
+
return await res.text();
|
|
6570
|
+
} catch (e) {
|
|
6571
|
+
console.error(e);
|
|
6572
|
+
return null;
|
|
6573
|
+
}
|
|
6574
|
+
}
|
|
6575
|
+
|
|
6576
|
+
// apps/desktop/cue-cli/src/helpers/graph-create.ts
|
|
6577
|
+
var import_auth10 = require("firebase/auth");
|
|
6578
|
+
async function createGraph(id) {
|
|
6579
|
+
const token = await (0, import_auth10.getAuth)().currentUser?.getIdToken();
|
|
6580
|
+
const dataSetUrl = SPARQL_ENDPOINT_EMULATOR.replace("/query", `/$/datasets`);
|
|
6581
|
+
const res = await fetch(dataSetUrl, {
|
|
6582
|
+
method: "POST",
|
|
6583
|
+
headers: {
|
|
6584
|
+
"x-project-id": id,
|
|
6585
|
+
authorization: `Bearer ${token}`,
|
|
6586
|
+
"content-type": "text/turtle"
|
|
6587
|
+
},
|
|
6588
|
+
body: _getTemplate(id)
|
|
6589
|
+
});
|
|
6590
|
+
if (!res.ok) {
|
|
6591
|
+
throw new Error(`Failed to create graph: ${res.statusText}`);
|
|
6592
|
+
}
|
|
6593
|
+
}
|
|
6594
|
+
function _getTemplate(partition, base = "fuseki-base-new/fuseki") {
|
|
6595
|
+
return `@prefix : <http://base/#> .
|
|
6596
|
+
@prefix fuseki: <http://jena.apache.org/fuseki#> .
|
|
6597
|
+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
|
6598
|
+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
|
6599
|
+
@prefix tdb2: <http://jena.apache.org/2016/tdb#> .
|
|
6600
|
+
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
|
|
6601
|
+
@prefix text: <http://jena.apache.org/text#> .
|
|
6602
|
+
@prefix qcy: <${qaecyPrefixes["qcy"]}> .
|
|
6603
|
+
@prefix qcye: <${qaecyPrefixes["qcy-e"]}> .
|
|
6604
|
+
|
|
6605
|
+
|
|
6606
|
+
:tdb_dataset_readwrite a tdb2:DatasetTDB2;
|
|
6607
|
+
tdb2:location "${base}/databases/${partition}/tdb";
|
|
6608
|
+
tdb2:unionDefaultGraph true ;
|
|
6609
|
+
tdb2:transactionMode "transactional" ;
|
|
6610
|
+
tdb2:transactionLogType "journal" ;
|
|
6611
|
+
tdb2:blockFileSize "32M" ;
|
|
6612
|
+
tdb2:blockCacheSize "1G" ;
|
|
6613
|
+
tdb2:fileMode "direct" .
|
|
6614
|
+
|
|
6615
|
+
:service_tdb_all a fuseki:Service;
|
|
6616
|
+
fuseki:name "${partition}" ;
|
|
6617
|
+
rdfs:label "TDB2 ${partition}";
|
|
6618
|
+
fuseki:dataset :tdb_dataset_readwrite;
|
|
6619
|
+
fuseki:allowedUsers "admin";
|
|
6620
|
+
fuseki:endpoint [ fuseki:name "update";
|
|
6621
|
+
fuseki:operation fuseki:update
|
|
6622
|
+
];
|
|
6623
|
+
fuseki:endpoint [ fuseki:name "query";
|
|
6624
|
+
fuseki:operation fuseki:query
|
|
6625
|
+
];
|
|
6626
|
+
fuseki:endpoint [ fuseki:name "get";
|
|
6627
|
+
fuseki:operation fuseki:gsp-r
|
|
6628
|
+
];
|
|
6629
|
+
fuseki:endpoint [ fuseki:name "shacl";
|
|
6630
|
+
fuseki:operation fuseki:shacl
|
|
6631
|
+
];
|
|
6632
|
+
fuseki:endpoint [ fuseki:name "data";
|
|
6633
|
+
fuseki:operation fuseki:gsp-rw
|
|
6634
|
+
];
|
|
6635
|
+
fuseki:endpoint [ fuseki:name "sparql";
|
|
6636
|
+
fuseki:operation fuseki:query
|
|
6637
|
+
].`;
|
|
6638
|
+
}
|
|
6639
|
+
|
|
6640
|
+
// apps/desktop/cue-cli/src/helpers/graph-upload.ts
|
|
6641
|
+
var import_auth11 = require("firebase/auth");
|
|
6642
|
+
var import_fs5 = require("fs");
|
|
6643
|
+
async function uploadToLocalGraph(id, filePath, mimeType = "application/n-quads", zipped = false) {
|
|
6644
|
+
const stream = (0, import_fs5.createReadStream)(filePath);
|
|
6645
|
+
const token = await (0, import_auth11.getAuth)().currentUser?.getIdToken();
|
|
6646
|
+
const dataUrl = SPARQL_ENDPOINT_EMULATOR.replace("/query", "/data");
|
|
6647
|
+
const headers = {
|
|
6648
|
+
"x-project-id": id,
|
|
6649
|
+
authorization: `Bearer ${token}`,
|
|
6650
|
+
"Content-Type": mimeType
|
|
5767
6651
|
};
|
|
6652
|
+
if (zipped) {
|
|
6653
|
+
headers["Content-Encoding"] = "gzip";
|
|
6654
|
+
}
|
|
6655
|
+
const res = await fetch(dataUrl, {
|
|
6656
|
+
method: "POST",
|
|
6657
|
+
headers,
|
|
6658
|
+
body: stream,
|
|
6659
|
+
duplex: "half"
|
|
6660
|
+
});
|
|
6661
|
+
const text = await res.text();
|
|
6662
|
+
console.log(text);
|
|
6663
|
+
if (!res.ok) {
|
|
6664
|
+
console.error(`Failed to upload graph: ${res.statusText}`);
|
|
6665
|
+
return;
|
|
6666
|
+
}
|
|
5768
6667
|
}
|
|
5769
6668
|
|
|
5770
|
-
// apps/desktop/cue-cli/src/
|
|
5771
|
-
|
|
5772
|
-
|
|
5773
|
-
|
|
5774
|
-
|
|
5775
|
-
|
|
5776
|
-
|
|
5777
|
-
|
|
5778
|
-
|
|
5779
|
-
|
|
5780
|
-
|
|
5781
|
-
|
|
5782
|
-
|
|
5783
|
-
|
|
5784
|
-
|
|
5785
|
-
|
|
5786
|
-
|
|
5787
|
-
|
|
5788
|
-
|
|
5789
|
-
|
|
5790
|
-
|
|
5791
|
-
|
|
5792
|
-
|
|
5793
|
-
|
|
5794
|
-
|
|
5795
|
-
|
|
5796
|
-
|
|
5797
|
-
|
|
5798
|
-
|
|
5799
|
-
|
|
5800
|
-
|
|
5801
|
-
md5: file.md5,
|
|
5802
|
-
blobName,
|
|
5803
|
-
blobNameLength: blobName?.length,
|
|
5804
|
-
blobNameIsUnusual: blobName && (blobName.length > 256 || /[^\w\-./]/.test(blobName)),
|
|
5805
|
-
errorCode: error?.code,
|
|
5806
|
-
errorMessage: error?.message,
|
|
5807
|
-
errorPayload: error,
|
|
5808
|
-
fileBufferSize: fileBuffer?.length,
|
|
5809
|
-
rawFileMetadataKeys: Object.keys(rawFileMetadata),
|
|
5810
|
-
rawFileMetadataLength: Object.keys(rawFileMetadata).length,
|
|
5811
|
-
attempt
|
|
5812
|
-
});
|
|
5813
|
-
reject(error);
|
|
5814
|
-
},
|
|
5815
|
-
() => resolve2()
|
|
5816
|
-
);
|
|
5817
|
-
if (!uploadTask) {
|
|
5818
|
-
console.error("[uploadFile] Upload task could not be created:", {
|
|
5819
|
-
filePath: file.fullPath,
|
|
5820
|
-
relativePath: file.relativePath,
|
|
5821
|
-
md5: file.md5,
|
|
5822
|
-
blobName: rawFileMetadata.blob_name,
|
|
5823
|
-
attempt
|
|
5824
|
-
});
|
|
5825
|
-
reject(new Error("Upload task could not be created"));
|
|
5826
|
-
}
|
|
5827
|
-
});
|
|
5828
|
-
lastError = null;
|
|
5829
|
-
break;
|
|
5830
|
-
} catch (err) {
|
|
5831
|
-
lastError = err;
|
|
5832
|
-
attempt++;
|
|
5833
|
-
if (attempt < maxRetries) {
|
|
5834
|
-
console.warn(`[uploadFile] Retry attempt ${attempt} for file: ${file.fullPath}`);
|
|
5835
|
-
await new Promise((res) => setTimeout(res, 1e3 * attempt));
|
|
6669
|
+
// apps/desktop/cue-cli/src/cue-cli-dump.ts
|
|
6670
|
+
async function dumpHandler(options) {
|
|
6671
|
+
const { space, verbose, emulators, jelly, query: query3, load } = options;
|
|
6672
|
+
try {
|
|
6673
|
+
const { isSuperAdmin } = await authenticate(emulators, space, options.key, verbose);
|
|
6674
|
+
if (!isSuperAdmin) {
|
|
6675
|
+
console.error("Sorry - this tool is only for super admin users");
|
|
6676
|
+
process.exit(1);
|
|
6677
|
+
}
|
|
6678
|
+
if (verbose)
|
|
6679
|
+
console.info("Dumping graph \u23F3");
|
|
6680
|
+
let filePath = "";
|
|
6681
|
+
if (jelly) {
|
|
6682
|
+
if (verbose)
|
|
6683
|
+
console.info("Setting: Jelly format");
|
|
6684
|
+
if (verbose)
|
|
6685
|
+
console.time("Downloaded graph \u2705");
|
|
6686
|
+
filePath = await dumpRdfGraphToFileJelly(space, emulators, verbose);
|
|
6687
|
+
if (verbose)
|
|
6688
|
+
console.timeEnd("Downloaded graph \u2705");
|
|
6689
|
+
} else {
|
|
6690
|
+
if (verbose)
|
|
6691
|
+
console.time("Downloaded and zipped graph \u2705");
|
|
6692
|
+
if (query3) {
|
|
6693
|
+
if (verbose)
|
|
6694
|
+
console.info("Setting: Construct query");
|
|
6695
|
+
filePath = await dumpRdfGraphToFileConstruct(space, emulators, verbose);
|
|
6696
|
+
} else {
|
|
6697
|
+
if (verbose)
|
|
6698
|
+
console.info("Setting: /data endpoint");
|
|
6699
|
+
filePath = await dumpRdfGraphToFile(space, emulators, verbose);
|
|
5836
6700
|
}
|
|
6701
|
+
if (verbose)
|
|
6702
|
+
console.timeEnd("Downloaded and zipped graph \u2705");
|
|
5837
6703
|
}
|
|
6704
|
+
console.info(`File written: ${filePath}`);
|
|
6705
|
+
if (load && !emulators) {
|
|
6706
|
+
if (verbose)
|
|
6707
|
+
console.info(`Creating local RDF graph with id: ${space} \u23F3`);
|
|
6708
|
+
await createGraph(space);
|
|
6709
|
+
if (verbose)
|
|
6710
|
+
console.info(`Created local RDF graph \u2705`);
|
|
6711
|
+
if (verbose)
|
|
6712
|
+
console.info(`Uploading file to local RDF graph \u23F3`);
|
|
6713
|
+
await uploadToLocalGraph(
|
|
6714
|
+
space,
|
|
6715
|
+
filePath,
|
|
6716
|
+
jelly ? "application/jelly+json" : "application/n-quads",
|
|
6717
|
+
jelly ? false : true
|
|
6718
|
+
);
|
|
6719
|
+
if (verbose)
|
|
6720
|
+
console.info(`Uploaded file to local RDF graph \u2705`);
|
|
6721
|
+
}
|
|
6722
|
+
} catch (err) {
|
|
6723
|
+
console.error("Error:", err);
|
|
6724
|
+
process.exit(1);
|
|
5838
6725
|
}
|
|
5839
|
-
if (lastError) {
|
|
5840
|
-
throw lastError;
|
|
5841
|
-
}
|
|
5842
|
-
return rawFileMetadata;
|
|
5843
6726
|
}
|
|
5844
6727
|
|
|
5845
|
-
// apps/desktop/cue-cli/src/helpers/
|
|
6728
|
+
// apps/desktop/cue-cli/src/helpers/repair-remote-ttl.ts
|
|
5846
6729
|
var import_storage6 = require("firebase/storage");
|
|
5847
|
-
async function
|
|
6730
|
+
async function repairRemoteTTL(space, subString, regex, substituteString) {
|
|
5848
6731
|
const firebase = CueFirebase.getInstance();
|
|
5849
|
-
const ttlMetadata = turtleFileMetadata(
|
|
5850
|
-
rawFileMetadata.blob_name,
|
|
5851
|
-
SERVICE_ID,
|
|
5852
|
-
file.locationUUID
|
|
5853
|
-
);
|
|
5854
|
-
const writer = fileLocationRaw(rawFileMetadata, file.size);
|
|
5855
|
-
const namespace = `${RDF_BASE}${ttlMetadata.space_id}/`;
|
|
5856
|
-
const triples = await serializeRDF(namespace, writer);
|
|
5857
6732
|
const storage = firebase.storageProcessed;
|
|
5858
|
-
|
|
5859
|
-
const
|
|
5860
|
-
|
|
6733
|
+
console.log("Fetching files from storage bucket 'triples' containing substring:", subString);
|
|
6734
|
+
const listResult = await (0, import_storage6.listAll)((0, import_storage6.ref)(storage, `${space}/triples`));
|
|
6735
|
+
for (const fileRef of listResult.items) {
|
|
6736
|
+
if (subString && !fileRef.name.match(subString))
|
|
6737
|
+
continue;
|
|
6738
|
+
const stream = await (0, import_storage6.getStream)(fileRef);
|
|
6739
|
+
const reader = stream.getReader();
|
|
6740
|
+
const chunks = [];
|
|
6741
|
+
let done = false;
|
|
6742
|
+
while (!done) {
|
|
6743
|
+
const { value, done: streamDone } = await reader.read();
|
|
6744
|
+
if (value) {
|
|
6745
|
+
chunks.push(value);
|
|
6746
|
+
}
|
|
6747
|
+
done = streamDone;
|
|
6748
|
+
}
|
|
6749
|
+
let fileContent = Buffer.concat(chunks).toString("utf8");
|
|
6750
|
+
let modified = false;
|
|
6751
|
+
if (substituteString && regex) {
|
|
6752
|
+
const newContent = fileContent.replace(regex, substituteString);
|
|
6753
|
+
if (newContent !== fileContent) {
|
|
6754
|
+
fileContent = newContent;
|
|
6755
|
+
modified = true;
|
|
6756
|
+
}
|
|
6757
|
+
}
|
|
6758
|
+
if (modified) {
|
|
6759
|
+
const buffer = Buffer.from(fileContent, "utf8");
|
|
6760
|
+
let existingMetadata = {};
|
|
6761
|
+
try {
|
|
6762
|
+
existingMetadata = await (0, import_storage6.getMetadata)(fileRef);
|
|
6763
|
+
} catch (err) {
|
|
6764
|
+
console.warn(`Could not fetch metadata for ${fileRef.name}, proceeding with default.`);
|
|
6765
|
+
}
|
|
6766
|
+
const customMetadata = { ...existingMetadata.customMetadata || {}, stored: "False" };
|
|
6767
|
+
const metadata = { customMetadata };
|
|
6768
|
+
await (0, import_storage6.uploadBytesResumable)((0, import_storage6.ref)(storage, `${space}/triples/${fileRef.name}`), buffer, metadata);
|
|
6769
|
+
console.log(`Fixed ${fileRef.name} \u2705`);
|
|
6770
|
+
} else {
|
|
6771
|
+
console.log(`No changes for ${fileRef.name}`);
|
|
6772
|
+
}
|
|
6773
|
+
}
|
|
6774
|
+
}
|
|
6775
|
+
|
|
6776
|
+
// apps/desktop/cue-cli/src/cue-cli-repair-ttl.ts
|
|
6777
|
+
async function repairTtlHandler(options) {
|
|
6778
|
+
const { space, verbose, emulators, processor, from, to } = options;
|
|
6779
|
+
try {
|
|
6780
|
+
const { isSuperAdmin } = await authenticate(emulators, space, options.key, verbose);
|
|
6781
|
+
if (!isSuperAdmin) {
|
|
6782
|
+
console.error("Sorry - this tool is only for super admin users");
|
|
6783
|
+
process.exit(1);
|
|
6784
|
+
}
|
|
5861
6785
|
if (verbose)
|
|
5862
|
-
console.info(
|
|
5863
|
-
|
|
5864
|
-
|
|
5865
|
-
|
|
6786
|
+
console.info(`Repairing TTL files for processor ${processor} \u23F3`);
|
|
6787
|
+
await repairRemoteTTL(space, processor, new RegExp(from, "g"), to);
|
|
6788
|
+
} catch (err) {
|
|
6789
|
+
console.error("Error:", err);
|
|
6790
|
+
process.exit(1);
|
|
5866
6791
|
}
|
|
5867
|
-
await (0, import_storage6.uploadBytes)(fileRef, new Uint8Array(Buffer.from(triples, "utf-8")), {
|
|
5868
|
-
customMetadata: ttlMetadata
|
|
5869
|
-
});
|
|
5870
6792
|
}
|
|
5871
6793
|
|
|
5872
6794
|
// apps/desktop/cue-cli/src/helpers/emit-idle.ts
|
|
@@ -5912,15 +6834,36 @@ async function publishMessage(topicName, data, useEmulator) {
|
|
|
5912
6834
|
// apps/desktop/cue-cli/src/cue-cli-sync.ts
|
|
5913
6835
|
var import_promises7 = require("fs/promises");
|
|
5914
6836
|
var import_fs6 = require("fs");
|
|
5915
|
-
var
|
|
6837
|
+
var import_path5 = require("path");
|
|
6838
|
+
var readline = __toESM(require("readline"));
|
|
6839
|
+
function askConfirm(question) {
|
|
6840
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
6841
|
+
return new Promise((resolve2) => {
|
|
6842
|
+
rl.question(question, (answer) => {
|
|
6843
|
+
rl.close();
|
|
6844
|
+
resolve2(answer.trim().toLowerCase() === "y");
|
|
6845
|
+
});
|
|
6846
|
+
});
|
|
6847
|
+
}
|
|
5916
6848
|
async function syncHandler(options) {
|
|
5917
6849
|
const { space, path, verbose, provider, emulators, zip } = options;
|
|
5918
6850
|
try {
|
|
5919
|
-
const
|
|
6851
|
+
const cue = new CueNode({
|
|
6852
|
+
apiKey: FIREBASE_CONFIG().apiKey,
|
|
6853
|
+
appId: FIREBASE_CONFIG().appId,
|
|
6854
|
+
measurementId: FIREBASE_CONFIG().measurementId,
|
|
6855
|
+
environment: emulators ? "emulator" : "production"
|
|
6856
|
+
});
|
|
6857
|
+
const key = options.key ?? process.env.CUE_API_KEY;
|
|
6858
|
+
if (!key) {
|
|
6859
|
+
console.error(
|
|
6860
|
+
"API key is required. Provide it via --key option or CUE_API_KEY environment variable."
|
|
6861
|
+
);
|
|
6862
|
+
process.exit(1);
|
|
6863
|
+
}
|
|
5920
6864
|
if (verbose)
|
|
5921
6865
|
console.info("Building sync base \u23F3");
|
|
5922
|
-
const
|
|
5923
|
-
const resolvedPath = (0, import_path4.resolve)(path);
|
|
6866
|
+
const resolvedPath = (0, import_path5.resolve)(path);
|
|
5924
6867
|
const pathStat = await (0, import_promises7.stat)(resolvedPath);
|
|
5925
6868
|
const isFile = pathStat.isFile();
|
|
5926
6869
|
let localFiles;
|
|
@@ -5928,18 +6871,20 @@ async function syncHandler(options) {
|
|
|
5928
6871
|
if (verbose)
|
|
5929
6872
|
console.info(`Path is a file, syncing single file: ${resolvedPath}`);
|
|
5930
6873
|
const md5 = await fromReadStream((0, import_fs6.createReadStream)(resolvedPath));
|
|
5931
|
-
const relativePath = (0,
|
|
6874
|
+
const relativePath = (0, import_path5.basename)(resolvedPath);
|
|
5932
6875
|
const contentUUID = contextBasedGuid(md5);
|
|
5933
6876
|
const locationUUID = generateFileUUID(relativePath, provider);
|
|
5934
|
-
localFiles = [
|
|
5935
|
-
|
|
5936
|
-
|
|
5937
|
-
|
|
5938
|
-
|
|
5939
|
-
|
|
5940
|
-
|
|
5941
|
-
|
|
5942
|
-
|
|
6877
|
+
localFiles = [
|
|
6878
|
+
{
|
|
6879
|
+
relativePath,
|
|
6880
|
+
fullPath: resolvedPath,
|
|
6881
|
+
md5,
|
|
6882
|
+
contentUUID,
|
|
6883
|
+
locationUUID,
|
|
6884
|
+
mtimeMs: pathStat.mtimeMs,
|
|
6885
|
+
size: pathStat.size
|
|
6886
|
+
}
|
|
6887
|
+
];
|
|
5943
6888
|
} else {
|
|
5944
6889
|
localFiles = await listLocalFiles(
|
|
5945
6890
|
path,
|
|
@@ -5951,89 +6896,33 @@ async function syncHandler(options) {
|
|
|
5951
6896
|
zip
|
|
5952
6897
|
);
|
|
5953
6898
|
}
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
console.info(`
|
|
5961
|
-
console.info(
|
|
5962
|
-
`Total files to sync: ${report.localNotOnRemote.length + report.localNotOnRemotePathOnly.length}`
|
|
5963
|
-
);
|
|
5964
|
-
console.info("");
|
|
5965
|
-
}
|
|
5966
|
-
let syncCount = report.syncCount;
|
|
5967
|
-
let syncSize = report.syncSize;
|
|
5968
|
-
if (report.synctPctCount !== 0 && verbose) {
|
|
5969
|
-
console.info(
|
|
5970
|
-
`Synced percentage: ${Math.round(
|
|
5971
|
-
report.synctPctCount * 100
|
|
5972
|
-
)} % ( ${syncCount}/${report.totalCount} files )`
|
|
5973
|
-
);
|
|
5974
|
-
console.info(
|
|
5975
|
-
`Synchronized size:: ${Math.round(
|
|
5976
|
-
report.synctPctSize * 100
|
|
5977
|
-
)}% ( ${fileSizePretty(syncSize)}/${fileSizePretty(report.totalSize)} )`
|
|
5978
|
-
);
|
|
5979
|
-
console.info("");
|
|
5980
|
-
}
|
|
5981
|
-
if (verbose && report.localNotOnRemote.length)
|
|
5982
|
-
console.info("Syncing missing files \u23F3");
|
|
5983
|
-
let rdfWritten = false;
|
|
5984
|
-
let failedUploads = 0;
|
|
5985
|
-
for (const file of report.localNotOnRemote) {
|
|
5986
|
-
let rawFileMetadata;
|
|
5987
|
-
try {
|
|
5988
|
-
rawFileMetadata = await uploadFile(file, space, userId, provider);
|
|
5989
|
-
await uploadFileRDF(file, rawFileMetadata, verbose);
|
|
5990
|
-
syncCount += 1;
|
|
5991
|
-
syncSize += file.size || 0;
|
|
5992
|
-
const pct = Math.floor(syncCount / report.totalCount * 100);
|
|
5993
|
-
if (verbose && report.totalCount > 0 && syncCount % Math.ceil(report.totalCount / 100) === 0) {
|
|
5994
|
-
console.info(
|
|
5995
|
-
`Progress: ${pct}% (${syncCount}/$${report.totalCount} files, ${fileSizePretty(syncSize)}/${fileSizePretty(
|
|
5996
|
-
report.totalSize
|
|
5997
|
-
)})`
|
|
5998
|
-
);
|
|
5999
|
-
}
|
|
6000
|
-
rdfWritten = true;
|
|
6001
|
-
} catch (err) {
|
|
6002
|
-
failedUploads += 1;
|
|
6003
|
-
console.error(`[syncHandler] Failed to upload file: ${file.fullPath}`);
|
|
6004
|
-
if (verbose) {
|
|
6005
|
-
console.error("[syncHandler] Upload error details:", err);
|
|
6006
|
-
}
|
|
6007
|
-
continue;
|
|
6008
|
-
}
|
|
6899
|
+
if (verbose)
|
|
6900
|
+
console.info("Built sync base \u2705\n");
|
|
6901
|
+
const costRecords = await cue.api.sync.scanCost(localFiles);
|
|
6902
|
+
const totalUnits = costRecords.reduce((sum, r) => sum + r.units, 0);
|
|
6903
|
+
console.info("Cost estimate:");
|
|
6904
|
+
for (const r of costRecords) {
|
|
6905
|
+
console.info(` .${r.ext}: ${r.count} file(s) \u2192 ${r.units} unit(s)`);
|
|
6009
6906
|
}
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
);
|
|
6015
|
-
|
|
6016
|
-
const rawFileMetadata = uploadedFileMetadata(
|
|
6017
|
-
file.relativePath,
|
|
6018
|
-
space,
|
|
6019
|
-
userId,
|
|
6020
|
-
file.md5,
|
|
6021
|
-
provider
|
|
6022
|
-
);
|
|
6023
|
-
await uploadFileRDF(file, rawFileMetadata, verbose);
|
|
6024
|
-
syncCount += 1;
|
|
6025
|
-
syncSize += file.size || 0;
|
|
6026
|
-
const pct = Math.floor(syncCount / report.totalCount * 100);
|
|
6027
|
-
if (verbose && report.totalCount > 0 && syncCount % Math.ceil(report.totalCount / 100) === 0) {
|
|
6028
|
-
console.info(
|
|
6029
|
-
`Progress: ${pct}% (${syncCount}/$${report.totalCount} files, ${fileSizePretty(syncSize)}/${fileSizePretty(
|
|
6030
|
-
report.totalSize
|
|
6031
|
-
)})`
|
|
6032
|
-
);
|
|
6033
|
-
}
|
|
6034
|
-
rdfWritten = true;
|
|
6907
|
+
console.info(` Total: ${totalUnits} unit(s) across ${localFiles.length} file(s)
|
|
6908
|
+
`);
|
|
6909
|
+
const confirmed = await askConfirm("Continue with sync? (y/n) ");
|
|
6910
|
+
if (!confirmed) {
|
|
6911
|
+
console.info("Sync cancelled.");
|
|
6912
|
+
process.exit(0);
|
|
6035
6913
|
}
|
|
6036
|
-
if (
|
|
6914
|
+
if (verbose)
|
|
6915
|
+
console.info("Authenticating \u23F3");
|
|
6916
|
+
const user = await cue.auth.signInWithApiKey(key, space);
|
|
6917
|
+
if (verbose)
|
|
6918
|
+
console.info("Authenticated \u2705\n");
|
|
6919
|
+
const result = await cue.api.sync.sync(localFiles, {
|
|
6920
|
+
spaceId: space,
|
|
6921
|
+
providerId: provider,
|
|
6922
|
+
userId: user.uid,
|
|
6923
|
+
verbose
|
|
6924
|
+
});
|
|
6925
|
+
if (result.rdfWritten && emulators) {
|
|
6037
6926
|
if (verbose) {
|
|
6038
6927
|
console.info("");
|
|
6039
6928
|
console.info(`Throwing RDF_WRITING_IDLE topic (only in emulators) \u23F3`);
|
|
@@ -6049,18 +6938,23 @@ async function syncHandler(options) {
|
|
|
6049
6938
|
console.warn("Continuing despite PubSub error in emulator mode...");
|
|
6050
6939
|
}
|
|
6051
6940
|
}
|
|
6941
|
+
const zipDeletePromise = zip && !isFile ? deleteUnzipped(path) : Promise.resolve();
|
|
6052
6942
|
await zipDeletePromise;
|
|
6053
6943
|
if (zip && verbose)
|
|
6054
6944
|
console.info("Cleaned up unzipped files \u2705");
|
|
6055
6945
|
if (verbose) {
|
|
6056
6946
|
console.info("");
|
|
6947
|
+
console.info(
|
|
6948
|
+
`Synced: ${result.syncCount}/${result.totalCount} files (${fileSizePretty(result.syncSize)}/${fileSizePretty(result.totalSize)})`
|
|
6949
|
+
);
|
|
6057
6950
|
console.info(`Sync finished \u{1F680}\u{1F680}\u{1F680}`);
|
|
6058
|
-
if (failedUploads > 0) {
|
|
6059
|
-
console.warn(`Total files failed to upload: ${failedUploads}`);
|
|
6951
|
+
if (result.failedUploads > 0) {
|
|
6952
|
+
console.warn(`Total files failed to upload: ${result.failedUploads}`);
|
|
6060
6953
|
}
|
|
6061
6954
|
}
|
|
6955
|
+
process.exit(0);
|
|
6062
6956
|
} catch (err) {
|
|
6063
|
-
console.error("
|
|
6957
|
+
console.error("[syncHandler] Unexpected error:", err);
|
|
6064
6958
|
process.exit(1);
|
|
6065
6959
|
}
|
|
6066
6960
|
}
|
|
@@ -6070,22 +6964,22 @@ var import_fs7 = require("fs");
|
|
|
6070
6964
|
var import_zlib2 = require("zlib");
|
|
6071
6965
|
|
|
6072
6966
|
// libs/js/rdf-tools/src/lib/nq-to-nt.ts
|
|
6073
|
-
var
|
|
6074
|
-
var { quad, defaultGraph } =
|
|
6967
|
+
var import_n36 = require("n3");
|
|
6968
|
+
var { quad, defaultGraph } = import_n36.DataFactory;
|
|
6075
6969
|
|
|
6076
6970
|
// libs/js/rdf-tools/src/lib/remove-rdf-star.ts
|
|
6077
|
-
var
|
|
6971
|
+
var import_n37 = require("n3");
|
|
6078
6972
|
var import_stream2 = require("stream");
|
|
6079
6973
|
function isRDFStarTerm(term) {
|
|
6080
6974
|
return term.termType === "Quad";
|
|
6081
6975
|
}
|
|
6082
6976
|
function removeRDFStar(inputStream, starCount) {
|
|
6083
|
-
const parser = new
|
|
6977
|
+
const parser = new import_n37.Parser({ format: "N-Quads*" });
|
|
6084
6978
|
const outputStream = new import_stream2.PassThrough();
|
|
6085
|
-
const writer = new
|
|
6979
|
+
const writer = new import_n37.StreamWriter({ format: "N-Quads" });
|
|
6086
6980
|
writer.pipe(outputStream);
|
|
6087
6981
|
let count = 0;
|
|
6088
|
-
parser.parse(inputStream, (error, quad2,
|
|
6982
|
+
parser.parse(inputStream, (error, quad2, prefixes4) => {
|
|
6089
6983
|
if (error) {
|
|
6090
6984
|
outputStream.emit("error", error);
|
|
6091
6985
|
writer.end();
|
|
@@ -6153,11 +7047,11 @@ async function utilRemoveRdfStarHandler(options) {
|
|
|
6153
7047
|
var import_fs8 = require("fs");
|
|
6154
7048
|
|
|
6155
7049
|
// libs/js/rdf-compare/src/lib/js-rdf-compare.ts
|
|
6156
|
-
var
|
|
7050
|
+
var import_n38 = require("n3");
|
|
6157
7051
|
var import_jsonld = require("jsonld");
|
|
6158
7052
|
function parseTurtle(content) {
|
|
6159
7053
|
return new Promise((resolve2, reject) => {
|
|
6160
|
-
const parser = new
|
|
7054
|
+
const parser = new import_n38.Parser({ format: "Turtle" });
|
|
6161
7055
|
const quads = [];
|
|
6162
7056
|
parser.parse(content, (error, quad2) => {
|
|
6163
7057
|
if (error)
|
|
@@ -6171,7 +7065,7 @@ function parseTurtle(content) {
|
|
|
6171
7065
|
}
|
|
6172
7066
|
function quadsToNQuads(quads) {
|
|
6173
7067
|
return new Promise((resolve2, reject) => {
|
|
6174
|
-
const writer = new
|
|
7068
|
+
const writer = new import_n38.Writer({ format: "N-Quads" });
|
|
6175
7069
|
writer.addQuads(quads);
|
|
6176
7070
|
writer.end((error, result) => {
|
|
6177
7071
|
if (error)
|
|
@@ -6248,10 +7142,10 @@ Triples only in file 2 (${result.triplesOnlyInFile2.size}):`);
|
|
|
6248
7142
|
// apps/desktop/cue-cli/src/main.ts
|
|
6249
7143
|
var packageJson;
|
|
6250
7144
|
try {
|
|
6251
|
-
packageJson = JSON.parse((0, import_fs9.readFileSync)((0,
|
|
7145
|
+
packageJson = JSON.parse((0, import_fs9.readFileSync)((0, import_path6.join)(__dirname, "package.json"), "utf8"));
|
|
6252
7146
|
} catch {
|
|
6253
7147
|
try {
|
|
6254
|
-
packageJson = JSON.parse((0, import_fs9.readFileSync)((0,
|
|
7148
|
+
packageJson = JSON.parse((0, import_fs9.readFileSync)((0, import_path6.join)(__dirname, "../package.json"), "utf8"));
|
|
6255
7149
|
} catch {
|
|
6256
7150
|
packageJson = { version: "0.0.0" };
|
|
6257
7151
|
console.warn("Could not find package.json, using fallback version");
|