@qaecy/cue-cli 0.0.33 → 0.0.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/main.js +205 -98
- package/package.json +1 -1
package/main.js
CHANGED
|
@@ -98,16 +98,28 @@ var import_path6 = require("path");
|
|
|
98
98
|
|
|
99
99
|
// apps/desktop/cue-cli/src/variables.ts
|
|
100
100
|
var import_path = require("path");
|
|
101
|
-
var EMULATOR_API_GATEWAY_PORT = process.env.CUE_EMULATOR_API_GATEWAY_PORT ?? "
|
|
101
|
+
var EMULATOR_API_GATEWAY_PORT = process.env.CUE_EMULATOR_API_GATEWAY_PORT ?? "18093";
|
|
102
102
|
var EMULATOR_API_GATEWAY_BASE = `http://localhost:${EMULATOR_API_GATEWAY_PORT}`;
|
|
103
103
|
var TOKEN_ENDPOINT_EMULATOR = `${EMULATOR_API_GATEWAY_BASE}/token`;
|
|
104
|
+
var EMULATOR_AUTH_PORT = process.env.CUE_EMULATOR_AUTH_PORT ?? "9099";
|
|
105
|
+
var EMULATOR_AUTH_HOST = `localhost:${EMULATOR_AUTH_PORT}`;
|
|
106
|
+
var EMULATOR_STORAGE_PORT = parseInt(process.env.CUE_EMULATOR_STORAGE_PORT ?? "9199", 10);
|
|
107
|
+
var EMULATOR_FIRESTORE_PORT = parseInt(process.env.CUE_EMULATOR_FIRESTORE_PORT ?? "8080", 10);
|
|
108
|
+
var getEmulatorEndpoints = () => ({
|
|
109
|
+
gatewayUrl: EMULATOR_API_GATEWAY_BASE,
|
|
110
|
+
tokenUrl: TOKEN_ENDPOINT_EMULATOR,
|
|
111
|
+
authEmulatorUrl: `http://${EMULATOR_AUTH_HOST}`,
|
|
112
|
+
storageEmulatorHost: "localhost",
|
|
113
|
+
storageEmulatorPort: EMULATOR_STORAGE_PORT,
|
|
114
|
+
firestoreEmulatorHost: "localhost",
|
|
115
|
+
firestoreEmulatorPort: EMULATOR_FIRESTORE_PORT
|
|
116
|
+
});
|
|
104
117
|
var EMULATOR_QLEVER_PORT = process.env.CUE_QLEVER_PORT ?? "8102";
|
|
105
118
|
var QLEVER_QUERY_ENDPOINT_EMULATOR = process.env.CUE_QLEVER_ENDPOINT ? `${process.env.CUE_QLEVER_ENDPOINT}/query` : `http://localhost:${EMULATOR_QLEVER_PORT}/query`;
|
|
106
119
|
var QLEVER_UPDATE_ENDPOINT_EMULATOR = process.env.CUE_QLEVER_ENDPOINT ? `${process.env.CUE_QLEVER_ENDPOINT}/update` : `http://localhost:${EMULATOR_QLEVER_PORT}/update`;
|
|
107
120
|
var SPARQL_ENDPOINT_EMULATOR = `${EMULATOR_API_GATEWAY_BASE}/triplestore/query`;
|
|
108
121
|
var SPARQL_ENDPOINT = "https://accessors-api-gateway-ueyeemwf2a-oa.a.run.app/triplestore/query";
|
|
109
122
|
var HASH_WORKER_PATH = (0, import_path.join)(__dirname, "hash-worker.js");
|
|
110
|
-
var PROJECT_ID = "qaecy-mvp-406413.appspot.com";
|
|
111
123
|
var FIREBASE_CONFIG = (useEmulator = false) => ({
|
|
112
124
|
apiKey: "AIzaSyCLhz5Wa3ZCERQZVurSt9bqupPeREALFLk",
|
|
113
125
|
appId: "1:151132927589:web:d11c64cc55bdc0f13ab88c",
|
|
@@ -4141,7 +4153,7 @@ var CueAuth = class {
|
|
|
4141
4153
|
// libs/js/cue-sdk/src/lib/api.ts
|
|
4142
4154
|
var ENDPOINT_SEARCH = "/assistant/search";
|
|
4143
4155
|
var ENDPOINT_SPARQL = "/triplestore/query";
|
|
4144
|
-
var ENDPOINT_CONSUMPTION = "/data-views/admin/consumption
|
|
4156
|
+
var ENDPOINT_CONSUMPTION = "/data-views/admin/consumption";
|
|
4145
4157
|
var CueApi = class {
|
|
4146
4158
|
constructor(_auth, _gatewayUrl, projects, sync) {
|
|
4147
4159
|
this._auth = _auth;
|
|
@@ -4201,10 +4213,14 @@ var CueApi = class {
|
|
|
4201
4213
|
}
|
|
4202
4214
|
return response.json();
|
|
4203
4215
|
}
|
|
4204
|
-
async getConsumption(projectId) {
|
|
4216
|
+
async getConsumption(projectId, tier) {
|
|
4205
4217
|
const headers = await this._authHeaders();
|
|
4206
4218
|
const response = await fetch(`${this._gatewayUrl}${ENDPOINT_CONSUMPTION}`, {
|
|
4207
|
-
headers: {
|
|
4219
|
+
headers: {
|
|
4220
|
+
...headers,
|
|
4221
|
+
"x-project-id": projectId,
|
|
4222
|
+
...tier ? { "x-tier": tier } : {}
|
|
4223
|
+
}
|
|
4208
4224
|
});
|
|
4209
4225
|
if (!response.ok) {
|
|
4210
4226
|
throw new Error(`Failed to fetch consumption: ${response.status} ${response.statusText}`);
|
|
@@ -4880,6 +4896,30 @@ var CueBlobStorage = class {
|
|
|
4880
4896
|
const result = await (0, import_storage3.listAll)(listRef);
|
|
4881
4897
|
return result.items.map((item) => item.name);
|
|
4882
4898
|
}
|
|
4899
|
+
/**
|
|
4900
|
+
* Download a file from the public bucket and return its contents as a string.
|
|
4901
|
+
*/
|
|
4902
|
+
async downloadPublic(blobName) {
|
|
4903
|
+
const fileRef = (0, import_storage3.ref)(this.options.storagePublic, blobName);
|
|
4904
|
+
const controller = new AbortController();
|
|
4905
|
+
const timeout = setTimeout(() => controller.abort(), 1e4);
|
|
4906
|
+
try {
|
|
4907
|
+
const [url, metadata] = await Promise.all([
|
|
4908
|
+
(0, import_storage3.getDownloadURL)(fileRef),
|
|
4909
|
+
(0, import_storage3.getMetadata)(fileRef)
|
|
4910
|
+
]);
|
|
4911
|
+
const cacheBustedUrl = `${url}&t=${encodeURIComponent(metadata.updated)}`;
|
|
4912
|
+
const res = await fetch(cacheBustedUrl, { signal: controller.signal });
|
|
4913
|
+
if (!res.ok)
|
|
4914
|
+
throw new Error(`HTTP ${res.status}`);
|
|
4915
|
+
return res.text();
|
|
4916
|
+
} catch (err) {
|
|
4917
|
+
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
4918
|
+
throw new Error(isTimeout ? `Download timed out: ${blobName}` : err instanceof Error ? err.message : String(err));
|
|
4919
|
+
} finally {
|
|
4920
|
+
clearTimeout(timeout);
|
|
4921
|
+
}
|
|
4922
|
+
}
|
|
4883
4923
|
};
|
|
4884
4924
|
|
|
4885
4925
|
// libs/js/cue-sdk/src/lib/sync.ts
|
|
@@ -5941,6 +5981,29 @@ var CueSyncApi = class {
|
|
|
5941
5981
|
_bindApi(api) {
|
|
5942
5982
|
this._api = api;
|
|
5943
5983
|
}
|
|
5984
|
+
/**
|
|
5985
|
+
* Flushes any pending metadata items from a previous interrupted sync.
|
|
5986
|
+
* Safe to call even when there is nothing new to upload.
|
|
5987
|
+
*/
|
|
5988
|
+
async flushPendingMetadata(spaceId, verbose) {
|
|
5989
|
+
const token = await this._auth.getToken();
|
|
5990
|
+
if (!token)
|
|
5991
|
+
throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
5992
|
+
const existing = await _loadPending(spaceId);
|
|
5993
|
+
if (!existing || existing.items.length === 0)
|
|
5994
|
+
return;
|
|
5995
|
+
console.info(`Trying to upload metadata (${existing.items.length} item(s))...`);
|
|
5996
|
+
if (verbose)
|
|
5997
|
+
console.info(`Flushing ${existing.items.length} pending file location(s) from previous sync \u23F3`);
|
|
5998
|
+
try {
|
|
5999
|
+
this._pendingSpaceId = spaceId;
|
|
6000
|
+
this._pendingItems = [];
|
|
6001
|
+
await this._flushBatch(existing.items, spaceId, token, verbose);
|
|
6002
|
+
console.info("Metadata uploaded \u2705");
|
|
6003
|
+
} catch (err) {
|
|
6004
|
+
throw new Error(`METADATA_SYNC_FAILED: ${err instanceof Error ? err.message : String(err)}`);
|
|
6005
|
+
}
|
|
6006
|
+
}
|
|
5944
6007
|
/**
|
|
5945
6008
|
* Returns a preview of what would be synced: cost breakdown for new files only,
|
|
5946
6009
|
* units required, and units still available. Use this before calling {@link sync}
|
|
@@ -5952,33 +6015,56 @@ var CueSyncApi = class {
|
|
|
5952
6015
|
if (!token)
|
|
5953
6016
|
throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
5954
6017
|
const graph = await this._getOrCreateGraph(spaceId, token);
|
|
5955
|
-
const
|
|
6018
|
+
const project = await this._projects.getProject(spaceId);
|
|
6019
|
+
const tier = project?.projectSettings?.tier ?? "l";
|
|
6020
|
+
const [remoteFiles, consumption, creditMap, tierNames] = await Promise.all([
|
|
5956
6021
|
this._listRemoteFiles(graph, spaceId, providerId, verbose),
|
|
5957
|
-
this._api?.getConsumption(spaceId) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
|
|
6022
|
+
this._api?.getConsumption(spaceId, tier) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance")),
|
|
6023
|
+
this._fetchUnitCreditMap(verbose),
|
|
6024
|
+
this._fetchTierNames()
|
|
5958
6025
|
]);
|
|
5959
6026
|
const report = await compareLocalRemote(localFiles, remoteFiles);
|
|
5960
6027
|
const toUpload = report.localNotOnRemote ?? [];
|
|
5961
6028
|
const costRecords = toUpload.length > 0 ? await this.scanCost(toUpload) : [];
|
|
5962
|
-
|
|
6029
|
+
let unitsToConsume = 0;
|
|
6030
|
+
let creditsToConsume = 0;
|
|
6031
|
+
for (const r of costRecords) {
|
|
6032
|
+
unitsToConsume += r.units;
|
|
6033
|
+
const tierMap = creditMap[tier];
|
|
6034
|
+
const creditPerUnit = tierMap?.[r.ext] ?? 1;
|
|
6035
|
+
if (verbose && tierMap && !(r.ext in tierMap)) {
|
|
6036
|
+
console.info(` Unknown format: .${r.ext} (using default rate of 1 credit/unit)`);
|
|
6037
|
+
}
|
|
6038
|
+
const credits = r.units * creditPerUnit;
|
|
6039
|
+
creditsToConsume += credits;
|
|
6040
|
+
r.credits = Math.round(credits);
|
|
6041
|
+
}
|
|
6042
|
+
const tierName = tierNames[tier] ?? tier;
|
|
5963
6043
|
return {
|
|
5964
6044
|
costRecords,
|
|
6045
|
+
tier,
|
|
6046
|
+
tierName,
|
|
5965
6047
|
unitsToConsume,
|
|
6048
|
+
creditsToConsume: Math.round(creditsToConsume),
|
|
6049
|
+
creditsAvailable: consumption.creditsAvailable,
|
|
5966
6050
|
unitsAvailable: consumption.unitsAvailable,
|
|
5967
6051
|
filesToUpload: toUpload.length,
|
|
5968
6052
|
totalLocalFiles: localFiles.length
|
|
5969
6053
|
};
|
|
5970
6054
|
}
|
|
5971
6055
|
async sync(localFiles, options) {
|
|
5972
|
-
const { spaceId, providerId, userId, verbose } = options;
|
|
6056
|
+
const { spaceId, providerId, userId, verbose, onProgress } = options;
|
|
5973
6057
|
const token = await this._auth.getToken();
|
|
5974
6058
|
if (!token)
|
|
5975
6059
|
throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
5976
6060
|
const graph = await this._getOrCreateGraph(spaceId, token);
|
|
5977
6061
|
if (verbose)
|
|
5978
6062
|
console.info("Listing remote files \u23F3");
|
|
6063
|
+
const project = await this._projects.getProject(spaceId);
|
|
6064
|
+
const tier = project?.projectSettings?.tier ?? "l";
|
|
5979
6065
|
const [remoteFiles, consumption] = await Promise.all([
|
|
5980
6066
|
this._listRemoteFiles(graph, spaceId, providerId, verbose),
|
|
5981
|
-
this._api?.getConsumption(spaceId) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
|
|
6067
|
+
this._api?.getConsumption(spaceId, tier) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
|
|
5982
6068
|
]);
|
|
5983
6069
|
const { unitsAvailable } = consumption;
|
|
5984
6070
|
const report = await compareLocalRemote(localFiles, remoteFiles);
|
|
@@ -6033,7 +6119,7 @@ var CueSyncApi = class {
|
|
|
6033
6119
|
rdfWritten = true;
|
|
6034
6120
|
syncCount += 1;
|
|
6035
6121
|
syncSize += file.size || 0;
|
|
6036
|
-
this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize,
|
|
6122
|
+
this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, onProgress);
|
|
6037
6123
|
} catch (err) {
|
|
6038
6124
|
failedUploads += 1;
|
|
6039
6125
|
console.error(`[CueSyncApi] Failed to upload file: ${file.fullPath}`);
|
|
@@ -6054,17 +6140,19 @@ var CueSyncApi = class {
|
|
|
6054
6140
|
rdfWritten = true;
|
|
6055
6141
|
syncCount += 1;
|
|
6056
6142
|
syncSize += file.size || 0;
|
|
6057
|
-
this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize,
|
|
6143
|
+
this._logProgress(syncCount, report.totalCount, syncSize, report.totalSize, onProgress);
|
|
6058
6144
|
}
|
|
6059
6145
|
await this._drainPending(verbose);
|
|
6060
6146
|
this._stopFlushTimer();
|
|
6147
|
+
const postSyncConsumption = await (this._api?.getConsumption(spaceId, tier) ?? Promise.resolve({ creditsAvailable: 0 }));
|
|
6061
6148
|
return {
|
|
6062
6149
|
syncCount,
|
|
6063
6150
|
syncSize,
|
|
6064
6151
|
failedUploads,
|
|
6065
6152
|
totalCount: report.totalCount,
|
|
6066
6153
|
totalSize: report.totalSize,
|
|
6067
|
-
rdfWritten
|
|
6154
|
+
rdfWritten,
|
|
6155
|
+
creditsAvailable: postSyncConsumption.creditsAvailable
|
|
6068
6156
|
};
|
|
6069
6157
|
}
|
|
6070
6158
|
async _getOrCreateGraph(spaceId, token) {
|
|
@@ -6090,9 +6178,15 @@ var CueSyncApi = class {
|
|
|
6090
6178
|
async _listRemoteFiles(graph, spaceId, providerId, verbose) {
|
|
6091
6179
|
if (verbose)
|
|
6092
6180
|
console.info(`Listing files in raw space: ${spaceId}`);
|
|
6181
|
+
const graphFilesWithTimeout = Promise.race([
|
|
6182
|
+
this._getGraphFiles(graph, providerId),
|
|
6183
|
+
new Promise(
|
|
6184
|
+
(_, reject) => setTimeout(() => reject(new Error("GRAPH_TIMEOUT: Knowledge graph query timed out")), 15e3)
|
|
6185
|
+
)
|
|
6186
|
+
]);
|
|
6093
6187
|
const [blobNames, locationMap] = await Promise.all([
|
|
6094
6188
|
this._blob.listRaw(spaceId),
|
|
6095
|
-
|
|
6189
|
+
graphFilesWithTimeout
|
|
6096
6190
|
]);
|
|
6097
6191
|
if (verbose)
|
|
6098
6192
|
console.info(`Found ${blobNames.length} files in raw store for space: ${spaceId}`);
|
|
@@ -6148,9 +6242,15 @@ WHERE {
|
|
|
6148
6242
|
this._pendingItems = [];
|
|
6149
6243
|
const existing = await _loadPending(spaceId);
|
|
6150
6244
|
if (existing && existing.items.length > 0) {
|
|
6245
|
+
console.info(`Trying to upload metadata from interrupted sync (${existing.items.length} item(s))...`);
|
|
6151
6246
|
if (verbose)
|
|
6152
6247
|
console.info(`Flushing ${existing.items.length} pending file location(s) from previous sync \u23F3`);
|
|
6153
|
-
|
|
6248
|
+
try {
|
|
6249
|
+
await this._flushBatch(existing.items, spaceId, token, verbose);
|
|
6250
|
+
console.info("Metadata uploaded \u2705");
|
|
6251
|
+
} catch (err) {
|
|
6252
|
+
throw new Error(`METADATA_SYNC_FAILED: ${err instanceof Error ? err.message : String(err)}`);
|
|
6253
|
+
}
|
|
6154
6254
|
}
|
|
6155
6255
|
const timer = setInterval(() => {
|
|
6156
6256
|
this._drainPending(verbose).catch(
|
|
@@ -6195,15 +6295,26 @@ WHERE {
|
|
|
6195
6295
|
}
|
|
6196
6296
|
}
|
|
6197
6297
|
async _postFssBatch(items, spaceId, token) {
|
|
6198
|
-
const
|
|
6199
|
-
|
|
6200
|
-
|
|
6201
|
-
|
|
6202
|
-
|
|
6203
|
-
"
|
|
6204
|
-
|
|
6205
|
-
|
|
6206
|
-
|
|
6298
|
+
const controller = new AbortController();
|
|
6299
|
+
const timeout = setTimeout(() => controller.abort(), 15e3);
|
|
6300
|
+
let response;
|
|
6301
|
+
try {
|
|
6302
|
+
response = await fetch(`${this._gatewayUrl}${ENDPOINT_FSS_BATCH}`, {
|
|
6303
|
+
method: "POST",
|
|
6304
|
+
headers: {
|
|
6305
|
+
Authorization: `Bearer ${token}`,
|
|
6306
|
+
"Content-Type": "application/json",
|
|
6307
|
+
"x-project-id": spaceId
|
|
6308
|
+
},
|
|
6309
|
+
body: JSON.stringify({ items }),
|
|
6310
|
+
signal: controller.signal
|
|
6311
|
+
});
|
|
6312
|
+
} catch (err) {
|
|
6313
|
+
const isTimeout = err instanceof Error && err.name === "AbortError";
|
|
6314
|
+
throw new Error(`File structure batch POST failed: ${isTimeout ? "request timed out" : err instanceof Error ? err.message : String(err)}`);
|
|
6315
|
+
} finally {
|
|
6316
|
+
clearTimeout(timeout);
|
|
6317
|
+
}
|
|
6207
6318
|
if (!response.ok) {
|
|
6208
6319
|
const body = await response.text().catch(() => "");
|
|
6209
6320
|
throw new Error(`File structure batch POST failed: ${response.status} ${response.statusText}${body ? ` \u2014 ${body}` : ""}`);
|
|
@@ -6252,21 +6363,41 @@ WHERE {
|
|
|
6252
6363
|
}
|
|
6253
6364
|
return Array.from(merged.values());
|
|
6254
6365
|
}
|
|
6255
|
-
|
|
6256
|
-
|
|
6257
|
-
|
|
6258
|
-
|
|
6366
|
+
async _fetchTierNames() {
|
|
6367
|
+
try {
|
|
6368
|
+
const text = await this._blob.downloadPublic("tier-names.json");
|
|
6369
|
+
return JSON.parse(text);
|
|
6370
|
+
} catch {
|
|
6371
|
+
return {};
|
|
6372
|
+
}
|
|
6373
|
+
}
|
|
6374
|
+
async _fetchUnitCreditMap(verbose) {
|
|
6375
|
+
let text;
|
|
6376
|
+
try {
|
|
6377
|
+
text = await this._blob.downloadPublic("unit-credit.json");
|
|
6378
|
+
} catch (err) {
|
|
6379
|
+
throw new Error(`Couldn't fetch credits table: ${err instanceof Error ? err.message : String(err)}`);
|
|
6380
|
+
}
|
|
6381
|
+
if (verbose)
|
|
6382
|
+
console.info(`Price file: ${text.length} bytes`);
|
|
6383
|
+
try {
|
|
6384
|
+
return JSON.parse(text);
|
|
6385
|
+
} catch {
|
|
6386
|
+
throw new Error(`Credits table is not valid JSON (${text.length} bytes)`);
|
|
6387
|
+
}
|
|
6388
|
+
}
|
|
6389
|
+
_logProgress(syncCount, totalCount, syncSize, totalSize, onProgress) {
|
|
6390
|
+
if (!onProgress || totalCount === 0)
|
|
6259
6391
|
return;
|
|
6260
6392
|
const pct = Math.floor(syncCount / totalCount * 100);
|
|
6261
|
-
|
|
6262
|
-
`Progress: ${pct}% (${syncCount}/${totalCount} files, ${syncSize}/${totalSize} bytes)`
|
|
6263
|
-
);
|
|
6393
|
+
onProgress({ percent: pct, syncCount, totalCount, syncSize, totalSize });
|
|
6264
6394
|
}
|
|
6265
6395
|
};
|
|
6266
6396
|
|
|
6267
6397
|
// libs/js/cue-sdk/src/lib/cue-node.ts
|
|
6268
6398
|
var BUCKET_RAW2 = "spaces_raw_eu_west6";
|
|
6269
6399
|
var BUCKET_PROCESSED2 = "spaces_processed_eu_west6";
|
|
6400
|
+
var BUCKET_PUBLIC2 = "cue_public_eu_west6";
|
|
6270
6401
|
var CueNode = class extends Cue {
|
|
6271
6402
|
constructor(config) {
|
|
6272
6403
|
super(config);
|
|
@@ -6274,13 +6405,15 @@ var CueNode = class extends Cue {
|
|
|
6274
6405
|
_buildApi(projects) {
|
|
6275
6406
|
const storageRaw = (0, import_storage4.getStorage)(this._app, BUCKET_RAW2);
|
|
6276
6407
|
const storageProcessed = (0, import_storage4.getStorage)(this._app, BUCKET_PROCESSED2);
|
|
6408
|
+
const storagePublic = (0, import_storage4.getStorage)(this._app, BUCKET_PUBLIC2);
|
|
6277
6409
|
if (this._isEmulator) {
|
|
6278
6410
|
const storageHost = this._endpoints.storageEmulatorHost;
|
|
6279
6411
|
const storagePort = this._endpoints.storageEmulatorPort;
|
|
6280
6412
|
(0, import_storage4.connectStorageEmulator)(storageRaw, storageHost, storagePort);
|
|
6281
6413
|
(0, import_storage4.connectStorageEmulator)(storageProcessed, storageHost, storagePort);
|
|
6414
|
+
(0, import_storage4.connectStorageEmulator)(storagePublic, storageHost, storagePort);
|
|
6282
6415
|
}
|
|
6283
|
-
const blob = new CueBlobStorage({ storageRaw, storageProcessed });
|
|
6416
|
+
const blob = new CueBlobStorage({ storageRaw, storageProcessed, storagePublic });
|
|
6284
6417
|
const syncApi = new CueSyncApi(this.auth, projects, blob, this._endpoints.gatewayUrl);
|
|
6285
6418
|
const api = new CueApi(this.auth, this._endpoints.gatewayUrl, projects, syncApi);
|
|
6286
6419
|
syncApi._bindApi(api);
|
|
@@ -6303,7 +6436,8 @@ async function authenticate(emulators, space, key, verbose = false) {
|
|
|
6303
6436
|
apiKey: FIREBASE_CONFIG().apiKey,
|
|
6304
6437
|
appId: FIREBASE_CONFIG().appId,
|
|
6305
6438
|
measurementId: FIREBASE_CONFIG().measurementId,
|
|
6306
|
-
environment: emulators ? "emulator" : "production"
|
|
6439
|
+
environment: emulators ? "emulator" : "production",
|
|
6440
|
+
...emulators ? { endpoints: getEmulatorEndpoints() } : {}
|
|
6307
6441
|
});
|
|
6308
6442
|
if (verbose)
|
|
6309
6443
|
console.info("Authenticating \u23F3");
|
|
@@ -6896,46 +7030,6 @@ async function repairTtlHandler(options) {
|
|
|
6896
7030
|
}
|
|
6897
7031
|
}
|
|
6898
7032
|
|
|
6899
|
-
// apps/desktop/cue-cli/src/helpers/emit-idle.ts
|
|
6900
|
-
var import_pubsub = require("@google-cloud/pubsub");
|
|
6901
|
-
async function emitIdle(spaceId, hoursBack, useEmulator) {
|
|
6902
|
-
const from = /* @__PURE__ */ new Date();
|
|
6903
|
-
from.setHours(from.getHours() - hoursBack);
|
|
6904
|
-
const message = {
|
|
6905
|
-
space_id: spaceId,
|
|
6906
|
-
first_write: from.toISOString(),
|
|
6907
|
-
last_write: (/* @__PURE__ */ new Date()).toISOString()
|
|
6908
|
-
};
|
|
6909
|
-
await publishMessage("RDF_WRITING_IDLE", message, useEmulator);
|
|
6910
|
-
}
|
|
6911
|
-
async function publishMessage(topicName, data, useEmulator) {
|
|
6912
|
-
if (useEmulator) {
|
|
6913
|
-
process.env.PUBSUB_EMULATOR_HOST = process.env.PUBSUB_EMULATOR_HOST || "localhost:8085";
|
|
6914
|
-
}
|
|
6915
|
-
const projectId = useEmulator ? "demo-project" : PROJECT_ID;
|
|
6916
|
-
const pubSubClient = new import_pubsub.PubSub({ projectId });
|
|
6917
|
-
const topic = pubSubClient.topic(topicName);
|
|
6918
|
-
const dataBuffer = Buffer.from(JSON.stringify(data));
|
|
6919
|
-
try {
|
|
6920
|
-
if (useEmulator) {
|
|
6921
|
-
const publishPromise = topic.publishMessage({ data: dataBuffer });
|
|
6922
|
-
const timeoutPromise = new Promise(
|
|
6923
|
-
(_, reject) => setTimeout(() => reject(new Error("PubSub publish timeout")), 5e3)
|
|
6924
|
-
);
|
|
6925
|
-
await Promise.race([publishPromise, timeoutPromise]);
|
|
6926
|
-
} else {
|
|
6927
|
-
await topic.publishMessage({ data: dataBuffer });
|
|
6928
|
-
}
|
|
6929
|
-
} catch (error) {
|
|
6930
|
-
console.error(`Error publishing message: ${error.message}`);
|
|
6931
|
-
if (useEmulator) {
|
|
6932
|
-
console.warn("Continuing despite PubSub error in emulator mode...");
|
|
6933
|
-
} else {
|
|
6934
|
-
throw error;
|
|
6935
|
-
}
|
|
6936
|
-
}
|
|
6937
|
-
}
|
|
6938
|
-
|
|
6939
7033
|
// apps/desktop/cue-cli/src/cue-cli-sync.ts
|
|
6940
7034
|
var import_promises7 = require("fs/promises");
|
|
6941
7035
|
var import_fs6 = require("fs");
|
|
@@ -6957,7 +7051,8 @@ async function syncHandler(options) {
|
|
|
6957
7051
|
apiKey: FIREBASE_CONFIG().apiKey,
|
|
6958
7052
|
appId: FIREBASE_CONFIG().appId,
|
|
6959
7053
|
measurementId: FIREBASE_CONFIG().measurementId,
|
|
6960
|
-
environment: emulators ? "emulator" : "production"
|
|
7054
|
+
environment: emulators ? "emulator" : "production",
|
|
7055
|
+
...emulators ? { endpoints: getEmulatorEndpoints() } : {}
|
|
6961
7056
|
});
|
|
6962
7057
|
const key = options.key ?? process.env.CUE_API_KEY;
|
|
6963
7058
|
if (!key) {
|
|
@@ -7005,7 +7100,14 @@ async function syncHandler(options) {
|
|
|
7005
7100
|
console.info("Built sync base \u2705\n");
|
|
7006
7101
|
if (verbose)
|
|
7007
7102
|
console.info("Authenticating \u23F3");
|
|
7008
|
-
|
|
7103
|
+
let user;
|
|
7104
|
+
try {
|
|
7105
|
+
user = await cue.auth.signInWithApiKey(key, space);
|
|
7106
|
+
} catch (err) {
|
|
7107
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
7108
|
+
console.error(`Couldn't authenticate. ${cause}`);
|
|
7109
|
+
process.exit(1);
|
|
7110
|
+
}
|
|
7009
7111
|
if (verbose)
|
|
7010
7112
|
console.info("Authenticated \u2705\n");
|
|
7011
7113
|
if (verbose)
|
|
@@ -7016,25 +7118,28 @@ async function syncHandler(options) {
|
|
|
7016
7118
|
userId: user.uid,
|
|
7017
7119
|
verbose
|
|
7018
7120
|
});
|
|
7121
|
+
console.info(`Project is configured at the ${preview.tierName} tier`);
|
|
7019
7122
|
console.info("Cost estimate (new files only):");
|
|
7020
7123
|
if (preview.costRecords.length === 0) {
|
|
7021
7124
|
console.info(" Nothing new to sync.");
|
|
7022
7125
|
} else {
|
|
7023
7126
|
for (const r of preview.costRecords) {
|
|
7024
|
-
|
|
7127
|
+
const creditsStr = r.credits !== void 0 ? ` (${Math.round(r.credits)} credits)` : "";
|
|
7128
|
+
console.info(` .${r.ext}: ${r.count} file(s)${creditsStr}`);
|
|
7025
7129
|
}
|
|
7026
7130
|
}
|
|
7027
7131
|
console.info(` New files: ${preview.filesToUpload}/${preview.totalLocalFiles}`);
|
|
7028
|
-
console.info(`
|
|
7029
|
-
console.info(`
|
|
7132
|
+
console.info(` Credits required: ${Math.round(preview.creditsToConsume)}`);
|
|
7133
|
+
console.info(` Credits available: ${Math.round(preview.creditsAvailable)}
|
|
7030
7134
|
`);
|
|
7135
|
+
await cue.api.sync.flushPendingMetadata(space, verbose);
|
|
7031
7136
|
if (preview.filesToUpload === 0) {
|
|
7032
7137
|
console.info("Everything is already synced.");
|
|
7033
7138
|
process.exit(0);
|
|
7034
7139
|
}
|
|
7035
|
-
if (preview.
|
|
7140
|
+
if (preview.creditsToConsume > preview.creditsAvailable) {
|
|
7036
7141
|
console.error(
|
|
7037
|
-
`Insufficient
|
|
7142
|
+
`Insufficient credits: ${Math.round(preview.creditsToConsume)} required, ${Math.round(preview.creditsAvailable)} available.`
|
|
7038
7143
|
);
|
|
7039
7144
|
process.exit(1);
|
|
7040
7145
|
}
|
|
@@ -7047,24 +7152,14 @@ async function syncHandler(options) {
|
|
|
7047
7152
|
spaceId: space,
|
|
7048
7153
|
providerId: provider,
|
|
7049
7154
|
userId: user.uid,
|
|
7050
|
-
verbose
|
|
7051
|
-
|
|
7052
|
-
|
|
7053
|
-
|
|
7054
|
-
|
|
7055
|
-
console.info(`Throwing RDF_WRITING_IDLE topic (only in emulators) \u23F3`);
|
|
7155
|
+
verbose,
|
|
7156
|
+
onProgress: ({ percent, syncCount, totalCount }) => {
|
|
7157
|
+
const filled = Math.round(percent / 5);
|
|
7158
|
+
const bar = "\u2588".repeat(filled) + "\u2591".repeat(20 - filled);
|
|
7159
|
+
process.stdout.write(`\r [${bar}] ${percent}% (${syncCount}/${totalCount} files) `);
|
|
7056
7160
|
}
|
|
7057
|
-
|
|
7058
|
-
|
|
7059
|
-
if (verbose)
|
|
7060
|
-
console.info(`Threw RDF_WRITING_IDLE topic \u2705`);
|
|
7061
|
-
} catch (error) {
|
|
7062
|
-
if (verbose)
|
|
7063
|
-
console.error(`Error throwing RDF_WRITING_IDLE topic: ${error instanceof Error ? error.message : String(error)}`);
|
|
7064
|
-
if (verbose)
|
|
7065
|
-
console.warn("Continuing despite PubSub error in emulator mode...");
|
|
7066
|
-
}
|
|
7067
|
-
}
|
|
7161
|
+
});
|
|
7162
|
+
process.stdout.write("\n");
|
|
7068
7163
|
const zipDeletePromise = zip && !isFile ? deleteUnzipped(path) : Promise.resolve();
|
|
7069
7164
|
await zipDeletePromise;
|
|
7070
7165
|
if (zip && verbose)
|
|
@@ -7074,6 +7169,7 @@ async function syncHandler(options) {
|
|
|
7074
7169
|
console.info(
|
|
7075
7170
|
`Synced: ${result.syncCount}/${result.totalCount} files (${fileSizePretty(result.syncSize)}/${fileSizePretty(result.totalSize)})`
|
|
7076
7171
|
);
|
|
7172
|
+
console.info(`Credits remaining: ${Math.round(result.creditsAvailable)}`);
|
|
7077
7173
|
console.info(`Sync finished \u{1F680}\u{1F680}\u{1F680}`);
|
|
7078
7174
|
if (result.failedUploads > 0) {
|
|
7079
7175
|
console.warn(`Total files failed to upload: ${result.failedUploads}`);
|
|
@@ -7081,7 +7177,18 @@ async function syncHandler(options) {
|
|
|
7081
7177
|
}
|
|
7082
7178
|
process.exit(0);
|
|
7083
7179
|
} catch (err) {
|
|
7084
|
-
|
|
7180
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
7181
|
+
if (msg.includes("GRAPH_TIMEOUT")) {
|
|
7182
|
+
console.error("Could not reach the knowledge graph. Make sure all required services are running.");
|
|
7183
|
+
} else if (msg.includes("METADATA_SYNC_FAILED")) {
|
|
7184
|
+
console.error("Metadata sync failed. Try again.");
|
|
7185
|
+
} else if (msg.includes("LEDGER_WRITE_FAILED") || msg.includes("File structure batch POST failed")) {
|
|
7186
|
+
console.error(
|
|
7187
|
+
"Failed communication with the knowledge graph. The files are up to date but metadata is not. Run the tool again to sync metadata. This is a swift job."
|
|
7188
|
+
);
|
|
7189
|
+
} else {
|
|
7190
|
+
console.error("[syncHandler] Unexpected error:", err);
|
|
7191
|
+
}
|
|
7085
7192
|
process.exit(1);
|
|
7086
7193
|
}
|
|
7087
7194
|
}
|