@qaecy/cue-sdk 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{cue-y-t8nSnt.js → cue-BA1pOKTu.js} +14 -10
- package/index.js +1 -1
- package/lib/api.d.ts +1 -1
- package/lib/models.d.ts +32 -2
- package/lib/sync.d.ts +7 -0
- package/node.js +239 -162
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { initializeApp as D } from "firebase/app";
|
|
2
|
-
import { getAuth as S, connectAuthEmulator as C, signInWithEmailAndPassword as T, GoogleAuthProvider as P, OAuthProvider as U, signInWithPopup as O, signInWithCustomToken as j, signOut as v, onAuthStateChanged as N, fetchSignInMethodsForEmail as K, linkWithPopup as R, unlink as $, reauthenticateWithCredential as f, EmailAuthProvider as m, updatePassword as q, linkWithCredential as
|
|
2
|
+
import { getAuth as S, connectAuthEmulator as C, signInWithEmailAndPassword as T, GoogleAuthProvider as P, OAuthProvider as U, signInWithPopup as O, signInWithCustomToken as j, signOut as v, onAuthStateChanged as N, fetchSignInMethodsForEmail as K, linkWithPopup as R, unlink as $, reauthenticateWithCredential as f, EmailAuthProvider as m, updatePassword as q, linkWithCredential as x, verifyBeforeUpdateEmail as H, sendEmailVerification as L } from "firebase/auth";
|
|
3
3
|
import { getFirestore as b, connectFirestoreEmulator as A, doc as _, collection as n, getDoc as y, setDoc as g, getDocs as h, query as l, where as d, serverTimestamp as F, increment as W, addDoc as z, deleteDoc as B, limit as J } from "firebase/firestore";
|
|
4
4
|
const M = "microsoft.com";
|
|
5
5
|
class V {
|
|
@@ -59,7 +59,7 @@ class V {
|
|
|
59
59
|
return this._auth;
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
|
-
const G = "/assistant/search", Q = "/triplestore/query", Y = "/data-views/admin/consumption
|
|
62
|
+
const G = "/assistant/search", Q = "/triplestore/query", Y = "/data-views/admin/consumption";
|
|
63
63
|
class X {
|
|
64
64
|
constructor(t, e, r, s) {
|
|
65
65
|
this._auth = t, this._gatewayUrl = e, this.projects = r, this.sync = s;
|
|
@@ -115,13 +115,17 @@ class X {
|
|
|
115
115
|
throw new Error(`SPARQL query failed: ${s.status} ${s.statusText}`);
|
|
116
116
|
return s.json();
|
|
117
117
|
}
|
|
118
|
-
async getConsumption(t) {
|
|
119
|
-
const
|
|
120
|
-
headers: {
|
|
118
|
+
async getConsumption(t, e) {
|
|
119
|
+
const r = await this._authHeaders(), s = await fetch(`${this._gatewayUrl}${Y}`, {
|
|
120
|
+
headers: {
|
|
121
|
+
...r,
|
|
122
|
+
"x-project-id": t,
|
|
123
|
+
...e ? { "x-tier": e } : {}
|
|
124
|
+
}
|
|
121
125
|
});
|
|
122
|
-
if (!
|
|
123
|
-
throw new Error(`Failed to fetch consumption: ${
|
|
124
|
-
return
|
|
126
|
+
if (!s.ok)
|
|
127
|
+
throw new Error(`Failed to fetch consumption: ${s.status} ${s.statusText}`);
|
|
128
|
+
return s.json();
|
|
125
129
|
}
|
|
126
130
|
}
|
|
127
131
|
const w = "projects";
|
|
@@ -250,7 +254,7 @@ class tt {
|
|
|
250
254
|
async addPassword(t) {
|
|
251
255
|
const e = this._requireUser();
|
|
252
256
|
if (!e.email) throw new Error("User has no e-mail");
|
|
253
|
-
await
|
|
257
|
+
await x(e, m.credential(e.email, t));
|
|
254
258
|
}
|
|
255
259
|
/** Requests an e-mail change. Sends a verification e-mail to the new address. */
|
|
256
260
|
async updateEmail(t, e) {
|
|
@@ -259,7 +263,7 @@ class tt {
|
|
|
259
263
|
await f(
|
|
260
264
|
r,
|
|
261
265
|
m.credential(r.email, e)
|
|
262
|
-
), await
|
|
266
|
+
), await H(r, t), await L(r);
|
|
263
267
|
}
|
|
264
268
|
/** Creates a new API key for the current user. */
|
|
265
269
|
async createAPIKey(t) {
|
package/index.js
CHANGED
package/lib/api.d.ts
CHANGED
|
@@ -24,5 +24,5 @@ export declare class CueApi {
|
|
|
24
24
|
* The user must be authenticated before calling this.
|
|
25
25
|
*/
|
|
26
26
|
sparql(query: string, projectId: string): Promise<unknown>;
|
|
27
|
-
getConsumption(projectId: string): Promise<UnitsConsumedDto>;
|
|
27
|
+
getConsumption(projectId: string, tier?: string): Promise<UnitsConsumedDto>;
|
|
28
28
|
}
|
package/lib/models.d.ts
CHANGED
|
@@ -63,6 +63,8 @@ export interface ProjectSettings {
|
|
|
63
63
|
type: string;
|
|
64
64
|
uri: string;
|
|
65
65
|
};
|
|
66
|
+
/** Processing tier determining credit costs. Defaults to "l" if not set. */
|
|
67
|
+
tier?: 's' | 'm' | 'l';
|
|
66
68
|
}
|
|
67
69
|
export interface ProjectData {
|
|
68
70
|
id: string;
|
|
@@ -85,6 +87,13 @@ export interface CreateProjectOptions {
|
|
|
85
87
|
/** Explicit project ID. Defaults to a new UUID. */
|
|
86
88
|
id?: string;
|
|
87
89
|
}
|
|
90
|
+
export interface SyncProgress {
|
|
91
|
+
percent: number;
|
|
92
|
+
syncCount: number;
|
|
93
|
+
totalCount: number;
|
|
94
|
+
syncSize: number;
|
|
95
|
+
totalSize: number;
|
|
96
|
+
}
|
|
88
97
|
export interface SyncOptions {
|
|
89
98
|
/** The project/space ID to sync into */
|
|
90
99
|
spaceId: string;
|
|
@@ -94,7 +103,14 @@ export interface SyncOptions {
|
|
|
94
103
|
userId: string;
|
|
95
104
|
/** Enable verbose logging */
|
|
96
105
|
verbose?: boolean;
|
|
97
|
-
|
|
106
|
+
/** Called whenever upload progress changes */
|
|
107
|
+
onProgress?: (progress: SyncProgress) => void;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Credit cost table fetched from the public bucket (`unit-credit.json`).
|
|
111
|
+
* Maps file extension → tier → credits per unit.
|
|
112
|
+
*/
|
|
113
|
+
export type UnitCreditMap = Partial<Record<'s' | 'm' | 'l', Record<string, number>>>;
|
|
98
114
|
/** Per-extension cost breakdown returned by {@link CueSyncApi.scanCost}. */
|
|
99
115
|
export interface ScanOutputRecord {
|
|
100
116
|
/** File extension (without leading dot), e.g. `"pdf"`, `"ifc"`. */
|
|
@@ -109,17 +125,29 @@ export interface ScanOutputRecord {
|
|
|
109
125
|
units: number;
|
|
110
126
|
/** Total size of files with this extension in megabytes. */
|
|
111
127
|
sizeMb: number;
|
|
128
|
+
/** Credits to consume for this extension (units × credit-per-unit for the project tier). */
|
|
129
|
+
credits?: number;
|
|
112
130
|
}
|
|
113
131
|
export interface UnitsConsumedDto {
|
|
132
|
+
creditsAvailable: number;
|
|
133
|
+
creditsConsumed: number;
|
|
134
|
+
filesProcessed: number;
|
|
114
135
|
unitsAvailable: number;
|
|
115
136
|
unitsConsumed: number;
|
|
116
|
-
filesProcessed: number;
|
|
117
137
|
}
|
|
118
138
|
export interface SyncPreview {
|
|
119
139
|
/** Per-extension cost breakdown for files not yet synced */
|
|
120
140
|
costRecords: ScanOutputRecord[];
|
|
141
|
+
/** Project tier used for cost calculation */
|
|
142
|
+
tier: 's' | 'm' | 'l';
|
|
143
|
+
/** Human-readable tier name from tier-names.json */
|
|
144
|
+
tierName: string;
|
|
121
145
|
/** Total units required for the new files */
|
|
122
146
|
unitsToConsume: number;
|
|
147
|
+
/** Total credits to consume (derived from unit-credit.json + project tier) */
|
|
148
|
+
creditsToConsume: number;
|
|
149
|
+
/** Credits currently available for this project */
|
|
150
|
+
creditsAvailable: number;
|
|
123
151
|
/** Units still available for this project */
|
|
124
152
|
unitsAvailable: number;
|
|
125
153
|
/** Number of new files that would be uploaded */
|
|
@@ -140,6 +168,8 @@ export interface SyncResult {
|
|
|
140
168
|
totalSize: number;
|
|
141
169
|
/** Whether any RDF metadata was written */
|
|
142
170
|
rdfWritten: boolean;
|
|
171
|
+
/** Updated credit balance after sync, fetched from the consumption endpoint */
|
|
172
|
+
creditsAvailable: number;
|
|
143
173
|
}
|
|
144
174
|
export interface ProfileSSOAccount {
|
|
145
175
|
id: string;
|
package/lib/sync.d.ts
CHANGED
|
@@ -17,6 +17,11 @@ export declare class CueSyncApi {
|
|
|
17
17
|
constructor(_auth: CueAuth, _projects: CueProjects, _blob: CueBlobStorage, _gatewayUrl: string);
|
|
18
18
|
/** @internal Injected by CueApi after construction to avoid circular dependency. */
|
|
19
19
|
_bindApi(api: CueApi): void;
|
|
20
|
+
/**
|
|
21
|
+
* Flushes any pending metadata items from a previous interrupted sync.
|
|
22
|
+
* Safe to call even when there is nothing new to upload.
|
|
23
|
+
*/
|
|
24
|
+
flushPendingMetadata(spaceId: string, verbose?: boolean): Promise<void>;
|
|
20
25
|
/**
|
|
21
26
|
* Returns a preview of what would be synced: cost breakdown for new files only,
|
|
22
27
|
* units required, and units still available. Use this before calling {@link sync}
|
|
@@ -41,5 +46,7 @@ export declare class CueSyncApi {
|
|
|
41
46
|
* shown to the user before or after calling {@link sync}.
|
|
42
47
|
*/
|
|
43
48
|
scanCost(localFiles: LocalFile[]): Promise<ScanOutputRecord[]>;
|
|
49
|
+
private _fetchTierNames;
|
|
50
|
+
private _fetchUnitCreditMap;
|
|
44
51
|
private _logProgress;
|
|
45
52
|
}
|
package/node.js
CHANGED
|
@@ -1,56 +1,56 @@
|
|
|
1
|
-
import { C as
|
|
2
|
-
import { b as
|
|
3
|
-
import { getStorage as
|
|
4
|
-
import { CueGraphDatabase as
|
|
5
|
-
import { readFile as
|
|
6
|
-
import { join as
|
|
7
|
-
import { pathToFileURL as
|
|
8
|
-
import { tmpdir as
|
|
9
|
-
import { compareLocalRemote as
|
|
10
|
-
import { uploadedFileMetadata as
|
|
11
|
-
import { qaecyPrefixes as
|
|
12
|
-
let
|
|
13
|
-
async function
|
|
14
|
-
const
|
|
15
|
-
await e.default({ module_or_path: t }),
|
|
1
|
+
import { C as z, a as D } from "./cue-BA1pOKTu.js";
|
|
2
|
+
import { b as yt, c as Pt, d as Et } from "./cue-BA1pOKTu.js";
|
|
3
|
+
import { getStorage as U, connectStorageEmulator as A } from "firebase/storage";
|
|
4
|
+
import { CueGraphDatabase as B, CueBlobStorage as x } from "js-databases";
|
|
5
|
+
import { readFile as b, writeFile as L, unlink as k } from "fs/promises";
|
|
6
|
+
import { join as S } from "path";
|
|
7
|
+
import { pathToFileURL as H } from "url";
|
|
8
|
+
import { tmpdir as G } from "os";
|
|
9
|
+
import { compareLocalRemote as j } from "js-sync-tools";
|
|
10
|
+
import { uploadedFileMetadata as J } from "js-file-metadata-helpers";
|
|
11
|
+
import { qaecyPrefixes as K } from "js/prefixes";
|
|
12
|
+
let N = null, I = null;
|
|
13
|
+
async function W() {
|
|
14
|
+
const u = S(__dirname, "assets", "wasm"), t = await b(S(u, "dir_scanner_wasm_bg.wasm")), e = await import(H(S(u, "dir_scanner_wasm.mjs")).href);
|
|
15
|
+
await e.default({ module_or_path: t }), N = e.scan;
|
|
16
16
|
}
|
|
17
|
-
const
|
|
18
|
-
function
|
|
19
|
-
return
|
|
17
|
+
const Y = "fuseki", Q = "/triplestore/query", V = "/sparql/query", X = "/triplestore/update", Z = "/sparql/update", tt = "/commands/file-system-structure/batch", O = 1e3, F = "cue:pending:";
|
|
18
|
+
function v(u) {
|
|
19
|
+
return S(G(), `cue-sync-pending-${u}.json`);
|
|
20
20
|
}
|
|
21
|
-
async function
|
|
21
|
+
async function R(u) {
|
|
22
22
|
if (typeof window < "u") {
|
|
23
|
-
const t = window.localStorage.getItem(`${
|
|
23
|
+
const t = window.localStorage.getItem(`${F}${u}`);
|
|
24
24
|
return t ? JSON.parse(t) : null;
|
|
25
25
|
}
|
|
26
26
|
try {
|
|
27
|
-
const t = await
|
|
27
|
+
const t = await b(v(u), "utf-8");
|
|
28
28
|
return JSON.parse(t);
|
|
29
29
|
} catch {
|
|
30
30
|
return null;
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
|
-
async function
|
|
34
|
-
const t = JSON.stringify(
|
|
33
|
+
async function q(u) {
|
|
34
|
+
const t = JSON.stringify(u);
|
|
35
35
|
if (typeof window < "u") {
|
|
36
|
-
window.localStorage.setItem(`${
|
|
36
|
+
window.localStorage.setItem(`${F}${u.spaceId}`, t);
|
|
37
37
|
return;
|
|
38
38
|
}
|
|
39
|
-
await
|
|
39
|
+
await L(v(u.spaceId), t, "utf-8");
|
|
40
40
|
}
|
|
41
|
-
async function
|
|
41
|
+
async function et(u) {
|
|
42
42
|
if (typeof window < "u") {
|
|
43
|
-
window.localStorage.removeItem(`${
|
|
43
|
+
window.localStorage.removeItem(`${F}${u}`);
|
|
44
44
|
return;
|
|
45
45
|
}
|
|
46
46
|
try {
|
|
47
|
-
await
|
|
47
|
+
await k(v(u));
|
|
48
48
|
} catch {
|
|
49
49
|
}
|
|
50
50
|
}
|
|
51
|
-
class
|
|
52
|
-
constructor(t, o, e,
|
|
53
|
-
this._auth = t, this._projects = o, this._blob = e, this._gatewayUrl =
|
|
51
|
+
class it {
|
|
52
|
+
constructor(t, o, e, n) {
|
|
53
|
+
this._auth = t, this._projects = o, this._blob = e, this._gatewayUrl = n;
|
|
54
54
|
}
|
|
55
55
|
_auth;
|
|
56
56
|
_projects;
|
|
@@ -65,97 +65,132 @@ class X {
|
|
|
65
65
|
_bindApi(t) {
|
|
66
66
|
this._api = t;
|
|
67
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* Flushes any pending metadata items from a previous interrupted sync.
|
|
70
|
+
* Safe to call even when there is nothing new to upload.
|
|
71
|
+
*/
|
|
72
|
+
async flushPendingMetadata(t, o) {
|
|
73
|
+
const e = await this._auth.getToken();
|
|
74
|
+
if (!e) throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
75
|
+
const n = await R(t);
|
|
76
|
+
if (!(!n || n.items.length === 0)) {
|
|
77
|
+
console.info(`Trying to upload metadata (${n.items.length} item(s))...`), o && console.info(`Flushing ${n.items.length} pending file location(s) from previous sync ⏳`);
|
|
78
|
+
try {
|
|
79
|
+
this._pendingSpaceId = t, this._pendingItems = [], await this._flushBatch(n.items, t, e, o), console.info("Metadata uploaded ✅");
|
|
80
|
+
} catch (s) {
|
|
81
|
+
throw new Error(`METADATA_SYNC_FAILED: ${s instanceof Error ? s.message : String(s)}`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
68
85
|
/**
|
|
69
86
|
* Returns a preview of what would be synced: cost breakdown for new files only,
|
|
70
87
|
* units required, and units still available. Use this before calling {@link sync}
|
|
71
88
|
* to show the user an accurate cost estimate.
|
|
72
89
|
*/
|
|
73
90
|
async previewSync(t, o) {
|
|
74
|
-
const { spaceId: e, providerId:
|
|
75
|
-
if (!
|
|
76
|
-
const r = await this._getOrCreateGraph(e,
|
|
77
|
-
this._listRemoteFiles(r, e,
|
|
78
|
-
this._api?.getConsumption(e) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
|
|
79
|
-
|
|
91
|
+
const { spaceId: e, providerId: n, verbose: s } = o, i = await this._auth.getToken();
|
|
92
|
+
if (!i) throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
93
|
+
const r = await this._getOrCreateGraph(e, i), c = (await this._projects.getProject(e))?.projectSettings?.tier ?? "l", [p, g, m, C] = await Promise.all([
|
|
94
|
+
this._listRemoteFiles(r, e, n, s),
|
|
95
|
+
this._api?.getConsumption(e, c) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance")),
|
|
96
|
+
this._fetchUnitCreditMap(s),
|
|
97
|
+
this._fetchTierNames()
|
|
98
|
+
]), l = (await j(t, p)).localNotOnRemote ?? [], _ = l.length > 0 ? await this.scanCost(l) : [];
|
|
99
|
+
let w = 0, y = 0;
|
|
100
|
+
for (const f of _) {
|
|
101
|
+
w += f.units;
|
|
102
|
+
const E = m[c], h = E?.[f.ext] ?? 1;
|
|
103
|
+
s && E && !(f.ext in E) && console.info(` Unknown format: .${f.ext} (using default rate of 1 credit/unit)`);
|
|
104
|
+
const d = f.units * h;
|
|
105
|
+
y += d, f.credits = Math.round(d);
|
|
106
|
+
}
|
|
107
|
+
const P = C[c] ?? c;
|
|
80
108
|
return {
|
|
81
|
-
costRecords:
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
109
|
+
costRecords: _,
|
|
110
|
+
tier: c,
|
|
111
|
+
tierName: P,
|
|
112
|
+
unitsToConsume: w,
|
|
113
|
+
creditsToConsume: Math.round(y),
|
|
114
|
+
creditsAvailable: g.creditsAvailable,
|
|
115
|
+
unitsAvailable: g.unitsAvailable,
|
|
116
|
+
filesToUpload: l.length,
|
|
85
117
|
totalLocalFiles: t.length
|
|
86
118
|
};
|
|
87
119
|
}
|
|
88
120
|
async sync(t, o) {
|
|
89
|
-
const { spaceId: e, providerId:
|
|
90
|
-
if (!
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
const
|
|
94
|
-
this._listRemoteFiles(
|
|
95
|
-
this._api?.getConsumption(e) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
|
|
96
|
-
]), { unitsAvailable:
|
|
97
|
-
|
|
98
|
-
`Total files to sync: ${(
|
|
121
|
+
const { spaceId: e, providerId: n, userId: s, verbose: i, onProgress: r } = o, a = await this._auth.getToken();
|
|
122
|
+
if (!a) throw new Error("Not authenticated. Call cue.auth.signIn() first.");
|
|
123
|
+
const c = await this._getOrCreateGraph(e, a);
|
|
124
|
+
i && console.info("Listing remote files ⏳");
|
|
125
|
+
const g = (await this._projects.getProject(e))?.projectSettings?.tier ?? "l", [m, C] = await Promise.all([
|
|
126
|
+
this._listRemoteFiles(c, e, n, i),
|
|
127
|
+
this._api?.getConsumption(e, g) ?? Promise.reject(new Error("CueSyncApi is not bound to a CueApi instance"))
|
|
128
|
+
]), { unitsAvailable: T } = C, l = await j(t, m);
|
|
129
|
+
i && (console.info(`Total local files: ${t.length}`), console.info(`Total remote files: ${m.length}`), console.info(
|
|
130
|
+
`Total files to sync: ${(l.localNotOnRemote?.length ?? 0) + l.localNotOnRemotePathOnly.length}`
|
|
99
131
|
));
|
|
100
|
-
let _ =
|
|
101
|
-
const
|
|
102
|
-
if (
|
|
103
|
-
const
|
|
104
|
-
if (
|
|
132
|
+
let _ = l.syncCount, w = l.syncSize, y = 0, P = !1;
|
|
133
|
+
const f = l.localNotOnRemote ?? [];
|
|
134
|
+
if (f.length > 0) {
|
|
135
|
+
const d = (await this.scanCost(f)).reduce(($, M) => $ + M.units, 0);
|
|
136
|
+
if (d > T)
|
|
105
137
|
throw new Error(
|
|
106
|
-
`Insufficient units: ${
|
|
138
|
+
`Insufficient units: ${d} units required but only ${T} available.`
|
|
107
139
|
);
|
|
108
140
|
}
|
|
109
|
-
await this._initPendingBatch(e,
|
|
110
|
-
for (const
|
|
141
|
+
await this._initPendingBatch(e, a, i), i && f.length && console.info("Syncing missing files ⏳");
|
|
142
|
+
for (const h of f)
|
|
111
143
|
try {
|
|
112
|
-
const
|
|
113
|
-
|
|
144
|
+
const d = J(
|
|
145
|
+
h.relativePath,
|
|
114
146
|
e,
|
|
115
147
|
s,
|
|
116
|
-
|
|
117
|
-
|
|
148
|
+
h.md5,
|
|
149
|
+
n
|
|
118
150
|
);
|
|
119
|
-
if (!
|
|
120
|
-
const
|
|
151
|
+
if (!d.blob_name) throw new Error(`blob_name missing for ${h.relativePath}`);
|
|
152
|
+
const $ = await b(h.fullPath);
|
|
121
153
|
await this._blob.uploadRaw(
|
|
122
|
-
|
|
123
|
-
new Uint8Array(
|
|
124
|
-
|
|
154
|
+
d.blob_name,
|
|
155
|
+
new Uint8Array($),
|
|
156
|
+
d
|
|
125
157
|
), await this._queueFileLocation({
|
|
126
|
-
relativePath:
|
|
127
|
-
md5:
|
|
128
|
-
size:
|
|
129
|
-
providerId:
|
|
158
|
+
relativePath: h.relativePath,
|
|
159
|
+
md5: h.md5,
|
|
160
|
+
size: h.size,
|
|
161
|
+
providerId: n,
|
|
130
162
|
fileContentExists: !1
|
|
131
|
-
}),
|
|
132
|
-
} catch (
|
|
133
|
-
|
|
163
|
+
}), P = !0, _ += 1, w += h.size || 0, this._logProgress(_, l.totalCount, w, l.totalSize, r);
|
|
164
|
+
} catch (d) {
|
|
165
|
+
y += 1, console.error(`[CueSyncApi] Failed to upload file: ${h.fullPath}`), i && console.error("[CueSyncApi] Upload error details:", d);
|
|
134
166
|
}
|
|
135
|
-
|
|
136
|
-
for (const
|
|
167
|
+
i && l.localNotOnRemotePathOnly.length && console.info(`Syncing missing file locations (on provider "${n}") ⏳`);
|
|
168
|
+
for (const h of l.localNotOnRemotePathOnly)
|
|
137
169
|
await this._queueFileLocation({
|
|
138
|
-
relativePath:
|
|
139
|
-
md5:
|
|
140
|
-
size:
|
|
141
|
-
providerId:
|
|
170
|
+
relativePath: h.relativePath,
|
|
171
|
+
md5: h.md5,
|
|
172
|
+
size: h.size,
|
|
173
|
+
providerId: n,
|
|
142
174
|
fileContentExists: !0
|
|
143
|
-
}),
|
|
144
|
-
|
|
175
|
+
}), P = !0, _ += 1, w += h.size || 0, this._logProgress(_, l.totalCount, w, l.totalSize, r);
|
|
176
|
+
await this._drainPending(i), this._stopFlushTimer();
|
|
177
|
+
const E = await (this._api?.getConsumption(e, g) ?? Promise.resolve({ creditsAvailable: 0 }));
|
|
178
|
+
return {
|
|
145
179
|
syncCount: _,
|
|
146
|
-
syncSize:
|
|
147
|
-
failedUploads:
|
|
148
|
-
totalCount:
|
|
149
|
-
totalSize:
|
|
150
|
-
rdfWritten:
|
|
180
|
+
syncSize: w,
|
|
181
|
+
failedUploads: y,
|
|
182
|
+
totalCount: l.totalCount,
|
|
183
|
+
totalSize: l.totalSize,
|
|
184
|
+
rdfWritten: P,
|
|
185
|
+
creditsAvailable: E.creditsAvailable
|
|
151
186
|
};
|
|
152
187
|
}
|
|
153
188
|
async _getOrCreateGraph(t, o) {
|
|
154
189
|
const e = this._graphMap.get(t);
|
|
155
190
|
if (e) return e;
|
|
156
|
-
const s = (await this._projects.getProject(t))?.projectSettings?.graph?.type ??
|
|
191
|
+
const s = (await this._projects.getProject(t))?.projectSettings?.graph?.type ?? Y, i = s === "qlever" ? `${this._gatewayUrl}${V}` : `${this._gatewayUrl}${Q}`, r = s === "qlever" ? `${this._gatewayUrl}${Z}` : `${this._gatewayUrl}${X}`, a = new B({
|
|
157
192
|
graphType: s,
|
|
158
|
-
queryEndpoint:
|
|
193
|
+
queryEndpoint: i,
|
|
159
194
|
updateEndpoint: r,
|
|
160
195
|
originalHeaders: {
|
|
161
196
|
"x-project-id": t,
|
|
@@ -164,31 +199,36 @@ class X {
|
|
|
164
199
|
});
|
|
165
200
|
return this._graphMap.set(t, a), a;
|
|
166
201
|
}
|
|
167
|
-
async _listRemoteFiles(t, o, e,
|
|
168
|
-
|
|
169
|
-
const
|
|
202
|
+
async _listRemoteFiles(t, o, e, n) {
|
|
203
|
+
n && console.info(`Listing files in raw space: ${o}`);
|
|
204
|
+
const s = Promise.race([
|
|
205
|
+
this._getGraphFiles(t, e),
|
|
206
|
+
new Promise(
|
|
207
|
+
(c, p) => setTimeout(() => p(new Error("GRAPH_TIMEOUT: Knowledge graph query timed out")), 15e3)
|
|
208
|
+
)
|
|
209
|
+
]), [i, r] = await Promise.all([
|
|
170
210
|
this._blob.listRaw(o),
|
|
171
|
-
|
|
211
|
+
s
|
|
172
212
|
]);
|
|
173
|
-
|
|
174
|
-
const
|
|
175
|
-
for (const
|
|
176
|
-
const
|
|
177
|
-
if (
|
|
178
|
-
|
|
213
|
+
n && console.info(`Found ${i.length} files in raw store for space: ${o}`);
|
|
214
|
+
const a = [];
|
|
215
|
+
for (const c of i) {
|
|
216
|
+
const p = c.substring(0, 36), g = r[p] ?? [];
|
|
217
|
+
if (g.length === 0)
|
|
218
|
+
n && console.warn(`No location data found for contentUUID: ${p}`), a.push({ contentUUID: p });
|
|
179
219
|
else
|
|
180
|
-
for (const
|
|
181
|
-
|
|
182
|
-
contentUUID:
|
|
183
|
-
locationUUID:
|
|
184
|
-
created:
|
|
185
|
-
size:
|
|
220
|
+
for (const m of g)
|
|
221
|
+
a.push({
|
|
222
|
+
contentUUID: p,
|
|
223
|
+
locationUUID: m.locationUUID,
|
|
224
|
+
created: m.created,
|
|
225
|
+
size: m.size
|
|
186
226
|
});
|
|
187
227
|
}
|
|
188
|
-
return
|
|
228
|
+
return a;
|
|
189
229
|
}
|
|
190
230
|
async _getGraphFiles(t, o) {
|
|
191
|
-
const e = `PREFIX qcy: <${
|
|
231
|
+
const e = `PREFIX qcy: <${K.qcy}>
|
|
192
232
|
SELECT ?fc ?loc ?created ?fp ?size
|
|
193
233
|
WHERE {
|
|
194
234
|
?fc a qcy:FileContent ;
|
|
@@ -197,58 +237,75 @@ WHERE {
|
|
|
197
237
|
?loc qcy:remoteProviderId "${o}" ;
|
|
198
238
|
qcy:dateCreated ?created ;
|
|
199
239
|
qcy:filePath ?fp .
|
|
200
|
-
}`,
|
|
201
|
-
if (typeof
|
|
202
|
-
for (const
|
|
203
|
-
const r =
|
|
204
|
-
s[a] || (s[a] = []), s[a].push({ locationUUID: r, created:
|
|
240
|
+
}`, n = await t.query(e, "application/sparql-results+json"), s = {};
|
|
241
|
+
if (typeof n == "string") return s;
|
|
242
|
+
for (const i of n.results.bindings) {
|
|
243
|
+
const r = i.loc.value.split("/").at(-1) ?? "", a = i.fc.value.split("/").at(-1) ?? "", c = i.created.value, p = i.size.value;
|
|
244
|
+
s[a] || (s[a] = []), s[a].push({ locationUUID: r, created: c, size: p });
|
|
205
245
|
}
|
|
206
246
|
return s;
|
|
207
247
|
}
|
|
208
248
|
async _initPendingBatch(t, o, e) {
|
|
209
249
|
this._pendingSpaceId = t, this._pendingItems = [];
|
|
210
|
-
const
|
|
211
|
-
|
|
250
|
+
const n = await R(t);
|
|
251
|
+
if (n && n.items.length > 0) {
|
|
252
|
+
console.info(`Trying to upload metadata from interrupted sync (${n.items.length} item(s))...`), e && console.info(`Flushing ${n.items.length} pending file location(s) from previous sync ⏳`);
|
|
253
|
+
try {
|
|
254
|
+
await this._flushBatch(n.items, t, o, e), console.info("Metadata uploaded ✅");
|
|
255
|
+
} catch (i) {
|
|
256
|
+
throw new Error(`METADATA_SYNC_FAILED: ${i instanceof Error ? i.message : String(i)}`);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
212
259
|
const s = setInterval(() => {
|
|
213
260
|
this._drainPending(e).catch(
|
|
214
|
-
(
|
|
261
|
+
(i) => console.error("[CueSyncApi] Periodic flush failed:", i)
|
|
215
262
|
);
|
|
216
263
|
}, 6e4);
|
|
217
264
|
typeof s == "object" && typeof s.unref == "function" && s.unref(), this._flushTimer = s;
|
|
218
265
|
}
|
|
219
266
|
async _queueFileLocation(t) {
|
|
220
|
-
this._pendingItems.push(t), this._pendingSpaceId && await
|
|
267
|
+
this._pendingItems.push(t), this._pendingSpaceId && await q({ spaceId: this._pendingSpaceId, items: this._pendingItems });
|
|
221
268
|
}
|
|
222
269
|
async _drainPending(t) {
|
|
223
270
|
if (!this._pendingSpaceId || this._pendingItems.length === 0) return;
|
|
224
271
|
const o = await this._auth.getToken();
|
|
225
272
|
o && await this._flushBatch(this._pendingItems, this._pendingSpaceId, o, t);
|
|
226
273
|
}
|
|
227
|
-
async _flushBatch(t, o, e,
|
|
274
|
+
async _flushBatch(t, o, e, n) {
|
|
228
275
|
const s = [...t];
|
|
229
|
-
this._pendingSpaceId === o && (this._pendingItems = []), await
|
|
276
|
+
this._pendingSpaceId === o && (this._pendingItems = []), await et(o);
|
|
230
277
|
try {
|
|
231
|
-
for (let
|
|
232
|
-
await this._postFssBatch(s.slice(
|
|
233
|
-
|
|
234
|
-
} catch (
|
|
278
|
+
for (let i = 0; i < s.length; i += O)
|
|
279
|
+
await this._postFssBatch(s.slice(i, i + O), o, e);
|
|
280
|
+
n && console.info(`Wrote ${s.length} file location(s) to commands API ✅`);
|
|
281
|
+
} catch (i) {
|
|
235
282
|
const r = [...s, ...this._pendingItems];
|
|
236
|
-
throw this._pendingItems = r, await
|
|
283
|
+
throw this._pendingItems = r, await q({ spaceId: o, items: r }), i;
|
|
237
284
|
}
|
|
238
285
|
}
|
|
239
286
|
async _postFssBatch(t, o, e) {
|
|
240
|
-
const
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
287
|
+
const n = new AbortController(), s = setTimeout(() => n.abort(), 15e3);
|
|
288
|
+
let i;
|
|
289
|
+
try {
|
|
290
|
+
i = await fetch(`${this._gatewayUrl}${tt}`, {
|
|
291
|
+
method: "POST",
|
|
292
|
+
headers: {
|
|
293
|
+
Authorization: `Bearer ${e}`,
|
|
294
|
+
"Content-Type": "application/json",
|
|
295
|
+
"x-project-id": o
|
|
296
|
+
},
|
|
297
|
+
body: JSON.stringify({ items: t }),
|
|
298
|
+
signal: n.signal
|
|
299
|
+
});
|
|
300
|
+
} catch (r) {
|
|
301
|
+
const a = r instanceof Error && r.name === "AbortError";
|
|
302
|
+
throw new Error(`File structure batch POST failed: ${a ? "request timed out" : r instanceof Error ? r.message : String(r)}`);
|
|
303
|
+
} finally {
|
|
304
|
+
clearTimeout(s);
|
|
305
|
+
}
|
|
249
306
|
if (!i.ok) {
|
|
250
|
-
const
|
|
251
|
-
throw new Error(`File structure batch POST failed: ${i.status} ${i.statusText}${
|
|
307
|
+
const r = await i.text().catch(() => "");
|
|
308
|
+
throw new Error(`File structure batch POST failed: ${i.status} ${i.statusText}${r ? ` — ${r}` : ""}`);
|
|
252
309
|
}
|
|
253
310
|
}
|
|
254
311
|
_stopFlushTimer() {
|
|
@@ -262,51 +319,71 @@ WHERE {
|
|
|
262
319
|
* shown to the user before or after calling {@link sync}.
|
|
263
320
|
*/
|
|
264
321
|
async scanCost(t) {
|
|
265
|
-
if (
|
|
322
|
+
if (I || (I = W()), await I, !N) throw new Error("WASM scan function not initialised");
|
|
266
323
|
const o = 200, e = /* @__PURE__ */ new Map();
|
|
267
|
-
for (let
|
|
268
|
-
const s = t.slice(
|
|
324
|
+
for (let n = 0; n < t.length; n += o) {
|
|
325
|
+
const s = t.slice(n, n + o), i = await Promise.all(
|
|
269
326
|
s.map(async (a) => ({
|
|
270
327
|
originalPath: a.relativePath,
|
|
271
|
-
data: new Uint8Array(await
|
|
328
|
+
data: new Uint8Array(await b(a.fullPath))
|
|
272
329
|
}))
|
|
273
|
-
), r =
|
|
330
|
+
), r = N(i);
|
|
274
331
|
for (const a of r) {
|
|
275
|
-
const
|
|
276
|
-
|
|
332
|
+
const c = e.get(a.ext);
|
|
333
|
+
c ? (c.count += a.count, c.units += a.units, c.sizeMb += a.sizeMb) : e.set(a.ext, { ...a });
|
|
277
334
|
}
|
|
278
335
|
}
|
|
279
336
|
return Array.from(e.values());
|
|
280
337
|
}
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
338
|
+
async _fetchTierNames() {
|
|
339
|
+
try {
|
|
340
|
+
const t = await this._blob.downloadPublic("tier-names.json");
|
|
341
|
+
return JSON.parse(t);
|
|
342
|
+
} catch {
|
|
343
|
+
return {};
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
async _fetchUnitCreditMap(t) {
|
|
347
|
+
let o;
|
|
348
|
+
try {
|
|
349
|
+
o = await this._blob.downloadPublic("unit-credit.json");
|
|
350
|
+
} catch (e) {
|
|
351
|
+
throw new Error(`Couldn't fetch credits table: ${e instanceof Error ? e.message : String(e)}`);
|
|
352
|
+
}
|
|
353
|
+
t && console.info(`Price file: ${o.length} bytes`);
|
|
354
|
+
try {
|
|
355
|
+
return JSON.parse(o);
|
|
356
|
+
} catch {
|
|
357
|
+
throw new Error(`Credits table is not valid JSON (${o.length} bytes)`);
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
_logProgress(t, o, e, n, s) {
|
|
361
|
+
if (!s || o === 0) return;
|
|
362
|
+
const i = Math.floor(t / o * 100);
|
|
363
|
+
s({ percent: i, syncCount: t, totalCount: o, syncSize: e, totalSize: n });
|
|
287
364
|
}
|
|
288
365
|
}
|
|
289
|
-
const
|
|
290
|
-
class
|
|
366
|
+
const ot = "spaces_raw_eu_west6", nt = "spaces_processed_eu_west6", st = "cue_public_eu_west6";
|
|
367
|
+
class mt extends z {
|
|
291
368
|
constructor(t) {
|
|
292
369
|
super(t);
|
|
293
370
|
}
|
|
294
371
|
_buildApi(t) {
|
|
295
|
-
const o =
|
|
372
|
+
const o = U(this._app, ot), e = U(this._app, nt), n = U(this._app, st);
|
|
296
373
|
if (this._isEmulator) {
|
|
297
|
-
const
|
|
298
|
-
|
|
374
|
+
const a = this._endpoints.storageEmulatorHost, c = this._endpoints.storageEmulatorPort;
|
|
375
|
+
A(o, a, c), A(e, a, c), A(n, a, c);
|
|
299
376
|
}
|
|
300
|
-
const
|
|
301
|
-
return
|
|
377
|
+
const s = new x({ storageRaw: o, storageProcessed: e, storagePublic: n }), i = new it(this.auth, t, s, this._endpoints.gatewayUrl), r = new D(this.auth, this._endpoints.gatewayUrl, t, i);
|
|
378
|
+
return i._bindApi(r), r;
|
|
302
379
|
}
|
|
303
380
|
}
|
|
304
381
|
export {
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
382
|
+
z as Cue,
|
|
383
|
+
D as CueApi,
|
|
384
|
+
yt as CueAuth,
|
|
385
|
+
mt as CueNode,
|
|
386
|
+
Pt as CueProfile,
|
|
387
|
+
Et as CueProjects,
|
|
388
|
+
it as CueSyncApi
|
|
312
389
|
};
|