core-3nweb-client-lib 0.43.7 → 0.43.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/api-defs/files.d.ts +82 -4
- package/build/core/storage/synced/obj-files.d.ts +1 -0
- package/build/core/storage/synced/obj-files.js +16 -0
- package/build/core/storage/synced/obj-status.d.ts +1 -0
- package/build/core/storage/synced/obj-status.js +18 -0
- package/build/core/storage/synced/storage.d.ts +6 -2
- package/build/core/storage/synced/storage.js +12 -17
- package/build/core/storage/synced/upsyncer.d.ts +10 -1
- package/build/core/storage/synced/upsyncer.js +72 -9
- package/build/core-ipc/file.d.ts +27 -4
- package/build/core-ipc/file.js +90 -94
- package/build/core-ipc/fs.js +68 -132
- package/build/lib-client/fs-utils/files.js +6 -0
- package/build/lib-client/objs-on-disk/obj-on-disk.d.ts +1 -0
- package/build/lib-client/objs-on-disk/obj-on-disk.js +8 -1
- package/build/lib-client/xsp-fs/common.d.ts +13 -21
- package/build/lib-client/xsp-fs/common.js +4 -16
- package/build/lib-client/xsp-fs/file-node.d.ts +3 -2
- package/build/lib-client/xsp-fs/file-node.js +17 -11
- package/build/lib-client/xsp-fs/file.d.ts +5 -0
- package/build/lib-client/xsp-fs/file.js +26 -13
- package/build/lib-client/xsp-fs/folder-node.d.ts +7 -1
- package/build/lib-client/xsp-fs/folder-node.js +18 -10
- package/build/lib-client/xsp-fs/fs.d.ts +5 -0
- package/build/lib-client/xsp-fs/fs.js +33 -26
- package/build/lib-client/xsp-fs/link-node.d.ts +3 -0
- package/build/lib-client/xsp-fs/link-node.js +5 -1
- package/build/lib-client/xsp-fs/node-in-fs.d.ts +20 -6
- package/build/lib-client/xsp-fs/node-in-fs.js +69 -14
- package/build/lib-client/xsp-fs/node-persistence.d.ts +1 -0
- package/build/lib-client/xsp-fs/node-persistence.js +4 -0
- package/build/lib-common/big-endian.js +2 -2
- package/build/lib-common/processes/labelled-exec-pools.js +1 -3
- package/build/protos/asmail.proto.js +1614 -1588
- package/build/protos/file.proto.js +1258 -541
- package/build/protos/fs.proto.js +1459 -1433
- package/package.json +4 -4
- package/protos/file.proto +31 -15
- package/protos/fs.proto +13 -22
|
@@ -43,6 +43,7 @@ declare namespace web3n.files {
|
|
|
43
43
|
notImplemented?: true;
|
|
44
44
|
attrsNotEnabledInFS?: true;
|
|
45
45
|
versionMismatch?: true;
|
|
46
|
+
storageIsNotSyncedType?: true;
|
|
46
47
|
isEndless?: true;
|
|
47
48
|
storageClosed?: true;
|
|
48
49
|
remoteNotSet?: true;
|
|
@@ -71,6 +72,7 @@ declare namespace web3n.files {
|
|
|
71
72
|
localVersion?: number;
|
|
72
73
|
remoteVersion?: number;
|
|
73
74
|
alreadyUploading?: true;
|
|
75
|
+
uploadTaskId?: number;
|
|
74
76
|
versionNotFound?: true;
|
|
75
77
|
childNeverUploaded?: true;
|
|
76
78
|
childName?: string;
|
|
@@ -147,8 +149,18 @@ declare namespace web3n.files {
|
|
|
147
149
|
*/
|
|
148
150
|
version?: number;
|
|
149
151
|
|
|
152
|
+
/**
|
|
153
|
+
* This gives a number of bytes that should be downloaded to have file
|
|
154
|
+
* completly on a device.
|
|
155
|
+
*/
|
|
156
|
+
bytesNeedDownload?: number;
|
|
157
|
+
|
|
158
|
+
versionSyncBranch?: SyncBranch;
|
|
159
|
+
|
|
150
160
|
}
|
|
151
161
|
|
|
162
|
+
type SyncBranch = 'local' | 'synced' | 'remote';
|
|
163
|
+
|
|
152
164
|
/**
|
|
153
165
|
* Sync status contains info about possible version branches with possible
|
|
154
166
|
* states:
|
|
@@ -400,7 +412,7 @@ declare namespace web3n.files {
|
|
|
400
412
|
*/
|
|
401
413
|
getByteSource(): Promise<FileByteSource>;
|
|
402
414
|
|
|
403
|
-
watch(observer: Observer<FileEvent|RemoteEvent>): () => void;
|
|
415
|
+
watch(observer: Observer<FileEvent|RemoteEvent|UploadEvent>): () => void;
|
|
404
416
|
|
|
405
417
|
}
|
|
406
418
|
|
|
@@ -464,6 +476,12 @@ declare namespace web3n.files {
|
|
|
464
476
|
|
|
465
477
|
interface ReadonlyFileVersionedAPI {
|
|
466
478
|
|
|
479
|
+
/**
|
|
480
|
+
* This returns a promise, resolvable to stats of the file.
|
|
481
|
+
* @param flags are optional flags to read archived or remote versions.
|
|
482
|
+
*/
|
|
483
|
+
stat(flags?: VersionedReadFlags): Promise<Stats>;
|
|
484
|
+
|
|
467
485
|
getXAttr(
|
|
468
486
|
xaName: string, flags?: VersionedReadFlags
|
|
469
487
|
): Promise<{ attr: any; version: number; }>;
|
|
@@ -516,6 +534,8 @@ declare namespace web3n.files {
|
|
|
516
534
|
listVersions(
|
|
517
535
|
flags?: VersionedReadFlags
|
|
518
536
|
): Promise<{ current?: number; archived?: number[]; }>;
|
|
537
|
+
// XXX
|
|
538
|
+
// ): Promise<{ current?: number; archived?: number[]; synced?: number; conflictingRemote?: number[]; }>;
|
|
519
539
|
|
|
520
540
|
/**
|
|
521
541
|
* File from synced storage has this api
|
|
@@ -639,6 +659,19 @@ declare namespace web3n.files {
|
|
|
639
659
|
|
|
640
660
|
interface WritableFileSyncAPI extends ReadonlyFileSyncAPI {
|
|
641
661
|
|
|
662
|
+
/**
|
|
663
|
+
* This starts/schedules an upload, if it hasn't been already.
|
|
664
|
+
* Upload in conflicting and behind state of sync requires explicit upload version.
|
|
665
|
+
* Undefined is returned when upload is not needed, e.g. version is already synced.
|
|
666
|
+
* Upload version and upload task id are returned together with an indicator of whether
|
|
667
|
+
* this call has started upload, or it has already been going on.
|
|
668
|
+
* Upload task id can be used to watch upload process.
|
|
669
|
+
* @param opts
|
|
670
|
+
*/
|
|
671
|
+
startUpload(
|
|
672
|
+
opts?: OptionsToUploadLocal
|
|
673
|
+
): Promise<{ uploadVersion: number; uploadTaskId: number; }|undefined>;
|
|
674
|
+
|
|
642
675
|
/**
|
|
643
676
|
* Upload in conflicting and behind state of sync requires explicit upload version.
|
|
644
677
|
* @param opts
|
|
@@ -730,16 +763,16 @@ declare namespace web3n.files {
|
|
|
730
763
|
readLink(path: string): Promise<SymLink>;
|
|
731
764
|
|
|
732
765
|
watchFolder(
|
|
733
|
-
path: string, observer: Observer<FolderEvent|RemoteEvent>
|
|
766
|
+
path: string, observer: Observer<FolderEvent|RemoteEvent|UploadEvent>
|
|
734
767
|
): () => void;
|
|
735
768
|
|
|
736
769
|
watchFile(
|
|
737
|
-
path: string, observer: Observer<FileEvent|RemoteEvent>
|
|
770
|
+
path: string, observer: Observer<FileEvent|RemoteEvent|UploadEvent>
|
|
738
771
|
): () => void;
|
|
739
772
|
|
|
740
773
|
watchTree(
|
|
741
774
|
path: string, depth: number|undefined,
|
|
742
|
-
observer: Observer<FolderEvent|FileEvent|RemoteEvent>
|
|
775
|
+
observer: Observer<FolderEvent|FileEvent|RemoteEvent|UploadEvent>
|
|
743
776
|
): () => void;
|
|
744
777
|
|
|
745
778
|
close(): Promise<void>;
|
|
@@ -1082,6 +1115,14 @@ declare namespace web3n.files {
|
|
|
1082
1115
|
|
|
1083
1116
|
interface ReadonlyFSVersionedAPI {
|
|
1084
1117
|
|
|
1118
|
+
/**
|
|
1119
|
+
* This returns a promise, resolvable to stats of an entity at a given
|
|
1120
|
+
* path.
|
|
1121
|
+
* @param path
|
|
1122
|
+
* @param flags are optional flags to read archived or remote versions.
|
|
1123
|
+
*/
|
|
1124
|
+
stat(path: string, flags?: VersionedReadFlags): Promise<Stats>;
|
|
1125
|
+
|
|
1085
1126
|
getXAttr(
|
|
1086
1127
|
path: string, xaName: string, flags?: VersionedReadFlags
|
|
1087
1128
|
): Promise<{ attr: any; version: number; }>;
|
|
@@ -1293,6 +1334,20 @@ declare namespace web3n.files {
|
|
|
1293
1334
|
|
|
1294
1335
|
interface WritableFSSyncAPI extends ReadonlyFSSyncAPI {
|
|
1295
1336
|
|
|
1337
|
+
/**
|
|
1338
|
+
* This starts/schedules an upload, if it hasn't been already.
|
|
1339
|
+
* Upload in conflicting and behind state of sync requires explicit upload version.
|
|
1340
|
+
* Undefined is returned when upload is not needed, e.g. version is already synced.
|
|
1341
|
+
* Upload version and upload task id are returned together with an indicator of whether
|
|
1342
|
+
* this call has started upload, or it has already been going on.
|
|
1343
|
+
* Upload task id can be used to watch upload process.
|
|
1344
|
+
* @param path
|
|
1345
|
+
* @param opts
|
|
1346
|
+
*/
|
|
1347
|
+
startUpload(
|
|
1348
|
+
path: string, opts?: OptionsToUploadLocal
|
|
1349
|
+
): Promise<{ uploadVersion: number; uploadTaskId: number; }|undefined>;
|
|
1350
|
+
|
|
1296
1351
|
/**
|
|
1297
1352
|
* Upload in conflicting and behind state of sync requires explicit upload version.
|
|
1298
1353
|
* @param path
|
|
@@ -1392,4 +1447,27 @@ declare namespace web3n.files {
|
|
|
1392
1447
|
newVersion: number;
|
|
1393
1448
|
}
|
|
1394
1449
|
|
|
1450
|
+
type UploadEvent = UploadStartEvent | UploadProgressEvent | UploadDoneEvent;
|
|
1451
|
+
|
|
1452
|
+
interface UploadEventBase extends FSEvent {
|
|
1453
|
+
uploadTaskId: number;
|
|
1454
|
+
localVersion: number;
|
|
1455
|
+
uploadVersion: number;
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
interface UploadStartEvent extends UploadEventBase {
|
|
1459
|
+
type: 'upload-started';
|
|
1460
|
+
totalBytesToUpload: number;
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
interface UploadProgressEvent extends UploadEventBase {
|
|
1464
|
+
type: 'upload-progress';
|
|
1465
|
+
totalBytesToUpload: number;
|
|
1466
|
+
bytesLeftToUpload: number;
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
interface UploadDoneEvent extends UploadEventBase {
|
|
1470
|
+
type: 'upload-done';
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1395
1473
|
}
|
|
@@ -90,5 +90,6 @@ export declare class SyncedObj {
|
|
|
90
90
|
statusObj(): ObjStatus;
|
|
91
91
|
recordRemovalUploadAndGC(): Promise<void>;
|
|
92
92
|
isRemoteVersionOnDisk(version: number): Promise<'complete' | 'partial' | 'none'>;
|
|
93
|
+
getNumOfBytesNeedingDownload(version: number): Promise<number | 'unknown'>;
|
|
93
94
|
downloadRemoteVersion(version: number): Promise<void>;
|
|
94
95
|
}
|
|
@@ -408,6 +408,22 @@ class SyncedObj {
|
|
|
408
408
|
const objVer = await this.instanceOfRemoteObjVer(version);
|
|
409
409
|
return (objVer.doesFileNeedDownload() ? 'complete' : 'partial');
|
|
410
410
|
}
|
|
411
|
+
async getNumOfBytesNeedingDownload(version) {
|
|
412
|
+
if (!this.status.isAmongRemote(version)) {
|
|
413
|
+
if (this.localVers.has(version)) {
|
|
414
|
+
return 0;
|
|
415
|
+
}
|
|
416
|
+
else {
|
|
417
|
+
throw (0, exceptions_1.makeObjVersionNotFoundExc)(this.objId, version);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
const verPath = this.remoteVerPath(version);
|
|
421
|
+
if (!(await isOnDisk(verPath))) {
|
|
422
|
+
return 'unknown';
|
|
423
|
+
}
|
|
424
|
+
const objVer = await this.instanceOfRemoteObjVer(version);
|
|
425
|
+
return objVer.numOfBytesNeedingDownload();
|
|
426
|
+
}
|
|
411
427
|
async downloadRemoteVersion(version) {
|
|
412
428
|
const objVer = await this.instanceOfRemoteObjVer(version);
|
|
413
429
|
await objVer.downloadMissingSections();
|
|
@@ -149,4 +149,5 @@ export declare class ObjStatus implements SyncedObjStatus, UploadStatusRecorder
|
|
|
149
149
|
isAmongRemote(version: number): boolean;
|
|
150
150
|
}
|
|
151
151
|
export declare function readAndCheckStatus(objFolder: string, objId: ObjId): Promise<ObjStatusInfo>;
|
|
152
|
+
export declare function countBytesIn(info: UploadInfo): number;
|
|
152
153
|
export {};
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
19
|
exports.ObjStatus = exports.STATUS_FILE_NAME = void 0;
|
|
20
20
|
exports.readAndCheckStatus = readAndCheckStatus;
|
|
21
|
+
exports.countBytesIn = countBytesIn;
|
|
21
22
|
const path_1 = require("path");
|
|
22
23
|
const exceptions_1 = require("../../../lib-client/xsp-fs/exceptions");
|
|
23
24
|
const json_saving_1 = require("../common/json-saving");
|
|
@@ -717,4 +718,21 @@ function uploadInStatus(upload) {
|
|
|
717
718
|
};
|
|
718
719
|
}
|
|
719
720
|
}
|
|
721
|
+
function countBytesIn(info) {
|
|
722
|
+
if ((info.type === 'removal') || !info.needUpload) {
|
|
723
|
+
return 0;
|
|
724
|
+
}
|
|
725
|
+
if (info.needUpload.type === 'ordered-whole') {
|
|
726
|
+
const { segsLeft, header } = info.needUpload;
|
|
727
|
+
return (segsLeft + (header !== null && header !== void 0 ? header : 0));
|
|
728
|
+
}
|
|
729
|
+
else {
|
|
730
|
+
const { header, newSegsLeft } = info.needUpload;
|
|
731
|
+
let count = (header !== null && header !== void 0 ? header : 0);
|
|
732
|
+
for (const { len } of newSegsLeft) {
|
|
733
|
+
count += len;
|
|
734
|
+
}
|
|
735
|
+
return count;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
720
738
|
Object.freeze(exports);
|
|
@@ -10,6 +10,7 @@ type FolderEvent = web3n.files.FolderEvent;
|
|
|
10
10
|
type FileEvent = web3n.files.FileEvent;
|
|
11
11
|
type SyncStatus = web3n.files.SyncStatus;
|
|
12
12
|
type OptionsToAdopteRemote = web3n.files.OptionsToAdopteRemote;
|
|
13
|
+
type UploadEvent = web3n.files.UploadEvent;
|
|
13
14
|
export declare class SyncedStore implements ISyncedStorage {
|
|
14
15
|
private readonly files;
|
|
15
16
|
private readonly remoteStorage;
|
|
@@ -37,10 +38,12 @@ export declare class SyncedStore implements ISyncedStorage {
|
|
|
37
38
|
status(objId: ObjId): Promise<SyncedObjStatus>;
|
|
38
39
|
adoptRemote(objId: ObjId, opts: OptionsToAdopteRemote | undefined): Promise<number | undefined>;
|
|
39
40
|
updateStatusInfo(objId: ObjId): Promise<SyncStatus>;
|
|
40
|
-
isObjOnDisk(objId: ObjId): Promise<boolean>;
|
|
41
41
|
isRemoteVersionOnDisk(objId: ObjId, version: number): Promise<'complete' | 'partial' | 'none'>;
|
|
42
42
|
download(objId: ObjId, version: number): Promise<void>;
|
|
43
|
-
|
|
43
|
+
startUpload(objId: ObjId, localVersion: number, uploadVersion: number, uploadHeader: UploadHeaderChange | undefined, createOnRemote: boolean, eventSink: (event: UploadEvent) => void): Promise<{
|
|
44
|
+
uploadTaskId: number;
|
|
45
|
+
completion: Promise<void>;
|
|
46
|
+
}>;
|
|
44
47
|
uploadObjRemoval(objId: ObjId): Promise<void>;
|
|
45
48
|
dropCachedLocalObjVersionsLessOrEqual(objId: ObjId, version: number): void;
|
|
46
49
|
archiveVersionOnServer(objId: ObjId, version: number): Promise<void>;
|
|
@@ -52,6 +55,7 @@ export declare class SyncedStore implements ISyncedStorage {
|
|
|
52
55
|
getObjSrcOfRemoteVersion(objId: ObjId, version: number): Promise<ObjSource>;
|
|
53
56
|
saveObj(objId: ObjId, version: number, encSub: Subscribe): Promise<void>;
|
|
54
57
|
removeObj(objId: string): Promise<void>;
|
|
58
|
+
getNumOfBytesNeedingDownload(objId: string, version: number): Promise<number | 'unknown'>;
|
|
55
59
|
close(): Promise<void>;
|
|
56
60
|
suspendNetworkActivity(): void;
|
|
57
61
|
resumeNetworkActivity(): void;
|
|
@@ -114,10 +114,6 @@ class SyncedStore {
|
|
|
114
114
|
throw exc;
|
|
115
115
|
}
|
|
116
116
|
}
|
|
117
|
-
async isObjOnDisk(objId) {
|
|
118
|
-
const obj = await this.files.findObj(objId);
|
|
119
|
-
return !!obj;
|
|
120
|
-
}
|
|
121
117
|
async isRemoteVersionOnDisk(objId, version) {
|
|
122
118
|
const obj = await this.getObjOrThrow(objId, true);
|
|
123
119
|
return obj.isRemoteVersionOnDisk(version);
|
|
@@ -126,20 +122,15 @@ class SyncedStore {
|
|
|
126
122
|
const obj = await this.getObjOrThrow(objId, true);
|
|
127
123
|
return obj.downloadRemoteVersion(version);
|
|
128
124
|
}
|
|
129
|
-
async
|
|
125
|
+
async startUpload(objId, localVersion, uploadVersion, uploadHeader, createOnRemote, eventSink) {
|
|
130
126
|
const obj = await this.getObjOrThrow(objId, true);
|
|
131
|
-
const
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
originalHeader: uploadHeader.localHeader
|
|
139
|
-
} : undefined));
|
|
140
|
-
if (localVersion > uploadVersion) {
|
|
141
|
-
await obj.removeLocalVersionFilesLessThan(localVersion);
|
|
142
|
-
}
|
|
127
|
+
const { completion, uploadTaskId } = await this.uploader.startUploadFromDisk(obj, localVersion, uploadVersion, uploadHeader, createOnRemote, eventSink);
|
|
128
|
+
return {
|
|
129
|
+
uploadTaskId,
|
|
130
|
+
completion: completion.then(() => {
|
|
131
|
+
this.dropCachedLocalObjVersionsLessOrEqual(objId, localVersion);
|
|
132
|
+
})
|
|
133
|
+
};
|
|
143
134
|
}
|
|
144
135
|
async uploadObjRemoval(objId) {
|
|
145
136
|
if (!objId) {
|
|
@@ -233,6 +224,10 @@ class SyncedStore {
|
|
|
233
224
|
}
|
|
234
225
|
await obj.removeCurrentVersion();
|
|
235
226
|
}
|
|
227
|
+
async getNumOfBytesNeedingDownload(objId, version) {
|
|
228
|
+
const obj = await this.getObjOrThrow(objId);
|
|
229
|
+
return obj.getNumOfBytesNeedingDownload(version);
|
|
230
|
+
}
|
|
236
231
|
async close() {
|
|
237
232
|
try {
|
|
238
233
|
await this.uploader.stop();
|
|
@@ -4,11 +4,16 @@ import { MonoTypeOperatorFunction } from "rxjs";
|
|
|
4
4
|
import { FileWrite } from "../../../lib-client/objs-on-disk/file-writing-proc";
|
|
5
5
|
import { LogError } from "../../../lib-client/logging/log-to-file";
|
|
6
6
|
import { NewVersionUpload } from "./obj-status";
|
|
7
|
+
import { UploadHeaderChange } from "../../../lib-client/xsp-fs/common";
|
|
8
|
+
type UploadEvent = web3n.files.UploadEvent;
|
|
7
9
|
export type FileWriteTapOperator = MonoTypeOperatorFunction<FileWrite[]>;
|
|
8
10
|
export declare class UpSyncer {
|
|
9
11
|
private readonly remoteStorage;
|
|
10
12
|
private readonly logError;
|
|
11
13
|
private readonly execPools;
|
|
14
|
+
private readonly tasksByObjIds;
|
|
15
|
+
private readonly tasksByIds;
|
|
16
|
+
private readonly syncedUploadStarts;
|
|
12
17
|
constructor(remoteStorage: StorageOwner, logError: LogError);
|
|
13
18
|
start(): void;
|
|
14
19
|
stop(): Promise<void>;
|
|
@@ -18,10 +23,14 @@ export declare class UpSyncer {
|
|
|
18
23
|
*/
|
|
19
24
|
tapFileWrite(obj: SyncedObj, isNew: boolean, newVersion: number, baseVersion?: number): FileWriteTapOperator;
|
|
20
25
|
removeCurrentVersionOf(obj: SyncedObj): Promise<void>;
|
|
21
|
-
|
|
26
|
+
startUploadFromDisk(obj: SyncedObj, localVersion: number, uploadVersion: number, uploadHeader: UploadHeaderChange | undefined, createOnRemote: boolean, eventSink: (event: UploadEvent) => void): Promise<{
|
|
27
|
+
completion: Promise<void>;
|
|
28
|
+
uploadTaskId: number;
|
|
29
|
+
}>;
|
|
22
30
|
}
|
|
23
31
|
export interface UploadStatusRecorder {
|
|
24
32
|
recordUploadStart(info: NewVersionUpload): Promise<void>;
|
|
25
33
|
recordUploadCancellation(info: NewVersionUpload): Promise<void>;
|
|
26
34
|
recordUploadInterimState(info: NewVersionUpload): Promise<void>;
|
|
27
35
|
}
|
|
36
|
+
export {};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/*
|
|
3
|
-
Copyright (C) 2020, 2022 3NSoft Inc.
|
|
3
|
+
Copyright (C) 2020, 2022, 2025 3NSoft Inc.
|
|
4
4
|
|
|
5
5
|
This program is free software: you can redistribute it and/or modify it under
|
|
6
6
|
the terms of the GNU General Public License as published by the Free Software
|
|
@@ -20,14 +20,19 @@ exports.UpSyncer = void 0;
|
|
|
20
20
|
const labelled_exec_pools_1 = require("../../../lib-common/processes/labelled-exec-pools");
|
|
21
21
|
const exceptions_1 = require("../../../lib-client/xsp-fs/exceptions");
|
|
22
22
|
const assert_1 = require("../../../lib-common/assert");
|
|
23
|
+
const obj_status_1 = require("./obj-status");
|
|
23
24
|
const deferred_1 = require("../../../lib-common/processes/deferred");
|
|
24
25
|
const buffer_utils_1 = require("../../../lib-common/buffer-utils");
|
|
26
|
+
const synced_1 = require("../../../lib-common/processes/synced");
|
|
25
27
|
const MAX_CHUNK_SIZE = 512 * 1024;
|
|
26
28
|
const MAX_FAST_UPLOAD = 2 * 1024 * 1024;
|
|
27
29
|
class UpSyncer {
|
|
28
30
|
constructor(remoteStorage, logError) {
|
|
29
31
|
this.remoteStorage = remoteStorage;
|
|
30
32
|
this.logError = logError;
|
|
33
|
+
this.tasksByObjIds = new Map();
|
|
34
|
+
this.tasksByIds = new Map();
|
|
35
|
+
this.syncedUploadStarts = new synced_1.NamedProcs();
|
|
31
36
|
this.execPools = new labelled_exec_pools_1.LabelledExecPools([
|
|
32
37
|
{ label: 'long', maxProcs: 1 },
|
|
33
38
|
{ label: 'fast', maxProcs: 1 }
|
|
@@ -62,17 +67,54 @@ class UpSyncer {
|
|
|
62
67
|
}
|
|
63
68
|
await obj.recordRemovalUploadAndGC();
|
|
64
69
|
}
|
|
65
|
-
async
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
|
|
70
|
+
async startUploadFromDisk(obj, localVersion, uploadVersion, uploadHeader, createOnRemote, eventSink) {
|
|
71
|
+
const foundTask = this.tasksByObjIds.get(obj.objId);
|
|
72
|
+
if (foundTask) {
|
|
73
|
+
throw (0, exceptions_1.makeFSSyncException)('', {
|
|
74
|
+
alreadyUploading: true,
|
|
75
|
+
uploadTaskId: foundTask.taskId
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
const uploadStart = this.syncedUploadStarts.latestTaskAtThisMoment(obj.objId);
|
|
79
|
+
if (uploadStart) {
|
|
80
|
+
return uploadStart;
|
|
81
|
+
}
|
|
82
|
+
return this.syncedUploadStarts.start(obj.objId, async () => {
|
|
83
|
+
let uploadTaskId;
|
|
84
|
+
do {
|
|
85
|
+
uploadTaskId = Math.floor(Number.MAX_SAFE_INTEGER * Math.random());
|
|
86
|
+
} while (this.tasksByIds.has(uploadTaskId));
|
|
87
|
+
const syncedBase = await obj.combineLocalBaseIfPresent(localVersion);
|
|
88
|
+
if (uploadHeader) {
|
|
89
|
+
await obj.saveUploadHeaderFile(uploadHeader);
|
|
90
|
+
}
|
|
91
|
+
const task = await UploadTask.for(obj, localVersion, uploadVersion, uploadHeader === null || uploadHeader === void 0 ? void 0 : uploadHeader.uploadHeader, syncedBase, createOnRemote, uploadTaskId, eventSink, this.remoteStorage, this.execPools, async () => {
|
|
92
|
+
if (this.tasksByIds.delete(task.taskId)) {
|
|
93
|
+
this.tasksByObjIds.delete(task.objId);
|
|
94
|
+
}
|
|
95
|
+
await obj.recordUploadCompletion(localVersion, uploadVersion, (uploadHeader ? {
|
|
96
|
+
newHeader: uploadHeader.uploadHeader,
|
|
97
|
+
originalHeader: uploadHeader.localHeader
|
|
98
|
+
} : undefined));
|
|
99
|
+
if (localVersion > uploadVersion) {
|
|
100
|
+
await obj.removeLocalVersionFilesLessThan(localVersion);
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
this.tasksByIds.set(uploadTaskId, task);
|
|
104
|
+
this.tasksByObjIds.set(task.objId, task);
|
|
105
|
+
const completion = task.completion();
|
|
106
|
+
completion.catch(noop);
|
|
107
|
+
this.execPools.add(task);
|
|
108
|
+
return { completion, uploadTaskId };
|
|
109
|
+
});
|
|
69
110
|
}
|
|
70
111
|
}
|
|
71
112
|
exports.UpSyncer = UpSyncer;
|
|
72
113
|
Object.freeze(UpSyncer.prototype);
|
|
73
114
|
Object.freeze(UpSyncer);
|
|
74
115
|
class UploadTask {
|
|
75
|
-
constructor(remoteStorage, objId, objStatus, src, execPools, info, uploadHeader) {
|
|
116
|
+
constructor(taskId, remoteStorage, objId, objStatus, src, execPools, info, uploadHeader, doAtCompletion, eventSink) {
|
|
117
|
+
this.taskId = taskId;
|
|
76
118
|
this.remoteStorage = remoteStorage;
|
|
77
119
|
this.objId = objId;
|
|
78
120
|
this.objStatus = objStatus;
|
|
@@ -80,11 +122,15 @@ class UploadTask {
|
|
|
80
122
|
this.execPools = execPools;
|
|
81
123
|
this.info = info;
|
|
82
124
|
this.uploadHeader = uploadHeader;
|
|
125
|
+
this.doAtCompletion = doAtCompletion;
|
|
126
|
+
this.eventSink = eventSink;
|
|
83
127
|
this.uploadCompletion = (0, deferred_1.defer)();
|
|
84
128
|
this.execLabel = executorLabelFor(this.info.needUpload);
|
|
129
|
+
this.totalBytesToUpload = (0, obj_status_1.countBytesIn)(this.info);
|
|
130
|
+
this.emitUploadEvent('upload-started', { totalBytesToUpload: this.totalBytesToUpload });
|
|
85
131
|
Object.seal(this);
|
|
86
132
|
}
|
|
87
|
-
static async for(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createObj, remoteStorage, execPools) {
|
|
133
|
+
static async for(obj, localVersion, uploadVersion, uploadHeader, syncedBase, createObj, taskId, eventSink, remoteStorage, execPools, doAtCompletion) {
|
|
88
134
|
const src = await obj.getObjSrcFromLocalAndSyncedBranch(localVersion);
|
|
89
135
|
let needUpload;
|
|
90
136
|
if (syncedBase) {
|
|
@@ -103,7 +149,7 @@ class UploadTask {
|
|
|
103
149
|
};
|
|
104
150
|
const objStatus = obj.statusObj();
|
|
105
151
|
await objStatus.recordUploadStart(info);
|
|
106
|
-
return new UploadTask(remoteStorage, obj.objId, objStatus, src, execPools, info, uploadHeader);
|
|
152
|
+
return new UploadTask(taskId, remoteStorage, obj.objId, objStatus, src, execPools, info, uploadHeader, doAtCompletion, eventSink);
|
|
107
153
|
}
|
|
108
154
|
neededExecutor() {
|
|
109
155
|
return (!this.info.needUpload ? undefined : this.execLabel);
|
|
@@ -111,6 +157,16 @@ class UploadTask {
|
|
|
111
157
|
completion() {
|
|
112
158
|
return this.uploadCompletion.promise;
|
|
113
159
|
}
|
|
160
|
+
emitUploadEvent(type, fields) {
|
|
161
|
+
this.eventSink({
|
|
162
|
+
type,
|
|
163
|
+
localVersion: this.info.localVersion,
|
|
164
|
+
uploadTaskId: this.taskId,
|
|
165
|
+
uploadVersion: this.info.uploadVersion,
|
|
166
|
+
path: '',
|
|
167
|
+
...fields
|
|
168
|
+
});
|
|
169
|
+
}
|
|
114
170
|
async process() {
|
|
115
171
|
if (!this.info.needUpload) {
|
|
116
172
|
return;
|
|
@@ -138,10 +194,17 @@ class UploadTask {
|
|
|
138
194
|
}
|
|
139
195
|
await this.objStatus.recordUploadInterimState(this.info);
|
|
140
196
|
if (this.info.needUpload) {
|
|
197
|
+
this.emitUploadEvent('upload-progress', {
|
|
198
|
+
totalBytesToUpload: this.totalBytesToUpload,
|
|
199
|
+
bytesLeftToUpload: (0, obj_status_1.countBytesIn)(this.info)
|
|
200
|
+
});
|
|
141
201
|
this.execPools.add(this);
|
|
142
202
|
}
|
|
143
203
|
else {
|
|
144
|
-
this.
|
|
204
|
+
await this.doAtCompletion().finally(() => {
|
|
205
|
+
this.uploadCompletion.resolve();
|
|
206
|
+
this.emitUploadEvent('upload-done', {});
|
|
207
|
+
});
|
|
145
208
|
}
|
|
146
209
|
}
|
|
147
210
|
catch (exc) {
|
package/build/core-ipc/file.d.ts
CHANGED
|
@@ -5,12 +5,11 @@ type ReadonlyFile = web3n.files.ReadonlyFile;
|
|
|
5
5
|
type ReadonlyFileVersionedAPI = web3n.files.ReadonlyFileVersionedAPI;
|
|
6
6
|
type WritableFile = web3n.files.WritableFile;
|
|
7
7
|
type WritableFileVersionedAPI = web3n.files.WritableFileVersionedAPI;
|
|
8
|
+
type WritableFileSyncAPI = web3n.files.WritableFileSyncAPI;
|
|
8
9
|
type File = web3n.files.File;
|
|
9
10
|
type Stats = web3n.files.Stats;
|
|
10
11
|
type SyncStatus = web3n.files.SyncStatus;
|
|
11
12
|
type XAttrsChanges = web3n.files.XAttrsChanges;
|
|
12
|
-
type FileEvent = web3n.files.FileEvent;
|
|
13
|
-
type RemoteEvent = web3n.files.RemoteEvent;
|
|
14
13
|
type OptionsToAdopteRemote = web3n.files.OptionsToAdopteRemote;
|
|
15
14
|
type OptionsToUploadLocal = web3n.files.OptionsToUploadLocal;
|
|
16
15
|
type VersionedReadFlags = web3n.files.VersionedReadFlags;
|
|
@@ -39,14 +38,18 @@ export declare namespace readBytes {
|
|
|
39
38
|
}
|
|
40
39
|
export declare function packJSON(json: any): EnvelopeBody;
|
|
41
40
|
export declare function unpackJSON(buf: EnvelopeBody): any;
|
|
42
|
-
export declare function
|
|
43
|
-
export declare function
|
|
41
|
+
export declare function packEvent<T extends object>(e: T): Buffer;
|
|
42
|
+
export declare function unpackEvent<T extends object>(buf: EnvelopeBody): T;
|
|
44
43
|
export interface VersionedReadFlagsMsg {
|
|
45
44
|
archivedVersion?: Value<number>;
|
|
46
45
|
remoteVersion?: Value<number>;
|
|
47
46
|
}
|
|
48
47
|
export declare function versionedReadFlagsFromMsg(msg: VersionedReadFlagsMsg | undefined): VersionedReadFlags | undefined;
|
|
49
48
|
export declare function versionedReadFlagsToMsg(flags: VersionedReadFlags | undefined): VersionedReadFlagsMsg | undefined;
|
|
49
|
+
export declare namespace vStat {
|
|
50
|
+
function wrapService(fn: ReadonlyFileVersionedAPI['stat']): ExposedFn;
|
|
51
|
+
function makeCaller(caller: Caller, objPath: string[]): ReadonlyFileVersionedAPI['stat'];
|
|
52
|
+
}
|
|
50
53
|
export declare namespace vGetXAttr {
|
|
51
54
|
interface Reply {
|
|
52
55
|
version: number;
|
|
@@ -150,6 +153,26 @@ export interface OptionsToUploadLocalMsg {
|
|
|
150
153
|
}
|
|
151
154
|
export declare function optionsToUploadLocalToMsg(opts?: OptionsToUploadLocal): OptionsToUploadLocalMsg | undefined;
|
|
152
155
|
export declare function optionsToUploadLocalFromMsg(opts?: OptionsToUploadLocalMsg): OptionsToUploadLocal | undefined;
|
|
156
|
+
export declare namespace vsStartUpload {
|
|
157
|
+
const requestType: ProtoType<{
|
|
158
|
+
opts?: OptionsToUploadLocalMsg;
|
|
159
|
+
}>;
|
|
160
|
+
const replyType: ProtoType<{
|
|
161
|
+
startedUpload?: {
|
|
162
|
+
uploadVersion: number;
|
|
163
|
+
uploadTaskId: number;
|
|
164
|
+
};
|
|
165
|
+
}>;
|
|
166
|
+
function wrapService(fn: WritableFileSyncAPI['startUpload']): ExposedFn;
|
|
167
|
+
function makeCaller(caller: Caller, objPath: string[]): WritableFileSyncAPI['startUpload'];
|
|
168
|
+
}
|
|
169
|
+
export declare namespace vsUpload {
|
|
170
|
+
const replyType: ProtoType<{
|
|
171
|
+
uploadedVersion?: Value<number>;
|
|
172
|
+
}>;
|
|
173
|
+
function wrapService(fn: WritableFileSyncAPI['upload']): ExposedFn;
|
|
174
|
+
function makeCaller(caller: Caller, objPath: string[]): WritableFileSyncAPI['upload'];
|
|
175
|
+
}
|
|
153
176
|
export interface OptionsToAdopteRemoteMsg {
|
|
154
177
|
dropLocalVer?: Value<boolean>;
|
|
155
178
|
remoteVersion?: Value<number>;
|