core-3nweb-client-lib 0.25.6 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/build/api-defs/files.d.ts +23 -20
- package/build/core/asmail/config/common.js +2 -2
- package/build/core/asmail/delivery/index.js +4 -3
- package/build/core/asmail/delivery/msg.js +5 -4
- package/build/core/asmail/delivery/per-recipient-wip.js +2 -2
- package/build/core/asmail/inbox/attachments/fs.js +6 -0
- package/build/core/asmail/inbox/cached-msgs.js +2 -2
- package/build/core/asmail/inbox/inbox-events.js +2 -1
- package/build/core/asmail/inbox/index.js +2 -2
- package/build/core/asmail/inbox/msg-downloader.js +2 -2
- package/build/core/asmail/inbox/msg-indexing.js +3 -3
- package/build/core/asmail/inbox/msg-on-disk.js +2 -2
- package/build/core/asmail/keyring/keyring-storage.js +2 -2
- package/build/core/asmail/sending-params/own-params.js +2 -2
- package/build/core/asmail/sending-params/params-from-others.js +2 -2
- package/build/core/id-manager.js +2 -2
- package/build/core/sign-in.d.ts +5 -4
- package/build/core/sign-in.js +9 -11
- package/build/core/sign-up.d.ts +1 -0
- package/build/core/sign-up.js +7 -3
- package/build/core/storage/common/json-saving.d.ts +21 -0
- package/build/core/storage/common/json-saving.js +82 -0
- package/build/core/storage/common/obj-info-file.d.ts +43 -0
- package/build/core/storage/common/obj-info-file.js +119 -3
- package/build/core/storage/index.js +1 -1
- package/build/core/storage/local/obj-files-gc.js +8 -6
- package/build/core/storage/local/obj-files.d.ts +3 -3
- package/build/core/storage/local/obj-files.js +9 -9
- package/build/core/storage/local/obj-status.d.ts +9 -25
- package/build/core/storage/local/obj-status.js +28 -110
- package/build/core/storage/local/storage.d.ts +8 -1
- package/build/core/storage/local/storage.js +10 -2
- package/build/core/storage/synced/downloader.js +6 -5
- package/build/core/storage/synced/obj-files-gc.d.ts +1 -0
- package/build/core/storage/synced/obj-files-gc.js +44 -5
- package/build/core/storage/synced/obj-files.d.ts +13 -20
- package/build/core/storage/synced/obj-files.js +70 -48
- package/build/core/storage/synced/obj-status.d.ts +74 -15
- package/build/core/storage/synced/obj-status.js +291 -107
- package/build/core/storage/synced/remote-events.js +32 -26
- package/build/core/storage/synced/storage.d.ts +11 -1
- package/build/core/storage/synced/storage.js +28 -3
- package/build/core/storage/synced/upsyncer.d.ts +8 -7
- package/build/core/storage/synced/upsyncer.js +211 -163
- package/build/ipc-via-protobuf/asmail-cap.js +15 -32
- package/build/ipc-via-protobuf/connector-clients-side.js +2 -2
- package/build/ipc-via-protobuf/file.js +22 -14
- package/build/ipc-via-protobuf/fs.js +31 -33
- package/build/ipc-via-protobuf/startup-cap.js +5 -5
- package/build/lib-client/3nstorage/exceptions.d.ts +9 -8
- package/build/lib-client/3nstorage/exceptions.js +18 -9
- package/build/lib-client/3nstorage/service.js +10 -6
- package/build/lib-client/3nstorage/xsp-fs/common.d.ts +18 -4
- package/build/lib-client/3nstorage/xsp-fs/common.js +6 -1
- package/build/lib-client/3nstorage/xsp-fs/file-node.js +3 -3
- package/build/lib-client/3nstorage/xsp-fs/file.js +4 -1
- package/build/lib-client/3nstorage/xsp-fs/folder-node.js +27 -13
- package/build/lib-client/3nstorage/xsp-fs/fs.d.ts +11 -6
- package/build/lib-client/3nstorage/xsp-fs/fs.js +189 -58
- package/build/lib-client/3nstorage/xsp-fs/node-in-fs.d.ts +4 -0
- package/build/lib-client/3nstorage/xsp-fs/node-in-fs.js +51 -24
- package/build/lib-client/3nstorage/xsp-fs/node-persistence.js +2 -2
- package/build/lib-client/3nstorage/xsp-fs/xsp-payload-v1.js +2 -2
- package/build/lib-client/3nstorage/xsp-fs/xsp-payload-v2.js +3 -3
- package/build/lib-client/cryptor/cryptor-in-worker.js +4 -4
- package/build/lib-client/cryptor/cryptor-wasm.js +1 -1
- package/build/lib-client/cryptor/cryptor.wasm +0 -0
- package/build/lib-client/cryptor/in-proc-wasm.js +2 -2
- package/build/lib-client/files.js +2 -0
- package/build/lib-client/fs-collection.js +3 -2
- package/build/lib-client/logging/log-to-file.js +4 -4
- package/build/lib-client/objs-on-disk/file-writing-proc.js +2 -2
- package/build/lib-client/objs-on-disk/obj-folders.js +2 -2
- package/build/lib-client/request-utils.js +2 -2
- package/build/lib-client/server-events.js +4 -3
- package/build/lib-client/ws-utils.js +2 -2
- package/build/lib-common/async-fs-node.js +4 -3
- package/build/lib-common/byte-streaming/wrapping.js +17 -17
- package/build/lib-common/exceptions/file.js +6 -1
- package/build/lib-common/ipc/generic-ipc.js +2 -2
- package/build/lib-common/json-utils.js +2 -1
- package/build/lib-common/objs-on-disk/obj-file.js +4 -3
- package/build/lib-common/objs-on-disk/utils.js +2 -2
- package/build/lib-common/processes/deferred.d.ts +6 -0
- package/build/lib-common/processes/deferred.js +30 -0
- package/build/lib-common/processes/labelled-exec-pools.d.ts +33 -0
- package/build/lib-common/processes/labelled-exec-pools.js +141 -0
- package/build/lib-common/processes/pressure.d.ts +7 -0
- package/build/lib-common/processes/pressure.js +56 -0
- package/build/lib-common/processes/sleep.d.ts +1 -0
- package/build/lib-common/processes/sleep.js +26 -0
- package/build/lib-common/{processes.d.ts → processes/synced.d.ts} +0 -40
- package/build/lib-common/{processes.js → processes/synced.js} +187 -204
- package/build/lib-common/processes/timeout.d.ts +1 -0
- package/build/lib-common/processes/timeout.js +51 -0
- package/build/lib-common/service-api/3nstorage/owner.d.ts +5 -4
- package/build/lib-common/service-api/3nstorage/owner.js +3 -2
- package/build/lib-common/utils-for-observables.d.ts +15 -1
- package/build/lib-common/utils-for-observables.js +68 -17
- package/build/protos/asmail.proto.js +404 -78
- package/build/protos/file.proto.js +370 -44
- package/build/protos/fs.proto.js +404 -78
- package/package.json +4 -4
- package/protos/file.proto +10 -2
- package/protos/fs.proto +2 -2
- package/build/core/storage/synced/upsync-status.d.ts +0 -41
- package/build/core/storage/synced/upsync-status.js +0 -158
|
Binary file
|
|
@@ -22,7 +22,7 @@ const fs_1 = require("fs");
|
|
|
22
22
|
const path_1 = require("path");
|
|
23
23
|
const protobuf_type_1 = require("../protobuf-type");
|
|
24
24
|
const cryptor_proto_1 = require("../../protos/cryptor.proto");
|
|
25
|
-
const
|
|
25
|
+
const deferred_1 = require("../../lib-common/processes/deferred");
|
|
26
26
|
const assert_1 = require("../../lib-common/assert");
|
|
27
27
|
const error_1 = require("../../lib-common/exceptions/error");
|
|
28
28
|
function wasmBytes() {
|
|
@@ -77,7 +77,7 @@ function makeInProcessWasmCryptor() {
|
|
|
77
77
|
}
|
|
78
78
|
catch (err) { }
|
|
79
79
|
}
|
|
80
|
-
deferred =
|
|
80
|
+
deferred = deferred_1.defer();
|
|
81
81
|
if (interim) {
|
|
82
82
|
interimSink = interim;
|
|
83
83
|
}
|
|
@@ -255,6 +255,7 @@ function wrapIntoVersionlessReadonlyFS(fs, type) {
|
|
|
255
255
|
stat: async (path) => {
|
|
256
256
|
const stats = await fs.stat(path);
|
|
257
257
|
delete stats.version;
|
|
258
|
+
delete stats.sync;
|
|
258
259
|
return stats;
|
|
259
260
|
},
|
|
260
261
|
readonlyFile: async (path) => toVersionlessReadonlyFile(await fs.readonlyFile(path)),
|
|
@@ -291,6 +292,7 @@ function toVersionlessReadonlyFile(f) {
|
|
|
291
292
|
stat: async () => {
|
|
292
293
|
const stats = await f.stat();
|
|
293
294
|
delete stats.version;
|
|
295
|
+
delete stats.sync;
|
|
294
296
|
return stats;
|
|
295
297
|
},
|
|
296
298
|
watch: f.watch.bind(f),
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/*
|
|
3
|
-
Copyright (C) 2017 - 2018 3NSoft Inc.
|
|
3
|
+
Copyright (C) 2017 - 2018, 2022 3NSoft Inc.
|
|
4
4
|
|
|
5
5
|
This program is free software: you can redistribute it and/or modify it under
|
|
6
6
|
the terms of the GNU General Public License as published by the Free Software
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
19
|
exports.makeFSCollection = exports.readonlyWrapFSCollection = void 0;
|
|
20
20
|
const rxjs_1 = require("rxjs");
|
|
21
|
+
const utils_for_observables_1 = require("../lib-common/utils-for-observables");
|
|
21
22
|
class FSItemsCollection {
|
|
22
23
|
constructor() {
|
|
23
24
|
this.items = new Map();
|
|
@@ -69,7 +70,7 @@ class FSItemsCollection {
|
|
|
69
70
|
return allItems;
|
|
70
71
|
}
|
|
71
72
|
watch(observer) {
|
|
72
|
-
const sub = this.change$.subscribe(observer);
|
|
73
|
+
const sub = this.change$.subscribe(utils_for_observables_1.toRxObserver(observer));
|
|
73
74
|
return () => sub.unsubscribe();
|
|
74
75
|
}
|
|
75
76
|
wrap() {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
19
|
exports.makeLogger = exports.LOGS_FOLDER = void 0;
|
|
20
20
|
const error_1 = require("../../lib-common/exceptions/error");
|
|
21
|
-
const
|
|
21
|
+
const synced_1 = require("../../lib-common/processes/synced");
|
|
22
22
|
const async_fs_node_1 = require("../../lib-common/async-fs-node");
|
|
23
23
|
const path_1 = require("path");
|
|
24
24
|
exports.LOGS_FOLDER = 'logs';
|
|
@@ -63,7 +63,7 @@ ${error_1.stringifyErr(err)}`;
|
|
|
63
63
|
console.error(err2);
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
|
-
const loggingProc = new
|
|
66
|
+
const loggingProc = new synced_1.SingleProc();
|
|
67
67
|
function appendLog(s, now, appDomain) {
|
|
68
68
|
return loggingProc.startOrChain(async () => {
|
|
69
69
|
const logFile = path_1.join(utilDir, exports.LOGS_FOLDER, logFileName(now, appDomain));
|
|
@@ -115,11 +115,11 @@ ${error_1.stringifyErr(err)}`;
|
|
|
115
115
|
}
|
|
116
116
|
function recordUnhandledRejectionsInProcess() {
|
|
117
117
|
process.on('unhandledRejection', (err, p) => {
|
|
118
|
-
console.
|
|
118
|
+
console.error(' -- Unhandled rejection of promise:', err);
|
|
119
119
|
logError(err, 'Unhandled rejection of promise');
|
|
120
120
|
});
|
|
121
121
|
process.on('uncaughtException', err => {
|
|
122
|
-
console.
|
|
122
|
+
console.error(' -- Unhandled exception:', err);
|
|
123
123
|
logError(err, 'Unhandled exception');
|
|
124
124
|
});
|
|
125
125
|
}
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
19
|
exports.FileWritingProc = void 0;
|
|
20
|
-
const
|
|
20
|
+
const pressure_1 = require("../../lib-common/processes/pressure");
|
|
21
21
|
const rxjs_1 = require("rxjs");
|
|
22
22
|
const operators_1 = require("rxjs/operators");
|
|
23
23
|
const buffer_utils_1 = require("../../lib-common/buffer-utils");
|
|
@@ -27,7 +27,7 @@ const BACKPRESSURE_BUF_SIZE = 5 * 1024 * 1024;
|
|
|
27
27
|
class FileWritingProc {
|
|
28
28
|
constructor(objFile) {
|
|
29
29
|
this.objFile = objFile;
|
|
30
|
-
this.pValve = new
|
|
30
|
+
this.pValve = new pressure_1.PressureValve();
|
|
31
31
|
this.buffer = { segs: [], size: 0 };
|
|
32
32
|
Object.seal(this);
|
|
33
33
|
}
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
19
|
exports.ObjFolders = exports.makeAlreadyExistExc = exports.makeNotFoundExc = void 0;
|
|
20
20
|
const fs = require("../../lib-common/async-fs-node");
|
|
21
|
-
const
|
|
21
|
+
const synced_1 = require("../../lib-common/processes/synced");
|
|
22
22
|
const xsp_files_1 = require("xsp-files");
|
|
23
23
|
const assert_1 = require("../../lib-common/assert");
|
|
24
24
|
const error_1 = require("../../lib-common/exceptions/error");
|
|
@@ -74,7 +74,7 @@ class ObjFolders {
|
|
|
74
74
|
/**
|
|
75
75
|
* Process ids here are values of the first section in each folder tree.
|
|
76
76
|
*/
|
|
77
|
-
this.syncProcs = new
|
|
77
|
+
this.syncProcs = new synced_1.NamedProcs();
|
|
78
78
|
this.saveCfg = () => writeCfgTo(this.path, {
|
|
79
79
|
charsInSplit: this.charsInSplit,
|
|
80
80
|
numOfSplits: this.numOfSplits,
|
|
@@ -22,7 +22,7 @@ const https = require("https");
|
|
|
22
22
|
const url_1 = require("url");
|
|
23
23
|
const rxjs_1 = require("rxjs");
|
|
24
24
|
const buffer_utils_1 = require("../lib-common/buffer-utils");
|
|
25
|
-
const
|
|
25
|
+
const deferred_1 = require("../lib-common/processes/deferred");
|
|
26
26
|
const operators_1 = require("rxjs/operators");
|
|
27
27
|
exports.SESSION_ID_HEADER = "X-Session-Id";
|
|
28
28
|
exports.CONTENT_TYPE_HEADER = 'Content-Type';
|
|
@@ -109,7 +109,7 @@ function isTimeoutErr(err, opts, currentAttempt) {
|
|
|
109
109
|
}
|
|
110
110
|
async function readAllBytesFrom(stream) {
|
|
111
111
|
const buf = new bytes_fifo_buffer_1.BytesFIFOBuffer();
|
|
112
|
-
const deferred =
|
|
112
|
+
const deferred = deferred_1.defer();
|
|
113
113
|
stream.on('error', e => deferred.reject(e));
|
|
114
114
|
stream.on('data', chunk => buf.push(chunk));
|
|
115
115
|
stream.on('end', () => deferred.resolve(buf.getBytes(undefined)));
|
|
@@ -17,7 +17,8 @@
|
|
|
17
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
18
|
exports.ServerEvents = void 0;
|
|
19
19
|
const rxjs_1 = require("rxjs");
|
|
20
|
-
const
|
|
20
|
+
const synced_1 = require("../lib-common/processes/synced");
|
|
21
|
+
const sleep_1 = require("../lib-common/processes/sleep");
|
|
21
22
|
const operators_1 = require("rxjs/operators");
|
|
22
23
|
const error_1 = require("../lib-common/exceptions/error");
|
|
23
24
|
class ServerEvents {
|
|
@@ -25,7 +26,7 @@ class ServerEvents {
|
|
|
25
26
|
this.subscribeToServer = subscribeToServer;
|
|
26
27
|
this.restartWaitSecs = restartWaitSecs;
|
|
27
28
|
this.server = undefined;
|
|
28
|
-
this.openningServer = new
|
|
29
|
+
this.openningServer = new synced_1.SingleProc();
|
|
29
30
|
Object.seal(this);
|
|
30
31
|
}
|
|
31
32
|
/**
|
|
@@ -106,7 +107,7 @@ class ServerEvents {
|
|
|
106
107
|
}
|
|
107
108
|
}
|
|
108
109
|
restartObservation(event) {
|
|
109
|
-
return rxjs_1.from(
|
|
110
|
+
return rxjs_1.from(sleep_1.sleep(this.restartWaitSecs * 1000))
|
|
110
111
|
.pipe(operators_1.mergeMap(() => this.observe(event)));
|
|
111
112
|
}
|
|
112
113
|
}
|
|
@@ -18,7 +18,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
18
18
|
exports.openSocket = void 0;
|
|
19
19
|
const WebSocket = require("ws");
|
|
20
20
|
const request_utils_1 = require("./request-utils");
|
|
21
|
-
const
|
|
21
|
+
const deferred_1 = require("../lib-common/processes/deferred");
|
|
22
22
|
const http_1 = require("../lib-common/exceptions/http");
|
|
23
23
|
const https_1 = require("https");
|
|
24
24
|
function openSocket(url, sessionId) {
|
|
@@ -28,7 +28,7 @@ function openSocket(url, sessionId) {
|
|
|
28
28
|
const headers = {};
|
|
29
29
|
headers[request_utils_1.SESSION_ID_HEADER] = sessionId;
|
|
30
30
|
const ws = new WebSocket(url, { headers, agent: https_1.globalAgent });
|
|
31
|
-
const opening =
|
|
31
|
+
const opening = deferred_1.defer();
|
|
32
32
|
const initOnError = err => opening.reject(http_1.makeConnectionException(url, undefined, `Cannot open websocket connection due to error: ${err.message}`));
|
|
33
33
|
const onNonOkReply = (req, res) => {
|
|
34
34
|
const errReply = {
|
|
@@ -20,7 +20,8 @@ exports.ensureFolderExists = exports.copyTree = exports.copyFile = exports.strea
|
|
|
20
20
|
const fs = require("fs");
|
|
21
21
|
const fs_1 = require("fs");
|
|
22
22
|
const file_1 = require("./exceptions/file");
|
|
23
|
-
const
|
|
23
|
+
const synced_1 = require("./processes/synced");
|
|
24
|
+
const deferred_1 = require("./processes/deferred");
|
|
24
25
|
const bytes_fifo_buffer_1 = require("./byte-streaming/bytes-fifo-buffer");
|
|
25
26
|
const buffer_utils_1 = require("./buffer-utils");
|
|
26
27
|
const path_1 = require("path");
|
|
@@ -527,11 +528,11 @@ async function streamToExistingFile(filePath, pos, len, src, bufSize) {
|
|
|
527
528
|
if ((typeof bufSize !== 'number') || (bufSize < 1024)) {
|
|
528
529
|
throw new Error('Illegal buffer size given: ' + bufSize);
|
|
529
530
|
}
|
|
530
|
-
const writeProc = new
|
|
531
|
+
const writeProc = new synced_1.SingleProc();
|
|
531
532
|
let bytesWritten = 0;
|
|
532
533
|
let bytesRead = 0;
|
|
533
534
|
const buf = new bytes_fifo_buffer_1.BytesFIFOBuffer();
|
|
534
|
-
let deferred =
|
|
535
|
+
let deferred = deferred_1.defer();
|
|
535
536
|
let doneReading = false;
|
|
536
537
|
const complete = (err) => {
|
|
537
538
|
if (!deferred) {
|
|
@@ -17,37 +17,37 @@
|
|
|
17
17
|
*/
|
|
18
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
19
|
exports.wrapAndSyncFileSource = exports.wrapAndSyncSource = exports.wrapAndSyncFileSink = void 0;
|
|
20
|
-
const
|
|
20
|
+
const synced_1 = require("../processes/synced");
|
|
21
21
|
function wrapAndSyncFileSink(sink) {
|
|
22
|
-
const syncProc = new
|
|
22
|
+
const syncProc = new synced_1.SingleProc();
|
|
23
23
|
const w = {
|
|
24
|
-
done:
|
|
25
|
-
getSize:
|
|
26
|
-
showLayout:
|
|
27
|
-
splice:
|
|
28
|
-
truncate:
|
|
24
|
+
done: synced_1.makeSyncedFunc(syncProc, sink, sink.done),
|
|
25
|
+
getSize: synced_1.makeSyncedFunc(syncProc, sink, sink.getSize),
|
|
26
|
+
showLayout: synced_1.makeSyncedFunc(syncProc, sink, sink.showLayout),
|
|
27
|
+
splice: synced_1.makeSyncedFunc(syncProc, sink, sink.splice),
|
|
28
|
+
truncate: synced_1.makeSyncedFunc(syncProc, sink, sink.truncate)
|
|
29
29
|
};
|
|
30
30
|
return w;
|
|
31
31
|
}
|
|
32
32
|
exports.wrapAndSyncFileSink = wrapAndSyncFileSink;
|
|
33
33
|
function wrapAndSyncSource(src) {
|
|
34
|
-
const syncProc = new
|
|
34
|
+
const syncProc = new synced_1.SingleProc();
|
|
35
35
|
const w = {
|
|
36
|
-
getPosition:
|
|
37
|
-
getSize:
|
|
38
|
-
read:
|
|
39
|
-
seek:
|
|
36
|
+
getPosition: synced_1.makeSyncedFunc(syncProc, src, src.getPosition),
|
|
37
|
+
getSize: synced_1.makeSyncedFunc(syncProc, src, src.getSize),
|
|
38
|
+
read: synced_1.makeSyncedFunc(syncProc, src, src.read),
|
|
39
|
+
seek: synced_1.makeSyncedFunc(syncProc, src, src.seek)
|
|
40
40
|
};
|
|
41
41
|
return w;
|
|
42
42
|
}
|
|
43
43
|
exports.wrapAndSyncSource = wrapAndSyncSource;
|
|
44
44
|
function wrapAndSyncFileSource(src) {
|
|
45
|
-
const syncProc = new
|
|
45
|
+
const syncProc = new synced_1.SingleProc();
|
|
46
46
|
const w = {
|
|
47
|
-
getPosition:
|
|
48
|
-
getSize:
|
|
49
|
-
read:
|
|
50
|
-
seek:
|
|
47
|
+
getPosition: synced_1.makeSyncedFunc(syncProc, src, src.getPosition),
|
|
48
|
+
getSize: synced_1.makeSyncedFunc(syncProc, src, src.getSize),
|
|
49
|
+
read: synced_1.makeSyncedFunc(syncProc, src, src.read),
|
|
50
|
+
seek: synced_1.makeSyncedFunc(syncProc, src, src.seek),
|
|
51
51
|
};
|
|
52
52
|
return w;
|
|
53
53
|
}
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
See the GNU General Public License for more details.
|
|
14
14
|
|
|
15
15
|
You should have received a copy of the GNU General Public License along with
|
|
16
|
-
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
17
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
19
|
exports.makeVersionMismatchExc = exports.makeNoAttrsExc = exports.ensureCorrectFS = exports.maskPathInExc = exports.makeFileException = exports.Code = void 0;
|
|
19
20
|
exports.Code = {
|
|
@@ -32,6 +33,7 @@ exports.Code = {
|
|
|
32
33
|
parsingError: 'parsing-error',
|
|
33
34
|
notImplemented: 'ENOSYS',
|
|
34
35
|
isEndless: 'is-endless',
|
|
36
|
+
storageClosed: 'storage-closed',
|
|
35
37
|
};
|
|
36
38
|
Object.freeze(exports.Code);
|
|
37
39
|
function makeFileException(code, path, cause) {
|
|
@@ -87,6 +89,9 @@ function makeFileException(code, path, cause) {
|
|
|
87
89
|
else if (code === exports.Code.isEndless) {
|
|
88
90
|
err.isEndless = true;
|
|
89
91
|
}
|
|
92
|
+
else if (code === exports.Code.storageClosed) {
|
|
93
|
+
err.storageClosed = true;
|
|
94
|
+
}
|
|
90
95
|
return err;
|
|
91
96
|
}
|
|
92
97
|
exports.makeFileException = makeFileException;
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
18
|
exports.SingleObserverWrap = exports.MultiObserverWrap = exports.makeSubscribingClient = exports.makeEventfulServer = exports.makeRequestServer = exports.makeRequestingClient = void 0;
|
|
19
19
|
const map_of_sets_1 = require("../map-of-sets");
|
|
20
|
-
const
|
|
20
|
+
const synced_1 = require("../processes/synced");
|
|
21
21
|
function toTransferrableError(e) {
|
|
22
22
|
const err = {
|
|
23
23
|
message: e.message,
|
|
@@ -556,7 +556,7 @@ class IpcEventChannels {
|
|
|
556
556
|
* These are all subscribed ipc channels.
|
|
557
557
|
*/
|
|
558
558
|
this.subscribedIpcChannels = new Set();
|
|
559
|
-
this.subscriptionProcs = new
|
|
559
|
+
this.subscriptionProcs = new synced_1.NamedProcs();
|
|
560
560
|
Object.freeze(this);
|
|
561
561
|
}
|
|
562
562
|
subscribeTo(ipcChannel) {
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
See the GNU General Public License for more details.
|
|
14
14
|
|
|
15
15
|
You should have received a copy of the GNU General Public License along with
|
|
16
|
-
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
17
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
19
|
exports.applyChangesToJSON = exports.deepFind = exports.deepEqual = exports.copy = void 0;
|
|
19
20
|
/**
|
|
@@ -20,7 +20,8 @@ exports.ObjVersionFile = void 0;
|
|
|
20
20
|
const fs = require("../async-fs-node");
|
|
21
21
|
const error_1 = require("../exceptions/error");
|
|
22
22
|
const bytes_equal_1 = require("../bytes-equal");
|
|
23
|
-
const
|
|
23
|
+
const synced_1 = require("../processes/synced");
|
|
24
|
+
const deferred_1 = require("../processes/deferred");
|
|
24
25
|
const buffer_utils_1 = require("../buffer-utils");
|
|
25
26
|
const v1_obj_file_format_1 = require("./v1-obj-file-format");
|
|
26
27
|
const fs_1 = require("fs");
|
|
@@ -30,7 +31,7 @@ class ObjVersionFile {
|
|
|
30
31
|
constructor(path, layout) {
|
|
31
32
|
this.path = path;
|
|
32
33
|
this.layout = layout;
|
|
33
|
-
this.writeProc = new
|
|
34
|
+
this.writeProc = new synced_1.SingleProc();
|
|
34
35
|
Object.seal(this);
|
|
35
36
|
}
|
|
36
37
|
static async forExisting(path) {
|
|
@@ -240,7 +241,7 @@ Object.freeze(ObjVersionFile.prototype);
|
|
|
240
241
|
Object.freeze(ObjVersionFile);
|
|
241
242
|
function noop() { }
|
|
242
243
|
async function pipeBytes(src, sink) {
|
|
243
|
-
const deferred =
|
|
244
|
+
const deferred = deferred_1.defer();
|
|
244
245
|
src.pipe(sink, { end: false });
|
|
245
246
|
src.on('error', (err) => {
|
|
246
247
|
deferred.reject(err);
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
19
|
exports.makeNoBaseObjPipe = exports.makeObjPipe = exports.chunksInOrderedStream = exports.streamToObjFile = exports.diffToLayout = void 0;
|
|
20
|
-
const
|
|
20
|
+
const deferred_1 = require("../processes/deferred");
|
|
21
21
|
const file_1 = require("../exceptions/file");
|
|
22
22
|
const assert_1 = require("../assert");
|
|
23
23
|
const bytes_fifo_buffer_1 = require("../byte-streaming/bytes-fifo-buffer");
|
|
@@ -42,7 +42,7 @@ function diffToLayout(diff) {
|
|
|
42
42
|
}
|
|
43
43
|
exports.diffToLayout = diffToLayout;
|
|
44
44
|
async function streamToObjFile(file, content, src, maxBufferLen) {
|
|
45
|
-
let deferred =
|
|
45
|
+
let deferred = deferred_1.defer();
|
|
46
46
|
const contentIter = content.values();
|
|
47
47
|
let chunk = undefined;
|
|
48
48
|
let buf = new bytes_fifo_buffer_1.BytesFIFOBuffer();
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
Copyright (C) 2015, 2017, 2019 - 2022 3NSoft Inc.
|
|
4
|
+
|
|
5
|
+
This program is free software: you can redistribute it and/or modify it under
|
|
6
|
+
the terms of the GNU General Public License as published by the Free Software
|
|
7
|
+
Foundation, either version 3 of the License, or (at your option) any later
|
|
8
|
+
version.
|
|
9
|
+
|
|
10
|
+
This program is distributed in the hope that it will be useful, but
|
|
11
|
+
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
13
|
+
See the GNU General Public License for more details.
|
|
14
|
+
|
|
15
|
+
You should have received a copy of the GNU General Public License along with
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.defer = void 0;
|
|
20
|
+
function defer() {
|
|
21
|
+
const d = {};
|
|
22
|
+
d.promise = new Promise((resolve, reject) => {
|
|
23
|
+
d.resolve = resolve;
|
|
24
|
+
d.reject = reject;
|
|
25
|
+
});
|
|
26
|
+
Object.freeze(d);
|
|
27
|
+
return d;
|
|
28
|
+
}
|
|
29
|
+
exports.defer = defer;
|
|
30
|
+
Object.freeze(exports);
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export interface Task<PoolLabel extends string> {
|
|
2
|
+
neededExecutor(): PoolLabel | undefined;
|
|
3
|
+
process(): Promise<void>;
|
|
4
|
+
cancel(): Promise<void>;
|
|
5
|
+
}
|
|
6
|
+
declare class ProcessingPool<PoolLabel extends string> {
|
|
7
|
+
readonly label: PoolLabel;
|
|
8
|
+
private readonly maxProcs;
|
|
9
|
+
private readonly logError?;
|
|
10
|
+
private readonly inProcess;
|
|
11
|
+
private readonly queue;
|
|
12
|
+
isRunning: boolean;
|
|
13
|
+
constructor(label: PoolLabel, maxProcs: number, logError?: ((err: any, msg?: string | undefined) => Promise<void>) | undefined);
|
|
14
|
+
add(task: Task<any>): void;
|
|
15
|
+
private processNextQueued;
|
|
16
|
+
pause(): void;
|
|
17
|
+
start(): void;
|
|
18
|
+
private readonly doOnError;
|
|
19
|
+
stop(): Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
export declare class LabelledExecPools<PoolLabel extends string> {
|
|
22
|
+
private readonly pools;
|
|
23
|
+
private isRunning;
|
|
24
|
+
constructor(setup: {
|
|
25
|
+
label: PoolLabel;
|
|
26
|
+
maxProcs: number;
|
|
27
|
+
}[], logError?: ProcessingPool<PoolLabel>['logError']);
|
|
28
|
+
add(task: Task<PoolLabel>, queueIfNotRunning?: boolean): void;
|
|
29
|
+
start(): void;
|
|
30
|
+
pause(): void;
|
|
31
|
+
stop(): Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
export {};
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
Copyright (C) 2022 3NSoft Inc.
|
|
4
|
+
|
|
5
|
+
This program is free software: you can redistribute it and/or modify it under
|
|
6
|
+
the terms of the GNU General Public License as published by the Free Software
|
|
7
|
+
Foundation, either version 3 of the License, or (at your option) any later
|
|
8
|
+
version.
|
|
9
|
+
|
|
10
|
+
This program is distributed in the hope that it will be useful, but
|
|
11
|
+
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
13
|
+
See the GNU General Public License for more details.
|
|
14
|
+
|
|
15
|
+
You should have received a copy of the GNU General Public License along with
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.LabelledExecPools = void 0;
|
|
20
|
+
const assert_1 = require("../assert");
|
|
21
|
+
class ProcessingPool {
|
|
22
|
+
constructor(label, maxProcs, logError) {
|
|
23
|
+
this.label = label;
|
|
24
|
+
this.maxProcs = maxProcs;
|
|
25
|
+
this.logError = logError;
|
|
26
|
+
this.inProcess = new Set();
|
|
27
|
+
this.queue = [];
|
|
28
|
+
this.isRunning = false;
|
|
29
|
+
this.doOnError = async (err) => {
|
|
30
|
+
if (this.logError) {
|
|
31
|
+
await this.logError(err, `Error in pool ${this.label}`);
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
Object.seal(this);
|
|
35
|
+
}
|
|
36
|
+
add(task) {
|
|
37
|
+
this.queue.push(task);
|
|
38
|
+
if (this.isRunning) {
|
|
39
|
+
this.processNextQueued();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
async processNextQueued() {
|
|
43
|
+
if (!this.isRunning || (this.inProcess.size >= this.maxProcs)) {
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
const task = this.queue.shift();
|
|
47
|
+
if (!task) {
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
if (task.neededExecutor() !== this.label) {
|
|
51
|
+
return this.processNextQueued();
|
|
52
|
+
}
|
|
53
|
+
let proc = undefined;
|
|
54
|
+
try {
|
|
55
|
+
proc = task.process().catch(this.doOnError);
|
|
56
|
+
this.inProcess.add(proc);
|
|
57
|
+
await proc;
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
await this.doOnError(err);
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
if (proc) {
|
|
64
|
+
this.inProcess.delete(proc);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return this.processNextQueued();
|
|
68
|
+
}
|
|
69
|
+
pause() {
|
|
70
|
+
this.isRunning = false;
|
|
71
|
+
}
|
|
72
|
+
start() {
|
|
73
|
+
this.isRunning = true;
|
|
74
|
+
const triggerCount = Math.min(this.maxProcs, this.queue.length);
|
|
75
|
+
for (let i = 0; i < triggerCount; i += 1) {
|
|
76
|
+
this.processNextQueued();
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
async stop() {
|
|
80
|
+
if (!this.isRunning) {
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
this.isRunning = false;
|
|
84
|
+
const cancellations = this.queue.splice(0, this.queue.length)
|
|
85
|
+
.map(t => t.cancel().catch(this.doOnError));
|
|
86
|
+
await Promise.all(cancellations.concat(Array.from(this.inProcess)));
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
Object.freeze(ProcessingPool.prototype);
|
|
90
|
+
Object.freeze(ProcessingPool);
|
|
91
|
+
class LabelledExecPools {
|
|
92
|
+
constructor(setup, logError) {
|
|
93
|
+
this.pools = new Map();
|
|
94
|
+
this.isRunning = false;
|
|
95
|
+
assert_1.assert(setup.length > 0);
|
|
96
|
+
for (const { label, maxProcs } of setup) {
|
|
97
|
+
const pool = new ProcessingPool(label, maxProcs, logError);
|
|
98
|
+
this.pools.set(pool.label, pool);
|
|
99
|
+
}
|
|
100
|
+
Object.seal(this);
|
|
101
|
+
}
|
|
102
|
+
add(task, queueIfNotRunning = false) {
|
|
103
|
+
if (!this.isRunning && !queueIfNotRunning) {
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const label = task.neededExecutor();
|
|
107
|
+
if (!label) {
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
const pool = this.pools.get(label);
|
|
111
|
+
if (!pool) {
|
|
112
|
+
throw new Error(`Task needs unknown pool ${label}`);
|
|
113
|
+
}
|
|
114
|
+
pool.add(task);
|
|
115
|
+
}
|
|
116
|
+
start() {
|
|
117
|
+
if (this.isRunning) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
this.isRunning = true;
|
|
121
|
+
for (const pool of this.pools.values()) {
|
|
122
|
+
pool.start();
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
pause() {
|
|
126
|
+
this.isRunning = false;
|
|
127
|
+
}
|
|
128
|
+
async stop() {
|
|
129
|
+
if (!this.isRunning) {
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
this.isRunning = false;
|
|
133
|
+
const cancelations = Array.from(this.pools.values())
|
|
134
|
+
.map(pool => pool.stop());
|
|
135
|
+
await Promise.all(cancelations);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
exports.LabelledExecPools = LabelledExecPools;
|
|
139
|
+
Object.freeze(ProcessingPool.prototype);
|
|
140
|
+
Object.freeze(ProcessingPool);
|
|
141
|
+
Object.freeze(exports);
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
Copyright (C) 2015, 2017, 2019 - 2022 3NSoft Inc.
|
|
4
|
+
|
|
5
|
+
This program is free software: you can redistribute it and/or modify it under
|
|
6
|
+
the terms of the GNU General Public License as published by the Free Software
|
|
7
|
+
Foundation, either version 3 of the License, or (at your option) any later
|
|
8
|
+
version.
|
|
9
|
+
|
|
10
|
+
This program is distributed in the hope that it will be useful, but
|
|
11
|
+
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
13
|
+
See the GNU General Public License for more details.
|
|
14
|
+
|
|
15
|
+
You should have received a copy of the GNU General Public License along with
|
|
16
|
+
this program. If not, see <http://www.gnu.org/licenses/>.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.PressureValve = void 0;
|
|
20
|
+
const error_1 = require("../exceptions/error");
|
|
21
|
+
const deferred_1 = require("./deferred");
|
|
22
|
+
class PressureValve {
|
|
23
|
+
constructor() {
|
|
24
|
+
this.stopper = undefined;
|
|
25
|
+
this.pressure = async () => {
|
|
26
|
+
if (!this.stopper) {
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
await this.stopper.promise;
|
|
30
|
+
};
|
|
31
|
+
Object.seal(this);
|
|
32
|
+
}
|
|
33
|
+
toggle(flag) {
|
|
34
|
+
if (flag) {
|
|
35
|
+
if (this.stopper) {
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
this.stopper = deferred_1.defer();
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
if (!this.stopper) {
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
this.stopper.resolve();
|
|
45
|
+
this.stopper = undefined;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
pressWithError(err) {
|
|
49
|
+
this.toggle(true);
|
|
50
|
+
this.stopper.reject(error_1.errWithCause(err, `Backpressure error`));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
exports.PressureValve = PressureValve;
|
|
54
|
+
Object.freeze(PressureValve.prototype);
|
|
55
|
+
Object.freeze(PressureValve);
|
|
56
|
+
Object.freeze(exports);
|