corebasic 1.0.232 → 1.0.234
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/libs/auth.js +3 -3
- package/dist/libs/dip/suffix/index.js +1 -1
- package/dist/libs/dip/suffix/suffix.js +2 -3
- package/dist/libs/dip/suffix/tests/policy.applySuffixPolicy.test.js +26 -99
- package/dist/libs/dip/suffix/tests/policy.test.js +1 -1
- package/dist/libs/dipper.js +14 -13
- package/dist/libs/elabase.js +36 -45
- package/dist/libs/features.js +5 -4
- package/dist/libs/kafka.js +2 -3
- package/dist/libs/messaging.js +2 -2
- package/dist/libs/utils.js +1 -1
- package/libs/auth.ts +3 -3
- package/libs/dip/suffix/index.ts +1 -1
- package/libs/dip/suffix/policy.ts +1 -1
- package/libs/dip/suffix/suffix.ts +3 -4
- package/libs/dip/suffix/tests/policy.applySuffixPolicy.test.ts +26 -180
- package/libs/dip/suffix/tests/policy.test.ts +1 -1
- package/libs/dip.ts +5 -5
- package/libs/dipper.ts +15 -14
- package/libs/elabase.ts +172 -126
- package/libs/features.ts +10 -9
- package/libs/kafka.ts +4 -4
- package/libs/messaging.ts +15 -13
- package/libs/utils.ts +1 -1
- package/package.json +1 -1
- package/tsconfig.json +16 -1
package/dist/libs/auth.js
CHANGED
|
@@ -7,7 +7,7 @@ import * as Utils from './utils.js';
|
|
|
7
7
|
import * as Session from './session.js';
|
|
8
8
|
let validateFn;
|
|
9
9
|
let validateErrMessage;
|
|
10
|
-
export const validate = (callback,
|
|
10
|
+
export const validate = (callback, _errMessage) => {
|
|
11
11
|
validateFn = callback;
|
|
12
12
|
};
|
|
13
13
|
export const start = (app, successCallback) => {
|
|
@@ -17,7 +17,7 @@ export const start = (app, successCallback) => {
|
|
|
17
17
|
return;
|
|
18
18
|
}
|
|
19
19
|
try {
|
|
20
|
-
let login = await attemptLogin(req
|
|
20
|
+
let login = await attemptLogin(req);
|
|
21
21
|
if (login.mode === 'verify' && login.success) {
|
|
22
22
|
let tokens = Session.generateAccessToken(login.userId, req.body.clientId ?? '');
|
|
23
23
|
let response = { ...login, tokens };
|
|
@@ -34,7 +34,7 @@ export const start = (app, successCallback) => {
|
|
|
34
34
|
}
|
|
35
35
|
});
|
|
36
36
|
};
|
|
37
|
-
async function attemptLogin(req
|
|
37
|
+
async function attemptLogin(req) {
|
|
38
38
|
let meta = { company: "GLOBAL", outlet: "GLOBAL" };
|
|
39
39
|
let expiry = 300000;
|
|
40
40
|
let userMob = req.body.mob ?? req.body.phone ?? '';
|
|
@@ -64,7 +64,7 @@ function flatten_value(target_key, value, flattened, ancestor_has_id) {
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
|
-
export function entries(target_key, query,
|
|
67
|
+
export function entries(target_key, query, _binary_slice) {
|
|
68
68
|
let result = flatten_id_values(target_key, query); // extract all valid _id
|
|
69
69
|
// let cows = result.map(v => extract_direct_id(v, binary_slice))
|
|
70
70
|
// // cows.sort_by(|a, b| a.as_ref().cmp(b.as_ref())); // sort for improved forward processing because kv engine keys are already sorted.
|
|
@@ -82,9 +82,8 @@ export async function suffix(doc, policies, insertMode, arg) {
|
|
|
82
82
|
if (!keys.length && !insertMode) {
|
|
83
83
|
ls = true;
|
|
84
84
|
for (const suffix of suffixes) {
|
|
85
|
-
const
|
|
86
|
-
|
|
87
|
-
const dirs = (await Dip.operation("ls", { db: arg?.db, collection: collection, suffix })).filter((dir) => !dir.startsWith('chunk-'));
|
|
85
|
+
for (const collection of arg.collection) {
|
|
86
|
+
const dirs = (await Dip.operation("ls", { db: arg.db, collection: collection, suffix })).filter((dir) => !dir.startsWith('chunk-'));
|
|
88
87
|
suffixAssociatedKeys[suffix] = suffixAssociatedKeys[suffix] ?? [];
|
|
89
88
|
suffixAssociatedKeys[suffix].push(...dirs);
|
|
90
89
|
}
|
|
@@ -1,20 +1,22 @@
|
|
|
1
1
|
import { describe, test } from "node:test";
|
|
2
2
|
import assert from "node:assert/strict";
|
|
3
3
|
import { applySuffixPolicy } from "../policy.js";
|
|
4
|
+
let insertMode = false;
|
|
5
|
+
const arg = { db: "test-db", collection: ["test-collection"] };
|
|
4
6
|
// -----------------------------------------------------------------------------
|
|
5
7
|
// applySuffixPolicy()
|
|
6
8
|
// -----------------------------------------------------------------------------
|
|
7
9
|
describe("applySuffixPolicy()", () => {
|
|
8
10
|
test("no collection", async () => {
|
|
9
11
|
const collections = {};
|
|
10
|
-
const result = await applySuffixPolicy(collections, "missing", {});
|
|
12
|
+
const result = await applySuffixPolicy(collections, "missing", {}, insertMode, arg);
|
|
11
13
|
assert.deepEqual(result, []);
|
|
12
14
|
});
|
|
13
15
|
test("collection without policy", async () => {
|
|
14
16
|
const collections = {
|
|
15
17
|
invoices: {}
|
|
16
18
|
};
|
|
17
|
-
const result = await applySuffixPolicy(collections, "invoices", {});
|
|
19
|
+
const result = await applySuffixPolicy(collections, "invoices", {}, insertMode, arg);
|
|
18
20
|
assert.deepEqual(result, []);
|
|
19
21
|
});
|
|
20
22
|
test("empty policy array", async () => {
|
|
@@ -23,7 +25,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
23
25
|
policy: []
|
|
24
26
|
}
|
|
25
27
|
};
|
|
26
|
-
const result = await applySuffixPolicy(collections, "invoices", {});
|
|
28
|
+
const result = await applySuffixPolicy(collections, "invoices", {}, insertMode, arg);
|
|
27
29
|
assert.deepEqual(result, []);
|
|
28
30
|
});
|
|
29
31
|
test("single policy object", async () => {
|
|
@@ -36,9 +38,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
36
38
|
}
|
|
37
39
|
}
|
|
38
40
|
};
|
|
39
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
40
|
-
type: "Sales"
|
|
41
|
-
});
|
|
41
|
+
const result = await applySuffixPolicy(collections, "invoices", { type: "Sales" }, insertMode, arg);
|
|
42
42
|
assert.deepEqual(result, [
|
|
43
43
|
"/Sales"
|
|
44
44
|
]);
|
|
@@ -55,12 +55,8 @@ describe("applySuffixPolicy()", () => {
|
|
|
55
55
|
]
|
|
56
56
|
}
|
|
57
57
|
};
|
|
58
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
59
|
-
|
|
60
|
-
});
|
|
61
|
-
assert.deepEqual(result, [
|
|
62
|
-
"/Sales"
|
|
63
|
-
]);
|
|
58
|
+
const result = await applySuffixPolicy(collections, "invoices", { type: "Sales" }, insertMode, arg);
|
|
59
|
+
assert.deepEqual(result, ["/Sales"]);
|
|
64
60
|
});
|
|
65
61
|
test("unconditional policy", async () => {
|
|
66
62
|
const collections = {
|
|
@@ -73,9 +69,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
73
69
|
}
|
|
74
70
|
}
|
|
75
71
|
};
|
|
76
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
77
|
-
type: "Sales"
|
|
78
|
-
});
|
|
72
|
+
const result = await applySuffixPolicy(collections, "invoices", { type: "Sales" }, insertMode, arg);
|
|
79
73
|
assert.deepEqual(result, [
|
|
80
74
|
"/Sales"
|
|
81
75
|
]);
|
|
@@ -93,10 +87,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
93
87
|
}
|
|
94
88
|
}
|
|
95
89
|
};
|
|
96
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
97
|
-
status: "Active",
|
|
98
|
-
type: "Sales"
|
|
99
|
-
});
|
|
90
|
+
const result = await applySuffixPolicy(collections, "invoices", { status: "Active", type: "Sales" }, insertMode, arg);
|
|
100
91
|
assert.deepEqual(result, [
|
|
101
92
|
"/Sales"
|
|
102
93
|
]);
|
|
@@ -114,10 +105,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
114
105
|
}
|
|
115
106
|
}
|
|
116
107
|
};
|
|
117
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
118
|
-
status: "Deleted",
|
|
119
|
-
type: "Sales"
|
|
120
|
-
});
|
|
108
|
+
const result = await applySuffixPolicy(collections, "invoices", { status: "Deleted", type: "Sales" }, insertMode, arg);
|
|
121
109
|
assert.deepEqual(result, []);
|
|
122
110
|
});
|
|
123
111
|
test("$in subset", async () => {
|
|
@@ -133,10 +121,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
133
121
|
}
|
|
134
122
|
}
|
|
135
123
|
};
|
|
136
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
137
|
-
status: { $in: ["Active", "Pending"] },
|
|
138
|
-
type: "Sales"
|
|
139
|
-
});
|
|
124
|
+
const result = await applySuffixPolicy(collections, "invoices", { status: { $in: ["Active", "Pending"] }, type: "Sales" }, insertMode, arg);
|
|
140
125
|
assert.deepEqual(result, [
|
|
141
126
|
"/Sales"
|
|
142
127
|
]);
|
|
@@ -164,11 +149,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
164
149
|
]
|
|
165
150
|
}
|
|
166
151
|
};
|
|
167
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
168
|
-
status: "Active",
|
|
169
|
-
type: "Sales",
|
|
170
|
-
region: "IN"
|
|
171
|
-
});
|
|
152
|
+
const result = await applySuffixPolicy(collections, "invoices", { status: "Active", type: "Sales", region: "IN" }, insertMode, arg);
|
|
172
153
|
assert.deepEqual(result.sort(), [
|
|
173
154
|
"/Sales",
|
|
174
155
|
"/IN"
|
|
@@ -197,11 +178,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
197
178
|
]
|
|
198
179
|
}
|
|
199
180
|
};
|
|
200
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
201
|
-
status: "Active",
|
|
202
|
-
type: "Sales",
|
|
203
|
-
region: "IN"
|
|
204
|
-
});
|
|
181
|
+
const result = await applySuffixPolicy(collections, "invoices", { status: "Active", type: "Sales", region: "IN" }, insertMode, arg);
|
|
205
182
|
assert.deepEqual(result, [
|
|
206
183
|
"/Sales"
|
|
207
184
|
]);
|
|
@@ -223,9 +200,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
223
200
|
]
|
|
224
201
|
}
|
|
225
202
|
};
|
|
226
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
227
|
-
type: "Sales"
|
|
228
|
-
});
|
|
203
|
+
const result = await applySuffixPolicy(collections, "invoices", { type: "Sales" }, insertMode, arg);
|
|
229
204
|
assert.deepEqual(result, [
|
|
230
205
|
"/Sales"
|
|
231
206
|
]);
|
|
@@ -243,9 +218,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
243
218
|
}
|
|
244
219
|
}
|
|
245
220
|
};
|
|
246
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
247
|
-
date: new Date(2026, 4, 3).getTime()
|
|
248
|
-
});
|
|
221
|
+
const result = await applySuffixPolicy(collections, "invoices", { date: new Date(2026, 4, 3).getTime() }, insertMode, arg);
|
|
249
222
|
assert.deepEqual(result, [
|
|
250
223
|
"/2026"
|
|
251
224
|
]);
|
|
@@ -264,10 +237,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
264
237
|
}
|
|
265
238
|
}
|
|
266
239
|
};
|
|
267
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
268
|
-
type: "Sales",
|
|
269
|
-
date: new Date(2026, 4, 3).getTime()
|
|
270
|
-
});
|
|
240
|
+
const result = await applySuffixPolicy(collections, "invoices", { type: "Sales", date: new Date(2026, 4, 3).getTime() }, insertMode, arg);
|
|
271
241
|
assert.deepEqual(result, [
|
|
272
242
|
"/Sales/2026"
|
|
273
243
|
]);
|
|
@@ -287,12 +257,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
287
257
|
}
|
|
288
258
|
}
|
|
289
259
|
};
|
|
290
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
291
|
-
age: {
|
|
292
|
-
$gt: 20
|
|
293
|
-
},
|
|
294
|
-
type: "Sales"
|
|
295
|
-
});
|
|
260
|
+
const result = await applySuffixPolicy(collections, "invoices", { age: { $gt: 20 }, type: "Sales" }, insertMode, arg);
|
|
296
261
|
assert.deepEqual(result, [
|
|
297
262
|
"/Sales"
|
|
298
263
|
]);
|
|
@@ -312,12 +277,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
312
277
|
}
|
|
313
278
|
}
|
|
314
279
|
};
|
|
315
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
316
|
-
age: {
|
|
317
|
-
$lt: 10
|
|
318
|
-
},
|
|
319
|
-
type: "Sales"
|
|
320
|
-
});
|
|
280
|
+
const result = await applySuffixPolicy(collections, "invoices", { age: { $lt: 10 }, type: "Sales" }, insertMode, arg);
|
|
321
281
|
assert.deepEqual(result, []);
|
|
322
282
|
});
|
|
323
283
|
test("$ne throws", async () => {
|
|
@@ -336,10 +296,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
336
296
|
}
|
|
337
297
|
};
|
|
338
298
|
await assert.rejects(async () => {
|
|
339
|
-
await applySuffixPolicy(collections, "invoices", {
|
|
340
|
-
status: "Active",
|
|
341
|
-
type: "Sales"
|
|
342
|
-
});
|
|
299
|
+
await applySuffixPolicy(collections, "invoices", { status: "Active", type: "Sales" }, insertMode, arg);
|
|
343
300
|
});
|
|
344
301
|
});
|
|
345
302
|
test("policy value inside query range", async () => {
|
|
@@ -355,13 +312,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
355
312
|
}
|
|
356
313
|
}
|
|
357
314
|
};
|
|
358
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
359
|
-
age: {
|
|
360
|
-
$gt: 5,
|
|
361
|
-
$lt: 15
|
|
362
|
-
},
|
|
363
|
-
type: "Sales"
|
|
364
|
-
});
|
|
315
|
+
const result = await applySuffixPolicy(collections, "invoices", { age: { $gt: 5, $lt: 15 }, type: "Sales" }, insertMode, arg);
|
|
365
316
|
assert.deepEqual(result, [
|
|
366
317
|
"/Sales"
|
|
367
318
|
]);
|
|
@@ -379,13 +330,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
379
330
|
}
|
|
380
331
|
}
|
|
381
332
|
};
|
|
382
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
383
|
-
age: {
|
|
384
|
-
$gt: 5,
|
|
385
|
-
$lt: 15
|
|
386
|
-
},
|
|
387
|
-
type: "Sales"
|
|
388
|
-
});
|
|
333
|
+
const result = await applySuffixPolicy(collections, "invoices", { age: { $gt: 5, $lt: 15 }, type: "Sales" }, insertMode, arg);
|
|
389
334
|
assert.deepEqual(result, []);
|
|
390
335
|
});
|
|
391
336
|
test("policy range intersects query value", async () => {
|
|
@@ -404,10 +349,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
404
349
|
}
|
|
405
350
|
}
|
|
406
351
|
};
|
|
407
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
408
|
-
age: { $in: [10] },
|
|
409
|
-
type: "Sales"
|
|
410
|
-
});
|
|
352
|
+
const result = await applySuffixPolicy(collections, "invoices", { age: { $in: [10] }, type: "Sales" }, insertMode, arg);
|
|
411
353
|
assert.deepEqual(result, [
|
|
412
354
|
"/Sales"
|
|
413
355
|
]);
|
|
@@ -428,10 +370,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
428
370
|
}
|
|
429
371
|
}
|
|
430
372
|
};
|
|
431
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
432
|
-
age: { $in: [10] },
|
|
433
|
-
type: "Sales"
|
|
434
|
-
});
|
|
373
|
+
const result = await applySuffixPolicy(collections, "invoices", { age: { $in: [10] }, type: "Sales" }, insertMode, arg);
|
|
435
374
|
assert.deepEqual(result, []);
|
|
436
375
|
});
|
|
437
376
|
test("policy range intersects query range", async () => {
|
|
@@ -450,13 +389,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
450
389
|
}
|
|
451
390
|
}
|
|
452
391
|
};
|
|
453
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
454
|
-
age: {
|
|
455
|
-
$gt: 20,
|
|
456
|
-
$lt: 40
|
|
457
|
-
},
|
|
458
|
-
type: "Sales"
|
|
459
|
-
});
|
|
392
|
+
const result = await applySuffixPolicy(collections, "invoices", { age: { $gt: 20, $lt: 40 }, type: "Sales" }, insertMode, arg);
|
|
460
393
|
assert.deepEqual(result, [
|
|
461
394
|
"/Sales"
|
|
462
395
|
]);
|
|
@@ -477,13 +410,7 @@ describe("applySuffixPolicy()", () => {
|
|
|
477
410
|
}
|
|
478
411
|
}
|
|
479
412
|
};
|
|
480
|
-
const result = await applySuffixPolicy(collections, "invoices", {
|
|
481
|
-
age: {
|
|
482
|
-
$gt: 10,
|
|
483
|
-
$lt: 20
|
|
484
|
-
},
|
|
485
|
-
type: "Sales"
|
|
486
|
-
});
|
|
413
|
+
const result = await applySuffixPolicy(collections, "invoices", { age: { $gt: 10, $lt: 20 }, type: "Sales" }, insertMode, arg);
|
|
487
414
|
assert.deepEqual(result, []);
|
|
488
415
|
});
|
|
489
416
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { describe, test } from "node:test";
|
|
2
2
|
import assert from "node:assert/strict";
|
|
3
|
-
import { matchPolicy, overlaps, contains, intersects
|
|
3
|
+
import { matchPolicy, overlaps, contains, intersects } from "../policy.js";
|
|
4
4
|
const defaultBounds = { values: [], from: -Infinity, to: Infinity, fromOp: "$gte", toOp: "$lte" };
|
|
5
5
|
/* ============================================================
|
|
6
6
|
* overlaps()
|
package/dist/libs/dipper.js
CHANGED
|
@@ -8,16 +8,16 @@ function hashStr(str) {
|
|
|
8
8
|
}
|
|
9
9
|
return hash;
|
|
10
10
|
}
|
|
11
|
-
function hash(string, digits) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
}
|
|
11
|
+
// function hash(string: string, digits: number) {
|
|
12
|
+
// digits = digits || 6;
|
|
13
|
+
// var m = Math.pow(10, digits+1) - 1;
|
|
14
|
+
// var phi = Math.pow(10, digits) / 2 - 1;
|
|
15
|
+
// var n = 0;
|
|
16
|
+
// for (var i = 0; i < string.length; i++) {
|
|
17
|
+
// n = (n + phi * string.charCodeAt(i)) % m;
|
|
18
|
+
// }
|
|
19
|
+
// return n.toString();
|
|
20
|
+
// }
|
|
21
21
|
let shards = [];
|
|
22
22
|
let shardCount = 20;
|
|
23
23
|
for (let i = 1; i <= shardCount; i++)
|
|
@@ -30,7 +30,7 @@ function shard(key) {
|
|
|
30
30
|
}
|
|
31
31
|
export let shard_hits = {};
|
|
32
32
|
async function postDip(url, arg) {
|
|
33
|
-
return (await axios.post(url, arg)).data;
|
|
33
|
+
return (await axios.post(String(url), arg)).data;
|
|
34
34
|
// curl
|
|
35
35
|
// -----
|
|
36
36
|
// const { data } = await curly.post(url, {
|
|
@@ -43,6 +43,7 @@ async function postDip(url, arg) {
|
|
|
43
43
|
// return data
|
|
44
44
|
}
|
|
45
45
|
export default async function dipper(arg) {
|
|
46
|
+
const mode = arg.insert || arg.update || arg.delete ? "query" : "command"; // TODO: batch
|
|
46
47
|
let result = [];
|
|
47
48
|
let _id = arg.query?._id ?? arg.insert?._id ?? undefined;
|
|
48
49
|
if (!_id) {
|
|
@@ -61,9 +62,9 @@ export default async function dipper(arg) {
|
|
|
61
62
|
result.push(await postDip(`http://${url}/execute`, arg));
|
|
62
63
|
}
|
|
63
64
|
}
|
|
64
|
-
if (
|
|
65
|
+
if (mode === "query")
|
|
65
66
|
result = result.reduce((partial, item) => partial.concat(item), []);
|
|
66
|
-
return
|
|
67
|
+
return mode === "query" ? result : result[0];
|
|
67
68
|
}
|
|
68
69
|
export const shard_stats = () => {
|
|
69
70
|
let total = Object.values(shard_hits).reduce((partial, value) => partial + value, 0);
|
package/dist/libs/elabase.js
CHANGED
|
@@ -6,12 +6,12 @@ import * as Cpp from './cpp.js';
|
|
|
6
6
|
import { applySuffixPolicy } from './dip/suffix/policy.js';
|
|
7
7
|
export const shard_stats = ShardStats;
|
|
8
8
|
let Events = {
|
|
9
|
-
send: function (
|
|
9
|
+
send: function (_topic, _data) { }
|
|
10
10
|
};
|
|
11
11
|
const url = process.env.DIP_URL || "http://127.0.0.1"; // Set DIP_URL to "http://dip" (Prod) or http://dip-dev (Dev) in CI/CD
|
|
12
12
|
let port = process.env.DIP_PORT || 9401;
|
|
13
13
|
let api = 'execute';
|
|
14
|
-
let fullurl = `${url}:${port}/${api}
|
|
14
|
+
// let fullurl = `${url}:${port}/${api}`
|
|
15
15
|
let Elabase = {
|
|
16
16
|
Engine: {
|
|
17
17
|
Lmdb: 1,
|
|
@@ -49,7 +49,7 @@ export class DipMeta {
|
|
|
49
49
|
Object.freeze(this);
|
|
50
50
|
}
|
|
51
51
|
}
|
|
52
|
-
export const isDipMeta = meta => meta instanceof DipMeta;
|
|
52
|
+
export const isDipMeta = (meta) => meta instanceof DipMeta;
|
|
53
53
|
export const globalMeta = (args) => {
|
|
54
54
|
return new DipMeta({ ...args, company: "GLOBAL", outlet: "GLOBAL" });
|
|
55
55
|
};
|
|
@@ -110,9 +110,9 @@ function generate_suffix(meta) {
|
|
|
110
110
|
company = company.length ? company : [""];
|
|
111
111
|
outlet = outlet.length ? outlet : [""];
|
|
112
112
|
suffix = suffix.length ? suffix : [""];
|
|
113
|
-
company = new Set(company);
|
|
114
|
-
outlet = new Set(outlet);
|
|
115
|
-
suffix = new Set(suffix);
|
|
113
|
+
company = [...new Set(company)];
|
|
114
|
+
outlet = [...new Set(outlet)];
|
|
115
|
+
suffix = [...new Set(suffix)];
|
|
116
116
|
// console.log(company, outlet, suffix)
|
|
117
117
|
let suffixes = [];
|
|
118
118
|
for (let c of company) {
|
|
@@ -132,8 +132,8 @@ let BATCH_COUNTER = 0n;
|
|
|
132
132
|
let batches = {};
|
|
133
133
|
// Ported
|
|
134
134
|
export function batchBegin() {
|
|
135
|
-
const uid = ++BATCH_COUNTER; // Safe. No skew in async ++BATCH_COUNTER
|
|
136
|
-
batches[uid
|
|
135
|
+
const uid = (++BATCH_COUNTER).toString(); // Safe. No skew in async ++BATCH_COUNTER. Single thread ++COUNTER is atomic in event loop sense. ⚠️ Breaks on worker_threads or cluster (multi-process)
|
|
136
|
+
batches[uid] = [];
|
|
137
137
|
return uid;
|
|
138
138
|
}
|
|
139
139
|
// Ported
|
|
@@ -173,8 +173,8 @@ export function batchSubmit(batch_id, arg) {
|
|
|
173
173
|
const port = arg?.port;
|
|
174
174
|
const timeout = arg?.timeout;
|
|
175
175
|
delete batches[batch_id];
|
|
176
|
-
return new Promise((resolve,
|
|
177
|
-
.then(() => execute({ "batch": txns, version: arg?.version, raw, engine, compression: arg?.compression, executor, syncMode, port, timeout }))
|
|
176
|
+
return new Promise((resolve, _reject) => resolve())
|
|
177
|
+
.then(() => execute({ "batch": txns, version: arg?.version, raw, engine, compression: arg?.compression, executor: executor ?? "native", syncMode: syncMode ?? false, port, timeout }))
|
|
178
178
|
.then(result => {
|
|
179
179
|
Events.send("Dip.batchSubmit", { response: result });
|
|
180
180
|
return result;
|
|
@@ -206,6 +206,7 @@ const COLLECTIONS_JSON = await (async () => {
|
|
|
206
206
|
console.error(e);
|
|
207
207
|
console.warn("Warn: Dip Failed to load collections.json. Semantics, topology and legality enforcement is not active");
|
|
208
208
|
}
|
|
209
|
+
return {};
|
|
209
210
|
})();
|
|
210
211
|
function init_collections_config() {
|
|
211
212
|
for (let collection in COLLECTIONS_JSON) {
|
|
@@ -224,7 +225,8 @@ export function excludeCollectionConfig(collection) {
|
|
|
224
225
|
export function includeCollectionConfig(collection) {
|
|
225
226
|
delete EXCLUDED_COLLECTIONS_CONFIG[collection];
|
|
226
227
|
}
|
|
227
|
-
function validate_topology(meta,
|
|
228
|
+
function validate_topology(meta, cols, fn_name) {
|
|
229
|
+
const collection = typeof cols === "string" ? cols : cols[0]; // TODO: arrays
|
|
228
230
|
let new_meta = meta;
|
|
229
231
|
let config = COLLECTIONS_JSON?.[collection];
|
|
230
232
|
if (!config) {
|
|
@@ -239,13 +241,14 @@ function validate_topology(meta, collection, fn_name) {
|
|
|
239
241
|
throw new Error(`Error: Incompatible meta.outlet config for collection: ${collection}${suffix ? ' suffix: ' + suffix : ''}${fn_name ? ` ${fn_name}` : ''}`);
|
|
240
242
|
}
|
|
241
243
|
check_core_meta(config, meta);
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
244
|
+
const meta_suffix = meta.suffix; // TODO: use loop
|
|
245
|
+
if (config.suffix && meta_suffix) {
|
|
246
|
+
if (!config.suffix[meta_suffix]) {
|
|
247
|
+
console.warn(`Warn: Missing suffix config for collection: ${collection} suffix:${meta_suffix}${fn_name ? ` ${fn_name}` : ''}`);
|
|
245
248
|
return new_meta;
|
|
246
249
|
}
|
|
247
|
-
check_core_meta(config.suffix[
|
|
248
|
-
new_meta = Object.assign({}, { ...config, suffix: undefined, company: undefined, outlet: undefined }, { ...config.suffix[
|
|
250
|
+
check_core_meta(config.suffix[meta_suffix], meta, meta_suffix);
|
|
251
|
+
new_meta = Object.assign({}, { ...config, suffix: undefined, company: undefined, outlet: undefined }, { ...config.suffix[meta_suffix], company: undefined, outlet: undefined }, meta);
|
|
249
252
|
}
|
|
250
253
|
else {
|
|
251
254
|
new_meta = Object.assign({}, { ...config, suffix: undefined, company: undefined, outlet: undefined }, { company: undefined, outlet: undefined }, meta);
|
|
@@ -267,7 +270,7 @@ export const insert = async (meta, collection, value, options, extras) => {
|
|
|
267
270
|
collection: typeof collection === "string" ? [collection] : collection,
|
|
268
271
|
insert: value,
|
|
269
272
|
options: options ? options : {},
|
|
270
|
-
syncMode: meta.syncMode,
|
|
273
|
+
syncMode: meta.syncMode ?? false,
|
|
271
274
|
db: extras?.db ?? meta?.db ?? db,
|
|
272
275
|
...(meta.DIP_URL ? { DIP_URL: meta.DIP_URL } : {}),
|
|
273
276
|
...(meta.DIP_DB ? { db: meta.DIP_DB } : {}),
|
|
@@ -290,7 +293,7 @@ export const insert = async (meta, collection, value, options, extras) => {
|
|
|
290
293
|
batches[meta.batch].push(arg);
|
|
291
294
|
return;
|
|
292
295
|
}
|
|
293
|
-
return new Promise((resolve,
|
|
296
|
+
return new Promise((resolve, _reject) => resolve())
|
|
294
297
|
.then(() => execute(arg))
|
|
295
298
|
.then(result => {
|
|
296
299
|
Events.send("Dip.insert", { collection: collectionArray(collection), response: result });
|
|
@@ -308,7 +311,7 @@ export const query = async (meta, collection, query, options, extras) => {
|
|
|
308
311
|
collection: typeof collection === "string" ? [collection] : collection,
|
|
309
312
|
query: query,
|
|
310
313
|
options: options ? options : {},
|
|
311
|
-
syncMode: meta.syncMode,
|
|
314
|
+
syncMode: meta.syncMode ?? false,
|
|
312
315
|
db: extras?.db ?? meta?.db ?? db,
|
|
313
316
|
...(meta.DIP_URL ? { DIP_URL: meta.DIP_URL } : {}),
|
|
314
317
|
...(meta.DIP_DB ? { db: meta.DIP_DB } : {}),
|
|
@@ -327,24 +330,12 @@ export const query = async (meta, collection, query, options, extras) => {
|
|
|
327
330
|
batches[meta.batch].push(arg);
|
|
328
331
|
return;
|
|
329
332
|
}
|
|
330
|
-
return new Promise((resolve,
|
|
333
|
+
return new Promise((resolve, _reject) => resolve())
|
|
331
334
|
.then(() => execute(arg));
|
|
332
335
|
};
|
|
333
336
|
function collectionArray(col) {
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
let newcol = [];
|
|
337
|
-
for (let i in col) {
|
|
338
|
-
if (!Utils.isEmpty(col[i]))
|
|
339
|
-
newcol.push(col[i]);
|
|
340
|
-
}
|
|
341
|
-
return newcol;
|
|
342
|
-
}
|
|
343
|
-
else if (type === '[object String]') {
|
|
344
|
-
if (!Utils.isEmpty(col))
|
|
345
|
-
return [col];
|
|
346
|
-
}
|
|
347
|
-
return [];
|
|
337
|
+
const collections = typeof col === 'string' ? [col] : Array.isArray(col) ? col : [];
|
|
338
|
+
return collections.filter(collection => collection.trim());
|
|
348
339
|
}
|
|
349
340
|
// Ported
|
|
350
341
|
export const update = async (meta, collection, query, update, options, extras) => {
|
|
@@ -358,7 +349,7 @@ export const update = async (meta, collection, query, update, options, extras) =
|
|
|
358
349
|
query: query,
|
|
359
350
|
update: update,
|
|
360
351
|
options: options ? options : {},
|
|
361
|
-
syncMode: meta.syncMode,
|
|
352
|
+
syncMode: meta.syncMode ?? false,
|
|
362
353
|
db: extras?.db ?? meta?.db ?? db,
|
|
363
354
|
...(meta.DIP_URL ? { DIP_URL: meta.DIP_URL } : {}),
|
|
364
355
|
...(meta.DIP_DB ? { db: meta.DIP_DB } : {}),
|
|
@@ -377,7 +368,7 @@ export const update = async (meta, collection, query, update, options, extras) =
|
|
|
377
368
|
batches[meta.batch].push(arg);
|
|
378
369
|
return;
|
|
379
370
|
}
|
|
380
|
-
return new Promise((resolve,
|
|
371
|
+
return new Promise((resolve, _reject) => resolve())
|
|
381
372
|
.then(() => execute(arg))
|
|
382
373
|
.then(result => {
|
|
383
374
|
Events.send("Dip.update", { collection: collectionArray(collection), response: result });
|
|
@@ -398,7 +389,7 @@ export const remove = async (meta, collection, query, options, extras) => {
|
|
|
398
389
|
query: query,
|
|
399
390
|
options: options ? options : {},
|
|
400
391
|
delete: true,
|
|
401
|
-
syncMode: meta.syncMode,
|
|
392
|
+
syncMode: meta.syncMode ?? false,
|
|
402
393
|
db: extras?.db ?? meta?.db ?? db,
|
|
403
394
|
...(meta.DIP_URL ? { DIP_URL: meta.DIP_URL } : {}),
|
|
404
395
|
...(meta.DIP_DB ? { db: meta.DIP_DB } : {}),
|
|
@@ -417,7 +408,7 @@ export const remove = async (meta, collection, query, options, extras) => {
|
|
|
417
408
|
batches[meta.batch].push(arg);
|
|
418
409
|
return;
|
|
419
410
|
}
|
|
420
|
-
return new Promise((resolve,
|
|
411
|
+
return new Promise((resolve, _reject) => resolve())
|
|
421
412
|
.then(() => execute(arg))
|
|
422
413
|
.then(result => {
|
|
423
414
|
Events.send("Dip.remove", { collection: collectionArray(collection), response: result });
|
|
@@ -438,7 +429,7 @@ function execute(arg) {
|
|
|
438
429
|
// update: { "$set": { "age": 21 } }
|
|
439
430
|
// _id: 1 // Optional. Used with insert
|
|
440
431
|
// }
|
|
441
|
-
arg.db = arg.db
|
|
432
|
+
arg.db = arg.db ? arg.db : db ? db : '';
|
|
442
433
|
arg.executor = arg.executor ?? "native";
|
|
443
434
|
arg.engine = arg.engine ?? internal.engineName;
|
|
444
435
|
if (arg.compression === undefined)
|
|
@@ -450,7 +441,7 @@ function execute(arg) {
|
|
|
450
441
|
return config.useDipper ? dipper(arg)
|
|
451
442
|
: axios
|
|
452
443
|
.post(arg.DIP_URL ?? fullurl, buildHybridRequest({ ...arg, DIP_URL: undefined }), { responseType: 'arraybuffer', timeout: timeout ?? 5000, headers: { "Content-Type": "application/dip" } })
|
|
453
|
-
.then(dip_result => {
|
|
444
|
+
.then((dip_result) => {
|
|
454
445
|
// console.log(res.status);
|
|
455
446
|
// console.log(JSON.stringify(res.header, null, 4));
|
|
456
447
|
// console.log(JSON.stringify(res.body, null, 4));
|
|
@@ -503,7 +494,7 @@ function parseBinaryResponse(result) {
|
|
|
503
494
|
// }
|
|
504
495
|
// parseWireFormat() equivalent
|
|
505
496
|
// =============================
|
|
506
|
-
let dataView = new DataView(arrayBuffer.buffer,
|
|
497
|
+
// let dataView = new DataView(arrayBuffer.buffer, arrayBuffer.byteOffset, arrayBuffer.byteLength);
|
|
507
498
|
let offset = 0;
|
|
508
499
|
// 1. Read the metadata size header
|
|
509
500
|
let vlq = Essentials.vlqToUint64(Essentials.sliceArrayBuffer(arrayBuffer, offset, 10));
|
|
@@ -519,7 +510,7 @@ function parseBinaryResponse(result) {
|
|
|
519
510
|
return { metadata: nativeResponse, content: content };
|
|
520
511
|
}
|
|
521
512
|
function setRawBatchIterator(arrayBuffer) {
|
|
522
|
-
arrayBuffer.metadata().items.forEach(batch => {
|
|
513
|
+
arrayBuffer.metadata().items.forEach((batch) => {
|
|
523
514
|
batch.kv = () => {
|
|
524
515
|
const batchItem = {
|
|
525
516
|
count: batch.count,
|
|
@@ -554,11 +545,11 @@ function convert_to_array_buffer(value) {
|
|
|
554
545
|
return Essentials.stringToUtf8ArrayBuffer(JSON.stringify(value));
|
|
555
546
|
}
|
|
556
547
|
export function newRaw() {
|
|
557
|
-
let obj = { buffers: [], size: 0, hashes: new Map() };
|
|
548
|
+
let obj = { buffers: [], size: 0, hashes: new Map(), add: (_value) => 0 };
|
|
558
549
|
obj.add = (value) => {
|
|
559
550
|
if (value === undefined)
|
|
560
551
|
return undefined;
|
|
561
|
-
const buf = (value instanceof
|
|
552
|
+
const buf = (value instanceof Buffer) ? value : convert_to_array_buffer(value);
|
|
562
553
|
const hash = Essentials.defaultHash(buf);
|
|
563
554
|
for (const item of obj.hashes.get(hash) ?? []) { // Conflict. Fallback to verify
|
|
564
555
|
if (Essentials.memcmpEqual(buf, item.buf))
|
|
@@ -641,7 +632,7 @@ function bufferToString(arrayBuffer) {
|
|
|
641
632
|
return Essentials.arrayBufferToString(arrayBuffer);
|
|
642
633
|
}
|
|
643
634
|
function getRawKeyValue(arrayBuffer, offset) {
|
|
644
|
-
let dataView = new DataView(arrayBuffer.buffer,
|
|
635
|
+
// let dataView = new DataView(arrayBuffer.buffer, arrayBuffer.byteOffset, arrayBuffer.byteLength);
|
|
645
636
|
let masterView = new Uint8Array(arrayBuffer);
|
|
646
637
|
// 1. Extract Key Size and Key
|
|
647
638
|
let vlq = Essentials.vlqToUint64(Essentials.sliceArrayBuffer(arrayBuffer, offset, 10));
|