@omegup/msync 0.1.29 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +1 -2
- package/index.esm.js +18 -42
- package/index.js +16 -41
- package/package.json +1 -1
package/index.d.ts
CHANGED
|
@@ -327,7 +327,6 @@ type View<V extends Model, K extends StrKey<V>> = {
|
|
|
327
327
|
};
|
|
328
328
|
|
|
329
329
|
declare const val: <T extends rawItem>(val: T) => Expr<T, unknown>;
|
|
330
|
-
declare const afterWriteTime: Expr<Timestamp, unknown>;
|
|
331
330
|
declare const current: Expr<Timestamp, unknown>;
|
|
332
331
|
declare const $let: <T, D, C, V extends RORec<string, jsonItem>>(vars: ExprsExact<V, D, C>, inExpr: Expr<T, D, C & V>) => Expr<T, D, C>;
|
|
333
332
|
declare const nil: Expr<null, unknown>;
|
|
@@ -681,5 +680,5 @@ declare const enablePreAndPostImages: <T extends doc>(coll: Collection<T>) => Pr
|
|
|
681
680
|
declare const prepare: (testName?: string) => Promise<MongoClient$1>;
|
|
682
681
|
declare const makeCol: <T extends ID>(docs: readonly OptionalUnlessRequiredId<T>[], database: Db, name?: string) => Promise<Collection<T>>;
|
|
683
682
|
|
|
684
|
-
export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, Expr, Field, Machine, Type, add,
|
|
683
|
+
export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, Expr, Field, Machine, Type, add, and, anyElementTrue, array, ceil, comp, concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, notNull, now, or, pair, prepare, rand, range, regex, root, set, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };
|
|
685
684
|
export type { Accumulators, Arr, AsLiteral, Delta, DeltaAccumulator, DeltaAccumulators, ExactKeys, ExprHKT, Exprs, ExprsExact, ExprsExactHKT, ExprsPart, ID, Loose, Merge, MergeArgs, MergeInto, MergeMapOArgs, Model, MongoTypeNames, N, NoRaw, NullToOBJ, O, OPick, OPickD, Patch, RONoRaw, RORec, RawStages, Rec, Replace, SnapshotStreamExecutionResult, StrKey, Strict, TS, WriteonlyCollection, doc, jsonPrim, notArr };
|
package/index.esm.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { UUID, Timestamp, MongoClient } from 'mongodb';
|
|
2
2
|
import crypto$1 from 'crypto';
|
|
3
3
|
import { canonicalize } from 'json-canonicalize';
|
|
4
4
|
import { SynchronousPromise } from 'synchronous-promise';
|
|
@@ -81,9 +81,6 @@ const val = (val) => asExpr({
|
|
|
81
81
|
? { $literal: val }
|
|
82
82
|
: val),
|
|
83
83
|
});
|
|
84
|
-
const afterWriteTime = asExpr({
|
|
85
|
-
raw: () => asExprRaw(new Timestamp(0xffffffffffffffffn)),
|
|
86
|
-
});
|
|
87
84
|
const current = asExpr({
|
|
88
85
|
raw: () => asExprRaw('$$CLUSTER_TIME'),
|
|
89
86
|
});
|
|
@@ -845,7 +842,7 @@ const $mergeX = (out, keys, f, map, ext) => {
|
|
|
845
842
|
const setDeleted = out.whenNotMatched === 'discard';
|
|
846
843
|
const replacer = map(field(omitPick().backward(spread(patch, {
|
|
847
844
|
_id: ['_id', f.of('_id').expr()],
|
|
848
|
-
touchedAt: ['touchedAt',
|
|
845
|
+
touchedAt: ['touchedAt', current],
|
|
849
846
|
}))));
|
|
850
847
|
const sss = setDeleted
|
|
851
848
|
? link()
|
|
@@ -889,7 +886,7 @@ const $mergeId = () => (out, keys, id, ext) => {
|
|
|
889
886
|
return $mergeX(out, keys, root().of('after'), or => {
|
|
890
887
|
return ite(eqTyped(root().of('after').expr(), nil), field(omRORec.backward(spread(mapExact(keys, () => nil), {
|
|
891
888
|
_id: ['_id', id],
|
|
892
|
-
touchedAt: ['touchedAt',
|
|
889
|
+
touchedAt: ['touchedAt', current],
|
|
893
890
|
}))), or);
|
|
894
891
|
}, ext);
|
|
895
892
|
};
|
|
@@ -911,7 +908,7 @@ const subMerge = (args, out, gid, extra, idPrefix, first) => {
|
|
|
911
908
|
const mapId = (k, v) => map1(k, v);
|
|
912
909
|
const F1 = {
|
|
913
910
|
_id: ['_id', to(idPrefix ? concat$1(val(idPrefix), $rand) : $rand)],
|
|
914
|
-
touchedAt: ['touchedAt', to(
|
|
911
|
+
touchedAt: ['touchedAt', to(current)],
|
|
915
912
|
};
|
|
916
913
|
const F2 = mapId(gid, to(gidPath));
|
|
917
914
|
const addExtraAndMerge = {
|
|
@@ -922,7 +919,7 @@ const subMerge = (args, out, gid, extra, idPrefix, first) => {
|
|
|
922
919
|
const addTSAndExtra = {
|
|
923
920
|
...mapExact0(e, to),
|
|
924
921
|
...(out.whenNotMatched === 'insert' ? { deletedAt: ['deletedAt', to(nil)] } : {}),
|
|
925
|
-
touchedAt: ['touchedAt', to(
|
|
922
|
+
touchedAt: ['touchedAt', to(current)],
|
|
926
923
|
};
|
|
927
924
|
const updater = set()(addTSAndExtra);
|
|
928
925
|
const whenMatched = getWhenMatched(out.whenNotMatched);
|
|
@@ -1540,7 +1537,7 @@ const $insertX = (out, expr, map, ext, extExpr) => {
|
|
|
1540
1537
|
raw: () => {
|
|
1541
1538
|
const replacer = map(mergeObjects(expr, field(mergeExpr(extExpr, {
|
|
1542
1539
|
deletedAt: ['deletedAt', nil],
|
|
1543
|
-
touchedAt: ['touchedAt',
|
|
1540
|
+
touchedAt: ['touchedAt', current],
|
|
1544
1541
|
}))));
|
|
1545
1542
|
return link()
|
|
1546
1543
|
.with($replaceWith_(replacer))
|
|
@@ -1560,7 +1557,7 @@ const $insertPart = (out, ext) => {
|
|
|
1560
1557
|
return $insertX(out, assertNotNull(root().of('after').expr()), x => ite(eq(root().of('after').expr())(nil), field(mergeExpr(translateOmit().forward(extExpr), {
|
|
1561
1558
|
deletedAt: ['deletedAt', current],
|
|
1562
1559
|
_id: ['_id', assertNotNull(root().of('before').of('_id').expr())],
|
|
1563
|
-
touchedAt: ['touchedAt',
|
|
1560
|
+
touchedAt: ['touchedAt', current],
|
|
1564
1561
|
})), x), ext, extExpr);
|
|
1565
1562
|
};
|
|
1566
1563
|
const $insert = (out) => $insertPart(out, {});
|
|
@@ -1610,7 +1607,6 @@ const addTeardown = (it, tr) => {
|
|
|
1610
1607
|
|
|
1611
1608
|
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
|
1612
1609
|
const maxTimestamp = new Timestamp(0xffffffffffffffffn);
|
|
1613
|
-
const isMax = (x) => x instanceof Timestamp && x.equals(maxTimestamp);
|
|
1614
1610
|
const getCurrentTimestamp = async (db) => {
|
|
1615
1611
|
const adminDb = db.admin();
|
|
1616
1612
|
const serverStatus = await adminDb.command({ serverStatus: 1 });
|
|
@@ -1633,9 +1629,9 @@ async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000
|
|
|
1633
1629
|
await sleep(pollMs);
|
|
1634
1630
|
}
|
|
1635
1631
|
}
|
|
1636
|
-
async function* tailOplog(db
|
|
1637
|
-
let lastTs =
|
|
1638
|
-
const reopenDelayMs =
|
|
1632
|
+
async function* tailOplog(db) {
|
|
1633
|
+
let lastTs = await getCurrentTimestamp(db);
|
|
1634
|
+
const reopenDelayMs = 250;
|
|
1639
1635
|
const coll = db.client.db('local').collection('oplog.rs');
|
|
1640
1636
|
while (true) {
|
|
1641
1637
|
const cursor = coll.find({
|
|
@@ -1649,14 +1645,12 @@ async function* tailOplog(db, opts) {
|
|
|
1649
1645
|
});
|
|
1650
1646
|
try {
|
|
1651
1647
|
for await (const doc of cursor) {
|
|
1652
|
-
lastTs = doc.ts;
|
|
1653
1648
|
if (doc.op === 'i' || '_id' in doc.o) {
|
|
1654
1649
|
const fields = new Set(Object.keys(doc.o));
|
|
1655
1650
|
fields.delete('_id');
|
|
1656
|
-
yield { fields, doc
|
|
1651
|
+
yield { fields, doc };
|
|
1657
1652
|
}
|
|
1658
1653
|
else {
|
|
1659
|
-
let changeTouched = false;
|
|
1660
1654
|
if (doc.o['$v'] !== 2) {
|
|
1661
1655
|
throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc)}`);
|
|
1662
1656
|
}
|
|
@@ -1665,21 +1659,19 @@ async function* tailOplog(db, opts) {
|
|
|
1665
1659
|
for (const updateOp in diff) {
|
|
1666
1660
|
if (['u', 'i', 'd'].includes(updateOp)) {
|
|
1667
1661
|
updatedFields.push(...Object.keys(diff[updateOp]));
|
|
1668
|
-
if (isMax(diff[updateOp]['touchedAt'])) {
|
|
1669
|
-
changeTouched = true;
|
|
1670
|
-
}
|
|
1671
1662
|
}
|
|
1672
1663
|
else if (updateOp.startsWith('s')) {
|
|
1673
1664
|
updatedFields.push(updateOp.slice(1));
|
|
1674
1665
|
}
|
|
1675
1666
|
}
|
|
1676
|
-
yield { fields: new Set(updatedFields), doc
|
|
1667
|
+
yield { fields: new Set(updatedFields), doc };
|
|
1677
1668
|
}
|
|
1678
1669
|
}
|
|
1679
1670
|
}
|
|
1680
1671
|
catch (e) {
|
|
1681
1672
|
log('oplog loop error, notifying watchers and reopening');
|
|
1682
1673
|
console.error(e);
|
|
1674
|
+
lastTs = await getCurrentTimestamp(db);
|
|
1683
1675
|
yield null;
|
|
1684
1676
|
}
|
|
1685
1677
|
finally {
|
|
@@ -1701,7 +1693,7 @@ const loop = async (db) => {
|
|
|
1701
1693
|
let notify = makePromise();
|
|
1702
1694
|
let batch = [];
|
|
1703
1695
|
const run = async () => {
|
|
1704
|
-
for await (const event of tailOplog(db
|
|
1696
|
+
for await (const event of tailOplog(db)) {
|
|
1705
1697
|
if (event?.fields.size === 0)
|
|
1706
1698
|
continue;
|
|
1707
1699
|
batch = event && batch ? [...batch, event] : null;
|
|
@@ -1728,23 +1720,6 @@ const loop = async (db) => {
|
|
|
1728
1720
|
}
|
|
1729
1721
|
continue;
|
|
1730
1722
|
}
|
|
1731
|
-
const groups = Object.groupBy(events.filter(e => e.changeTouched), ev => ev.doc.ns);
|
|
1732
|
-
for (const [ns, evs] of Object.entries(groups)) {
|
|
1733
|
-
if (!evs)
|
|
1734
|
-
continue;
|
|
1735
|
-
const [dbName, collName] = ns.split('.');
|
|
1736
|
-
if (dbName !== db.databaseName)
|
|
1737
|
-
continue;
|
|
1738
|
-
const coll = db.collection(collName);
|
|
1739
|
-
coll
|
|
1740
|
-
.bulkWrite(evs.map((e) => ({
|
|
1741
|
-
updateOne: {
|
|
1742
|
-
filter: { _id: e.doc.o['_id'] ?? e.doc.o2?._id },
|
|
1743
|
-
update: { $set: { touchedAt: e.doc.ts } },
|
|
1744
|
-
},
|
|
1745
|
-
})))
|
|
1746
|
-
.catch(() => { });
|
|
1747
|
-
}
|
|
1748
1723
|
for (const { fields, doc } of events) {
|
|
1749
1724
|
const m = watchers.get(doc.ns);
|
|
1750
1725
|
if (!m)
|
|
@@ -1812,6 +1787,7 @@ const actions = {
|
|
|
1812
1787
|
],
|
|
1813
1788
|
};
|
|
1814
1789
|
|
|
1790
|
+
const previous = (ts) => new Timestamp({ t: ts.high - 60, i: 0 });
|
|
1815
1791
|
const getFirstStages = (view, needs) => {
|
|
1816
1792
|
const { projection, hardMatch: pre, match } = view;
|
|
1817
1793
|
const projectInput = projection && $project_(spread(projection, {
|
|
@@ -1822,7 +1798,7 @@ const getFirstStages = (view, needs) => {
|
|
|
1822
1798
|
const hardMatch = removeNotYetSynchronizedFields ? $and(pre, ...removeNotYetSynchronizedFields) : pre;
|
|
1823
1799
|
const firstStages = (lastTS, keepNulls = false) => {
|
|
1824
1800
|
const hardQuery = $and(lastTS
|
|
1825
|
-
? root().of('touchedAt').has($gtTs(lastTS.ts))
|
|
1801
|
+
? root().of('touchedAt').has($gtTs(previous(lastTS.ts)))
|
|
1826
1802
|
: root().of('deletedAt').has($eq(null)), lastTS ? null : match && $expr(match), keepNulls ? pre : hardMatch);
|
|
1827
1803
|
const ln = link()
|
|
1828
1804
|
.with($match_(hardQuery));
|
|
@@ -2129,7 +2105,7 @@ const executes$1 = (view, input, streamName, needs) => {
|
|
|
2129
2105
|
}));
|
|
2130
2106
|
const notDeleted = root().of('deletedAt').has($eq(null));
|
|
2131
2107
|
const stages = (lastTS) => {
|
|
2132
|
-
const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(lastTS.ts)), hardMatch, notDeleted, match && $expr(match));
|
|
2108
|
+
const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(previous(lastTS.ts))), hardMatch, notDeleted, match && $expr(match));
|
|
2133
2109
|
const ln = link().with($match_(hardQuery));
|
|
2134
2110
|
return (projectInput ? ln.with(projectInput) : ln).with(input);
|
|
2135
2111
|
};
|
|
@@ -2250,4 +2226,4 @@ const executes = (view, input, needs) => {
|
|
|
2250
2226
|
};
|
|
2251
2227
|
const single = (view, needs = {}) => pipe(input => executes(view, input, needs), emptyDelta(), concatDelta, emptyDelta);
|
|
2252
2228
|
|
|
2253
|
-
export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, Field, Machine, add,
|
|
2229
|
+
export { $accumulator, $and, $countDict, $entries, $eq, $exists, $expr, $getField, $group, $groupId, $groupMerge, $group_, $gt, $gtTs, $gte, $gteTs, $ifNull, $in, $insert, $insertPart, $insertX, $keys, $let, $lookup, $lt, $lte, $map, $map0, $map1, $match, $matchDelta, $merge, $merge2, $mergeId, $mergePart, $merge_, $ne, $nin, $nor, $or, $outerLookup, $pushDict, $rand, $reduce, $replaceWith, $set, $simpleInsert, $simpleMerge, $simpleMergePart, $sum, $type, $unwind, $unwindDelta, Field, Machine, add, and, anyElementTrue, array, ceil, comp, concat$1 as concat, concatArray, createIndex, ctx, current, dateAdd, dateDiff, dateLt, datePart, dayAndMonthPart, divide, enablePreAndPostImages, eq, eqTyped, except, exprMapVal, field, fieldF, fieldM, filter, filterDefined, first$1 as first, firstSure, floor, from, func, getWhenMatched, getWhenMatchedForMerge, gt, gte, inArray, isArray, ite, last, log, lt, lte, makeCol, map1, mapVal, max, maxDate, mergeExact, mergeExact0, mergeExpr, mergeObjects, minDate, monthPart, multiply, ne, nil, noop, not, notNull, now, or, pair, prepare, rand, range, regex, root, set, setField, single, size, slice, sortArray, staging, startOf, str, sub, subtract, to, toInt, val, weekPart, wrap, year };
|
package/index.js
CHANGED
|
@@ -83,9 +83,6 @@ const val = (val) => asExpr({
|
|
|
83
83
|
? { $literal: val }
|
|
84
84
|
: val),
|
|
85
85
|
});
|
|
86
|
-
const afterWriteTime = asExpr({
|
|
87
|
-
raw: () => asExprRaw(new mongodb.Timestamp(0xffffffffffffffffn)),
|
|
88
|
-
});
|
|
89
86
|
const current = asExpr({
|
|
90
87
|
raw: () => asExprRaw('$$CLUSTER_TIME'),
|
|
91
88
|
});
|
|
@@ -847,7 +844,7 @@ const $mergeX = (out, keys, f, map, ext) => {
|
|
|
847
844
|
const setDeleted = out.whenNotMatched === 'discard';
|
|
848
845
|
const replacer = map(field(omitPick().backward(spread(patch, {
|
|
849
846
|
_id: ['_id', f.of('_id').expr()],
|
|
850
|
-
touchedAt: ['touchedAt',
|
|
847
|
+
touchedAt: ['touchedAt', current],
|
|
851
848
|
}))));
|
|
852
849
|
const sss = setDeleted
|
|
853
850
|
? link()
|
|
@@ -891,7 +888,7 @@ const $mergeId = () => (out, keys, id, ext) => {
|
|
|
891
888
|
return $mergeX(out, keys, root().of('after'), or => {
|
|
892
889
|
return ite(eqTyped(root().of('after').expr(), nil), field(omRORec.backward(spread(mapExact(keys, () => nil), {
|
|
893
890
|
_id: ['_id', id],
|
|
894
|
-
touchedAt: ['touchedAt',
|
|
891
|
+
touchedAt: ['touchedAt', current],
|
|
895
892
|
}))), or);
|
|
896
893
|
}, ext);
|
|
897
894
|
};
|
|
@@ -913,7 +910,7 @@ const subMerge = (args, out, gid, extra, idPrefix, first) => {
|
|
|
913
910
|
const mapId = (k, v) => map1(k, v);
|
|
914
911
|
const F1 = {
|
|
915
912
|
_id: ['_id', to(idPrefix ? concat$1(val(idPrefix), $rand) : $rand)],
|
|
916
|
-
touchedAt: ['touchedAt', to(
|
|
913
|
+
touchedAt: ['touchedAt', to(current)],
|
|
917
914
|
};
|
|
918
915
|
const F2 = mapId(gid, to(gidPath));
|
|
919
916
|
const addExtraAndMerge = {
|
|
@@ -924,7 +921,7 @@ const subMerge = (args, out, gid, extra, idPrefix, first) => {
|
|
|
924
921
|
const addTSAndExtra = {
|
|
925
922
|
...mapExact0(e, to),
|
|
926
923
|
...(out.whenNotMatched === 'insert' ? { deletedAt: ['deletedAt', to(nil)] } : {}),
|
|
927
|
-
touchedAt: ['touchedAt', to(
|
|
924
|
+
touchedAt: ['touchedAt', to(current)],
|
|
928
925
|
};
|
|
929
926
|
const updater = set()(addTSAndExtra);
|
|
930
927
|
const whenMatched = getWhenMatched(out.whenNotMatched);
|
|
@@ -1542,7 +1539,7 @@ const $insertX = (out, expr, map, ext, extExpr) => {
|
|
|
1542
1539
|
raw: () => {
|
|
1543
1540
|
const replacer = map(mergeObjects(expr, field(mergeExpr(extExpr, {
|
|
1544
1541
|
deletedAt: ['deletedAt', nil],
|
|
1545
|
-
touchedAt: ['touchedAt',
|
|
1542
|
+
touchedAt: ['touchedAt', current],
|
|
1546
1543
|
}))));
|
|
1547
1544
|
return link()
|
|
1548
1545
|
.with($replaceWith_(replacer))
|
|
@@ -1562,7 +1559,7 @@ const $insertPart = (out, ext) => {
|
|
|
1562
1559
|
return $insertX(out, assertNotNull(root().of('after').expr()), x => ite(eq(root().of('after').expr())(nil), field(mergeExpr(translateOmit().forward(extExpr), {
|
|
1563
1560
|
deletedAt: ['deletedAt', current],
|
|
1564
1561
|
_id: ['_id', assertNotNull(root().of('before').of('_id').expr())],
|
|
1565
|
-
touchedAt: ['touchedAt',
|
|
1562
|
+
touchedAt: ['touchedAt', current],
|
|
1566
1563
|
})), x), ext, extExpr);
|
|
1567
1564
|
};
|
|
1568
1565
|
const $insert = (out) => $insertPart(out, {});
|
|
@@ -1612,7 +1609,6 @@ const addTeardown = (it, tr) => {
|
|
|
1612
1609
|
|
|
1613
1610
|
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
|
1614
1611
|
const maxTimestamp = new mongodb.Timestamp(0xffffffffffffffffn);
|
|
1615
|
-
const isMax = (x) => x instanceof mongodb.Timestamp && x.equals(maxTimestamp);
|
|
1616
1612
|
const getCurrentTimestamp = async (db) => {
|
|
1617
1613
|
const adminDb = db.admin();
|
|
1618
1614
|
const serverStatus = await adminDb.command({ serverStatus: 1 });
|
|
@@ -1635,9 +1631,9 @@ async function waitUntilStablePast(db, oplogTs, { pollMs = 0, timeoutMs = 10_000
|
|
|
1635
1631
|
await sleep(pollMs);
|
|
1636
1632
|
}
|
|
1637
1633
|
}
|
|
1638
|
-
async function* tailOplog(db
|
|
1639
|
-
let lastTs =
|
|
1640
|
-
const reopenDelayMs =
|
|
1634
|
+
async function* tailOplog(db) {
|
|
1635
|
+
let lastTs = await getCurrentTimestamp(db);
|
|
1636
|
+
const reopenDelayMs = 250;
|
|
1641
1637
|
const coll = db.client.db('local').collection('oplog.rs');
|
|
1642
1638
|
while (true) {
|
|
1643
1639
|
const cursor = coll.find({
|
|
@@ -1651,14 +1647,12 @@ async function* tailOplog(db, opts) {
|
|
|
1651
1647
|
});
|
|
1652
1648
|
try {
|
|
1653
1649
|
for await (const doc of cursor) {
|
|
1654
|
-
lastTs = doc.ts;
|
|
1655
1650
|
if (doc.op === 'i' || '_id' in doc.o) {
|
|
1656
1651
|
const fields = new Set(Object.keys(doc.o));
|
|
1657
1652
|
fields.delete('_id');
|
|
1658
|
-
yield { fields, doc
|
|
1653
|
+
yield { fields, doc };
|
|
1659
1654
|
}
|
|
1660
1655
|
else {
|
|
1661
|
-
let changeTouched = false;
|
|
1662
1656
|
if (doc.o['$v'] !== 2) {
|
|
1663
1657
|
throw new Error(`Expected update with $v: 2, got ${JSON.stringify(doc)}`);
|
|
1664
1658
|
}
|
|
@@ -1667,21 +1661,19 @@ async function* tailOplog(db, opts) {
|
|
|
1667
1661
|
for (const updateOp in diff) {
|
|
1668
1662
|
if (['u', 'i', 'd'].includes(updateOp)) {
|
|
1669
1663
|
updatedFields.push(...Object.keys(diff[updateOp]));
|
|
1670
|
-
if (isMax(diff[updateOp]['touchedAt'])) {
|
|
1671
|
-
changeTouched = true;
|
|
1672
|
-
}
|
|
1673
1664
|
}
|
|
1674
1665
|
else if (updateOp.startsWith('s')) {
|
|
1675
1666
|
updatedFields.push(updateOp.slice(1));
|
|
1676
1667
|
}
|
|
1677
1668
|
}
|
|
1678
|
-
yield { fields: new Set(updatedFields), doc
|
|
1669
|
+
yield { fields: new Set(updatedFields), doc };
|
|
1679
1670
|
}
|
|
1680
1671
|
}
|
|
1681
1672
|
}
|
|
1682
1673
|
catch (e) {
|
|
1683
1674
|
log('oplog loop error, notifying watchers and reopening');
|
|
1684
1675
|
console.error(e);
|
|
1676
|
+
lastTs = await getCurrentTimestamp(db);
|
|
1685
1677
|
yield null;
|
|
1686
1678
|
}
|
|
1687
1679
|
finally {
|
|
@@ -1703,7 +1695,7 @@ const loop = async (db) => {
|
|
|
1703
1695
|
let notify = makePromise();
|
|
1704
1696
|
let batch = [];
|
|
1705
1697
|
const run = async () => {
|
|
1706
|
-
for await (const event of tailOplog(db
|
|
1698
|
+
for await (const event of tailOplog(db)) {
|
|
1707
1699
|
if (event?.fields.size === 0)
|
|
1708
1700
|
continue;
|
|
1709
1701
|
batch = event && batch ? [...batch, event] : null;
|
|
@@ -1730,23 +1722,6 @@ const loop = async (db) => {
|
|
|
1730
1722
|
}
|
|
1731
1723
|
continue;
|
|
1732
1724
|
}
|
|
1733
|
-
const groups = Object.groupBy(events.filter(e => e.changeTouched), ev => ev.doc.ns);
|
|
1734
|
-
for (const [ns, evs] of Object.entries(groups)) {
|
|
1735
|
-
if (!evs)
|
|
1736
|
-
continue;
|
|
1737
|
-
const [dbName, collName] = ns.split('.');
|
|
1738
|
-
if (dbName !== db.databaseName)
|
|
1739
|
-
continue;
|
|
1740
|
-
const coll = db.collection(collName);
|
|
1741
|
-
coll
|
|
1742
|
-
.bulkWrite(evs.map((e) => ({
|
|
1743
|
-
updateOne: {
|
|
1744
|
-
filter: { _id: e.doc.o['_id'] ?? e.doc.o2?._id },
|
|
1745
|
-
update: { $set: { touchedAt: e.doc.ts } },
|
|
1746
|
-
},
|
|
1747
|
-
})))
|
|
1748
|
-
.catch(() => { });
|
|
1749
|
-
}
|
|
1750
1725
|
for (const { fields, doc } of events) {
|
|
1751
1726
|
const m = watchers.get(doc.ns);
|
|
1752
1727
|
if (!m)
|
|
@@ -1814,6 +1789,7 @@ const actions = {
|
|
|
1814
1789
|
],
|
|
1815
1790
|
};
|
|
1816
1791
|
|
|
1792
|
+
const previous = (ts) => new mongodb.Timestamp({ t: ts.high - 60, i: 0 });
|
|
1817
1793
|
const getFirstStages = (view, needs) => {
|
|
1818
1794
|
const { projection, hardMatch: pre, match } = view;
|
|
1819
1795
|
const projectInput = projection && $project_(spread(projection, {
|
|
@@ -1824,7 +1800,7 @@ const getFirstStages = (view, needs) => {
|
|
|
1824
1800
|
const hardMatch = removeNotYetSynchronizedFields ? $and(pre, ...removeNotYetSynchronizedFields) : pre;
|
|
1825
1801
|
const firstStages = (lastTS, keepNulls = false) => {
|
|
1826
1802
|
const hardQuery = $and(lastTS
|
|
1827
|
-
? root().of('touchedAt').has($gtTs(lastTS.ts))
|
|
1803
|
+
? root().of('touchedAt').has($gtTs(previous(lastTS.ts)))
|
|
1828
1804
|
: root().of('deletedAt').has($eq(null)), lastTS ? null : match && $expr(match), keepNulls ? pre : hardMatch);
|
|
1829
1805
|
const ln = link()
|
|
1830
1806
|
.with($match_(hardQuery));
|
|
@@ -2131,7 +2107,7 @@ const executes$1 = (view, input, streamName, needs) => {
|
|
|
2131
2107
|
}));
|
|
2132
2108
|
const notDeleted = root().of('deletedAt').has($eq(null));
|
|
2133
2109
|
const stages = (lastTS) => {
|
|
2134
|
-
const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(lastTS.ts)), hardMatch, notDeleted, match && $expr(match));
|
|
2110
|
+
const hardQuery = $and(lastTS && root().of('touchedAt').has($gteTs(previous(lastTS.ts))), hardMatch, notDeleted, match && $expr(match));
|
|
2135
2111
|
const ln = link().with($match_(hardQuery));
|
|
2136
2112
|
return (projectInput ? ln.with(projectInput) : ln).with(input);
|
|
2137
2113
|
};
|
|
@@ -2308,7 +2284,6 @@ exports.$unwindDelta = $unwindDelta;
|
|
|
2308
2284
|
exports.Field = Field;
|
|
2309
2285
|
exports.Machine = Machine;
|
|
2310
2286
|
exports.add = add;
|
|
2311
|
-
exports.afterWriteTime = afterWriteTime;
|
|
2312
2287
|
exports.and = and;
|
|
2313
2288
|
exports.anyElementTrue = anyElementTrue;
|
|
2314
2289
|
exports.array = array;
|