@rocicorp/zero 1.12.0-canary.12 → 1.12.0-canary.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/replicache/src/btree/diff.d.ts.map +1 -1
- package/out/replicache/src/btree/diff.js +1 -2
- package/out/replicache/src/btree/diff.js.map +1 -1
- package/out/replicache/src/btree/read.d.ts +31 -1
- package/out/replicache/src/btree/read.d.ts.map +1 -1
- package/out/replicache/src/btree/read.js +74 -19
- package/out/replicache/src/btree/read.js.map +1 -1
- package/out/replicache/src/btree/splice.d.ts.map +1 -1
- package/out/replicache/src/btree/splice.js +2 -1
- package/out/replicache/src/btree/splice.js.map +1 -1
- package/out/replicache/src/btree/write.d.ts.map +1 -1
- package/out/replicache/src/btree/write.js +2 -1
- package/out/replicache/src/btree/write.js.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.d.ts +1 -0
- package/out/replicache/src/persist/collect-idb-databases.d.ts.map +1 -1
- package/out/replicache/src/persist/collect-idb-databases.js +13 -5
- package/out/replicache/src/persist/collect-idb-databases.js.map +1 -1
- package/out/replicache/src/scan-iterator.js +1 -1
- package/out/replicache/src/sync/request-id.js +2 -2
- package/out/replicache/src/sync/request-id.js.map +1 -1
- package/out/shared/src/bigint-json.d.ts.map +1 -1
- package/out/shared/src/bigint-json.js +1 -0
- package/out/shared/src/bigint-json.js.map +1 -1
- package/out/shared/src/btree-set.d.ts +21 -0
- package/out/shared/src/btree-set.d.ts.map +1 -1
- package/out/shared/src/btree-set.js +54 -0
- package/out/shared/src/btree-set.js.map +1 -1
- package/out/shared/src/headers.js +20 -0
- package/out/shared/src/headers.js.map +1 -0
- package/out/shared/src/natural-runs.d.ts +191 -0
- package/out/shared/src/natural-runs.d.ts.map +1 -0
- package/out/shared/src/natural-runs.js +337 -0
- package/out/shared/src/natural-runs.js.map +1 -0
- package/out/shared/src/random-uint64.d.ts.map +1 -1
- package/out/shared/src/random-uint64.js +2 -2
- package/out/shared/src/random-uint64.js.map +1 -1
- package/out/shared/src/random-values.d.ts +19 -1
- package/out/shared/src/random-values.d.ts.map +1 -1
- package/out/shared/src/random-values.js +38 -4
- package/out/shared/src/random-values.js.map +1 -1
- package/out/shared/src/tuple-map.d.ts +22 -0
- package/out/shared/src/tuple-map.d.ts.map +1 -0
- package/out/shared/src/tuple-map.js +48 -0
- package/out/shared/src/tuple-map.js.map +1 -0
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/auth/auth.d.ts.map +1 -1
- package/out/zero-cache/src/auth/auth.js +5 -0
- package/out/zero-cache/src/auth/auth.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +21 -0
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +47 -9
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/custom-queries/transform-query.js +2 -2
- package/out/zero-cache/src/custom-queries/transform-query.js.map +1 -1
- package/out/zero-cache/src/db/pg-to-lite.d.ts.map +1 -1
- package/out/zero-cache/src/db/pg-to-lite.js +3 -2
- package/out/zero-cache/src/db/pg-to-lite.js.map +1 -1
- package/out/zero-cache/src/db/pg-type-parser.js +55 -2
- package/out/zero-cache/src/db/pg-type-parser.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +5 -2
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/inspector-delegate.js +6 -29
- package/out/zero-cache/src/server/inspector-delegate.js.map +1 -1
- package/out/zero-cache/src/server/main.js +5 -0
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/replicator.js +3 -2
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/syncer.js +5 -4
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/server/worker-dispatcher.d.ts.map +1 -1
- package/out/zero-cache/src/server/worker-dispatcher.js +2 -1
- package/out/zero-cache/src/server/worker-dispatcher.js.map +1 -1
- package/out/zero-cache/src/services/admin-auth.d.ts +22 -0
- package/out/zero-cache/src/services/admin-auth.d.ts.map +1 -0
- package/out/zero-cache/src/services/admin-auth.js +42 -0
- package/out/zero-cache/src/services/admin-auth.js.map +1 -0
- package/out/zero-cache/src/services/auth-attempt-counter.d.ts +33 -0
- package/out/zero-cache/src/services/auth-attempt-counter.d.ts.map +1 -0
- package/out/zero-cache/src/services/auth-attempt-counter.js +61 -0
- package/out/zero-cache/src/services/auth-attempt-counter.js.map +1 -0
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +110 -13
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/ddl.d.ts +13 -7
- package/out/zero-cache/src/services/change-source/pg/schema/ddl.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/ddl.js +38 -5
- package/out/zero-cache/src/services/change-source/pg/schema/ddl.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js +10 -1
- package/out/zero-cache/src/services/change-source/pg/schema/init.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts +4 -0
- package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js +10 -3
- package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +23 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +2 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js +10 -3
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscriber.js.map +1 -1
- package/out/zero-cache/src/services/heapz.d.ts.map +1 -1
- package/out/zero-cache/src/services/heapz.js +4 -5
- package/out/zero-cache/src/services/heapz.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/error.d.ts +5 -0
- package/out/zero-cache/src/services/mutagen/error.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/error.js +7 -1
- package/out/zero-cache/src/services/mutagen/error.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js +1 -1
- package/out/zero-cache/src/services/mutagen/pusher.js.map +1 -1
- package/out/zero-cache/src/services/plannerz.d.ts.map +1 -1
- package/out/zero-cache/src/services/plannerz.js +3 -7
- package/out/zero-cache/src/services/plannerz.js.map +1 -1
- package/out/zero-cache/src/services/profz.d.ts.map +1 -1
- package/out/zero-cache/src/services/profz.js +4 -13
- package/out/zero-cache/src/services/profz.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-processor.js +29 -2
- package/out/zero-cache/src/services/replicator/change-processor.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +3 -3
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/statz.d.ts.map +1 -1
- package/out/zero-cache/src/services/statz.js +2 -7
- package/out/zero-cache/src/services/statz.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js +22 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/connection-context-manager.js +40 -2
- package/out/zero-cache/src/services/view-syncer/connection-context-manager.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.js +54 -17
- package/out/zero-cache/src/services/view-syncer/cvr-store.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +11 -0
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-auth-limiter.js +107 -0
- package/out/zero-cache/src/services/view-syncer/inspect-auth-limiter.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js +11 -6
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +2 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/row-record-cache.js +16 -14
- package/out/zero-cache/src/services/view-syncer/row-record-cache.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/schema/cvr.js +4 -1
- package/out/zero-cache/src/services/view-syncer/schema/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/schema/init.js +3 -0
- package/out/zero-cache/src/services/view-syncer/schema/init.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +89 -51
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/client-group-log-id.d.ts +9 -0
- package/out/zero-cache/src/types/client-group-log-id.d.ts.map +1 -0
- package/out/zero-cache/src/types/client-group-log-id.js +16 -0
- package/out/zero-cache/src/types/client-group-log-id.js.map +1 -0
- package/out/zero-cache/src/types/lite.d.ts.map +1 -1
- package/out/zero-cache/src/types/lite.js +1 -0
- package/out/zero-cache/src/types/lite.js.map +1 -1
- package/out/zero-cache/src/types/pg.d.ts +13 -2
- package/out/zero-cache/src/types/pg.d.ts.map +1 -1
- package/out/zero-cache/src/types/pg.js +2 -2
- package/out/zero-cache/src/types/pg.js.map +1 -1
- package/out/zero-cache/src/types/poke-chunk.d.ts +25 -4
- package/out/zero-cache/src/types/poke-chunk.d.ts.map +1 -1
- package/out/zero-cache/src/types/poke-chunk.js +46 -5
- package/out/zero-cache/src/types/poke-chunk.js.map +1 -1
- package/out/zero-cache/src/workers/connection.d.ts.map +1 -1
- package/out/zero-cache/src/workers/connection.js +10 -3
- package/out/zero-cache/src/workers/connection.js.map +1 -1
- package/out/zero-cache/src/workers/rehome.d.ts +10 -0
- package/out/zero-cache/src/workers/rehome.d.ts.map +1 -1
- package/out/zero-cache/src/workers/rehome.js +4 -2
- package/out/zero-cache/src/workers/rehome.js.map +1 -1
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js +3 -2
- package/out/zero-cache/src/workers/syncer-ws-message-handler.js.map +1 -1
- package/out/zero-cache/src/workers/syncer.js +15 -15
- package/out/zero-cache/src/workers/syncer.js.map +1 -1
- package/out/zero-client/src/client/inspector/inspector.d.ts +4 -0
- package/out/zero-client/src/client/inspector/inspector.d.ts.map +1 -1
- package/out/zero-client/src/client/inspector/inspector.js +4 -0
- package/out/zero-client/src/client/inspector/inspector.js.map +1 -1
- package/out/zero-client/src/client/inspector/lazy-inspector.d.ts +10 -2
- package/out/zero-client/src/client/inspector/lazy-inspector.d.ts.map +1 -1
- package/out/zero-client/src/client/inspector/lazy-inspector.js +38 -31
- package/out/zero-client/src/client/inspector/lazy-inspector.js.map +1 -1
- package/out/zero-client/src/client/inspector/query.d.ts +2 -2
- package/out/zero-client/src/client/inspector/query.d.ts.map +1 -1
- package/out/zero-client/src/client/inspector/query.js +4 -4
- package/out/zero-client/src/client/inspector/query.js.map +1 -1
- package/out/zero-client/src/client/ivm-branch.d.ts +3 -0
- package/out/zero-client/src/client/ivm-branch.d.ts.map +1 -1
- package/out/zero-client/src/client/ivm-branch.js +10 -5
- package/out/zero-client/src/client/ivm-branch.js.map +1 -1
- package/out/zero-client/src/client/reconnect-backoff.d.ts +31 -0
- package/out/zero-client/src/client/reconnect-backoff.d.ts.map +1 -0
- package/out/zero-client/src/client/reconnect-backoff.js +44 -0
- package/out/zero-client/src/client/reconnect-backoff.js.map +1 -0
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +7 -0
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-client/src/util/nanoid.js +2 -2
- package/out/zero-client/src/util/nanoid.js.map +1 -1
- package/out/zero-protocol/src/up.d.ts +42 -42
- package/out/zero-server/src/process-mutations.d.ts +10 -0
- package/out/zero-server/src/process-mutations.d.ts.map +1 -1
- package/out/zero-server/src/process-mutations.js +39 -6
- package/out/zero-server/src/process-mutations.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +89 -6
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/planner/planner-connection.d.ts.map +1 -1
- package/out/zql/src/planner/planner-connection.js +37 -3
- package/out/zql/src/planner/planner-connection.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diff.d.ts","sourceRoot":"","sources":["../../../../../replicache/src/btree/diff.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"diff.d.ts","sourceRoot":"","sources":["../../../../../replicache/src/btree/diff.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,WAAW,CAAC;AAEzC,wBAAgB,IAAI,CAClB,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,SAAS,GAChB,OAAO,CAAC,YAAY,CAAC,CAEvB"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { asyncIterableToArray } from "../async-iterable-to-array.js";
|
|
2
1
|
//#region ../replicache/src/btree/diff.ts
|
|
3
2
|
function diff(oldMap, newMap) {
|
|
4
|
-
return
|
|
3
|
+
return newMap.diff(oldMap);
|
|
5
4
|
}
|
|
6
5
|
//#endregion
|
|
7
6
|
export { diff };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diff.js","names":[],"sources":["../../../../../replicache/src/btree/diff.ts"],"sourcesContent":["import
|
|
1
|
+
{"version":3,"file":"diff.js","names":[],"sources":["../../../../../replicache/src/btree/diff.ts"],"sourcesContent":["import type {InternalDiff} from './node.ts';\nimport type {BTreeRead} from './read.ts';\n\nexport function diff(\n oldMap: BTreeRead,\n newMap: BTreeRead,\n): Promise<InternalDiff> {\n return newMap.diff(oldMap);\n}\n"],"mappings":";AAGA,SAAgB,KACd,QACA,QACuB;CACvB,OAAO,OAAO,KAAK,MAAM;AAC3B"}
|
|
@@ -37,10 +37,40 @@ export declare class BTreeRead implements AsyncIterable<Entry<FrozenJSONValue>>
|
|
|
37
37
|
* or explicitly accepts that extra work.
|
|
38
38
|
*/
|
|
39
39
|
scan(fromKey: string, prefetch: boolean): AsyncIterableIterator<Entry<FrozenJSONValue>>;
|
|
40
|
+
/**
|
|
41
|
+
* Calls `visitor` with each entry from `fromKey` on, in key order, until it
|
|
42
|
+
* returns `false` or the entries run out.
|
|
43
|
+
*
|
|
44
|
+
* This is for reading a long range to its end. {@link scan} is an async
|
|
45
|
+
* generator, so every entry it produces crosses a microtask boundary and is
|
|
46
|
+
* yielded again at each level of the tree above it. That is what lets a
|
|
47
|
+
* caller stop anywhere and pay only for what it read, but over a whole range
|
|
48
|
+
* it costs more than the read does, and more again on an engine that
|
|
49
|
+
* compiles async generators to state machines, as Hermes does. A visit
|
|
50
|
+
* awaits once per node and calls `visitor` for a leaf's entries in a plain
|
|
51
|
+
* loop.
|
|
52
|
+
*
|
|
53
|
+
* The tree must not be written to during the visit. `scan` notices when the
|
|
54
|
+
* root changes and restarts from the key it had reached. `visit` does not
|
|
55
|
+
* check for that and does not restart.
|
|
56
|
+
*
|
|
57
|
+
* @param prefetch Read sibling nodes in parallel before visiting them so
|
|
58
|
+
* stores can batch the reads, as for {@link scan}. A visit that stops early
|
|
59
|
+
* has then read nodes it never visits.
|
|
60
|
+
*/
|
|
61
|
+
visit(fromKey: string, prefetch: boolean, visitor: (entry: Entry<FrozenJSONValue>) => boolean): Promise<void>;
|
|
40
62
|
keys(): AsyncIterableIterator<string>;
|
|
41
63
|
entries(): AsyncIterableIterator<Entry<FrozenJSONValue>>;
|
|
42
64
|
[Symbol.asyncIterator](): AsyncIterableIterator<Entry<FrozenJSONValue>>;
|
|
43
|
-
|
|
65
|
+
/**
|
|
66
|
+
* The operations that turn `last` into this tree, in key order.
|
|
67
|
+
*
|
|
68
|
+
* This builds the array itself rather than yielding into one. Every caller
|
|
69
|
+
* wants all of the diff, and yielding it out of nested async generators
|
|
70
|
+
* costs more than computing it: each operation crosses a microtask boundary
|
|
71
|
+
* once per level of the tree, which on a first sync is every row.
|
|
72
|
+
*/
|
|
73
|
+
diff(last: BTreeRead): Promise<InternalDiffOperation[]>;
|
|
44
74
|
}
|
|
45
75
|
export declare function allEntriesAsDiff(map: BTreeRead, op: 'add' | 'del'): Promise<InternalDiff>;
|
|
46
76
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"read.d.ts","sourceRoot":"","sources":["../../../../../replicache/src/btree/read.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"read.d.ts","sourceRoot":"","sources":["../../../../../replicache/src/btree/read.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,6BAA6B,CAAC;AAGtD,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,KAAK,aAAa,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAC,KAAK,IAAI,EAAY,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAC,YAAY,EAAE,gBAAgB,EAAC,MAAM,WAAW,CAAC;AAC9D,OAAO,EACL,KAAK,KAAK,EACV,KAAK,YAAY,EACjB,KAAK,qBAAqB,EAU3B,MAAM,WAAW,CAAC;AASnB,KAAK,aAAa,GAAG,IAAI,CAAC,OAAO,aAAa,CAAC,CAAC;AAEhD;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,KAAK,CAAC;AAEnC,qBAAa,SAAU,YAAW,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACrE,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,YAAY,GAAG,gBAAgB,CAAC,CACzD;IAEZ,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC;IAClC,SAAS,CAAC,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC;IACjD,QAAQ,EAAE,IAAI,CAAC;IACf,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,MAAM,CAAC;IACpD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAEjC,YACE,OAAO,EAAE,IAAI,EACb,aAAa,EAAE,aAAa,EAC5B,IAAI,GAAE,IAAgB,EACtB,YAAY,GAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,MAAuB,EAC3D,eAAe,SAAmB,EAOnC;IAEK,OAAO,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,YAAY,GAAG,gBAAgB,CAAC,CAwBlE;IAEK,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAO3D;IAEK,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAIvC;IAEK,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAQhC;IAED;;;;;;;;;;OAUG;IACH,IAAI,CACF,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,OAAO,GAChB,qBAAqB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAY/C;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,KAAK,CACT,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,OAAO,EACjB,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,KAAK,OAAO,GAClD,OAAO,CAAC,IAAI,CAAC,CAEf;IAEM,IAAI,IAAI,qBAAqB,CAAC,MAAM,CAAC,CAG3C;IAEM,OAAO,IAAI,qBAAqB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAG9D;IAED,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAEtE;IAED;;;;;;;OAOG;IACG,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAQ5D;CACF;AAqPD,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,SAAS,EACd,EAAE,EAAE,KAAK,GAAG,KAAK,GAChB,OAAO,CAAC,YAAY,CAAC,CAmBvB"}
|
|
@@ -3,6 +3,7 @@ import { emptyHash } from "../hash.js";
|
|
|
3
3
|
import { binarySearch, binarySearchFound, emptyDataNodeImpl, findLeaf, isDataNodeImpl, newNodeImpl, parseBTreeNode } from "./node.js";
|
|
4
4
|
import { getSizeOfEntry } from "../../../shared/src/size-of-value.js";
|
|
5
5
|
import { computeSplices } from "./splice.js";
|
|
6
|
+
import { compareUTF8 } from "compare-utf8";
|
|
6
7
|
var BTreeRead = class {
|
|
7
8
|
_cache = /* @__PURE__ */ new Map();
|
|
8
9
|
_dagRead;
|
|
@@ -61,6 +62,30 @@ var BTreeRead = class {
|
|
|
61
62
|
return [level, isMutable ? entries.slice() : entries];
|
|
62
63
|
}, prefetch);
|
|
63
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* Calls `visitor` with each entry from `fromKey` on, in key order, until it
|
|
67
|
+
* returns `false` or the entries run out.
|
|
68
|
+
*
|
|
69
|
+
* This is for reading a long range to its end. {@link scan} is an async
|
|
70
|
+
* generator, so every entry it produces crosses a microtask boundary and is
|
|
71
|
+
* yielded again at each level of the tree above it. That is what lets a
|
|
72
|
+
* caller stop anywhere and pay only for what it read, but over a whole range
|
|
73
|
+
* it costs more than the read does, and more again on an engine that
|
|
74
|
+
* compiles async generators to state machines, as Hermes does. A visit
|
|
75
|
+
* awaits once per node and calls `visitor` for a leaf's entries in a plain
|
|
76
|
+
* loop.
|
|
77
|
+
*
|
|
78
|
+
* The tree must not be written to during the visit. `scan` notices when the
|
|
79
|
+
* root changes and restarts from the key it had reached. `visit` does not
|
|
80
|
+
* check for that and does not restart.
|
|
81
|
+
*
|
|
82
|
+
* @param prefetch Read sibling nodes in parallel before visiting them so
|
|
83
|
+
* stores can batch the reads, as for {@link scan}. A visit that stops early
|
|
84
|
+
* has then read nodes it never visits.
|
|
85
|
+
*/
|
|
86
|
+
async visit(fromKey, prefetch, visitor) {
|
|
87
|
+
await visitNode(this, this.rootHash, fromKey, prefetch, visitor);
|
|
88
|
+
}
|
|
64
89
|
async *keys() {
|
|
65
90
|
yield* (await this.getNode(this.rootHash)).keys(this);
|
|
66
91
|
}
|
|
@@ -70,31 +95,41 @@ var BTreeRead = class {
|
|
|
70
95
|
[Symbol.asyncIterator]() {
|
|
71
96
|
return this.entries();
|
|
72
97
|
}
|
|
73
|
-
|
|
98
|
+
/**
|
|
99
|
+
* The operations that turn `last` into this tree, in key order.
|
|
100
|
+
*
|
|
101
|
+
* This builds the array itself rather than yielding into one. Every caller
|
|
102
|
+
* wants all of the diff, and yielding it out of nested async generators
|
|
103
|
+
* costs more than computing it: each operation crosses a microtask boundary
|
|
104
|
+
* once per level of the tree, which on a first sync is every row.
|
|
105
|
+
*/
|
|
106
|
+
async diff(last) {
|
|
74
107
|
const [currentNode, lastNode] = await Promise.all([this.getNode(this.rootHash), last.getNode(last.rootHash)]);
|
|
75
|
-
|
|
108
|
+
const diff = [];
|
|
109
|
+
await diffNodes(lastNode, currentNode, last, this, diff);
|
|
110
|
+
return diff;
|
|
76
111
|
}
|
|
77
112
|
};
|
|
78
|
-
async function
|
|
113
|
+
async function diffNodes(last, current, lastTree, currentTree, diff) {
|
|
79
114
|
if (last.level > current.level) {
|
|
80
|
-
|
|
115
|
+
await diffNodes(await last.getCompositeChildren(0, last.entries.length, lastTree), current, lastTree, currentTree, diff);
|
|
81
116
|
return;
|
|
82
117
|
}
|
|
83
118
|
if (current.level > last.level) {
|
|
84
|
-
|
|
119
|
+
await diffNodes(last, await current.getCompositeChildren(0, current.entries.length, currentTree), lastTree, currentTree, diff);
|
|
85
120
|
return;
|
|
86
121
|
}
|
|
87
122
|
if (isDataNodeImpl(last) && isDataNodeImpl(current)) {
|
|
88
|
-
|
|
123
|
+
diffEntries(last.entries, current.entries, diff);
|
|
89
124
|
return;
|
|
90
125
|
}
|
|
91
126
|
const initialSplices = computeSplices(last.entries, current.entries);
|
|
92
127
|
for (const splice of initialSplices) {
|
|
93
128
|
const [lastChild, currentChild] = await Promise.all([last.getCompositeChildren(splice[0], splice[1], lastTree), current.getCompositeChildren(splice[3], splice[2], currentTree)]);
|
|
94
|
-
|
|
129
|
+
await diffNodes(lastChild, currentChild, lastTree, currentTree, diff);
|
|
95
130
|
}
|
|
96
131
|
}
|
|
97
|
-
function
|
|
132
|
+
function diffEntries(lastEntries, currentEntries, diff) {
|
|
98
133
|
const lastLength = lastEntries.length;
|
|
99
134
|
const currentLength = currentEntries.length;
|
|
100
135
|
let i = 0;
|
|
@@ -103,40 +138,60 @@ function* diffEntries(lastEntries, currentEntries) {
|
|
|
103
138
|
const lastKey = lastEntries[i][0];
|
|
104
139
|
const currentKey = currentEntries[j][0];
|
|
105
140
|
if (lastKey === currentKey) {
|
|
106
|
-
if (!deepEqual(lastEntries[i][1], currentEntries[j][1]))
|
|
141
|
+
if (!deepEqual(lastEntries[i][1], currentEntries[j][1])) diff.push({
|
|
107
142
|
op: "change",
|
|
108
143
|
key: lastKey,
|
|
109
144
|
oldValue: lastEntries[i][1],
|
|
110
145
|
newValue: currentEntries[j][1]
|
|
111
|
-
};
|
|
146
|
+
});
|
|
112
147
|
i++;
|
|
113
148
|
j++;
|
|
114
|
-
} else if (lastKey <
|
|
115
|
-
|
|
149
|
+
} else if (compareUTF8(lastKey, currentKey) < 0) {
|
|
150
|
+
diff.push({
|
|
116
151
|
op: "del",
|
|
117
152
|
key: lastKey,
|
|
118
153
|
oldValue: lastEntries[i][1]
|
|
119
|
-
};
|
|
154
|
+
});
|
|
120
155
|
i++;
|
|
121
156
|
} else {
|
|
122
|
-
|
|
157
|
+
diff.push({
|
|
123
158
|
op: "add",
|
|
124
159
|
key: currentKey,
|
|
125
160
|
newValue: currentEntries[j][1]
|
|
126
|
-
};
|
|
161
|
+
});
|
|
127
162
|
j++;
|
|
128
163
|
}
|
|
129
164
|
}
|
|
130
|
-
for (; i < lastLength; i++)
|
|
165
|
+
for (; i < lastLength; i++) diff.push({
|
|
131
166
|
op: "del",
|
|
132
167
|
key: lastEntries[i][0],
|
|
133
168
|
oldValue: lastEntries[i][1]
|
|
134
|
-
};
|
|
135
|
-
for (; j < currentLength; j++)
|
|
169
|
+
});
|
|
170
|
+
for (; j < currentLength; j++) diff.push({
|
|
136
171
|
op: "add",
|
|
137
172
|
key: currentEntries[j][0],
|
|
138
173
|
newValue: currentEntries[j][1]
|
|
139
|
-
};
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Visits the entries from `fromKey` on in the subtree at `hash`. Returns
|
|
178
|
+
* whether to go on to the entries after this subtree.
|
|
179
|
+
*/
|
|
180
|
+
async function visitNode(tree, hash, fromKey, prefetch, visitor) {
|
|
181
|
+
const node = await tree.getNode(hash);
|
|
182
|
+
let i = fromKey ? binarySearch(fromKey, node.entries) : 0;
|
|
183
|
+
if (isDataNodeImpl(node)) {
|
|
184
|
+
const { entries } = node;
|
|
185
|
+
for (; i < entries.length; i++) if (!visitor(entries[i])) return false;
|
|
186
|
+
return true;
|
|
187
|
+
}
|
|
188
|
+
const { entries } = node;
|
|
189
|
+
if (prefetch) await Promise.all(entries.slice(i).map((entry) => tree.getNode(entry[1]).catch(() => {})));
|
|
190
|
+
for (; i < entries.length; i++) {
|
|
191
|
+
if (!await visitNode(tree, entries[i][1], fromKey, prefetch, visitor)) return false;
|
|
192
|
+
fromKey = "";
|
|
193
|
+
}
|
|
194
|
+
return true;
|
|
140
195
|
}
|
|
141
196
|
async function* scanForHash(expectedRootHash, getRootHash, hash, fromKey, readNode, prefetch) {
|
|
142
197
|
if (hash === emptyHash) return;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"read.js","names":[],"sources":["../../../../../replicache/src/btree/read.ts"],"sourcesContent":["import type {Enum} from '../../../shared/src/enum.ts';\nimport {deepEqual} from '../../../shared/src/json.ts';\nimport {getSizeOfEntry} from '../../../shared/src/size-of-value.ts';\nimport type {Read} from '../dag/store.ts';\nimport type * as FormatVersion from '../format-version-enum.ts';\nimport type {FrozenJSONValue} from '../frozen-json.ts';\nimport {type Hash, emptyHash} from '../hash.ts';\nimport type {DataNodeImpl, InternalNodeImpl} from './node.ts';\nimport {\n type Entry,\n type InternalDiff,\n type InternalDiffOperation,\n NODE_ENTRIES,\n NODE_LEVEL,\n binarySearch,\n binarySearchFound,\n emptyDataNodeImpl,\n findLeaf,\n isDataNodeImpl,\n newNodeImpl,\n parseBTreeNode,\n} from './node.ts';\nimport {\n SPLICE_ADDED,\n SPLICE_AT,\n SPLICE_FROM,\n SPLICE_REMOVED,\n computeSplices,\n} from './splice.ts';\n\ntype FormatVersion = Enum<typeof FormatVersion>;\n\n/**\n * The size of the header of a node. (If we had compile time\n * constants we would have used that).\n *\n * There is a test ensuring this is correct.\n */\nexport const NODE_HEADER_SIZE = 11;\n\nexport class BTreeRead implements AsyncIterable<Entry<FrozenJSONValue>> {\n protected readonly _cache: Map<Hash, DataNodeImpl | InternalNodeImpl> =\n new Map();\n\n protected readonly _dagRead: Read;\n protected readonly _formatVersion: FormatVersion;\n rootHash: Hash;\n readonly getEntrySize: <K, V>(k: K, v: V) => number;\n readonly chunkHeaderSize: number;\n\n constructor(\n dagRead: Read,\n formatVersion: FormatVersion,\n root: Hash = emptyHash,\n getEntrySize: <K, V>(k: K, v: V) => number = getSizeOfEntry,\n chunkHeaderSize = NODE_HEADER_SIZE,\n ) {\n this._dagRead = dagRead;\n this._formatVersion = formatVersion;\n this.rootHash = root;\n this.getEntrySize = getEntrySize;\n this.chunkHeaderSize = chunkHeaderSize;\n }\n\n async getNode(hash: Hash): Promise<DataNodeImpl | InternalNodeImpl> {\n if (hash === emptyHash) {\n return emptyDataNodeImpl;\n }\n\n const cached = this._cache.get(hash);\n if (cached) {\n return cached;\n }\n\n const chunk = await this._dagRead.mustGetChunk(hash);\n const data = parseBTreeNode(\n chunk.data,\n this._formatVersion,\n this.getEntrySize,\n );\n const impl = newNodeImpl(\n data[NODE_ENTRIES] as Entry<FrozenJSONValue>[],\n hash,\n data[NODE_LEVEL],\n false,\n );\n this._cache.set(hash, impl);\n return impl;\n }\n\n async get(key: string): Promise<FrozenJSONValue | undefined> {\n const leaf = await findLeaf(key, this.rootHash, this, this.rootHash);\n const index = binarySearch(key, leaf.entries);\n if (!binarySearchFound(index, leaf.entries, key)) {\n return undefined;\n }\n return leaf.entries[index][1];\n }\n\n async has(key: string): Promise<boolean> {\n const leaf = await findLeaf(key, this.rootHash, this, this.rootHash);\n const index = binarySearch(key, leaf.entries);\n return binarySearchFound(index, leaf.entries, key);\n }\n\n async isEmpty(): Promise<boolean> {\n const {rootHash} = this;\n const node = await this.getNode(this.rootHash);\n // The root hash has changed, so the tree has been modified.\n if (this.rootHash !== rootHash) {\n return this.isEmpty();\n }\n return node.entries.length === 0;\n }\n\n /**\n * We don't do any encoding of the key in the map, so we have no way of\n * determining from an entry.key alone whether it is a regular key or an\n * encoded IndexKey in an index map. Without encoding regular map keys the\n * caller has to deal with encoding and decoding the keys for the index map.\n *\n * @param prefetch Read sibling nodes in parallel before traversing them so\n * stores can batch the reads. This can read nodes that the caller never\n * visits. Enable it only when the caller expects to consume most of the scan\n * or explicitly accepts that extra work.\n */\n scan(\n fromKey: string,\n prefetch: boolean,\n ): AsyncIterableIterator<Entry<FrozenJSONValue>> {\n return scanForHash(\n this.rootHash,\n () => this.rootHash,\n this.rootHash,\n fromKey,\n async hash => {\n const {level, isMutable, entries} = await this.getNode(hash);\n return [level, isMutable ? entries.slice() : entries];\n },\n prefetch,\n );\n }\n\n async *keys(): AsyncIterableIterator<string> {\n const node = await this.getNode(this.rootHash);\n yield* node.keys(this);\n }\n\n async *entries(): AsyncIterableIterator<Entry<FrozenJSONValue>> {\n const node = await this.getNode(this.rootHash);\n yield* node.entriesIter(this);\n }\n\n [Symbol.asyncIterator](): AsyncIterableIterator<Entry<FrozenJSONValue>> {\n return this.entries();\n }\n\n async *diff(last: BTreeRead): AsyncIterableIterator<InternalDiffOperation> {\n const [currentNode, lastNode] = await Promise.all([\n this.getNode(this.rootHash),\n last.getNode(last.rootHash),\n ]);\n yield* diffNodes(lastNode, currentNode, last, this);\n }\n}\n\nasync function* diffNodes(\n last: InternalNodeImpl | DataNodeImpl,\n current: InternalNodeImpl | DataNodeImpl,\n lastTree: BTreeRead,\n currentTree: BTreeRead,\n): AsyncIterableIterator<InternalDiffOperation> {\n if (last.level > current.level) {\n // merge all of last's children into a new node\n // We know last is an internal node because level > 0.\n const lastChild = (await (last as InternalNodeImpl).getCompositeChildren(\n 0,\n last.entries.length,\n lastTree,\n )) as InternalNodeImpl;\n yield* diffNodes(lastChild, current, lastTree, currentTree);\n return;\n }\n\n if (current.level > last.level) {\n // We know current is an internal node because level > 0.\n const currentChild = (await (\n current as InternalNodeImpl\n ).getCompositeChildren(\n 0,\n current.entries.length,\n currentTree,\n )) as InternalNodeImpl;\n yield* diffNodes(last, currentChild, lastTree, currentTree);\n return;\n }\n\n if (isDataNodeImpl(last) && isDataNodeImpl(current)) {\n yield* diffEntries(\n (last as DataNodeImpl).entries,\n (current as DataNodeImpl).entries,\n );\n return;\n }\n\n // Now we have two internal nodes with the same level. We compute the diff as\n // splices for the internal node entries. We then flatten these and call diff\n // recursively.\n const initialSplices = computeSplices(\n (last as InternalNodeImpl).entries,\n (current as InternalNodeImpl).entries,\n );\n for (const splice of initialSplices) {\n const [lastChild, currentChild] = await Promise.all([\n (last as InternalNodeImpl).getCompositeChildren(\n splice[SPLICE_AT],\n splice[SPLICE_REMOVED],\n lastTree,\n ),\n (current as InternalNodeImpl).getCompositeChildren(\n splice[SPLICE_FROM],\n splice[SPLICE_ADDED],\n currentTree,\n ),\n ]);\n yield* diffNodes(lastChild, currentChild, lastTree, currentTree);\n }\n}\n\nfunction* diffEntries(\n lastEntries: readonly Entry<FrozenJSONValue>[],\n currentEntries: readonly Entry<FrozenJSONValue>[],\n): IterableIterator<InternalDiffOperation> {\n const lastLength = lastEntries.length;\n const currentLength = currentEntries.length;\n let i = 0;\n let j = 0;\n while (i < lastLength && j < currentLength) {\n const lastKey = lastEntries[i][0];\n const currentKey = currentEntries[j][0];\n if (lastKey === currentKey) {\n if (!deepEqual(lastEntries[i][1], currentEntries[j][1])) {\n yield {\n op: 'change',\n key: lastKey,\n oldValue: lastEntries[i][1],\n newValue: currentEntries[j][1],\n };\n }\n i++;\n j++;\n } else if (lastKey < currentKey) {\n yield {\n op: 'del',\n key: lastKey,\n oldValue: lastEntries[i][1],\n };\n i++;\n } else {\n yield {\n op: 'add',\n key: currentKey,\n newValue: currentEntries[j][1],\n };\n j++;\n }\n }\n for (; i < lastLength; i++) {\n yield {\n op: 'del',\n key: lastEntries[i][0],\n oldValue: lastEntries[i][1],\n };\n }\n for (; j < currentLength; j++) {\n yield {\n op: 'add',\n key: currentEntries[j][0],\n newValue: currentEntries[j][1],\n };\n }\n}\n\n// Redefine the type here to allow the optional size in the tuple.\ntype ReadNodeResult = readonly [\n level: number,\n data: readonly Entry<FrozenJSONValue>[] | readonly Entry<Hash>[],\n];\n\ntype ReadNode = (hash: Hash) => Promise<ReadNodeResult>;\n\nasync function* scanForHash(\n expectedRootHash: Hash,\n getRootHash: () => Hash,\n hash: Hash,\n fromKey: string,\n readNode: ReadNode,\n prefetch: boolean,\n): AsyncIterableIterator<Entry<FrozenJSONValue>> {\n if (hash === emptyHash) {\n return;\n }\n\n const data = await readNode(hash);\n const entries = data[NODE_ENTRIES];\n let i = 0;\n if (fromKey) {\n i = binarySearch(fromKey, entries);\n }\n if (data[NODE_LEVEL] > 0) {\n if (prefetch) {\n // Preload children in parallel so the backing store can coalesce their\n // reads. This can waste work when a caller stops before visiting every\n // child, so callers must explicitly opt in when they expect to consume\n // the full scan. Zero currently accepts that tradeoff: startup scans the\n // entire cached entity range into IVM memory before becoming ready (see\n // `ZeroRep.init` in packages/zero-client/src/client/zero-rep.ts), and\n // client views are intended to remain small. If views commonly outgrow\n // the 100 MB `LAZY_STORE_SOURCE_CHUNK_CACHE_SIZE_LIMIT`, reconsider the\n // preload window, although batching may still be a useful general\n // optimization. Longer term, the Replicache/IVM boundary should also be\n // revisited because materializing entities into `MemorySource`\n // duplicates B-tree storage.\n //\n // The serial scan below preserves traversal order and remains the\n // authoritative read, so speculative failures are ignored here.\n await Promise.all(\n entries\n .slice(i)\n .map(entry =>\n readNode((entry as Entry<Hash>)[1]).catch(() => undefined),\n ),\n );\n }\n for (; i < entries.length; i++) {\n yield* scanForHash(\n expectedRootHash,\n getRootHash,\n (entries[i] as Entry<Hash>)[1],\n fromKey,\n readNode,\n prefetch,\n );\n fromKey = '';\n }\n } else {\n for (; i < entries.length; i++) {\n const rootHash = getRootHash();\n // If rootHash changed then we start a new iterator from the key.\n if (expectedRootHash !== rootHash) {\n yield* scanForHash(\n rootHash,\n getRootHash,\n rootHash,\n entries[i][0],\n readNode,\n prefetch,\n );\n return;\n }\n yield entries[i] as Entry<FrozenJSONValue>;\n }\n }\n}\n\nexport async function allEntriesAsDiff(\n map: BTreeRead,\n op: 'add' | 'del',\n): Promise<InternalDiff> {\n const diff: InternalDiffOperation[] = [];\n const make: (entry: Entry<FrozenJSONValue>) => InternalDiffOperation =\n op === 'add'\n ? entry => ({\n op: 'add',\n key: entry[0],\n newValue: entry[1],\n })\n : entry => ({\n op: 'del',\n key: entry[0],\n oldValue: entry[1],\n });\n\n for await (const entry of map.entries()) {\n diff.push(make(entry));\n }\n return diff;\n}\n"],"mappings":";;;;;AAwCA,IAAa,YAAb,MAAwE;CACtE,yBACE,IAAI,IAAI;CAEV;CACA;CACA;CACA;CACA;CAEA,YACE,SACA,eACA,OAAa,WACb,eAA6C,gBAC7C,kBAAA,IACA;EACA,KAAK,WAAW;EAChB,KAAK,iBAAiB;EACtB,KAAK,WAAW;EAChB,KAAK,eAAe;EACpB,KAAK,kBAAkB;CACzB;CAEA,MAAM,QAAQ,MAAsD;EAClE,IAAI,SAAS,WACX,OAAO;EAGT,MAAM,SAAS,KAAK,OAAO,IAAI,IAAI;EACnC,IAAI,QACF,OAAO;EAGT,MAAM,QAAQ,MAAM,KAAK,SAAS,aAAa,IAAI;EACnD,MAAM,OAAO,eACX,MAAM,MACN,KAAK,gBACL,KAAK,YACP;EACA,MAAM,OAAO,YACX,KAAA,IACA,MACA,KAAA,IACA,KACF;EACA,KAAK,OAAO,IAAI,MAAM,IAAI;EAC1B,OAAO;CACT;CAEA,MAAM,IAAI,KAAmD;EAC3D,MAAM,OAAO,MAAM,SAAS,KAAK,KAAK,UAAU,MAAM,KAAK,QAAQ;EACnE,MAAM,QAAQ,aAAa,KAAK,KAAK,OAAO;EAC5C,IAAI,CAAC,kBAAkB,OAAO,KAAK,SAAS,GAAG,GAC7C;EAEF,OAAO,KAAK,QAAQ,MAAM,CAAC;CAC7B;CAEA,MAAM,IAAI,KAA+B;EACvC,MAAM,OAAO,MAAM,SAAS,KAAK,KAAK,UAAU,MAAM,KAAK,QAAQ;EACnE,MAAM,QAAQ,aAAa,KAAK,KAAK,OAAO;EAC5C,OAAO,kBAAkB,OAAO,KAAK,SAAS,GAAG;CACnD;CAEA,MAAM,UAA4B;EAChC,MAAM,EAAC,aAAY;EACnB,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,QAAQ;EAE7C,IAAI,KAAK,aAAa,UACpB,OAAO,KAAK,QAAQ;EAEtB,OAAO,KAAK,QAAQ,WAAW;CACjC;;;;;;;;;;;;CAaA,KACE,SACA,UAC+C;EAC/C,OAAO,YACL,KAAK,gBACC,KAAK,UACX,KAAK,UACL,SACA,OAAM,SAAQ;GACZ,MAAM,EAAC,OAAO,WAAW,YAAW,MAAM,KAAK,QAAQ,IAAI;GAC3D,OAAO,CAAC,OAAO,YAAY,QAAQ,MAAM,IAAI,OAAO;EACtD,GACA,QACF;CACF;CAEA,OAAO,OAAsC;EAE3C,QAAO,MADY,KAAK,QAAQ,KAAK,QAAQ,EAAA,CACjC,KAAK,IAAI;CACvB;CAEA,OAAO,UAAyD;EAE9D,QAAO,MADY,KAAK,QAAQ,KAAK,QAAQ,EAAA,CACjC,YAAY,IAAI;CAC9B;CAEA,CAAC,OAAO,iBAAgE;EACtE,OAAO,KAAK,QAAQ;CACtB;CAEA,OAAO,KAAK,MAA+D;EACzE,MAAM,CAAC,aAAa,YAAY,MAAM,QAAQ,IAAI,CAChD,KAAK,QAAQ,KAAK,QAAQ,GAC1B,KAAK,QAAQ,KAAK,QAAQ,CAC5B,CAAC;EACD,OAAO,UAAU,UAAU,aAAa,MAAM,IAAI;CACpD;AACF;AAEA,gBAAgB,UACd,MACA,SACA,UACA,aAC8C;CAC9C,IAAI,KAAK,QAAQ,QAAQ,OAAO;EAQ9B,OAAO,UAAU,MALS,KAA0B,qBAClD,GACA,KAAK,QAAQ,QACb,QACF,GAC4B,SAAS,UAAU,WAAW;EAC1D;CACF;CAEA,IAAI,QAAQ,QAAQ,KAAK,OAAO;EAS9B,OAAO,UAAU,MAAM,MANrB,QACA,qBACA,GACA,QAAQ,QAAQ,QAChB,WACF,GACqC,UAAU,WAAW;EAC1D;CACF;CAEA,IAAI,eAAe,IAAI,KAAK,eAAe,OAAO,GAAG;EACnD,OAAO,YACJ,KAAsB,SACtB,QAAyB,OAC5B;EACA;CACF;CAKA,MAAM,iBAAiB,eACpB,KAA0B,SAC1B,QAA6B,OAChC;CACA,KAAK,MAAM,UAAU,gBAAgB;EACnC,MAAM,CAAC,WAAW,gBAAgB,MAAM,QAAQ,IAAI,CACjD,KAA0B,qBACzB,OAAA,IACA,OAAA,IACA,QACF,GACC,QAA6B,qBAC5B,OAAA,IACA,OAAA,IACA,WACF,CACF,CAAC;EACD,OAAO,UAAU,WAAW,cAAc,UAAU,WAAW;CACjE;AACF;AAEA,UAAU,YACR,aACA,gBACyC;CACzC,MAAM,aAAa,YAAY;CAC/B,MAAM,gBAAgB,eAAe;CACrC,IAAI,IAAI;CACR,IAAI,IAAI;CACR,OAAO,IAAI,cAAc,IAAI,eAAe;EAC1C,MAAM,UAAU,YAAY,EAAE,CAAC;EAC/B,MAAM,aAAa,eAAe,EAAE,CAAC;EACrC,IAAI,YAAY,YAAY;GAC1B,IAAI,CAAC,UAAU,YAAY,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC,EAAE,GACpD,MAAM;IACJ,IAAI;IACJ,KAAK;IACL,UAAU,YAAY,EAAE,CAAC;IACzB,UAAU,eAAe,EAAE,CAAC;GAC9B;GAEF;GACA;EACF,OAAO,IAAI,UAAU,YAAY;GAC/B,MAAM;IACJ,IAAI;IACJ,KAAK;IACL,UAAU,YAAY,EAAE,CAAC;GAC3B;GACA;EACF,OAAO;GACL,MAAM;IACJ,IAAI;IACJ,KAAK;IACL,UAAU,eAAe,EAAE,CAAC;GAC9B;GACA;EACF;CACF;CACA,OAAO,IAAI,YAAY,KACrB,MAAM;EACJ,IAAI;EACJ,KAAK,YAAY,EAAE,CAAC;EACpB,UAAU,YAAY,EAAE,CAAC;CAC3B;CAEF,OAAO,IAAI,eAAe,KACxB,MAAM;EACJ,IAAI;EACJ,KAAK,eAAe,EAAE,CAAC;EACvB,UAAU,eAAe,EAAE,CAAC;CAC9B;AAEJ;AAUA,gBAAgB,YACd,kBACA,aACA,MACA,SACA,UACA,UAC+C;CAC/C,IAAI,SAAS,WACX;CAGF,MAAM,OAAO,MAAM,SAAS,IAAI;CAChC,MAAM,UAAU,KAAA;CAChB,IAAI,IAAI;CACR,IAAI,SACF,IAAI,aAAa,SAAS,OAAO;CAEnC,IAAI,KAAA,KAAmB,GAAG;EACxB,IAAI,UAgBF,MAAM,QAAQ,IACZ,QACG,MAAM,CAAC,CAAC,CACR,KAAI,UACH,SAAU,MAAsB,EAAE,CAAC,CAAC,YAAY,KAAA,CAAS,CAC3D,CACJ;EAEF,OAAO,IAAI,QAAQ,QAAQ,KAAK;GAC9B,OAAO,YACL,kBACA,aACC,QAAQ,EAAE,CAAiB,IAC5B,SACA,UACA,QACF;GACA,UAAU;EACZ;CACF,OACE,OAAO,IAAI,QAAQ,QAAQ,KAAK;EAC9B,MAAM,WAAW,YAAY;EAE7B,IAAI,qBAAqB,UAAU;GACjC,OAAO,YACL,UACA,aACA,UACA,QAAQ,EAAE,CAAC,IACX,UACA,QACF;GACA;EACF;EACA,MAAM,QAAQ;CAChB;AAEJ;AAEA,eAAsB,iBACpB,KACA,IACuB;CACvB,MAAM,OAAgC,CAAC;CACvC,MAAM,OACJ,OAAO,SACH,WAAU;EACR,IAAI;EACJ,KAAK,MAAM;EACX,UAAU,MAAM;CAClB,MACA,WAAU;EACR,IAAI;EACJ,KAAK,MAAM;EACX,UAAU,MAAM;CAClB;CAEN,WAAW,MAAM,SAAS,IAAI,QAAQ,GACpC,KAAK,KAAK,KAAK,KAAK,CAAC;CAEvB,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"read.js","names":[],"sources":["../../../../../replicache/src/btree/read.ts"],"sourcesContent":["import {compareUTF8} from 'compare-utf8';\nimport type {Enum} from '../../../shared/src/enum.ts';\nimport {deepEqual} from '../../../shared/src/json.ts';\nimport {getSizeOfEntry} from '../../../shared/src/size-of-value.ts';\nimport type {Read} from '../dag/store.ts';\nimport type * as FormatVersion from '../format-version-enum.ts';\nimport type {FrozenJSONValue} from '../frozen-json.ts';\nimport {type Hash, emptyHash} from '../hash.ts';\nimport type {DataNodeImpl, InternalNodeImpl} from './node.ts';\nimport {\n type Entry,\n type InternalDiff,\n type InternalDiffOperation,\n NODE_ENTRIES,\n NODE_LEVEL,\n binarySearch,\n binarySearchFound,\n emptyDataNodeImpl,\n findLeaf,\n isDataNodeImpl,\n newNodeImpl,\n parseBTreeNode,\n} from './node.ts';\nimport {\n SPLICE_ADDED,\n SPLICE_AT,\n SPLICE_FROM,\n SPLICE_REMOVED,\n computeSplices,\n} from './splice.ts';\n\ntype FormatVersion = Enum<typeof FormatVersion>;\n\n/**\n * The size of the header of a node. (If we had compile time\n * constants we would have used that).\n *\n * There is a test ensuring this is correct.\n */\nexport const NODE_HEADER_SIZE = 11;\n\nexport class BTreeRead implements AsyncIterable<Entry<FrozenJSONValue>> {\n protected readonly _cache: Map<Hash, DataNodeImpl | InternalNodeImpl> =\n new Map();\n\n protected readonly _dagRead: Read;\n protected readonly _formatVersion: FormatVersion;\n rootHash: Hash;\n readonly getEntrySize: <K, V>(k: K, v: V) => number;\n readonly chunkHeaderSize: number;\n\n constructor(\n dagRead: Read,\n formatVersion: FormatVersion,\n root: Hash = emptyHash,\n getEntrySize: <K, V>(k: K, v: V) => number = getSizeOfEntry,\n chunkHeaderSize = NODE_HEADER_SIZE,\n ) {\n this._dagRead = dagRead;\n this._formatVersion = formatVersion;\n this.rootHash = root;\n this.getEntrySize = getEntrySize;\n this.chunkHeaderSize = chunkHeaderSize;\n }\n\n async getNode(hash: Hash): Promise<DataNodeImpl | InternalNodeImpl> {\n if (hash === emptyHash) {\n return emptyDataNodeImpl;\n }\n\n const cached = this._cache.get(hash);\n if (cached) {\n return cached;\n }\n\n const chunk = await this._dagRead.mustGetChunk(hash);\n const data = parseBTreeNode(\n chunk.data,\n this._formatVersion,\n this.getEntrySize,\n );\n const impl = newNodeImpl(\n data[NODE_ENTRIES] as Entry<FrozenJSONValue>[],\n hash,\n data[NODE_LEVEL],\n false,\n );\n this._cache.set(hash, impl);\n return impl;\n }\n\n async get(key: string): Promise<FrozenJSONValue | undefined> {\n const leaf = await findLeaf(key, this.rootHash, this, this.rootHash);\n const index = binarySearch(key, leaf.entries);\n if (!binarySearchFound(index, leaf.entries, key)) {\n return undefined;\n }\n return leaf.entries[index][1];\n }\n\n async has(key: string): Promise<boolean> {\n const leaf = await findLeaf(key, this.rootHash, this, this.rootHash);\n const index = binarySearch(key, leaf.entries);\n return binarySearchFound(index, leaf.entries, key);\n }\n\n async isEmpty(): Promise<boolean> {\n const {rootHash} = this;\n const node = await this.getNode(this.rootHash);\n // The root hash has changed, so the tree has been modified.\n if (this.rootHash !== rootHash) {\n return this.isEmpty();\n }\n return node.entries.length === 0;\n }\n\n /**\n * We don't do any encoding of the key in the map, so we have no way of\n * determining from an entry.key alone whether it is a regular key or an\n * encoded IndexKey in an index map. Without encoding regular map keys the\n * caller has to deal with encoding and decoding the keys for the index map.\n *\n * @param prefetch Read sibling nodes in parallel before traversing them so\n * stores can batch the reads. This can read nodes that the caller never\n * visits. Enable it only when the caller expects to consume most of the scan\n * or explicitly accepts that extra work.\n */\n scan(\n fromKey: string,\n prefetch: boolean,\n ): AsyncIterableIterator<Entry<FrozenJSONValue>> {\n return scanForHash(\n this.rootHash,\n () => this.rootHash,\n this.rootHash,\n fromKey,\n async hash => {\n const {level, isMutable, entries} = await this.getNode(hash);\n return [level, isMutable ? entries.slice() : entries];\n },\n prefetch,\n );\n }\n\n /**\n * Calls `visitor` with each entry from `fromKey` on, in key order, until it\n * returns `false` or the entries run out.\n *\n * This is for reading a long range to its end. {@link scan} is an async\n * generator, so every entry it produces crosses a microtask boundary and is\n * yielded again at each level of the tree above it. That is what lets a\n * caller stop anywhere and pay only for what it read, but over a whole range\n * it costs more than the read does, and more again on an engine that\n * compiles async generators to state machines, as Hermes does. A visit\n * awaits once per node and calls `visitor` for a leaf's entries in a plain\n * loop.\n *\n * The tree must not be written to during the visit. `scan` notices when the\n * root changes and restarts from the key it had reached. `visit` does not\n * check for that and does not restart.\n *\n * @param prefetch Read sibling nodes in parallel before visiting them so\n * stores can batch the reads, as for {@link scan}. A visit that stops early\n * has then read nodes it never visits.\n */\n async visit(\n fromKey: string,\n prefetch: boolean,\n visitor: (entry: Entry<FrozenJSONValue>) => boolean,\n ): Promise<void> {\n await visitNode(this, this.rootHash, fromKey, prefetch, visitor);\n }\n\n async *keys(): AsyncIterableIterator<string> {\n const node = await this.getNode(this.rootHash);\n yield* node.keys(this);\n }\n\n async *entries(): AsyncIterableIterator<Entry<FrozenJSONValue>> {\n const node = await this.getNode(this.rootHash);\n yield* node.entriesIter(this);\n }\n\n [Symbol.asyncIterator](): AsyncIterableIterator<Entry<FrozenJSONValue>> {\n return this.entries();\n }\n\n /**\n * The operations that turn `last` into this tree, in key order.\n *\n * This builds the array itself rather than yielding into one. Every caller\n * wants all of the diff, and yielding it out of nested async generators\n * costs more than computing it: each operation crosses a microtask boundary\n * once per level of the tree, which on a first sync is every row.\n */\n async diff(last: BTreeRead): Promise<InternalDiffOperation[]> {\n const [currentNode, lastNode] = await Promise.all([\n this.getNode(this.rootHash),\n last.getNode(last.rootHash),\n ]);\n const diff: InternalDiffOperation[] = [];\n await diffNodes(lastNode, currentNode, last, this, diff);\n return diff;\n }\n}\n\nasync function diffNodes(\n last: InternalNodeImpl | DataNodeImpl,\n current: InternalNodeImpl | DataNodeImpl,\n lastTree: BTreeRead,\n currentTree: BTreeRead,\n diff: InternalDiffOperation[],\n): Promise<void> {\n if (last.level > current.level) {\n // merge all of last's children into a new node\n // We know last is an internal node because level > 0.\n const lastChild = (await (last as InternalNodeImpl).getCompositeChildren(\n 0,\n last.entries.length,\n lastTree,\n )) as InternalNodeImpl;\n await diffNodes(lastChild, current, lastTree, currentTree, diff);\n return;\n }\n\n if (current.level > last.level) {\n // We know current is an internal node because level > 0.\n const currentChild = (await (\n current as InternalNodeImpl\n ).getCompositeChildren(\n 0,\n current.entries.length,\n currentTree,\n )) as InternalNodeImpl;\n await diffNodes(last, currentChild, lastTree, currentTree, diff);\n return;\n }\n\n if (isDataNodeImpl(last) && isDataNodeImpl(current)) {\n diffEntries(\n (last as DataNodeImpl).entries,\n (current as DataNodeImpl).entries,\n diff,\n );\n return;\n }\n\n // Now we have two internal nodes with the same level. We compute the diff as\n // splices for the internal node entries. We then flatten these and call diff\n // recursively.\n const initialSplices = computeSplices(\n (last as InternalNodeImpl).entries,\n (current as InternalNodeImpl).entries,\n );\n for (const splice of initialSplices) {\n const [lastChild, currentChild] = await Promise.all([\n (last as InternalNodeImpl).getCompositeChildren(\n splice[SPLICE_AT],\n splice[SPLICE_REMOVED],\n lastTree,\n ),\n (current as InternalNodeImpl).getCompositeChildren(\n splice[SPLICE_FROM],\n splice[SPLICE_ADDED],\n currentTree,\n ),\n ]);\n await diffNodes(lastChild, currentChild, lastTree, currentTree, diff);\n }\n}\n\nfunction diffEntries(\n lastEntries: readonly Entry<FrozenJSONValue>[],\n currentEntries: readonly Entry<FrozenJSONValue>[],\n diff: InternalDiffOperation[],\n): void {\n const lastLength = lastEntries.length;\n const currentLength = currentEntries.length;\n let i = 0;\n let j = 0;\n while (i < lastLength && j < currentLength) {\n const lastKey = lastEntries[i][0];\n const currentKey = currentEntries[j][0];\n if (lastKey === currentKey) {\n if (!deepEqual(lastEntries[i][1], currentEntries[j][1])) {\n diff.push({\n op: 'change',\n key: lastKey,\n oldValue: lastEntries[i][1],\n newValue: currentEntries[j][1],\n });\n }\n i++;\n j++;\n } else if (compareUTF8(lastKey, currentKey) < 0) {\n diff.push({\n op: 'del',\n key: lastKey,\n oldValue: lastEntries[i][1],\n });\n i++;\n } else {\n diff.push({\n op: 'add',\n key: currentKey,\n newValue: currentEntries[j][1],\n });\n j++;\n }\n }\n for (; i < lastLength; i++) {\n diff.push({\n op: 'del',\n key: lastEntries[i][0],\n oldValue: lastEntries[i][1],\n });\n }\n for (; j < currentLength; j++) {\n diff.push({\n op: 'add',\n key: currentEntries[j][0],\n newValue: currentEntries[j][1],\n });\n }\n}\n\n// Redefine the type here to allow the optional size in the tuple.\ntype ReadNodeResult = readonly [\n level: number,\n data: readonly Entry<FrozenJSONValue>[] | readonly Entry<Hash>[],\n];\n\ntype ReadNode = (hash: Hash) => Promise<ReadNodeResult>;\n\n/**\n * Visits the entries from `fromKey` on in the subtree at `hash`. Returns\n * whether to go on to the entries after this subtree.\n */\nasync function visitNode(\n tree: BTreeRead,\n hash: Hash,\n fromKey: string,\n prefetch: boolean,\n visitor: (entry: Entry<FrozenJSONValue>) => boolean,\n): Promise<boolean> {\n const node = await tree.getNode(hash);\n let i = fromKey ? binarySearch(fromKey, node.entries) : 0;\n if (isDataNodeImpl(node)) {\n const {entries} = node;\n for (; i < entries.length; i++) {\n if (!visitor(entries[i])) {\n return false;\n }\n }\n return true;\n }\n\n const {entries} = node;\n if (prefetch) {\n // A failure here is ignored, as in `scanForHash`: the loop below reads\n // each child again and reports it.\n await Promise.all(\n entries.slice(i).map(entry => tree.getNode(entry[1]).catch(() => {})),\n );\n }\n for (; i < entries.length; i++) {\n if (!(await visitNode(tree, entries[i][1], fromKey, prefetch, visitor))) {\n return false;\n }\n // Only the first subtree can hold keys before `fromKey`.\n fromKey = '';\n }\n return true;\n}\n\nasync function* scanForHash(\n expectedRootHash: Hash,\n getRootHash: () => Hash,\n hash: Hash,\n fromKey: string,\n readNode: ReadNode,\n prefetch: boolean,\n): AsyncIterableIterator<Entry<FrozenJSONValue>> {\n if (hash === emptyHash) {\n return;\n }\n\n const data = await readNode(hash);\n const entries = data[NODE_ENTRIES];\n let i = 0;\n if (fromKey) {\n i = binarySearch(fromKey, entries);\n }\n if (data[NODE_LEVEL] > 0) {\n if (prefetch) {\n // Preload children in parallel so the backing store can coalesce their\n // reads. This can waste work when a caller stops before visiting every\n // child, so callers must explicitly opt in when they expect to consume\n // the full scan. Zero currently accepts that tradeoff: startup scans the\n // entire cached entity range into IVM memory before becoming ready (see\n // `ZeroRep.init` in packages/zero-client/src/client/zero-rep.ts), and\n // client views are intended to remain small. If views commonly outgrow\n // the 100 MB `LAZY_STORE_SOURCE_CHUNK_CACHE_SIZE_LIMIT`, reconsider the\n // preload window, although batching may still be a useful general\n // optimization. Longer term, the Replicache/IVM boundary should also be\n // revisited because materializing entities into `MemorySource`\n // duplicates B-tree storage.\n //\n // The serial scan below preserves traversal order and remains the\n // authoritative read, so speculative failures are ignored here.\n await Promise.all(\n entries\n .slice(i)\n .map(entry =>\n readNode((entry as Entry<Hash>)[1]).catch(() => undefined),\n ),\n );\n }\n for (; i < entries.length; i++) {\n yield* scanForHash(\n expectedRootHash,\n getRootHash,\n (entries[i] as Entry<Hash>)[1],\n fromKey,\n readNode,\n prefetch,\n );\n fromKey = '';\n }\n } else {\n for (; i < entries.length; i++) {\n const rootHash = getRootHash();\n // If rootHash changed then we start a new iterator from the key.\n if (expectedRootHash !== rootHash) {\n yield* scanForHash(\n rootHash,\n getRootHash,\n rootHash,\n entries[i][0],\n readNode,\n prefetch,\n );\n return;\n }\n yield entries[i] as Entry<FrozenJSONValue>;\n }\n }\n}\n\nexport async function allEntriesAsDiff(\n map: BTreeRead,\n op: 'add' | 'del',\n): Promise<InternalDiff> {\n const diff: InternalDiffOperation[] = [];\n const make: (entry: Entry<FrozenJSONValue>) => InternalDiffOperation =\n op === 'add'\n ? entry => ({\n op: 'add',\n key: entry[0],\n newValue: entry[1],\n })\n : entry => ({\n op: 'del',\n key: entry[0],\n oldValue: entry[1],\n });\n\n for await (const entry of map.entries()) {\n diff.push(make(entry));\n }\n return diff;\n}\n"],"mappings":";;;;;;AAyCA,IAAa,YAAb,MAAwE;CACtE,yBACE,IAAI,IAAI;CAEV;CACA;CACA;CACA;CACA;CAEA,YACE,SACA,eACA,OAAa,WACb,eAA6C,gBAC7C,kBAAA,IACA;EACA,KAAK,WAAW;EAChB,KAAK,iBAAiB;EACtB,KAAK,WAAW;EAChB,KAAK,eAAe;EACpB,KAAK,kBAAkB;CACzB;CAEA,MAAM,QAAQ,MAAsD;EAClE,IAAI,SAAS,WACX,OAAO;EAGT,MAAM,SAAS,KAAK,OAAO,IAAI,IAAI;EACnC,IAAI,QACF,OAAO;EAGT,MAAM,QAAQ,MAAM,KAAK,SAAS,aAAa,IAAI;EACnD,MAAM,OAAO,eACX,MAAM,MACN,KAAK,gBACL,KAAK,YACP;EACA,MAAM,OAAO,YACX,KAAA,IACA,MACA,KAAA,IACA,KACF;EACA,KAAK,OAAO,IAAI,MAAM,IAAI;EAC1B,OAAO;CACT;CAEA,MAAM,IAAI,KAAmD;EAC3D,MAAM,OAAO,MAAM,SAAS,KAAK,KAAK,UAAU,MAAM,KAAK,QAAQ;EACnE,MAAM,QAAQ,aAAa,KAAK,KAAK,OAAO;EAC5C,IAAI,CAAC,kBAAkB,OAAO,KAAK,SAAS,GAAG,GAC7C;EAEF,OAAO,KAAK,QAAQ,MAAM,CAAC;CAC7B;CAEA,MAAM,IAAI,KAA+B;EACvC,MAAM,OAAO,MAAM,SAAS,KAAK,KAAK,UAAU,MAAM,KAAK,QAAQ;EACnE,MAAM,QAAQ,aAAa,KAAK,KAAK,OAAO;EAC5C,OAAO,kBAAkB,OAAO,KAAK,SAAS,GAAG;CACnD;CAEA,MAAM,UAA4B;EAChC,MAAM,EAAC,aAAY;EACnB,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,QAAQ;EAE7C,IAAI,KAAK,aAAa,UACpB,OAAO,KAAK,QAAQ;EAEtB,OAAO,KAAK,QAAQ,WAAW;CACjC;;;;;;;;;;;;CAaA,KACE,SACA,UAC+C;EAC/C,OAAO,YACL,KAAK,gBACC,KAAK,UACX,KAAK,UACL,SACA,OAAM,SAAQ;GACZ,MAAM,EAAC,OAAO,WAAW,YAAW,MAAM,KAAK,QAAQ,IAAI;GAC3D,OAAO,CAAC,OAAO,YAAY,QAAQ,MAAM,IAAI,OAAO;EACtD,GACA,QACF;CACF;;;;;;;;;;;;;;;;;;;;;;CAuBA,MAAM,MACJ,SACA,UACA,SACe;EACf,MAAM,UAAU,MAAM,KAAK,UAAU,SAAS,UAAU,OAAO;CACjE;CAEA,OAAO,OAAsC;EAE3C,QAAO,MADY,KAAK,QAAQ,KAAK,QAAQ,EAAA,CACjC,KAAK,IAAI;CACvB;CAEA,OAAO,UAAyD;EAE9D,QAAO,MADY,KAAK,QAAQ,KAAK,QAAQ,EAAA,CACjC,YAAY,IAAI;CAC9B;CAEA,CAAC,OAAO,iBAAgE;EACtE,OAAO,KAAK,QAAQ;CACtB;;;;;;;;;CAUA,MAAM,KAAK,MAAmD;EAC5D,MAAM,CAAC,aAAa,YAAY,MAAM,QAAQ,IAAI,CAChD,KAAK,QAAQ,KAAK,QAAQ,GAC1B,KAAK,QAAQ,KAAK,QAAQ,CAC5B,CAAC;EACD,MAAM,OAAgC,CAAC;EACvC,MAAM,UAAU,UAAU,aAAa,MAAM,MAAM,IAAI;EACvD,OAAO;CACT;AACF;AAEA,eAAe,UACb,MACA,SACA,UACA,aACA,MACe;CACf,IAAI,KAAK,QAAQ,QAAQ,OAAO;EAQ9B,MAAM,UAAU,MALU,KAA0B,qBAClD,GACA,KAAK,QAAQ,QACb,QACF,GAC2B,SAAS,UAAU,aAAa,IAAI;EAC/D;CACF;CAEA,IAAI,QAAQ,QAAQ,KAAK,OAAO;EAS9B,MAAM,UAAU,MAAM,MANpB,QACA,qBACA,GACA,QAAQ,QAAQ,QAChB,WACF,GACoC,UAAU,aAAa,IAAI;EAC/D;CACF;CAEA,IAAI,eAAe,IAAI,KAAK,eAAe,OAAO,GAAG;EACnD,YACG,KAAsB,SACtB,QAAyB,SAC1B,IACF;EACA;CACF;CAKA,MAAM,iBAAiB,eACpB,KAA0B,SAC1B,QAA6B,OAChC;CACA,KAAK,MAAM,UAAU,gBAAgB;EACnC,MAAM,CAAC,WAAW,gBAAgB,MAAM,QAAQ,IAAI,CACjD,KAA0B,qBACzB,OAAA,IACA,OAAA,IACA,QACF,GACC,QAA6B,qBAC5B,OAAA,IACA,OAAA,IACA,WACF,CACF,CAAC;EACD,MAAM,UAAU,WAAW,cAAc,UAAU,aAAa,IAAI;CACtE;AACF;AAEA,SAAS,YACP,aACA,gBACA,MACM;CACN,MAAM,aAAa,YAAY;CAC/B,MAAM,gBAAgB,eAAe;CACrC,IAAI,IAAI;CACR,IAAI,IAAI;CACR,OAAO,IAAI,cAAc,IAAI,eAAe;EAC1C,MAAM,UAAU,YAAY,EAAE,CAAC;EAC/B,MAAM,aAAa,eAAe,EAAE,CAAC;EACrC,IAAI,YAAY,YAAY;GAC1B,IAAI,CAAC,UAAU,YAAY,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC,EAAE,GACpD,KAAK,KAAK;IACR,IAAI;IACJ,KAAK;IACL,UAAU,YAAY,EAAE,CAAC;IACzB,UAAU,eAAe,EAAE,CAAC;GAC9B,CAAC;GAEH;GACA;EACF,OAAO,IAAI,YAAY,SAAS,UAAU,IAAI,GAAG;GAC/C,KAAK,KAAK;IACR,IAAI;IACJ,KAAK;IACL,UAAU,YAAY,EAAE,CAAC;GAC3B,CAAC;GACD;EACF,OAAO;GACL,KAAK,KAAK;IACR,IAAI;IACJ,KAAK;IACL,UAAU,eAAe,EAAE,CAAC;GAC9B,CAAC;GACD;EACF;CACF;CACA,OAAO,IAAI,YAAY,KACrB,KAAK,KAAK;EACR,IAAI;EACJ,KAAK,YAAY,EAAE,CAAC;EACpB,UAAU,YAAY,EAAE,CAAC;CAC3B,CAAC;CAEH,OAAO,IAAI,eAAe,KACxB,KAAK,KAAK;EACR,IAAI;EACJ,KAAK,eAAe,EAAE,CAAC;EACvB,UAAU,eAAe,EAAE,CAAC;CAC9B,CAAC;AAEL;;;;;AAcA,eAAe,UACb,MACA,MACA,SACA,UACA,SACkB;CAClB,MAAM,OAAO,MAAM,KAAK,QAAQ,IAAI;CACpC,IAAI,IAAI,UAAU,aAAa,SAAS,KAAK,OAAO,IAAI;CACxD,IAAI,eAAe,IAAI,GAAG;EACxB,MAAM,EAAC,YAAW;EAClB,OAAO,IAAI,QAAQ,QAAQ,KACzB,IAAI,CAAC,QAAQ,QAAQ,EAAE,GACrB,OAAO;EAGX,OAAO;CACT;CAEA,MAAM,EAAC,YAAW;CAClB,IAAI,UAGF,MAAM,QAAQ,IACZ,QAAQ,MAAM,CAAC,CAAC,CAAC,KAAI,UAAS,KAAK,QAAQ,MAAM,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CACtE;CAEF,OAAO,IAAI,QAAQ,QAAQ,KAAK;EAC9B,IAAI,CAAE,MAAM,UAAU,MAAM,QAAQ,EAAE,CAAC,IAAI,SAAS,UAAU,OAAO,GACnE,OAAO;EAGT,UAAU;CACZ;CACA,OAAO;AACT;AAEA,gBAAgB,YACd,kBACA,aACA,MACA,SACA,UACA,UAC+C;CAC/C,IAAI,SAAS,WACX;CAGF,MAAM,OAAO,MAAM,SAAS,IAAI;CAChC,MAAM,UAAU,KAAA;CAChB,IAAI,IAAI;CACR,IAAI,SACF,IAAI,aAAa,SAAS,OAAO;CAEnC,IAAI,KAAA,KAAmB,GAAG;EACxB,IAAI,UAgBF,MAAM,QAAQ,IACZ,QACG,MAAM,CAAC,CAAC,CACR,KAAI,UACH,SAAU,MAAsB,EAAE,CAAC,CAAC,YAAY,KAAA,CAAS,CAC3D,CACJ;EAEF,OAAO,IAAI,QAAQ,QAAQ,KAAK;GAC9B,OAAO,YACL,kBACA,aACC,QAAQ,EAAE,CAAiB,IAC5B,SACA,UACA,QACF;GACA,UAAU;EACZ;CACF,OACE,OAAO,IAAI,QAAQ,QAAQ,KAAK;EAC9B,MAAM,WAAW,YAAY;EAE7B,IAAI,qBAAqB,UAAU;GACjC,OAAO,YACL,UACA,aACA,UACA,QAAQ,EAAE,CAAC,IACX,UACA,QACF;GACA;EACF;EACA,MAAM,QAAQ;CAChB;AAEJ;AAEA,eAAsB,iBACpB,KACA,IACuB;CACvB,MAAM,OAAgC,CAAC;CACvC,MAAM,OACJ,OAAO,SACH,WAAU;EACR,IAAI;EACJ,KAAK,MAAM;EACX,UAAU,MAAM;CAClB,MACA,WAAU;EACR,IAAI;EACJ,KAAK,MAAM;EACX,UAAU,MAAM;CAClB;CAEN,WAAW,MAAM,SAAS,IAAI,QAAQ,GACpC,KAAK,KAAK,KAAK,KAAK,CAAC;CAEvB,OAAO;AACT"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"splice.d.ts","sourceRoot":"","sources":["../../../../../replicache/src/btree/splice.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"splice.d.ts","sourceRoot":"","sources":["../../../../../replicache/src/btree/splice.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,MAAM,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;AAGhF,eAAO,MAAM,SAAS,IAAI,CAAC;AAC3B,eAAO,MAAM,cAAc,IAAI,CAAC;AAChC,eAAO,MAAM,YAAY,IAAI,CAAC;AAC9B,eAAO,MAAM,WAAW,IAAI,CAAC;AAK7B,KAAK,KAAK,CAAC,CAAC,IAAI,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;AAErE,wBAAiB,cAAc,CAAC,CAAC,EAC/B,QAAQ,EAAE,SAAS,KAAK,CAAC,CAAC,CAAC,EAAE,EAC7B,OAAO,EAAE,SAAS,KAAK,CAAC,CAAC,CAAC,EAAE,GAC3B,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAgFzB"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { deepEqual } from "../../../shared/src/json.js";
|
|
2
|
+
import { compareUTF8 } from "compare-utf8";
|
|
2
3
|
//#region ../replicache/src/btree/splice.ts
|
|
3
4
|
var SPLICE_UNASSIGNED = -1;
|
|
4
5
|
var KEY = 0;
|
|
@@ -33,7 +34,7 @@ function* computeSplices(previous, current) {
|
|
|
33
34
|
}
|
|
34
35
|
previousIndex++;
|
|
35
36
|
currentIndex++;
|
|
36
|
-
} else if (previous[previousIndex][KEY]
|
|
37
|
+
} else if (compareUTF8(previous[previousIndex][KEY], current[currentIndex][KEY]) < 0) {
|
|
37
38
|
if (!splice) splice = newSplice();
|
|
38
39
|
splice[1]++;
|
|
39
40
|
previousIndex++;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"splice.js","names":[],"sources":["../../../../../replicache/src/btree/splice.ts"],"sourcesContent":["import {deepEqual, type ReadonlyJSONValue} from '../../../shared/src/json.ts';\n\nexport type Splice = [at: number, removed: number, added: number, from: number];\n\nconst SPLICE_UNASSIGNED = -1;\nexport const SPLICE_AT = 0;\nexport const SPLICE_REMOVED = 1;\nexport const SPLICE_ADDED = 2;\nexport const SPLICE_FROM = 3;\n\nconst KEY = 0;\nconst VALUE = 1;\n\ntype Entry<V> = readonly [key: string, value: V, ...rest: unknown[]];\n\nexport function* computeSplices<T>(\n previous: readonly Entry<T>[],\n current: readonly Entry<T>[],\n): Generator<Splice, void> {\n let previousIndex = 0;\n let currentIndex = 0;\n let splice: Splice | undefined;\n\n function ensureAssigned(splice: Splice, index: number): void {\n if (splice[SPLICE_FROM] === SPLICE_UNASSIGNED) {\n splice[SPLICE_FROM] = index;\n }\n }\n\n function newSplice(): Splice {\n return [previousIndex, 0, 0, SPLICE_UNASSIGNED];\n }\n\n while (previousIndex < previous.length && currentIndex < current.length) {\n if (previous[previousIndex][KEY] === current[currentIndex][KEY]) {\n if (\n deepEqual(\n // These are really Hash | InternalValue\n previous[previousIndex][VALUE] as ReadonlyJSONValue,\n current[currentIndex][VALUE] as ReadonlyJSONValue,\n )\n ) {\n if (splice) {\n ensureAssigned(splice, 0);\n yield splice;\n splice = undefined;\n }\n } else {\n if (!splice) {\n splice = newSplice();\n }\n splice[SPLICE_ADDED]++;\n splice[SPLICE_REMOVED]++;\n ensureAssigned(splice, currentIndex);\n }\n previousIndex++;\n currentIndex++;\n } else if (previous[previousIndex][KEY]
|
|
1
|
+
{"version":3,"file":"splice.js","names":[],"sources":["../../../../../replicache/src/btree/splice.ts"],"sourcesContent":["import {compareUTF8} from 'compare-utf8';\nimport {deepEqual, type ReadonlyJSONValue} from '../../../shared/src/json.ts';\n\nexport type Splice = [at: number, removed: number, added: number, from: number];\n\nconst SPLICE_UNASSIGNED = -1;\nexport const SPLICE_AT = 0;\nexport const SPLICE_REMOVED = 1;\nexport const SPLICE_ADDED = 2;\nexport const SPLICE_FROM = 3;\n\nconst KEY = 0;\nconst VALUE = 1;\n\ntype Entry<V> = readonly [key: string, value: V, ...rest: unknown[]];\n\nexport function* computeSplices<T>(\n previous: readonly Entry<T>[],\n current: readonly Entry<T>[],\n): Generator<Splice, void> {\n let previousIndex = 0;\n let currentIndex = 0;\n let splice: Splice | undefined;\n\n function ensureAssigned(splice: Splice, index: number): void {\n if (splice[SPLICE_FROM] === SPLICE_UNASSIGNED) {\n splice[SPLICE_FROM] = index;\n }\n }\n\n function newSplice(): Splice {\n return [previousIndex, 0, 0, SPLICE_UNASSIGNED];\n }\n\n while (previousIndex < previous.length && currentIndex < current.length) {\n if (previous[previousIndex][KEY] === current[currentIndex][KEY]) {\n if (\n deepEqual(\n // These are really Hash | InternalValue\n previous[previousIndex][VALUE] as ReadonlyJSONValue,\n current[currentIndex][VALUE] as ReadonlyJSONValue,\n )\n ) {\n if (splice) {\n ensureAssigned(splice, 0);\n yield splice;\n splice = undefined;\n }\n } else {\n if (!splice) {\n splice = newSplice();\n }\n splice[SPLICE_ADDED]++;\n splice[SPLICE_REMOVED]++;\n ensureAssigned(splice, currentIndex);\n }\n previousIndex++;\n currentIndex++;\n } else if (\n compareUTF8(previous[previousIndex][KEY], current[currentIndex][KEY]) < 0\n ) {\n // previous was removed\n if (!splice) {\n splice = newSplice();\n }\n splice[SPLICE_REMOVED]++;\n\n previousIndex++;\n } else {\n // current was added\n if (!splice) {\n splice = newSplice();\n }\n splice[SPLICE_ADDED]++;\n ensureAssigned(splice, currentIndex);\n\n currentIndex++;\n }\n }\n\n if (currentIndex < current.length) {\n if (!splice) {\n splice = newSplice();\n }\n splice[SPLICE_ADDED] += current.length - currentIndex;\n ensureAssigned(splice, currentIndex);\n }\n\n if (previousIndex < previous.length) {\n if (!splice) {\n splice = newSplice();\n }\n splice[SPLICE_REMOVED] += previous.length - previousIndex;\n }\n\n if (splice) {\n ensureAssigned(splice, 0);\n yield splice;\n }\n}\n"],"mappings":";;;AAKA,IAAM,oBAAoB;AAM1B,IAAM,MAAM;AACZ,IAAM,QAAQ;AAId,UAAiB,eACf,UACA,SACyB;CACzB,IAAI,gBAAgB;CACpB,IAAI,eAAe;CACnB,IAAI;CAEJ,SAAS,eAAe,QAAgB,OAAqB;EAC3D,IAAI,OAAA,OAAwB,mBAC1B,OAAA,KAAsB;CAE1B;CAEA,SAAS,YAAoB;EAC3B,OAAO;GAAC;GAAe;GAAG;GAAG;EAAiB;CAChD;CAEA,OAAO,gBAAgB,SAAS,UAAU,eAAe,QAAQ,QAC/D,IAAI,SAAS,cAAc,CAAC,SAAS,QAAQ,aAAa,CAAC,MAAM;EAC/D,IACE,UAEE,SAAS,cAAc,CAAC,QACxB,QAAQ,aAAa,CAAC,MACxB,GAEI;OAAA,QAAQ;IACV,eAAe,QAAQ,CAAC;IACxB,MAAM;IACN,SAAS,KAAA;GACX;SACK;GACL,IAAI,CAAC,QACH,SAAS,UAAU;GAErB,OAAA,EAAoB;GACpB,OAAA,EAAsB;GACtB,eAAe,QAAQ,YAAY;EACrC;EACA;EACA;CACF,OAAO,IACL,YAAY,SAAS,cAAc,CAAC,MAAM,QAAQ,aAAa,CAAC,IAAI,IAAI,GACxE;EAEA,IAAI,CAAC,QACH,SAAS,UAAU;EAErB,OAAA,EAAsB;EAEtB;CACF,OAAO;EAEL,IAAI,CAAC,QACH,SAAS,UAAU;EAErB,OAAA,EAAoB;EACpB,eAAe,QAAQ,YAAY;EAEnC;CACF;CAGF,IAAI,eAAe,QAAQ,QAAQ;EACjC,IAAI,CAAC,QACH,SAAS,UAAU;EAErB,OAAA,MAAwB,QAAQ,SAAS;EACzC,eAAe,QAAQ,YAAY;CACrC;CAEA,IAAI,gBAAgB,SAAS,QAAQ;EACnC,IAAI,CAAC,QACH,SAAS,UAAU;EAErB,OAAA,MAA0B,SAAS,SAAS;CAC9C;CAEA,IAAI,QAAQ;EACV,eAAe,QAAQ,CAAC;EACxB,MAAM;CACR;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"write.d.ts","sourceRoot":"","sources":["../../../../../replicache/src/btree/write.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"write.d.ts","sourceRoot":"","sources":["../../../../../replicache/src/btree/write.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,6BAA6B,CAAC;AAItD,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,KAAK,aAAa,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAC,KAAK,IAAI,EAA2B,MAAM,YAAY,CAAC;AAC/D,OAAO,EACL,YAAY,EACZ,KAAK,KAAK,EACV,gBAAgB,EAOjB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAC,SAAS,EAAC,MAAM,WAAW,CAAC;AAEpC,KAAK,aAAa,GAAG,IAAI,CAAC,OAAO,aAAa,CAAC,CAAC;AAEhD,qBAAa,UAAW,SAAQ,SAAS;;IAoBvC,UAAkB,QAAQ,EAAE,KAAK,CAAC;IAElC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,YACE,QAAQ,EAAE,KAAK,EACf,aAAa,EAAE,aAAa,EAC5B,IAAI,GAAE,IAAgB,EACtB,OAAO,SAAW,EAClB,OAAO,SAAY,EACnB,YAAY,GAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,MAAuB,EAC3D,eAAe,CAAC,EAAE,MAAM,EAMzB;IAQD,UAAU,CAAC,IAAI,EAAE,YAAY,GAAG,gBAAgB,GAAG,IAAI,CAKtD;IAED,mBAAmB,CACjB,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAC3B,KAAK,EAAE,MAAM,GACZ,gBAAgB,CAIlB;IAED,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC,eAAe,CAAC,EAAE,GAAG,YAAY,CAI/D;IAED,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,eAAe,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,CAAC;IAC5E,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAAC;IACrE,WAAW,CACT,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,eAAe,CAAC,EAAE,EACjD,KAAK,EAAE,MAAM,GACZ,gBAAgB,GAAG,YAAY,CAAC;IAUnC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CA2BtD;IAED;;;;;OAKG;IACH,OAAO,CACL,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,GACzD,OAAO,CAAC,IAAI,CAAC,CA2Gf;IAED,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAmBjC;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAKrB;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAwBrB;CACF"}
|
|
@@ -4,6 +4,7 @@ import { toRefs } from "../dag/chunk.js";
|
|
|
4
4
|
import { DataNodeImpl, InternalNodeImpl, createNewInternalEntryForNode, emptyDataNode, isDataNodeImpl, newNodeImpl, partition, toChunkData } from "./node.js";
|
|
5
5
|
import { getSizeOfEntry } from "../../../shared/src/size-of-value.js";
|
|
6
6
|
import { BTreeRead } from "./read.js";
|
|
7
|
+
import { compareUTF8 } from "compare-utf8";
|
|
7
8
|
import { Lock } from "@rocicorp/lock";
|
|
8
9
|
//#region ../replicache/src/btree/write.ts
|
|
9
10
|
var BTreeWrite = class extends BTreeRead {
|
|
@@ -87,7 +88,7 @@ var BTreeWrite = class extends BTreeRead {
|
|
|
87
88
|
return this.#lock.withLock(async () => {
|
|
88
89
|
if (entries.length === 0) return;
|
|
89
90
|
const sizedEntries = entries.map(([k, v], i) => {
|
|
90
|
-
if (i > 0) assert(entries[i - 1][0] <
|
|
91
|
+
if (i > 0) assert(compareUTF8(entries[i - 1][0], k) < 0, `putMany entries must be sorted and unique`);
|
|
91
92
|
return [
|
|
92
93
|
k,
|
|
93
94
|
v,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"write.js","names":[],"sources":["../../../../../replicache/src/btree/write.ts"],"sourcesContent":["import {Lock} from '@rocicorp/lock';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../shared/src/enum.ts';\nimport type {ReadonlyJSONValue} from '../../../shared/src/json.ts';\nimport {getSizeOfEntry} from '../../../shared/src/size-of-value.ts';\nimport {type Chunk, type CreateChunk, toRefs} from '../dag/chunk.ts';\nimport type {Write} from '../dag/store.ts';\nimport type * as FormatVersion from '../format-version-enum.ts';\nimport type {FrozenJSONValue} from '../frozen-json.ts';\nimport {type Hash, emptyHash, newRandomHash} from '../hash.ts';\nimport {\n DataNodeImpl,\n type Entry,\n InternalNodeImpl,\n createNewInternalEntryForNode,\n emptyDataNode,\n isDataNodeImpl,\n newNodeImpl,\n partition,\n toChunkData,\n} from './node.ts';\nimport {BTreeRead} from './read.ts';\n\ntype FormatVersion = Enum<typeof FormatVersion>;\n\nexport class BTreeWrite extends BTreeRead {\n /**\n * This rw lock is used to ensure we do not mutate the btree in parallel. It\n * would be a problem if we didn't have the lock in cases like this:\n *\n * ```ts\n * const p1 = tree.put('a', 0);\n * const p2 = tree.put('b', 1);\n * await p1;\n * await p2;\n * ```\n *\n * because both `p1` and `p2` would start from the old root hash but a put\n * changes the root hash so the two concurrent puts would lead to only one of\n * them actually working, and it is not deterministic which one would finish\n * last.\n */\n readonly #lock = new Lock();\n readonly #modified: Map<Hash, DataNodeImpl | InternalNodeImpl> = new Map();\n\n declare protected _dagRead: Write;\n\n readonly minSize: number;\n readonly maxSize: number;\n\n constructor(\n dagWrite: Write,\n formatVersion: FormatVersion,\n root: Hash = emptyHash,\n minSize = 8 * 1024,\n maxSize = 16 * 1024,\n getEntrySize: <K, V>(k: K, v: V) => number = getSizeOfEntry,\n chunkHeaderSize?: number,\n ) {\n super(dagWrite, formatVersion, root, getEntrySize, chunkHeaderSize);\n\n this.minSize = minSize;\n this.maxSize = maxSize;\n }\n\n #addToModified(node: DataNodeImpl | InternalNodeImpl): void {\n assert(node.isMutable, 'Expected node to be mutable');\n this.#modified.set(node.hash, node);\n this._cache.set(node.hash, node);\n }\n\n updateNode(node: DataNodeImpl | InternalNodeImpl): void {\n assert(node.isMutable, 'Expected node to be mutable');\n this.#modified.delete(node.hash);\n node.hash = newRandomHash();\n this.#addToModified(node);\n }\n\n newInternalNodeImpl(\n entries: Array<Entry<Hash>>,\n level: number,\n ): InternalNodeImpl {\n const n = new InternalNodeImpl(entries, newRandomHash(), level, true);\n this.#addToModified(n);\n return n;\n }\n\n newDataNodeImpl(entries: Entry<FrozenJSONValue>[]): DataNodeImpl {\n const n = new DataNodeImpl(entries, newRandomHash(), true);\n this.#addToModified(n);\n return n;\n }\n\n newNodeImpl(entries: Entry<FrozenJSONValue>[], level: number): DataNodeImpl;\n newNodeImpl(entries: Entry<Hash>[], level: number): InternalNodeImpl;\n newNodeImpl(\n entries: Entry<Hash>[] | Entry<FrozenJSONValue>[],\n level: number,\n ): InternalNodeImpl | DataNodeImpl;\n newNodeImpl(\n entries: Entry<Hash>[] | Entry<FrozenJSONValue>[],\n level: number,\n ): InternalNodeImpl | DataNodeImpl {\n const n = newNodeImpl(entries, newRandomHash(), level, true);\n this.#addToModified(n);\n return n;\n }\n\n put(key: string, value: FrozenJSONValue): Promise<void> {\n return this.#lock.withLock(async () => {\n const oldRootNode = await this.getNode(this.rootHash);\n const entrySize = this.getEntrySize(key, value);\n const rootNode = await oldRootNode.set(key, value, entrySize, this);\n\n // We do the rebalancing in the parent so we need to do it here as well.\n if (rootNode.getChildNodeSize(this) > this.maxSize) {\n const headerSize = this.chunkHeaderSize;\n const {level} = rootNode;\n const entries = partition(\n rootNode.entries,\n value => value[2],\n this.minSize - headerSize,\n this.maxSize - headerSize,\n entries => {\n const node = this.newNodeImpl(entries, level);\n return createNewInternalEntryForNode(node, this.getEntrySize);\n },\n );\n const newRoot = this.newInternalNodeImpl(entries, level + 1);\n this.rootHash = newRoot.hash;\n return;\n }\n\n this.rootHash = rootNode.hash;\n });\n }\n\n /**\n * Inserts multiple key-value pairs into the BTree efficiently.\n * The entries array must be sorted by key.\n * @param entries - Array of [key, value] tuples, must be sorted by key\n * @returns Promise that resolves when all entries are inserted\n */\n putMany(\n entries: ReadonlyArray<readonly [string, FrozenJSONValue]>,\n ): Promise<void> {\n return this.#lock.withLock(async () => {\n if (entries.length === 0) {\n return;\n }\n\n // Validate sortedness and convert to sized entries in one pass\n const sizedEntries: Entry<FrozenJSONValue>[] = entries.map(\n ([k, v], i) => {\n if (i > 0) {\n assert(\n entries[i - 1][0] < k,\n `putMany entries must be sorted and unique`,\n );\n }\n return [k, v, this.getEntrySize(k, v)];\n },\n );\n\n // Compute content size constraints\n const headerSize = this.chunkHeaderSize;\n const contentMin = this.minSize - headerSize;\n const contentMax = this.maxSize - headerSize;\n\n // Fast path: if tree is empty, use bulk loading algorithm\n if (this.rootHash === emptyHash) {\n // Build leaf nodes\n const leafPartitions = partition(\n sizedEntries,\n e => e[2],\n contentMin,\n contentMax,\n entries => entries,\n );\n\n if (leafPartitions.length === 0) {\n return;\n }\n\n if (leafPartitions.length === 1) {\n const leaf = this.newDataNodeImpl(leafPartitions[0]);\n this.rootHash = leaf.hash;\n return;\n }\n\n // Build tree bottom-up - reuse array to avoid allocations\n let currentLevel: Array<DataNodeImpl | InternalNodeImpl | Entry<Hash>> =\n leafPartitions.map(entries => this.newDataNodeImpl(entries));\n let level = 0;\n\n while (currentLevel.length > 1) {\n level++;\n\n // Create entries pointing to current level nodes - reuse array\n const parentEntriesLength = currentLevel.length;\n for (let i = 0; i < parentEntriesLength; i++) {\n currentLevel[i] = createNewInternalEntryForNode(\n currentLevel[i] as DataNodeImpl | InternalNodeImpl,\n this.getEntrySize,\n );\n }\n\n // Partition parent entries\n const parentPartitions = partition(\n currentLevel as Entry<Hash>[],\n e => e[2],\n contentMin,\n contentMax,\n entries => entries,\n );\n\n // Create internal nodes\n currentLevel = parentPartitions.map(entries =>\n this.newInternalNodeImpl(entries, level),\n );\n }\n\n this.rootHash = (\n currentLevel[0] as DataNodeImpl | InternalNodeImpl\n ).hash;\n return;\n }\n\n // Slow path: merge with existing tree\n const oldRootNode = await this.getNode(this.rootHash);\n const rootNode = await oldRootNode.putMany(sizedEntries, this);\n\n // We do the rebalancing in the parent so we need to do it here as well.\n if (rootNode.getChildNodeSize(this) > this.maxSize) {\n const {level} = rootNode;\n const entries: Entry<Hash>[] = partition(\n rootNode.entries,\n e => e[2],\n contentMin,\n contentMax,\n partitionEntries => {\n const node = this.newNodeImpl(partitionEntries, level);\n return createNewInternalEntryForNode(node, this.getEntrySize);\n },\n );\n const newRoot = this.newInternalNodeImpl(entries, level + 1);\n this.rootHash = newRoot.hash;\n return;\n }\n\n this.rootHash = rootNode.hash;\n });\n }\n\n del(key: string): Promise<boolean> {\n return this.#lock.withLock(async () => {\n const oldRootNode = await this.getNode(this.rootHash);\n const newRootNode = await oldRootNode.del(key, this);\n\n // No need to rebalance here since if root gets too small there is nothing\n // we can do about that.\n const found = this.rootHash !== newRootNode.hash;\n if (found) {\n // Flatten one layer.\n if (newRootNode.level > 0 && newRootNode.entries.length === 1) {\n this.rootHash = (newRootNode as InternalNodeImpl).entries[0][1];\n } else {\n this.rootHash = newRootNode.hash;\n }\n }\n\n return found;\n });\n }\n\n clear(): Promise<void> {\n return this.#lock.withLock(() => {\n this.#modified.clear();\n this.rootHash = emptyHash;\n });\n }\n\n flush(): Promise<Hash> {\n return this.#lock.withLock(async () => {\n const dagWrite = this._dagRead;\n\n if (this.rootHash === emptyHash) {\n // Write a chunk for the empty tree.\n const chunk = dagWrite.createChunk(emptyDataNode, []);\n await dagWrite.putChunk(chunk as Chunk<ReadonlyJSONValue>);\n return chunk.hash;\n }\n\n const newChunks: Chunk[] = [];\n const newRoot = gatherNewChunks(\n this.rootHash,\n newChunks,\n dagWrite.createChunk,\n this.#modified,\n this._formatVersion,\n );\n await Promise.all(newChunks.map(chunk => dagWrite.putChunk(chunk)));\n this.#modified.clear();\n this.rootHash = newRoot;\n return newRoot;\n });\n }\n}\n\nfunction gatherNewChunks(\n hash: Hash,\n newChunks: Chunk[],\n createChunk: CreateChunk,\n modified: Map<Hash, DataNodeImpl | InternalNodeImpl>,\n formatVersion: FormatVersion,\n): Hash {\n const node = modified.get(hash);\n if (node === undefined) {\n // Not modified, use the original.\n return hash;\n }\n\n if (isDataNodeImpl(node)) {\n const chunk = createChunk(toChunkData(node, formatVersion), []);\n newChunks.push(chunk);\n return chunk.hash;\n }\n\n // The BTree cannot have duplicate keys so the child entry hashes are unique.\n // No need fot a set to dedupe here.\n const refs: Hash[] = [];\n const {entries} = node;\n for (let i = 0; i < entries.length; i++) {\n const entry = entries[i];\n const childHash = entry[1];\n const newChildHash = gatherNewChunks(\n childHash,\n newChunks,\n createChunk,\n modified,\n formatVersion,\n );\n if (newChildHash !== childHash) {\n // MUTATES the entries!\n // Hashes do not change the size of the entry because all hashes have the same length\n entries[i] = [entry[0], newChildHash, entry[2]];\n }\n refs.push(newChildHash);\n }\n const chunk = createChunk(toChunkData(node, formatVersion), toRefs(refs));\n newChunks.push(chunk);\n return chunk.hash;\n}\n"],"mappings":";;;;;;;;AAyBA,IAAa,aAAb,cAAgC,UAAU;;;;;;;;;;;;;;;;;CAiBxC,QAAiB,IAAI,KAAK;CAC1B,4BAAiE,IAAI,IAAI;CAIzE;CACA;CAEA,YACE,UACA,eACA,OAAa,WACb,UAAU,MACV,UAAU,OACV,eAA6C,gBAC7C,iBACA;EACA,MAAM,UAAU,eAAe,MAAM,cAAc,eAAe;EAElE,KAAK,UAAU;EACf,KAAK,UAAU;CACjB;CAEA,eAAe,MAA6C;EAC1D,OAAO,KAAK,WAAW,6BAA6B;EACpD,KAAK,UAAU,IAAI,KAAK,MAAM,IAAI;EAClC,KAAK,OAAO,IAAI,KAAK,MAAM,IAAI;CACjC;CAEA,WAAW,MAA6C;EACtD,OAAO,KAAK,WAAW,6BAA6B;EACpD,KAAK,UAAU,OAAO,KAAK,IAAI;EAC/B,KAAK,OAAO,cAAc;EAC1B,KAAK,eAAe,IAAI;CAC1B;CAEA,oBACE,SACA,OACkB;EAClB,MAAM,IAAI,IAAI,iBAAiB,SAAS,cAAc,GAAG,OAAO,IAAI;EACpE,KAAK,eAAe,CAAC;EACrB,OAAO;CACT;CAEA,gBAAgB,SAAiD;EAC/D,MAAM,IAAI,IAAI,aAAa,SAAS,cAAc,GAAG,IAAI;EACzD,KAAK,eAAe,CAAC;EACrB,OAAO;CACT;CAQA,YACE,SACA,OACiC;EACjC,MAAM,IAAI,YAAY,SAAS,cAAc,GAAG,OAAO,IAAI;EAC3D,KAAK,eAAe,CAAC;EACrB,OAAO;CACT;CAEA,IAAI,KAAa,OAAuC;EACtD,OAAO,KAAK,MAAM,SAAS,YAAY;GACrC,MAAM,cAAc,MAAM,KAAK,QAAQ,KAAK,QAAQ;GACpD,MAAM,YAAY,KAAK,aAAa,KAAK,KAAK;GAC9C,MAAM,WAAW,MAAM,YAAY,IAAI,KAAK,OAAO,WAAW,IAAI;GAGlE,IAAI,SAAS,iBAAiB,IAAI,IAAI,KAAK,SAAS;IAClD,MAAM,aAAa,KAAK;IACxB,MAAM,EAAC,UAAS;IAChB,MAAM,UAAU,UACd,SAAS,UACT,UAAS,MAAM,IACf,KAAK,UAAU,YACf,KAAK,UAAU,aACf,YAAW;KACT,MAAM,OAAO,KAAK,YAAY,SAAS,KAAK;KAC5C,OAAO,8BAA8B,MAAM,KAAK,YAAY;IAC9D,CACF;IACA,MAAM,UAAU,KAAK,oBAAoB,SAAS,QAAQ,CAAC;IAC3D,KAAK,WAAW,QAAQ;IACxB;GACF;GAEA,KAAK,WAAW,SAAS;EAC3B,CAAC;CACH;;;;;;;CAQA,QACE,SACe;EACf,OAAO,KAAK,MAAM,SAAS,YAAY;GACrC,IAAI,QAAQ,WAAW,GACrB;GAIF,MAAM,eAAyC,QAAQ,KACpD,CAAC,GAAG,IAAI,MAAM;IACb,IAAI,IAAI,GACN,OACE,QAAQ,IAAI,EAAE,CAAC,KAAK,GACpB,2CACF;IAEF,OAAO;KAAC;KAAG;KAAG,KAAK,aAAa,GAAG,CAAC;IAAC;GACvC,CACF;GAGA,MAAM,aAAa,KAAK;GACxB,MAAM,aAAa,KAAK,UAAU;GAClC,MAAM,aAAa,KAAK,UAAU;GAGlC,IAAI,KAAK,aAAa,WAAW;IAE/B,MAAM,iBAAiB,UACrB,eACA,MAAK,EAAE,IACP,YACA,aACA,YAAW,OACb;IAEA,IAAI,eAAe,WAAW,GAC5B;IAGF,IAAI,eAAe,WAAW,GAAG;KAC/B,MAAM,OAAO,KAAK,gBAAgB,eAAe,EAAE;KACnD,KAAK,WAAW,KAAK;KACrB;IACF;IAGA,IAAI,eACF,eAAe,KAAI,YAAW,KAAK,gBAAgB,OAAO,CAAC;IAC7D,IAAI,QAAQ;IAEZ,OAAO,aAAa,SAAS,GAAG;KAC9B;KAGA,MAAM,sBAAsB,aAAa;KACzC,KAAK,IAAI,IAAI,GAAG,IAAI,qBAAqB,KACvC,aAAa,KAAK,8BAChB,aAAa,IACb,KAAK,YACP;KAaF,eATyB,UACvB,eACA,MAAK,EAAE,IACP,YACA,aACA,YAAW,OAIE,CAAA,CAAiB,KAAI,YAClC,KAAK,oBAAoB,SAAS,KAAK,CACzC;IACF;IAEA,KAAK,WACH,aAAa,EAAE,CACf;IACF;GACF;GAIA,MAAM,WAAW,OAAM,MADG,KAAK,QAAQ,KAAK,QAAQ,EAAA,CACjB,QAAQ,cAAc,IAAI;GAG7D,IAAI,SAAS,iBAAiB,IAAI,IAAI,KAAK,SAAS;IAClD,MAAM,EAAC,UAAS;IAChB,MAAM,UAAyB,UAC7B,SAAS,UACT,MAAK,EAAE,IACP,YACA,aACA,qBAAoB;KAClB,MAAM,OAAO,KAAK,YAAY,kBAAkB,KAAK;KACrD,OAAO,8BAA8B,MAAM,KAAK,YAAY;IAC9D,CACF;IACA,MAAM,UAAU,KAAK,oBAAoB,SAAS,QAAQ,CAAC;IAC3D,KAAK,WAAW,QAAQ;IACxB;GACF;GAEA,KAAK,WAAW,SAAS;EAC3B,CAAC;CACH;CAEA,IAAI,KAA+B;EACjC,OAAO,KAAK,MAAM,SAAS,YAAY;GAErC,MAAM,cAAc,OAAM,MADA,KAAK,QAAQ,KAAK,QAAQ,EAAA,CACd,IAAI,KAAK,IAAI;GAInD,MAAM,QAAQ,KAAK,aAAa,YAAY;GAC5C,IAAI,OAAO;IAET,IAAI,YAAY,QAAQ,KAAK,YAAY,QAAQ,WAAW,GAC1D,KAAK,WAAY,YAAiC,QAAQ,EAAE,CAAC;SAE7D,KAAK,WAAW,YAAY;GAEhC;GAEA,OAAO;EACT,CAAC;CACH;CAEA,QAAuB;EACrB,OAAO,KAAK,MAAM,eAAe;GAC/B,KAAK,UAAU,MAAM;GACrB,KAAK,WAAW;EAClB,CAAC;CACH;CAEA,QAAuB;EACrB,OAAO,KAAK,MAAM,SAAS,YAAY;GACrC,MAAM,WAAW,KAAK;GAEtB,IAAI,KAAK,aAAa,WAAW;IAE/B,MAAM,QAAQ,SAAS,YAAY,eAAe,CAAC,CAAC;IACpD,MAAM,SAAS,SAAS,KAAiC;IACzD,OAAO,MAAM;GACf;GAEA,MAAM,YAAqB,CAAC;GAC5B,MAAM,UAAU,gBACd,KAAK,UACL,WACA,SAAS,aACT,KAAK,WACL,KAAK,cACP;GACA,MAAM,QAAQ,IAAI,UAAU,KAAI,UAAS,SAAS,SAAS,KAAK,CAAC,CAAC;GAClE,KAAK,UAAU,MAAM;GACrB,KAAK,WAAW;GAChB,OAAO;EACT,CAAC;CACH;AACF;AAEA,SAAS,gBACP,MACA,WACA,aACA,UACA,eACM;CACN,MAAM,OAAO,SAAS,IAAI,IAAI;CAC9B,IAAI,SAAS,KAAA,GAEX,OAAO;CAGT,IAAI,eAAe,IAAI,GAAG;EACxB,MAAM,QAAQ,YAAY,YAAY,MAAM,aAAa,GAAG,CAAC,CAAC;EAC9D,UAAU,KAAK,KAAK;EACpB,OAAO,MAAM;CACf;CAIA,MAAM,OAAe,CAAC;CACtB,MAAM,EAAC,YAAW;CAClB,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,QAAQ,QAAQ;EACtB,MAAM,YAAY,MAAM;EACxB,MAAM,eAAe,gBACnB,WACA,WACA,aACA,UACA,aACF;EACA,IAAI,iBAAiB,WAGnB,QAAQ,KAAK;GAAC,MAAM;GAAI;GAAc,MAAM;EAAE;EAEhD,KAAK,KAAK,YAAY;CACxB;CACA,MAAM,QAAQ,YAAY,YAAY,MAAM,aAAa,GAAG,OAAO,IAAI,CAAC;CACxE,UAAU,KAAK,KAAK;CACpB,OAAO,MAAM;AACf"}
|
|
1
|
+
{"version":3,"file":"write.js","names":[],"sources":["../../../../../replicache/src/btree/write.ts"],"sourcesContent":["import {Lock} from '@rocicorp/lock';\nimport {compareUTF8} from 'compare-utf8';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../shared/src/enum.ts';\nimport type {ReadonlyJSONValue} from '../../../shared/src/json.ts';\nimport {getSizeOfEntry} from '../../../shared/src/size-of-value.ts';\nimport {type Chunk, type CreateChunk, toRefs} from '../dag/chunk.ts';\nimport type {Write} from '../dag/store.ts';\nimport type * as FormatVersion from '../format-version-enum.ts';\nimport type {FrozenJSONValue} from '../frozen-json.ts';\nimport {type Hash, emptyHash, newRandomHash} from '../hash.ts';\nimport {\n DataNodeImpl,\n type Entry,\n InternalNodeImpl,\n createNewInternalEntryForNode,\n emptyDataNode,\n isDataNodeImpl,\n newNodeImpl,\n partition,\n toChunkData,\n} from './node.ts';\nimport {BTreeRead} from './read.ts';\n\ntype FormatVersion = Enum<typeof FormatVersion>;\n\nexport class BTreeWrite extends BTreeRead {\n /**\n * This rw lock is used to ensure we do not mutate the btree in parallel. It\n * would be a problem if we didn't have the lock in cases like this:\n *\n * ```ts\n * const p1 = tree.put('a', 0);\n * const p2 = tree.put('b', 1);\n * await p1;\n * await p2;\n * ```\n *\n * because both `p1` and `p2` would start from the old root hash but a put\n * changes the root hash so the two concurrent puts would lead to only one of\n * them actually working, and it is not deterministic which one would finish\n * last.\n */\n readonly #lock = new Lock();\n readonly #modified: Map<Hash, DataNodeImpl | InternalNodeImpl> = new Map();\n\n declare protected _dagRead: Write;\n\n readonly minSize: number;\n readonly maxSize: number;\n\n constructor(\n dagWrite: Write,\n formatVersion: FormatVersion,\n root: Hash = emptyHash,\n minSize = 8 * 1024,\n maxSize = 16 * 1024,\n getEntrySize: <K, V>(k: K, v: V) => number = getSizeOfEntry,\n chunkHeaderSize?: number,\n ) {\n super(dagWrite, formatVersion, root, getEntrySize, chunkHeaderSize);\n\n this.minSize = minSize;\n this.maxSize = maxSize;\n }\n\n #addToModified(node: DataNodeImpl | InternalNodeImpl): void {\n assert(node.isMutable, 'Expected node to be mutable');\n this.#modified.set(node.hash, node);\n this._cache.set(node.hash, node);\n }\n\n updateNode(node: DataNodeImpl | InternalNodeImpl): void {\n assert(node.isMutable, 'Expected node to be mutable');\n this.#modified.delete(node.hash);\n node.hash = newRandomHash();\n this.#addToModified(node);\n }\n\n newInternalNodeImpl(\n entries: Array<Entry<Hash>>,\n level: number,\n ): InternalNodeImpl {\n const n = new InternalNodeImpl(entries, newRandomHash(), level, true);\n this.#addToModified(n);\n return n;\n }\n\n newDataNodeImpl(entries: Entry<FrozenJSONValue>[]): DataNodeImpl {\n const n = new DataNodeImpl(entries, newRandomHash(), true);\n this.#addToModified(n);\n return n;\n }\n\n newNodeImpl(entries: Entry<FrozenJSONValue>[], level: number): DataNodeImpl;\n newNodeImpl(entries: Entry<Hash>[], level: number): InternalNodeImpl;\n newNodeImpl(\n entries: Entry<Hash>[] | Entry<FrozenJSONValue>[],\n level: number,\n ): InternalNodeImpl | DataNodeImpl;\n newNodeImpl(\n entries: Entry<Hash>[] | Entry<FrozenJSONValue>[],\n level: number,\n ): InternalNodeImpl | DataNodeImpl {\n const n = newNodeImpl(entries, newRandomHash(), level, true);\n this.#addToModified(n);\n return n;\n }\n\n put(key: string, value: FrozenJSONValue): Promise<void> {\n return this.#lock.withLock(async () => {\n const oldRootNode = await this.getNode(this.rootHash);\n const entrySize = this.getEntrySize(key, value);\n const rootNode = await oldRootNode.set(key, value, entrySize, this);\n\n // We do the rebalancing in the parent so we need to do it here as well.\n if (rootNode.getChildNodeSize(this) > this.maxSize) {\n const headerSize = this.chunkHeaderSize;\n const {level} = rootNode;\n const entries = partition(\n rootNode.entries,\n value => value[2],\n this.minSize - headerSize,\n this.maxSize - headerSize,\n entries => {\n const node = this.newNodeImpl(entries, level);\n return createNewInternalEntryForNode(node, this.getEntrySize);\n },\n );\n const newRoot = this.newInternalNodeImpl(entries, level + 1);\n this.rootHash = newRoot.hash;\n return;\n }\n\n this.rootHash = rootNode.hash;\n });\n }\n\n /**\n * Inserts multiple key-value pairs into the BTree efficiently.\n * The entries array must be sorted by key.\n * @param entries - Array of [key, value] tuples, must be sorted by key\n * @returns Promise that resolves when all entries are inserted\n */\n putMany(\n entries: ReadonlyArray<readonly [string, FrozenJSONValue]>,\n ): Promise<void> {\n return this.#lock.withLock(async () => {\n if (entries.length === 0) {\n return;\n }\n\n // Validate sortedness and convert to sized entries in one pass\n const sizedEntries: Entry<FrozenJSONValue>[] = entries.map(\n ([k, v], i) => {\n if (i > 0) {\n assert(\n compareUTF8(entries[i - 1][0], k) < 0,\n `putMany entries must be sorted and unique`,\n );\n }\n return [k, v, this.getEntrySize(k, v)];\n },\n );\n\n // Compute content size constraints\n const headerSize = this.chunkHeaderSize;\n const contentMin = this.minSize - headerSize;\n const contentMax = this.maxSize - headerSize;\n\n // Fast path: if tree is empty, use bulk loading algorithm\n if (this.rootHash === emptyHash) {\n // Build leaf nodes\n const leafPartitions = partition(\n sizedEntries,\n e => e[2],\n contentMin,\n contentMax,\n entries => entries,\n );\n\n if (leafPartitions.length === 0) {\n return;\n }\n\n if (leafPartitions.length === 1) {\n const leaf = this.newDataNodeImpl(leafPartitions[0]);\n this.rootHash = leaf.hash;\n return;\n }\n\n // Build tree bottom-up - reuse array to avoid allocations\n let currentLevel: Array<DataNodeImpl | InternalNodeImpl | Entry<Hash>> =\n leafPartitions.map(entries => this.newDataNodeImpl(entries));\n let level = 0;\n\n while (currentLevel.length > 1) {\n level++;\n\n // Create entries pointing to current level nodes - reuse array\n const parentEntriesLength = currentLevel.length;\n for (let i = 0; i < parentEntriesLength; i++) {\n currentLevel[i] = createNewInternalEntryForNode(\n currentLevel[i] as DataNodeImpl | InternalNodeImpl,\n this.getEntrySize,\n );\n }\n\n // Partition parent entries\n const parentPartitions = partition(\n currentLevel as Entry<Hash>[],\n e => e[2],\n contentMin,\n contentMax,\n entries => entries,\n );\n\n // Create internal nodes\n currentLevel = parentPartitions.map(entries =>\n this.newInternalNodeImpl(entries, level),\n );\n }\n\n this.rootHash = (\n currentLevel[0] as DataNodeImpl | InternalNodeImpl\n ).hash;\n return;\n }\n\n // Slow path: merge with existing tree\n const oldRootNode = await this.getNode(this.rootHash);\n const rootNode = await oldRootNode.putMany(sizedEntries, this);\n\n // We do the rebalancing in the parent so we need to do it here as well.\n if (rootNode.getChildNodeSize(this) > this.maxSize) {\n const {level} = rootNode;\n const entries: Entry<Hash>[] = partition(\n rootNode.entries,\n e => e[2],\n contentMin,\n contentMax,\n partitionEntries => {\n const node = this.newNodeImpl(partitionEntries, level);\n return createNewInternalEntryForNode(node, this.getEntrySize);\n },\n );\n const newRoot = this.newInternalNodeImpl(entries, level + 1);\n this.rootHash = newRoot.hash;\n return;\n }\n\n this.rootHash = rootNode.hash;\n });\n }\n\n del(key: string): Promise<boolean> {\n return this.#lock.withLock(async () => {\n const oldRootNode = await this.getNode(this.rootHash);\n const newRootNode = await oldRootNode.del(key, this);\n\n // No need to rebalance here since if root gets too small there is nothing\n // we can do about that.\n const found = this.rootHash !== newRootNode.hash;\n if (found) {\n // Flatten one layer.\n if (newRootNode.level > 0 && newRootNode.entries.length === 1) {\n this.rootHash = (newRootNode as InternalNodeImpl).entries[0][1];\n } else {\n this.rootHash = newRootNode.hash;\n }\n }\n\n return found;\n });\n }\n\n clear(): Promise<void> {\n return this.#lock.withLock(() => {\n this.#modified.clear();\n this.rootHash = emptyHash;\n });\n }\n\n flush(): Promise<Hash> {\n return this.#lock.withLock(async () => {\n const dagWrite = this._dagRead;\n\n if (this.rootHash === emptyHash) {\n // Write a chunk for the empty tree.\n const chunk = dagWrite.createChunk(emptyDataNode, []);\n await dagWrite.putChunk(chunk as Chunk<ReadonlyJSONValue>);\n return chunk.hash;\n }\n\n const newChunks: Chunk[] = [];\n const newRoot = gatherNewChunks(\n this.rootHash,\n newChunks,\n dagWrite.createChunk,\n this.#modified,\n this._formatVersion,\n );\n await Promise.all(newChunks.map(chunk => dagWrite.putChunk(chunk)));\n this.#modified.clear();\n this.rootHash = newRoot;\n return newRoot;\n });\n }\n}\n\nfunction gatherNewChunks(\n hash: Hash,\n newChunks: Chunk[],\n createChunk: CreateChunk,\n modified: Map<Hash, DataNodeImpl | InternalNodeImpl>,\n formatVersion: FormatVersion,\n): Hash {\n const node = modified.get(hash);\n if (node === undefined) {\n // Not modified, use the original.\n return hash;\n }\n\n if (isDataNodeImpl(node)) {\n const chunk = createChunk(toChunkData(node, formatVersion), []);\n newChunks.push(chunk);\n return chunk.hash;\n }\n\n // The BTree cannot have duplicate keys so the child entry hashes are unique.\n // No need fot a set to dedupe here.\n const refs: Hash[] = [];\n const {entries} = node;\n for (let i = 0; i < entries.length; i++) {\n const entry = entries[i];\n const childHash = entry[1];\n const newChildHash = gatherNewChunks(\n childHash,\n newChunks,\n createChunk,\n modified,\n formatVersion,\n );\n if (newChildHash !== childHash) {\n // MUTATES the entries!\n // Hashes do not change the size of the entry because all hashes have the same length\n entries[i] = [entry[0], newChildHash, entry[2]];\n }\n refs.push(newChildHash);\n }\n const chunk = createChunk(toChunkData(node, formatVersion), toRefs(refs));\n newChunks.push(chunk);\n return chunk.hash;\n}\n"],"mappings":";;;;;;;;;AA0BA,IAAa,aAAb,cAAgC,UAAU;;;;;;;;;;;;;;;;;CAiBxC,QAAiB,IAAI,KAAK;CAC1B,4BAAiE,IAAI,IAAI;CAIzE;CACA;CAEA,YACE,UACA,eACA,OAAa,WACb,UAAU,MACV,UAAU,OACV,eAA6C,gBAC7C,iBACA;EACA,MAAM,UAAU,eAAe,MAAM,cAAc,eAAe;EAElE,KAAK,UAAU;EACf,KAAK,UAAU;CACjB;CAEA,eAAe,MAA6C;EAC1D,OAAO,KAAK,WAAW,6BAA6B;EACpD,KAAK,UAAU,IAAI,KAAK,MAAM,IAAI;EAClC,KAAK,OAAO,IAAI,KAAK,MAAM,IAAI;CACjC;CAEA,WAAW,MAA6C;EACtD,OAAO,KAAK,WAAW,6BAA6B;EACpD,KAAK,UAAU,OAAO,KAAK,IAAI;EAC/B,KAAK,OAAO,cAAc;EAC1B,KAAK,eAAe,IAAI;CAC1B;CAEA,oBACE,SACA,OACkB;EAClB,MAAM,IAAI,IAAI,iBAAiB,SAAS,cAAc,GAAG,OAAO,IAAI;EACpE,KAAK,eAAe,CAAC;EACrB,OAAO;CACT;CAEA,gBAAgB,SAAiD;EAC/D,MAAM,IAAI,IAAI,aAAa,SAAS,cAAc,GAAG,IAAI;EACzD,KAAK,eAAe,CAAC;EACrB,OAAO;CACT;CAQA,YACE,SACA,OACiC;EACjC,MAAM,IAAI,YAAY,SAAS,cAAc,GAAG,OAAO,IAAI;EAC3D,KAAK,eAAe,CAAC;EACrB,OAAO;CACT;CAEA,IAAI,KAAa,OAAuC;EACtD,OAAO,KAAK,MAAM,SAAS,YAAY;GACrC,MAAM,cAAc,MAAM,KAAK,QAAQ,KAAK,QAAQ;GACpD,MAAM,YAAY,KAAK,aAAa,KAAK,KAAK;GAC9C,MAAM,WAAW,MAAM,YAAY,IAAI,KAAK,OAAO,WAAW,IAAI;GAGlE,IAAI,SAAS,iBAAiB,IAAI,IAAI,KAAK,SAAS;IAClD,MAAM,aAAa,KAAK;IACxB,MAAM,EAAC,UAAS;IAChB,MAAM,UAAU,UACd,SAAS,UACT,UAAS,MAAM,IACf,KAAK,UAAU,YACf,KAAK,UAAU,aACf,YAAW;KACT,MAAM,OAAO,KAAK,YAAY,SAAS,KAAK;KAC5C,OAAO,8BAA8B,MAAM,KAAK,YAAY;IAC9D,CACF;IACA,MAAM,UAAU,KAAK,oBAAoB,SAAS,QAAQ,CAAC;IAC3D,KAAK,WAAW,QAAQ;IACxB;GACF;GAEA,KAAK,WAAW,SAAS;EAC3B,CAAC;CACH;;;;;;;CAQA,QACE,SACe;EACf,OAAO,KAAK,MAAM,SAAS,YAAY;GACrC,IAAI,QAAQ,WAAW,GACrB;GAIF,MAAM,eAAyC,QAAQ,KACpD,CAAC,GAAG,IAAI,MAAM;IACb,IAAI,IAAI,GACN,OACE,YAAY,QAAQ,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,GACpC,2CACF;IAEF,OAAO;KAAC;KAAG;KAAG,KAAK,aAAa,GAAG,CAAC;IAAC;GACvC,CACF;GAGA,MAAM,aAAa,KAAK;GACxB,MAAM,aAAa,KAAK,UAAU;GAClC,MAAM,aAAa,KAAK,UAAU;GAGlC,IAAI,KAAK,aAAa,WAAW;IAE/B,MAAM,iBAAiB,UACrB,eACA,MAAK,EAAE,IACP,YACA,aACA,YAAW,OACb;IAEA,IAAI,eAAe,WAAW,GAC5B;IAGF,IAAI,eAAe,WAAW,GAAG;KAC/B,MAAM,OAAO,KAAK,gBAAgB,eAAe,EAAE;KACnD,KAAK,WAAW,KAAK;KACrB;IACF;IAGA,IAAI,eACF,eAAe,KAAI,YAAW,KAAK,gBAAgB,OAAO,CAAC;IAC7D,IAAI,QAAQ;IAEZ,OAAO,aAAa,SAAS,GAAG;KAC9B;KAGA,MAAM,sBAAsB,aAAa;KACzC,KAAK,IAAI,IAAI,GAAG,IAAI,qBAAqB,KACvC,aAAa,KAAK,8BAChB,aAAa,IACb,KAAK,YACP;KAaF,eATyB,UACvB,eACA,MAAK,EAAE,IACP,YACA,aACA,YAAW,OAIE,CAAA,CAAiB,KAAI,YAClC,KAAK,oBAAoB,SAAS,KAAK,CACzC;IACF;IAEA,KAAK,WACH,aAAa,EAAE,CACf;IACF;GACF;GAIA,MAAM,WAAW,OAAM,MADG,KAAK,QAAQ,KAAK,QAAQ,EAAA,CACjB,QAAQ,cAAc,IAAI;GAG7D,IAAI,SAAS,iBAAiB,IAAI,IAAI,KAAK,SAAS;IAClD,MAAM,EAAC,UAAS;IAChB,MAAM,UAAyB,UAC7B,SAAS,UACT,MAAK,EAAE,IACP,YACA,aACA,qBAAoB;KAClB,MAAM,OAAO,KAAK,YAAY,kBAAkB,KAAK;KACrD,OAAO,8BAA8B,MAAM,KAAK,YAAY;IAC9D,CACF;IACA,MAAM,UAAU,KAAK,oBAAoB,SAAS,QAAQ,CAAC;IAC3D,KAAK,WAAW,QAAQ;IACxB;GACF;GAEA,KAAK,WAAW,SAAS;EAC3B,CAAC;CACH;CAEA,IAAI,KAA+B;EACjC,OAAO,KAAK,MAAM,SAAS,YAAY;GAErC,MAAM,cAAc,OAAM,MADA,KAAK,QAAQ,KAAK,QAAQ,EAAA,CACd,IAAI,KAAK,IAAI;GAInD,MAAM,QAAQ,KAAK,aAAa,YAAY;GAC5C,IAAI,OAAO;IAET,IAAI,YAAY,QAAQ,KAAK,YAAY,QAAQ,WAAW,GAC1D,KAAK,WAAY,YAAiC,QAAQ,EAAE,CAAC;SAE7D,KAAK,WAAW,YAAY;GAEhC;GAEA,OAAO;EACT,CAAC;CACH;CAEA,QAAuB;EACrB,OAAO,KAAK,MAAM,eAAe;GAC/B,KAAK,UAAU,MAAM;GACrB,KAAK,WAAW;EAClB,CAAC;CACH;CAEA,QAAuB;EACrB,OAAO,KAAK,MAAM,SAAS,YAAY;GACrC,MAAM,WAAW,KAAK;GAEtB,IAAI,KAAK,aAAa,WAAW;IAE/B,MAAM,QAAQ,SAAS,YAAY,eAAe,CAAC,CAAC;IACpD,MAAM,SAAS,SAAS,KAAiC;IACzD,OAAO,MAAM;GACf;GAEA,MAAM,YAAqB,CAAC;GAC5B,MAAM,UAAU,gBACd,KAAK,UACL,WACA,SAAS,aACT,KAAK,WACL,KAAK,cACP;GACA,MAAM,QAAQ,IAAI,UAAU,KAAI,UAAS,SAAS,SAAS,KAAK,CAAC,CAAC;GAClE,KAAK,UAAU,MAAM;GACrB,KAAK,WAAW;GAChB,OAAO;EACT,CAAC;CACH;AACF;AAEA,SAAS,gBACP,MACA,WACA,aACA,UACA,eACM;CACN,MAAM,OAAO,SAAS,IAAI,IAAI;CAC9B,IAAI,SAAS,KAAA,GAEX,OAAO;CAGT,IAAI,eAAe,IAAI,GAAG;EACxB,MAAM,QAAQ,YAAY,YAAY,MAAM,aAAa,GAAG,CAAC,CAAC;EAC9D,UAAU,KAAK,KAAK;EACpB,OAAO,MAAM;CACf;CAIA,MAAM,OAAe,CAAC;CACtB,MAAM,EAAC,YAAW;CAClB,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,QAAQ,QAAQ;EACtB,MAAM,YAAY,MAAM;EACxB,MAAM,eAAe,gBACnB,WACA,WACA,aACA,UACA,aACF;EACA,IAAI,iBAAiB,WAGnB,QAAQ,KAAK;GAAC,MAAM;GAAI;GAAc,MAAM;EAAE;EAEhD,KAAK,KAAK,YAAY;CACxB;CACA,MAAM,QAAQ,YAAY,YAAY,MAAM,aAAa,GAAG,OAAO,IAAI,CAAC;CACxE,UAAU,KAAK,KAAK;CACpB,OAAO,MAAM;AACf"}
|
|
@@ -23,6 +23,7 @@ export declare function dropDatabaseInternal(name: string, idbDatabasesStore: ID
|
|
|
23
23
|
* Creates the dag store used to inspect and update a database during
|
|
24
24
|
* collection. Callers can override it to install store hooks, for example so
|
|
25
25
|
* a corrupt ref count found while writing deleted clients is recovered from.
|
|
26
|
+
* Collection closes every store it creates once it is done with it.
|
|
26
27
|
*/
|
|
27
28
|
export type NewDagStore = (name: string, kvCreateStore: CreateStore) => Store;
|
|
28
29
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collect-idb-databases.d.ts","sourceRoot":"","sources":["../../../../../replicache/src/persist/collect-idb-databases.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAC,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"collect-idb-databases.d.ts","sourceRoot":"","sources":["../../../../../replicache/src/persist/collect-idb-databases.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAC,MAAM,kBAAkB,CAAC;AAKpE,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,iBAAiB,CAAC;AAgB3C,OAAO,KAAK,EAAC,WAAW,EAAE,SAAS,EAAE,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAE1E,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,uBAAuB,CAAC;AAM/D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAGnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAE3D;;GAEG;AACH,eAAO,MAAM,oBAAoB,QAAsB,CAAC;AAExD;;GAEG;AACH,eAAO,MAAM,yBAAyB,QAAgB,CAAC;AAEvD,wBAAgB,uBAAuB,CACrC,iBAAiB,EAAE,iBAAiB,EACpC,eAAe,EAAE,aAAa,EAC9B,eAAe,EAAE,MAAM,EACvB,mBAAmB,EAAE,MAAM,EAC3B,MAAM,EAAE,MAAM,EACd,sBAAsB,EAAE,OAAO,EAC/B,gBAAgB,EAAE,gBAAgB,EAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,WAAW,EACnB,WAAW,GAAE,WAAgC,EAC7C,gBAAgB,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,mBAAmB,KAAK,IAAI,CAAC,GAAG,SAAS,GACtE,IAAI,CA0BN;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,iBAAiB,EAAE,iBAAiB,EACpC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,aAAa,EAC9B,sBAAsB,EAAE,OAAO,EAC/B,gBAAgB,EAAE,gBAAgB,EAClC,WAAW,GAAE,WAAgC,GAC5C,OAAO,CAAC,IAAI,CAAC,CA2Ef;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,MAAM,EACZ,iBAAiB,EAAE,iBAAiB,EACpC,WAAW,EAAE,SAAS,iBAIvB;AA6BD;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,WAAW,KAAK,KAAK,CAAC;AAgD9E;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC;;;OAGG;IACH,OAAO,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,aAAa,GAAG,SAAS,CAAC;IACpD;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAChC;;;;;;;;;;;;;;OAcG;IACH,QAAQ,CAAC,EAAE,OAAO,EAAE,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF;;;;;;GAMG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,mBAAmB,iBAe5E;AAED;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC;IACpE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,OAAO,EAAE,CAAC;CACnB,CAAC,CAED;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,CAAC,EAAE,EAAE,iBAAiB,KAAK,OAAO,EAC7C,IAAI,CAAC,EAAE,mBAAmB,GACzB,OAAO,CAAC;IACT,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,OAAO,EAAE,CAAC;CACnB,CAAC,CAqBD;AA0CD;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,CAAC,EAAE,mBAAmB;aAvFvD,MAAM,EAAE;YACT,OAAO,EAAE;GAwFlB"}
|
|
@@ -52,12 +52,15 @@ async function collectIDBDatabases(idbDatabasesStore, now, maxAge, kvStoreProvid
|
|
|
52
52
|
if (deletedClientsToRemove.length > 0) {
|
|
53
53
|
let newDeletedClients = deletedClientsToRemove;
|
|
54
54
|
for (const name of dbNamesToKeep) {
|
|
55
|
+
const perdag = newDagStore(name, kvStoreProvider.create);
|
|
55
56
|
let dbDeletedClients;
|
|
56
57
|
try {
|
|
57
|
-
dbDeletedClients = await withWrite(
|
|
58
|
+
dbDeletedClients = await withWrite(perdag, (dagWrite) => addDeletedClients(dagWrite, deletedClientsToRemove));
|
|
58
59
|
} catch (e) {
|
|
59
60
|
if (!(e instanceof InvalidRefCountError)) throw e;
|
|
60
61
|
continue;
|
|
62
|
+
} finally {
|
|
63
|
+
await closeIgnoringErrors(perdag);
|
|
61
64
|
}
|
|
62
65
|
newDeletedClients = mergeDeletedClients(newDeletedClients, dbDeletedClients);
|
|
63
66
|
}
|
|
@@ -90,10 +93,15 @@ function defaultNewDagStore(name, kvCreateStore) {
|
|
|
90
93
|
* If any client has a recent heartbeat or there are pending mutations we
|
|
91
94
|
* return `[false]`. Otherwise we return `[true, deletedClients]`.
|
|
92
95
|
*/
|
|
93
|
-
function gatherDatabaseInfoForCollect(db, now, maxAge, enableMutationRecovery, kvCreateStore, newDagStore) {
|
|
96
|
+
async function gatherDatabaseInfoForCollect(db, now, maxAge, enableMutationRecovery, kvCreateStore, newDagStore) {
|
|
94
97
|
if (db.replicacheFormatVersion > 7) return [false];
|
|
95
98
|
assert(db.replicacheFormatVersion === 5 || db.replicacheFormatVersion === 6 || db.replicacheFormatVersion === 7, () => `Expected replicacheFormatVersion to be DD31, V6, or V7, got ${db.replicacheFormatVersion}`);
|
|
96
|
-
|
|
99
|
+
const perdag = newDagStore(db.name, kvCreateStore);
|
|
100
|
+
try {
|
|
101
|
+
return await canDatabaseBeCollectedAndGetDeletedClientIDs(enableMutationRecovery, perdag, now, maxAge);
|
|
102
|
+
} finally {
|
|
103
|
+
await closeIgnoringErrors(perdag);
|
|
104
|
+
}
|
|
97
105
|
}
|
|
98
106
|
/**
|
|
99
107
|
* Drops the specified database, from memory as well as from the configured
|
|
@@ -178,8 +186,8 @@ function dropTargets(lc, opts) {
|
|
|
178
186
|
return targets;
|
|
179
187
|
}
|
|
180
188
|
/**
|
|
181
|
-
* A store that failed to open may also fail to close; the drop's
|
|
182
|
-
* is what the caller needs.
|
|
189
|
+
* A store that failed to open may also fail to close; the drop's or the
|
|
190
|
+
* collection's own result is what the caller needs.
|
|
183
191
|
*/
|
|
184
192
|
function closeIgnoringErrors(store) {
|
|
185
193
|
return store.close().catch(() => void 0);
|