datahike-browser-tests 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.circleci/config.yml +405 -0
- package/.circleci/scripts/gen_ci.clj +194 -0
- package/.cirrus.yml +60 -0
- package/.clj-kondo/babashka/sci/config.edn +1 -0
- package/.clj-kondo/babashka/sci/sci/core.clj +9 -0
- package/.clj-kondo/config.edn +95 -0
- package/.dir-locals.el +2 -0
- package/.github/FUNDING.yml +3 -0
- package/.github/ISSUE_TEMPLATE/1-bug-report.yml +68 -0
- package/.github/ISSUE_TEMPLATE/2-feature-request.yml +28 -0
- package/.github/ISSUE_TEMPLATE/config.yml +6 -0
- package/.github/pull_request_template.md +24 -0
- package/.github/workflows/native-image.yml +84 -0
- package/LICENSE +203 -0
- package/README.md +273 -0
- package/bb/deps.edn +9 -0
- package/bb/resources/github-fingerprints +3 -0
- package/bb/resources/native-image-tests/run-bb-pod-tests.clj +162 -0
- package/bb/resources/native-image-tests/run-libdatahike-tests +12 -0
- package/bb/resources/native-image-tests/run-native-image-tests +74 -0
- package/bb/resources/native-image-tests/run-python-tests +22 -0
- package/bb/resources/native-image-tests/testconfig.attr-refs.edn +6 -0
- package/bb/resources/native-image-tests/testconfig.edn +5 -0
- package/bb/resources/template/.settings/org.eclipse.jdt.apt.core.prefs +2 -0
- package/bb/resources/template/.settings/org.eclipse.jdt.core.prefs +9 -0
- package/bb/resources/template/.settings/org.eclipse.m2e.core.prefs +4 -0
- package/bb/resources/template/pom.xml +22 -0
- package/bb/src/tools/build.clj +132 -0
- package/bb/src/tools/clj_kondo.clj +32 -0
- package/bb/src/tools/deploy.clj +26 -0
- package/bb/src/tools/examples.clj +19 -0
- package/bb/src/tools/npm.clj +100 -0
- package/bb/src/tools/python.clj +14 -0
- package/bb/src/tools/release.clj +94 -0
- package/bb/src/tools/test.clj +148 -0
- package/bb/src/tools/version.clj +47 -0
- package/bb.edn +269 -0
- package/benchmark/src/benchmark/cli.clj +195 -0
- package/benchmark/src/benchmark/compare.clj +157 -0
- package/benchmark/src/benchmark/config.clj +316 -0
- package/benchmark/src/benchmark/measure.clj +187 -0
- package/benchmark/src/benchmark/store.clj +190 -0
- package/benchmark/test/benchmark/measure_test.clj +156 -0
- package/build.clj +30 -0
- package/config.edn +49 -0
- package/deps.edn +138 -0
- package/dev/sandbox.clj +82 -0
- package/dev/sandbox.cljs +127 -0
- package/dev/sandbox_benchmarks.clj +27 -0
- package/dev/sandbox_client.clj +87 -0
- package/dev/sandbox_transact_bench.clj +109 -0
- package/dev/user.clj +79 -0
- package/doc/README.md +96 -0
- package/doc/adl/README.md +6 -0
- package/doc/adl/adr-000-adr.org +28 -0
- package/doc/adl/adr-001-attribute-references.org +15 -0
- package/doc/adl/adr-002-build-tooling.org +54 -0
- package/doc/adl/adr-003-db-meta-data.md +52 -0
- package/doc/adl/adr-004-github-flow.md +40 -0
- package/doc/adl/adr-XYZ-template.md +30 -0
- package/doc/adl/index.org +3 -0
- package/doc/assets/datahike-logo.svg +3 -0
- package/doc/assets/datahiking-invoice.org +85 -0
- package/doc/assets/hhtree2.png +0 -0
- package/doc/assets/network_topology.svg +624 -0
- package/doc/assets/perf.png +0 -0
- package/doc/assets/schema_mindmap.mm +132 -0
- package/doc/assets/schema_mindmap.svg +970 -0
- package/doc/assets/temporal_index.mm +74 -0
- package/doc/backend-development.md +78 -0
- package/doc/bb-pod.md +89 -0
- package/doc/benchmarking.md +360 -0
- package/doc/bindings/edn-conversion.md +383 -0
- package/doc/cli.md +162 -0
- package/doc/cljdoc.edn +27 -0
- package/doc/cljs-support.md +133 -0
- package/doc/config.md +406 -0
- package/doc/contributing.md +114 -0
- package/doc/datalog-vs-sql.md +210 -0
- package/doc/datomic_differences.md +109 -0
- package/doc/development/pull-api-ns.md +186 -0
- package/doc/development/pull-frame-state-diagram.jpg +0 -0
- package/doc/distributed.md +566 -0
- package/doc/entity_spec.md +92 -0
- package/doc/gc.md +273 -0
- package/doc/java-api.md +808 -0
- package/doc/javascript-api.md +421 -0
- package/doc/libdatahike.md +86 -0
- package/doc/logging_and_error_handling.md +43 -0
- package/doc/norms.md +66 -0
- package/doc/schema-migration.md +85 -0
- package/doc/schema.md +287 -0
- package/doc/storage-backends.md +363 -0
- package/doc/store-id-refactoring.md +596 -0
- package/doc/time_variance.md +325 -0
- package/doc/unstructured.md +167 -0
- package/doc/versioning.md +261 -0
- package/examples/basic/README.md +19 -0
- package/examples/basic/deps.edn +6 -0
- package/examples/basic/docker-compose.yml +13 -0
- package/examples/basic/src/examples/core.clj +60 -0
- package/examples/basic/src/examples/schema.clj +155 -0
- package/examples/basic/src/examples/store.clj +60 -0
- package/examples/basic/src/examples/time_travel.clj +185 -0
- package/examples/java/.settings/org.eclipse.core.resources.prefs +3 -0
- package/examples/java/.settings/org.eclipse.jdt.apt.core.prefs +2 -0
- package/examples/java/.settings/org.eclipse.jdt.core.prefs +9 -0
- package/examples/java/.settings/org.eclipse.m2e.core.prefs +4 -0
- package/examples/java/README.md +162 -0
- package/examples/java/pom.xml +62 -0
- package/examples/java/src/main/java/examples/QuickStart.java +115 -0
- package/examples/java/src/main/java/examples/SchemaExample.java +148 -0
- package/examples/java/src/main/java/examples/TimeTravelExample.java +121 -0
- package/flake.lock +27 -0
- package/flake.nix +27 -0
- package/http-server/datahike/http/middleware.clj +75 -0
- package/http-server/datahike/http/server.clj +269 -0
- package/java/src/datahike/java/Database.java +274 -0
- package/java/src/datahike/java/Datahike.java +281 -0
- package/java/src/datahike/java/DatahikeGeneratedTest.java +349 -0
- package/java/src/datahike/java/DatahikeTest.java +370 -0
- package/java/src/datahike/java/EDN.java +170 -0
- package/java/src/datahike/java/IEntity.java +11 -0
- package/java/src/datahike/java/Keywords.java +161 -0
- package/java/src/datahike/java/SchemaFlexibility.java +52 -0
- package/java/src/datahike/java/Util.java +219 -0
- package/karma.conf.js +19 -0
- package/libdatahike/compile-cpp +7 -0
- package/libdatahike/src/datahike/impl/LibDatahikeBase.java +203 -0
- package/libdatahike/src/datahike/impl/libdatahike.clj +59 -0
- package/libdatahike/src/test_cpp.cpp +61 -0
- package/npm-package/PUBLISHING.md +140 -0
- package/npm-package/README.md +226 -0
- package/npm-package/package.template.json +34 -0
- package/npm-package/test-isomorphic.ts +281 -0
- package/npm-package/test.js +557 -0
- package/npm-package/typescript-test.ts +70 -0
- package/package.json +16 -0
- package/pydatahike/README.md +569 -0
- package/pydatahike/pyproject.toml +91 -0
- package/pydatahike/setup.py +42 -0
- package/pydatahike/src/datahike/__init__.py +134 -0
- package/pydatahike/src/datahike/_native.py +250 -0
- package/pydatahike/src/datahike/_version.py +2 -0
- package/pydatahike/src/datahike/database.py +722 -0
- package/pydatahike/src/datahike/edn.py +311 -0
- package/pydatahike/src/datahike/py.typed +0 -0
- package/pydatahike/tests/conftest.py +17 -0
- package/pydatahike/tests/test_basic.py +170 -0
- package/pydatahike/tests/test_database.py +51 -0
- package/pydatahike/tests/test_edn_conversion.py +299 -0
- package/pydatahike/tests/test_query.py +99 -0
- package/pydatahike/tests/test_schema.py +55 -0
- package/resources/clj-kondo.exports/io.replikativ/datahike/config.edn +5 -0
- package/resources/example_server.edn +4 -0
- package/shadow-cljs.edn +56 -0
- package/src/data_readers.clj +7 -0
- package/src/datahike/api/impl.cljc +176 -0
- package/src/datahike/api/specification.cljc +633 -0
- package/src/datahike/api/types.cljc +261 -0
- package/src/datahike/api.cljc +41 -0
- package/src/datahike/array.cljc +99 -0
- package/src/datahike/cli.clj +166 -0
- package/src/datahike/cljs.cljs +6 -0
- package/src/datahike/codegen/cli.clj +406 -0
- package/src/datahike/codegen/clj_kondo.clj +291 -0
- package/src/datahike/codegen/java.clj +403 -0
- package/src/datahike/codegen/naming.cljc +33 -0
- package/src/datahike/codegen/native.clj +559 -0
- package/src/datahike/codegen/pod.clj +488 -0
- package/src/datahike/codegen/python.clj +838 -0
- package/src/datahike/codegen/report.clj +55 -0
- package/src/datahike/codegen/typescript.clj +262 -0
- package/src/datahike/codegen/validation.clj +145 -0
- package/src/datahike/config.cljc +294 -0
- package/src/datahike/connections.cljc +16 -0
- package/src/datahike/connector.cljc +265 -0
- package/src/datahike/constants.cljc +142 -0
- package/src/datahike/core.cljc +297 -0
- package/src/datahike/datom.cljc +459 -0
- package/src/datahike/db/interface.cljc +119 -0
- package/src/datahike/db/search.cljc +305 -0
- package/src/datahike/db/transaction.cljc +937 -0
- package/src/datahike/db/utils.cljc +338 -0
- package/src/datahike/db.cljc +956 -0
- package/src/datahike/experimental/unstructured.cljc +126 -0
- package/src/datahike/experimental/versioning.cljc +172 -0
- package/src/datahike/externs.js +31 -0
- package/src/datahike/gc.cljc +69 -0
- package/src/datahike/http/client.clj +188 -0
- package/src/datahike/http/writer.clj +79 -0
- package/src/datahike/impl/entity.cljc +218 -0
- package/src/datahike/index/interface.cljc +93 -0
- package/src/datahike/index/persistent_set.cljc +469 -0
- package/src/datahike/index/utils.cljc +44 -0
- package/src/datahike/index.cljc +32 -0
- package/src/datahike/js/api.cljs +172 -0
- package/src/datahike/js/api_macros.clj +22 -0
- package/src/datahike/js.cljs +163 -0
- package/src/datahike/json.cljc +209 -0
- package/src/datahike/lru.cljc +146 -0
- package/src/datahike/migrate.clj +39 -0
- package/src/datahike/norm/norm.clj +245 -0
- package/src/datahike/online_gc.cljc +252 -0
- package/src/datahike/pod.clj +155 -0
- package/src/datahike/pull_api.cljc +325 -0
- package/src/datahike/query.cljc +1945 -0
- package/src/datahike/query_stats.cljc +88 -0
- package/src/datahike/readers.cljc +62 -0
- package/src/datahike/remote.cljc +218 -0
- package/src/datahike/schema.cljc +228 -0
- package/src/datahike/schema_cache.cljc +42 -0
- package/src/datahike/spec.cljc +101 -0
- package/src/datahike/store.cljc +80 -0
- package/src/datahike/tools.cljc +308 -0
- package/src/datahike/transit.cljc +80 -0
- package/src/datahike/writer.cljc +239 -0
- package/src/datahike/writing.cljc +362 -0
- package/src/deps.cljs +1 -0
- package/src-hitchhiker-tree/datahike/index/hitchhiker_tree/insert.cljc +76 -0
- package/src-hitchhiker-tree/datahike/index/hitchhiker_tree/upsert.cljc +128 -0
- package/src-hitchhiker-tree/datahike/index/hitchhiker_tree.cljc +213 -0
- package/test/datahike/backward_compatibility_test/src/backward_test.clj +37 -0
- package/test/datahike/integration_test/config_record_file_test.clj +14 -0
- package/test/datahike/integration_test/config_record_test.clj +14 -0
- package/test/datahike/integration_test/depr_config_uri_test.clj +15 -0
- package/test/datahike/integration_test/return_map_test.clj +62 -0
- package/test/datahike/integration_test.cljc +67 -0
- package/test/datahike/norm/norm_test.clj +124 -0
- package/test/datahike/norm/resources/naming-and-sorting-test/001-a1-example.edn +5 -0
- package/test/datahike/norm/resources/naming-and-sorting-test/002-a2-example.edn +5 -0
- package/test/datahike/norm/resources/naming-and-sorting-test/003-tx-fn-test.edn +1 -0
- package/test/datahike/norm/resources/naming-and-sorting-test/004-tx-data-and-tx-fn-test.edn +5 -0
- package/test/datahike/norm/resources/naming-and-sorting-test/01-transact-basic-characters.edn +2 -0
- package/test/datahike/norm/resources/naming-and-sorting-test/02 add occupation.edn +5 -0
- package/test/datahike/norm/resources/naming-and-sorting-test/checksums.edn +12 -0
- package/test/datahike/norm/resources/simple-test/001-a1-example.edn +5 -0
- package/test/datahike/norm/resources/simple-test/002-a2-example.edn +5 -0
- package/test/datahike/norm/resources/simple-test/checksums.edn +4 -0
- package/test/datahike/norm/resources/tx-data-and-tx-fn-test/first/001-a1-example.edn +5 -0
- package/test/datahike/norm/resources/tx-data-and-tx-fn-test/first/002-a2-example.edn +5 -0
- package/test/datahike/norm/resources/tx-data-and-tx-fn-test/first/003-tx-fn-test.edn +1 -0
- package/test/datahike/norm/resources/tx-data-and-tx-fn-test/first/checksums.edn +6 -0
- package/test/datahike/norm/resources/tx-data-and-tx-fn-test/second/004-tx-data-and-tx-fn-test.edn +5 -0
- package/test/datahike/norm/resources/tx-data-and-tx-fn-test/second/checksums.edn +2 -0
- package/test/datahike/norm/resources/tx-fn-test/first/001-a1-example.edn +5 -0
- package/test/datahike/norm/resources/tx-fn-test/first/002-a2-example.edn +5 -0
- package/test/datahike/norm/resources/tx-fn-test/first/checksums.edn +4 -0
- package/test/datahike/norm/resources/tx-fn-test/second/003-tx-fn-test.edn +1 -0
- package/test/datahike/norm/resources/tx-fn-test/second/checksums.edn +2 -0
- package/test/datahike/test/api_test.cljc +895 -0
- package/test/datahike/test/array_test.cljc +40 -0
- package/test/datahike/test/attribute_refs/datoms_test.cljc +140 -0
- package/test/datahike/test/attribute_refs/db_test.cljc +42 -0
- package/test/datahike/test/attribute_refs/differences_test.cljc +515 -0
- package/test/datahike/test/attribute_refs/entity_test.cljc +89 -0
- package/test/datahike/test/attribute_refs/pull_api_test.cljc +320 -0
- package/test/datahike/test/attribute_refs/query_find_specs_test.cljc +59 -0
- package/test/datahike/test/attribute_refs/query_fns_test.cljc +130 -0
- package/test/datahike/test/attribute_refs/query_interop_test.cljc +47 -0
- package/test/datahike/test/attribute_refs/query_not_test.cljc +193 -0
- package/test/datahike/test/attribute_refs/query_or_test.cljc +137 -0
- package/test/datahike/test/attribute_refs/query_pull_test.cljc +156 -0
- package/test/datahike/test/attribute_refs/query_rules_test.cljc +176 -0
- package/test/datahike/test/attribute_refs/query_test.cljc +241 -0
- package/test/datahike/test/attribute_refs/temporal_search.cljc +22 -0
- package/test/datahike/test/attribute_refs/transact_test.cljc +220 -0
- package/test/datahike/test/attribute_refs/utils.cljc +128 -0
- package/test/datahike/test/cache_test.cljc +38 -0
- package/test/datahike/test/components_test.cljc +92 -0
- package/test/datahike/test/config_test.cljc +158 -0
- package/test/datahike/test/core_test.cljc +105 -0
- package/test/datahike/test/datom_test.cljc +44 -0
- package/test/datahike/test/db_test.cljc +54 -0
- package/test/datahike/test/entity_spec_test.cljc +159 -0
- package/test/datahike/test/entity_test.cljc +103 -0
- package/test/datahike/test/explode_test.cljc +143 -0
- package/test/datahike/test/filter_test.cljc +75 -0
- package/test/datahike/test/gc_test.cljc +159 -0
- package/test/datahike/test/http/server_test.clj +192 -0
- package/test/datahike/test/http/writer_test.clj +86 -0
- package/test/datahike/test/ident_test.cljc +32 -0
- package/test/datahike/test/index_test.cljc +345 -0
- package/test/datahike/test/insert.cljc +125 -0
- package/test/datahike/test/java_bindings_test.clj +6 -0
- package/test/datahike/test/listen_test.cljc +41 -0
- package/test/datahike/test/lookup_refs_test.cljc +266 -0
- package/test/datahike/test/lru_test.cljc +27 -0
- package/test/datahike/test/migrate_test.clj +297 -0
- package/test/datahike/test/model/core.cljc +376 -0
- package/test/datahike/test/model/invariant.cljc +142 -0
- package/test/datahike/test/model/rng.cljc +82 -0
- package/test/datahike/test/model_test.clj +217 -0
- package/test/datahike/test/nodejs_test.cljs +262 -0
- package/test/datahike/test/online_gc_test.cljc +475 -0
- package/test/datahike/test/pod_test.clj +369 -0
- package/test/datahike/test/pull_api_test.cljc +474 -0
- package/test/datahike/test/purge_test.cljc +144 -0
- package/test/datahike/test/query_aggregates_test.cljc +101 -0
- package/test/datahike/test/query_find_specs_test.cljc +52 -0
- package/test/datahike/test/query_fns_test.cljc +523 -0
- package/test/datahike/test/query_interop_test.cljc +47 -0
- package/test/datahike/test/query_not_test.cljc +189 -0
- package/test/datahike/test/query_or_test.cljc +158 -0
- package/test/datahike/test/query_pull_test.cljc +147 -0
- package/test/datahike/test/query_rules_test.cljc +248 -0
- package/test/datahike/test/query_stats_test.cljc +218 -0
- package/test/datahike/test/query_test.cljc +984 -0
- package/test/datahike/test/schema_test.cljc +424 -0
- package/test/datahike/test/specification_test.cljc +30 -0
- package/test/datahike/test/store_test.cljc +78 -0
- package/test/datahike/test/stress_test.cljc +57 -0
- package/test/datahike/test/time_variance_test.cljc +518 -0
- package/test/datahike/test/tools_test.clj +134 -0
- package/test/datahike/test/transact_test.cljc +518 -0
- package/test/datahike/test/tuples_test.cljc +564 -0
- package/test/datahike/test/unstructured_test.cljc +291 -0
- package/test/datahike/test/upsert_impl_test.cljc +205 -0
- package/test/datahike/test/upsert_test.cljc +363 -0
- package/test/datahike/test/utils.cljc +110 -0
- package/test/datahike/test/validation_test.cljc +48 -0
- package/test/datahike/test/versioning_test.cljc +56 -0
- package/test/datahike/test.cljc +66 -0
- package/tests.edn +24 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
(ns ^:no-doc datahike.lru
|
|
2
|
+
(:require [#?(:clj clojure.core.cache :cljs cljs.cache) :refer [defcache CacheProtocol]]
|
|
3
|
+
#?(:clj clojure.data.priority-map
|
|
4
|
+
:cljs tailrecursion.priority-map)))
|
|
5
|
+
|
|
6
|
+
(declare assoc-lru cleanup-lru)
|
|
7
|
+
|
|
8
|
+
#?(:cljs
|
|
9
|
+
(deftype LRU [key-value gen-key key-gen gen limit]
|
|
10
|
+
IAssociative
|
|
11
|
+
(-assoc [this k v] (assoc-lru this k v))
|
|
12
|
+
(-contains-key? [_ k] (-contains-key? key-value k))
|
|
13
|
+
ILookup
|
|
14
|
+
(-lookup [_ k] (-lookup key-value k nil))
|
|
15
|
+
(-lookup [_ k nf] (-lookup key-value k nf))
|
|
16
|
+
IPrintWithWriter
|
|
17
|
+
(-pr-writer [_ writer opts]
|
|
18
|
+
(-pr-writer (persistent! key-value) writer opts)))
|
|
19
|
+
:clj
|
|
20
|
+
(deftype LRU [^clojure.lang.Associative key-value gen-key key-gen gen limit]
|
|
21
|
+
clojure.lang.ILookup
|
|
22
|
+
(valAt [_ k] (.valAt key-value k))
|
|
23
|
+
(valAt [_ k not-found] (.valAt key-value k not-found))
|
|
24
|
+
clojure.lang.Associative
|
|
25
|
+
(containsKey [_ k] (.containsKey key-value k))
|
|
26
|
+
(entryAt [_ k] (.entryAt key-value k))
|
|
27
|
+
(assoc [this k v] (assoc-lru this k v))))
|
|
28
|
+
|
|
29
|
+
(defn assoc-lru [^LRU lru k v]
|
|
30
|
+
(let [key-value (.-key-value lru)
|
|
31
|
+
gen-key (.-gen-key lru)
|
|
32
|
+
key-gen (.-key-gen lru)
|
|
33
|
+
gen (.-gen lru)
|
|
34
|
+
limit (.-limit lru)]
|
|
35
|
+
(if-let [g (key-gen k nil)]
|
|
36
|
+
(->LRU key-value
|
|
37
|
+
(-> gen-key
|
|
38
|
+
(dissoc g)
|
|
39
|
+
(assoc gen k))
|
|
40
|
+
(assoc key-gen k gen)
|
|
41
|
+
(inc gen)
|
|
42
|
+
limit)
|
|
43
|
+
(cleanup-lru
|
|
44
|
+
(->LRU (assoc key-value k v)
|
|
45
|
+
(assoc gen-key gen k)
|
|
46
|
+
(assoc key-gen k gen)
|
|
47
|
+
(inc gen)
|
|
48
|
+
limit)))))
|
|
49
|
+
|
|
50
|
+
(defn cleanup-lru [^LRU lru]
|
|
51
|
+
(if (> (count (.-key-value lru)) (.-limit lru))
|
|
52
|
+
(let [key-value (.-key-value lru)
|
|
53
|
+
gen-key (.-gen-key lru)
|
|
54
|
+
key-gen (.-key-gen lru)
|
|
55
|
+
gen (.-gen lru)
|
|
56
|
+
limit (.-limit lru)
|
|
57
|
+
[g k] (first gen-key)]
|
|
58
|
+
(->LRU (dissoc key-value k)
|
|
59
|
+
(dissoc gen-key g)
|
|
60
|
+
(dissoc key-gen k)
|
|
61
|
+
gen
|
|
62
|
+
limit))
|
|
63
|
+
lru))
|
|
64
|
+
|
|
65
|
+
(defn lru [limit]
|
|
66
|
+
(->LRU {} (sorted-map) {} 0 limit))
|
|
67
|
+
|
|
68
|
+
(defcache LRUDatomCache [cache lru counts n-total-datoms tick datom-limit]
|
|
69
|
+
CacheProtocol
|
|
70
|
+
(lookup [_ item]
|
|
71
|
+
(get cache item))
|
|
72
|
+
(lookup [_ item not-found]
|
|
73
|
+
(get cache item not-found))
|
|
74
|
+
(has? [_ item]
|
|
75
|
+
(contains? cache item))
|
|
76
|
+
(hit [_ item]
|
|
77
|
+
(let [tick+ (inc tick)]
|
|
78
|
+
(LRUDatomCache. cache
|
|
79
|
+
(if (contains? cache item)
|
|
80
|
+
(assoc lru item tick+)
|
|
81
|
+
lru)
|
|
82
|
+
counts
|
|
83
|
+
n-total-datoms
|
|
84
|
+
tick+
|
|
85
|
+
datom-limit)))
|
|
86
|
+
(miss [this item result]
|
|
87
|
+
(let [tick+ (inc tick)
|
|
88
|
+
n-new-datoms (count result)
|
|
89
|
+
new-size (+ n-total-datoms n-new-datoms)
|
|
90
|
+
[c l n s] (if (contains? lru item)
|
|
91
|
+
[(dissoc cache item)
|
|
92
|
+
(dissoc lru item)
|
|
93
|
+
(dissoc counts item)
|
|
94
|
+
(- new-size (get counts item))]
|
|
95
|
+
[cache lru counts new-size])
|
|
96
|
+
[c l n s] (loop [c c l l n n s s]
|
|
97
|
+
(if (> s datom-limit)
|
|
98
|
+
(let [k (first (peek lru))]
|
|
99
|
+
(if-let [x (get n k)]
|
|
100
|
+
(recur (dissoc c k)
|
|
101
|
+
(dissoc l k)
|
|
102
|
+
(dissoc n k)
|
|
103
|
+
(- s x))
|
|
104
|
+
[c l n s]))
|
|
105
|
+
[c l n s]))]
|
|
106
|
+
(LRUDatomCache. (assoc c item result)
|
|
107
|
+
(assoc l item tick+)
|
|
108
|
+
(assoc n item n-new-datoms)
|
|
109
|
+
s
|
|
110
|
+
tick+
|
|
111
|
+
datom-limit)))
|
|
112
|
+
(evict [this key]
|
|
113
|
+
(if (contains? cache key)
|
|
114
|
+
(LRUDatomCache. (dissoc cache key)
|
|
115
|
+
(dissoc lru key)
|
|
116
|
+
(dissoc counts key)
|
|
117
|
+
(- n-total-datoms (get counts key))
|
|
118
|
+
(inc tick)
|
|
119
|
+
datom-limit)
|
|
120
|
+
this))
|
|
121
|
+
(seed [_ base]
|
|
122
|
+
(LRUDatomCache. base
|
|
123
|
+
(into #?(:clj (clojure.data.priority-map/priority-map)
|
|
124
|
+
:cljs (tailrecursion.priority-map/priority-map))
|
|
125
|
+
(map #(vector % 0)
|
|
126
|
+
(keys base)))
|
|
127
|
+
(into {}
|
|
128
|
+
(map #(vector % (count (get base %)))
|
|
129
|
+
(keys base)))
|
|
130
|
+
0
|
|
131
|
+
0
|
|
132
|
+
datom-limit))
|
|
133
|
+
Object
|
|
134
|
+
(toString [_]
|
|
135
|
+
(str cache \, \space lru \, \space counts \, \space n-total-datoms \, \space tick \, \space datom-limit)))
|
|
136
|
+
|
|
137
|
+
(defn lru-datom-cache-factory
|
|
138
|
+
"Returns an LRU cache with the cache and usage-table initialied to `base` --
|
|
139
|
+
each entry is initialized with the same usage value.
|
|
140
|
+
This function takes an optional `:threshold` argument that defines the maximum number
|
|
141
|
+
of elements in the cache before the LRU semantics apply (default is 32)."
|
|
142
|
+
[base & {threshold :threshold :or {threshold 32}}]
|
|
143
|
+
{:pre [(number? threshold) (< 0 threshold)
|
|
144
|
+
(map? base)]}
|
|
145
|
+
#?(:clj (atom (clojure.core.cache/seed (LRUDatomCache. {} (clojure.data.priority-map/priority-map) {} 0 0 threshold) base))
|
|
146
|
+
:cljs (atom (cljs.cache/seed (LRUDatomCache. {} (tailrecursion.priority-map/priority-map) {} 0 0 threshold) base))))
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
(ns ^:no-doc datahike.migrate
|
|
2
|
+
(:require [datahike.api :as api]
|
|
3
|
+
[datahike.constants :as c]
|
|
4
|
+
[datahike.datom :as d]
|
|
5
|
+
[datahike.db :as db]
|
|
6
|
+
[clj-cbor.core :as cbor]))
|
|
7
|
+
|
|
8
|
+
(defn export-db
|
|
9
|
+
"Export the database in a flat-file of datoms at path.
|
|
10
|
+
Intended as a temporary solution, pending developments in Wanderung."
|
|
11
|
+
[conn path]
|
|
12
|
+
(let [db @conn
|
|
13
|
+
cfg (:config db)]
|
|
14
|
+
(cbor/spit-all path (cond->> (sort-by
|
|
15
|
+
(juxt d/datom-tx :e)
|
|
16
|
+
(api/datoms (if (:keep-history? cfg) (api/history db) db) :eavt))
|
|
17
|
+
(:attribute-refs? cfg) (remove #(= (d/datom-tx %) c/tx0))
|
|
18
|
+
true (map seq)))))
|
|
19
|
+
|
|
20
|
+
(defn update-max-tx
|
|
21
|
+
"Find bigest tx in datoms and update max-tx of db.
|
|
22
|
+
Note: the last tx might not be the biggest if the db
|
|
23
|
+
has been imported before."
|
|
24
|
+
[db datoms]
|
|
25
|
+
(assoc db :max-tx (reduce #(max %1 (nth %2 3)) 0 datoms)))
|
|
26
|
+
|
|
27
|
+
(defn- instance-to-date [v]
|
|
28
|
+
(if (instance? java.time.Instant v) (java.util.Date/from v) v))
|
|
29
|
+
|
|
30
|
+
(defn import-db
|
|
31
|
+
"Import a flat-file of datoms at path into your database.
|
|
32
|
+
Intended as a temporary solution, pending developments in Wanderung."
|
|
33
|
+
[conn path]
|
|
34
|
+
(println "Preparing import of" path "in batches of 1000")
|
|
35
|
+
(let [datoms (->> (cbor/slurp-all path)
|
|
36
|
+
(map #(-> (apply d/datom %) (update :v instance-to-date))))]
|
|
37
|
+
(swap! conn update-max-tx datoms)
|
|
38
|
+
(print "Importing ")
|
|
39
|
+
(api/transact conn (vec datoms))))
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
(ns datahike.norm.norm
|
|
2
|
+
(:require
|
|
3
|
+
[clojure.java.io :as io]
|
|
4
|
+
[clojure.string :as string]
|
|
5
|
+
[clojure.pprint :as pp]
|
|
6
|
+
[clojure.data :as data]
|
|
7
|
+
[clojure.edn :as edn]
|
|
8
|
+
[clojure.spec.alpha :as s]
|
|
9
|
+
[taoensso.timbre :as log]
|
|
10
|
+
[hasch.core :as h]
|
|
11
|
+
[hasch.platform :as hp]
|
|
12
|
+
[datahike.api :as d]
|
|
13
|
+
[datahike.tools :as dt])
|
|
14
|
+
(:import
|
|
15
|
+
[java.io File]
|
|
16
|
+
[java.util.jar JarFile JarEntry]
|
|
17
|
+
[java.net URL]))
|
|
18
|
+
|
|
19
|
+
(def checksums-file "checksums.edn")
|
|
20
|
+
|
|
21
|
+
(defn- attribute-installed? [conn attr]
|
|
22
|
+
(some? (d/entity @conn [:db/ident attr])))
|
|
23
|
+
|
|
24
|
+
(defn- ensure-norm-attribute! [conn]
|
|
25
|
+
(if-not (attribute-installed? conn :tx/norm)
|
|
26
|
+
(:db-after (d/transact conn {:tx-data [{:db/ident :tx/norm
|
|
27
|
+
:db/valueType :db.type/keyword
|
|
28
|
+
:db/cardinality :db.cardinality/one}]}))
|
|
29
|
+
@conn))
|
|
30
|
+
|
|
31
|
+
(defn- norm-installed? [db norm]
|
|
32
|
+
(->> {:query '[:find (count ?t) .
|
|
33
|
+
:in $ ?tn
|
|
34
|
+
:where
|
|
35
|
+
[_ :tx/norm ?tn ?t]]
|
|
36
|
+
:args [db norm]}
|
|
37
|
+
d/q
|
|
38
|
+
some?))
|
|
39
|
+
|
|
40
|
+
(defn- get-jar [resource]
|
|
41
|
+
(-> (.getPath resource)
|
|
42
|
+
(string/split #"!" 2)
|
|
43
|
+
first
|
|
44
|
+
(subs 5)
|
|
45
|
+
JarFile.))
|
|
46
|
+
|
|
47
|
+
(defmulti ^:private retrieve-file-list
|
|
48
|
+
(fn [file-or-resource] (type file-or-resource)))
|
|
49
|
+
|
|
50
|
+
(defmethod ^:private retrieve-file-list File [file]
|
|
51
|
+
(if (.exists file)
|
|
52
|
+
(let [migration-files (file-seq file)
|
|
53
|
+
xf (comp
|
|
54
|
+
(filter #(.isFile %))
|
|
55
|
+
(filter #(string/ends-with? (.getPath %) ".edn")))]
|
|
56
|
+
(into [] xf migration-files))
|
|
57
|
+
(dt/raise (format "Norms folder %s does not exist." (str file)) {:folder file})))
|
|
58
|
+
|
|
59
|
+
(defmethod ^:private retrieve-file-list URL [resource]
|
|
60
|
+
(if resource
|
|
61
|
+
(let [abs-path (.getPath resource)
|
|
62
|
+
last-path-segment (-> abs-path (string/split #"/") peek)]
|
|
63
|
+
(if (string/starts-with? abs-path "file:")
|
|
64
|
+
(->> (get-jar resource)
|
|
65
|
+
.entries
|
|
66
|
+
enumeration-seq
|
|
67
|
+
(filter #(and (string/starts-with? (.getName %) last-path-segment)
|
|
68
|
+
(not (.isDirectory %))
|
|
69
|
+
(string/ends-with? % ".edn"))))
|
|
70
|
+
(->> (file-seq (io/file abs-path))
|
|
71
|
+
(filter #(not (.isDirectory %))))))
|
|
72
|
+
(dt/raise "Resource does not exist." {:resource (str resource)})))
|
|
73
|
+
|
|
74
|
+
(defmethod ^:private retrieve-file-list :default [arg]
|
|
75
|
+
(dt/raise "Can only read a File or a URL (resource)" {:arg arg :type (type arg)}))
|
|
76
|
+
|
|
77
|
+
(defn- filter-file-list [file-list]
|
|
78
|
+
(filter #(and (string/ends-with? % ".edn")
|
|
79
|
+
(not (string/ends-with? (.getName %) checksums-file)))
|
|
80
|
+
file-list))
|
|
81
|
+
|
|
82
|
+
(defn filename->keyword [filename]
|
|
83
|
+
(-> filename
|
|
84
|
+
(string/replace #" " "-")
|
|
85
|
+
(keyword)))
|
|
86
|
+
|
|
87
|
+
(defmulti ^:private read-edn-file
|
|
88
|
+
(fn [file-or-entry _file-or-resource] (type file-or-entry)))
|
|
89
|
+
|
|
90
|
+
(defmethod ^:private read-edn-file File [f _file]
|
|
91
|
+
(when (not (.exists f))
|
|
92
|
+
(dt/raise "Failed reading file because it does not exist" {:filename (str f)}))
|
|
93
|
+
[(-> (slurp f)
|
|
94
|
+
edn/read-string)
|
|
95
|
+
{:name (.getName f)
|
|
96
|
+
:norm (filename->keyword (.getName f))}])
|
|
97
|
+
|
|
98
|
+
(defmethod ^:private read-edn-file JarEntry [entry resource]
|
|
99
|
+
(when (nil? resource)
|
|
100
|
+
(dt/raise "Failed reading resource because it does not exist" {:resource (str resource)}))
|
|
101
|
+
(let [file-name (-> (.getName entry)
|
|
102
|
+
(string/split #"/")
|
|
103
|
+
peek)]
|
|
104
|
+
[(-> (get-jar resource)
|
|
105
|
+
(.getInputStream entry)
|
|
106
|
+
slurp
|
|
107
|
+
edn/read-string)
|
|
108
|
+
{:name file-name
|
|
109
|
+
:norm (filename->keyword file-name)}]))
|
|
110
|
+
|
|
111
|
+
(defmethod ^:private read-edn-file :default [t _]
|
|
112
|
+
(dt/raise "Can not handle argument" {:type (type t) :arg t}))
|
|
113
|
+
|
|
114
|
+
(defn- read-norm-files [norm-list file-or-resource]
|
|
115
|
+
(->> norm-list
|
|
116
|
+
(map (fn [f]
|
|
117
|
+
(let [[content metadata] (read-edn-file f file-or-resource)]
|
|
118
|
+
(merge content metadata))))
|
|
119
|
+
(sort-by :norm)))
|
|
120
|
+
|
|
121
|
+
(defn- compute-checksums [norm-files]
|
|
122
|
+
(->> norm-files
|
|
123
|
+
(reduce (fn [m {:keys [norm] :as content}]
|
|
124
|
+
(assoc m
|
|
125
|
+
norm
|
|
126
|
+
(-> (select-keys content [:tx-data :tx-fn])
|
|
127
|
+
h/edn-hash
|
|
128
|
+
hp/hash->str)))
|
|
129
|
+
{})))
|
|
130
|
+
|
|
131
|
+
(s/def ::tx-data vector?)
|
|
132
|
+
(s/def ::tx-fn symbol?)
|
|
133
|
+
(s/def ::norm-map (s/keys :opt-un [::tx-data ::tx-fn]))
|
|
134
|
+
(defn- validate-norm [norm]
|
|
135
|
+
(if (s/valid? ::norm-map norm)
|
|
136
|
+
(log/debug "Norm validated" {:norm-map norm})
|
|
137
|
+
(let [res (s/explain-data ::norm-map norm)]
|
|
138
|
+
(dt/raise "Invalid norm" {:validation-error res}))))
|
|
139
|
+
|
|
140
|
+
(defn- neutral-fn [_] [])
|
|
141
|
+
|
|
142
|
+
(defn- transact-norms [conn norm-list]
|
|
143
|
+
(let [db (ensure-norm-attribute! conn)]
|
|
144
|
+
(log/info "Checking migrations ...")
|
|
145
|
+
(doseq [{:keys [norm tx-data tx-fn]
|
|
146
|
+
:as norm-map
|
|
147
|
+
:or {tx-data []
|
|
148
|
+
tx-fn 'datahike.norm.norm/neutral-fn}}
|
|
149
|
+
norm-list]
|
|
150
|
+
(log/info "Checking migration" norm)
|
|
151
|
+
(validate-norm norm-map)
|
|
152
|
+
(when-not (norm-installed? db norm)
|
|
153
|
+
(log/info "Running migration")
|
|
154
|
+
(d/transact conn {:tx-data (vec (concat [{:tx/norm norm}]
|
|
155
|
+
tx-data
|
|
156
|
+
((var-get (requiring-resolve tx-fn)) conn)))})
|
|
157
|
+
(log/info "Done")))))
|
|
158
|
+
|
|
159
|
+
(defn- diff-checksums [checksums edn-content]
|
|
160
|
+
(let [diff (data/diff checksums edn-content)]
|
|
161
|
+
(when-not (every? nil? (butlast diff))
|
|
162
|
+
(dt/raise "Deviation of the checksums found. Migration aborted." {:diff diff}))))
|
|
163
|
+
|
|
164
|
+
(defmulti verify-checksums
|
|
165
|
+
(fn [file-or-resource] (type file-or-resource)))
|
|
166
|
+
|
|
167
|
+
(defmethod verify-checksums File [file]
|
|
168
|
+
(let [norm-list (-> (retrieve-file-list file)
|
|
169
|
+
filter-file-list
|
|
170
|
+
(read-norm-files file))
|
|
171
|
+
edn-content (-> (io/file (io/file file) checksums-file)
|
|
172
|
+
(read-edn-file file)
|
|
173
|
+
first)]
|
|
174
|
+
(diff-checksums (compute-checksums norm-list)
|
|
175
|
+
edn-content)))
|
|
176
|
+
|
|
177
|
+
(defmethod verify-checksums URL [resource]
|
|
178
|
+
(let [file-list (retrieve-file-list resource)
|
|
179
|
+
norm-list (-> (filter-file-list file-list)
|
|
180
|
+
(read-norm-files resource))
|
|
181
|
+
edn-content (-> (->> file-list
|
|
182
|
+
(filter #(-> (.getName %) (string/ends-with? checksums-file)))
|
|
183
|
+
first)
|
|
184
|
+
(read-edn-file resource)
|
|
185
|
+
first)]
|
|
186
|
+
(diff-checksums (compute-checksums norm-list)
|
|
187
|
+
edn-content)))
|
|
188
|
+
|
|
189
|
+
(defmulti ^:private ensure-norms
|
|
190
|
+
(fn [_conn file-or-resource] (type file-or-resource)))
|
|
191
|
+
|
|
192
|
+
(defmethod ^:private ensure-norms File [conn file]
|
|
193
|
+
(let [norm-list (-> (retrieve-file-list file)
|
|
194
|
+
filter-file-list
|
|
195
|
+
(read-norm-files file))]
|
|
196
|
+
(transact-norms conn norm-list)))
|
|
197
|
+
|
|
198
|
+
(defmethod ^:private ensure-norms URL [conn resource]
|
|
199
|
+
(let [file-list (retrieve-file-list resource)
|
|
200
|
+
norm-list (-> (filter-file-list file-list)
|
|
201
|
+
(read-norm-files resource))]
|
|
202
|
+
(transact-norms conn norm-list)))
|
|
203
|
+
|
|
204
|
+
(defn ensure-norms!
|
|
205
|
+
"Takes Datahike-connection and optional a java.io.File object
|
|
206
|
+
or java.net.URL to specify the location of your norms.
|
|
207
|
+
Defaults to the resource `migrations`.
|
|
208
|
+
Returns nil when successful and throws exception when not.
|
|
209
|
+
|
|
210
|
+
Ensures your norms are present on your Datahike database.
|
|
211
|
+
All the edn-files in this folder and its subfolders are
|
|
212
|
+
considered migration-files aka norms and will be transacted
|
|
213
|
+
ordered by their names into your database. All norms that
|
|
214
|
+
are successfully transacted will have an attribute that
|
|
215
|
+
marks them as migrated and they will not be applied twice."
|
|
216
|
+
([conn]
|
|
217
|
+
(ensure-norms! conn (io/resource "migrations")))
|
|
218
|
+
([conn file-or-resource]
|
|
219
|
+
(ensure-norms conn file-or-resource)))
|
|
220
|
+
|
|
221
|
+
(defn update-checksums!
|
|
222
|
+
"Optionally takes a folder as string. Defaults to the
|
|
223
|
+
folder `resources/migrations`.
|
|
224
|
+
Returns nil when successful and throws exception when not.
|
|
225
|
+
|
|
226
|
+
All the edn-files in the folder and its subfolders are
|
|
227
|
+
considered migration-files aka norms. For each of
|
|
228
|
+
these norms a checksum will be computed and written to
|
|
229
|
+
the file `checksums.edn`. Each time this fn is run,
|
|
230
|
+
the `checksums.edn` will be overwritten with the current
|
|
231
|
+
values.
|
|
232
|
+
This prevents inadvertent migrations of your database
|
|
233
|
+
when used in conjunction with a VCS. A merge-conflict
|
|
234
|
+
should be raised when trying to merge a checksums.edn
|
|
235
|
+
with stale data."
|
|
236
|
+
([]
|
|
237
|
+
(update-checksums! "resources/migrations"))
|
|
238
|
+
([^String norms-folder]
|
|
239
|
+
(let [file (io/file norms-folder)]
|
|
240
|
+
(-> (retrieve-file-list file)
|
|
241
|
+
filter-file-list
|
|
242
|
+
(read-norm-files file)
|
|
243
|
+
compute-checksums
|
|
244
|
+
(#(spit (io/file norms-folder checksums-file)
|
|
245
|
+
(with-out-str (pp/pprint %))))))))
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
(ns datahike.online-gc
|
|
2
|
+
"Online garbage collection for freed index nodes.
|
|
3
|
+
|
|
4
|
+
Runs incrementally during transaction commits with configurable grace period.
|
|
5
|
+
Addresses are marked as freed during transient operations (PSS mutations),
|
|
6
|
+
then deleted after a grace period to ensure concurrent readers don't crash.
|
|
7
|
+
|
|
8
|
+
Usage:
|
|
9
|
+
;; In datahike config
|
|
10
|
+
{:online-gc {:enabled? true
|
|
11
|
+
:grace-period-ms 60000 ;; 1 minute default
|
|
12
|
+
:max-batch 1000}} ;; Max addresses per GC run
|
|
13
|
+
|
|
14
|
+
;; For bulk imports (no concurrent readers)
|
|
15
|
+
{:online-gc {:enabled? true
|
|
16
|
+
:grace-period-ms 0 ;; No grace period
|
|
17
|
+
:max-batch 10000}}
|
|
18
|
+
|
|
19
|
+
;; Background GC (optional)
|
|
20
|
+
(def stop-ch (start-background-gc! store {...opts...}))
|
|
21
|
+
;; Later: (async/close! stop-ch)"
|
|
22
|
+
(:require [konserve.core :as k]
|
|
23
|
+
[konserve.utils :refer [multi-key-capable?]]
|
|
24
|
+
#?@(:clj [[clojure.core.cache.wrapped :as wrapped]]
|
|
25
|
+
:cljs [[cljs.cache.wrapped :as wrapped]])
|
|
26
|
+
[taoensso.timbre :refer [debug trace warn]]
|
|
27
|
+
[clojure.core.async :as async]
|
|
28
|
+
[superv.async #?(:clj :refer :cljs :refer-macros) [go-try- <?-]]
|
|
29
|
+
#?(:cljs [clojure.core.async :refer-macros [go]]))
|
|
30
|
+
#?(:clj (:import [java.util Date])))
|
|
31
|
+
|
|
32
|
+
(defn get-time [d]
|
|
33
|
+
#?(:clj (.getTime ^Date d)
|
|
34
|
+
:cljs (.getTime d)))
|
|
35
|
+
|
|
36
|
+
(defn get-and-clear-eligible-freed!
|
|
37
|
+
"Get ALL freed addresses eligible for deletion based on grace period.
|
|
38
|
+
|
|
39
|
+
Arguments:
|
|
40
|
+
store - The datahike store
|
|
41
|
+
grace-period-ms - Minimum age before deletion (milliseconds)
|
|
42
|
+
|
|
43
|
+
Returns: [addresses-to-delete remaining-freed]
|
|
44
|
+
- addresses-to-delete: vector of all addresses ready to delete
|
|
45
|
+
- remaining-freed: vector of [addr ts] pairs still in grace period"
|
|
46
|
+
[store grace-period-ms]
|
|
47
|
+
(let [freed-atom (-> store :storage :freed-addresses)
|
|
48
|
+
freed-set-atom (-> store :storage :freed-set)
|
|
49
|
+
now #?(:clj (Date.) :cljs (js/Date.))
|
|
50
|
+
cutoff-time (- (get-time now) grace-period-ms)]
|
|
51
|
+
(if freed-atom
|
|
52
|
+
(let [result (atom nil)]
|
|
53
|
+
(swap! freed-atom
|
|
54
|
+
(fn [freed-pairs]
|
|
55
|
+
(let [eligible (filterv (fn [[_addr ts]]
|
|
56
|
+
(<= (get-time ts) cutoff-time))
|
|
57
|
+
freed-pairs)
|
|
58
|
+
remaining (filterv (fn [[_addr ts]]
|
|
59
|
+
(> (get-time ts) cutoff-time))
|
|
60
|
+
freed-pairs)]
|
|
61
|
+
(reset! result [(mapv first eligible) remaining])
|
|
62
|
+
remaining)))
|
|
63
|
+
;; Also clear the freed-set and freed-stacks of eligible addresses
|
|
64
|
+
(when freed-set-atom
|
|
65
|
+
(let [[eligible _] @result]
|
|
66
|
+
(swap! freed-set-atom #(reduce disj % eligible))
|
|
67
|
+
(when-let [freed-stacks-atom (-> store :storage :freed-stacks)]
|
|
68
|
+
(swap! freed-stacks-atom #(reduce dissoc % eligible)))))
|
|
69
|
+
@result)
|
|
70
|
+
;; No freed-addresses atom (shouldn't happen with CachedStorage)
|
|
71
|
+
[[] []])))
|
|
72
|
+
|
|
73
|
+
(defn recycle-freed-addresses!
|
|
74
|
+
"Add freed addresses to the storage freelist for reuse instead of deleting.
|
|
75
|
+
Evicts addresses from cache to prevent stale reads.
|
|
76
|
+
The actual LMDB entries are overwritten when the address is reused.
|
|
77
|
+
|
|
78
|
+
Arguments:
|
|
79
|
+
store - The datahike store
|
|
80
|
+
addresses - Vector of addresses to recycle
|
|
81
|
+
|
|
82
|
+
Returns: Number of addresses recycled"
|
|
83
|
+
[store addresses]
|
|
84
|
+
(if (empty? addresses)
|
|
85
|
+
0
|
|
86
|
+
(let [cache (-> store :storage :cache)
|
|
87
|
+
freelist (-> store :storage :freelist)]
|
|
88
|
+
(trace "Recycling" (count addresses) "freed addresses to freelist")
|
|
89
|
+
;; Evict from cache to prevent stale reads
|
|
90
|
+
(doseq [addr addresses]
|
|
91
|
+
(wrapped/evict cache addr))
|
|
92
|
+
;; Add all addresses to the freelist for reuse
|
|
93
|
+
(swap! freelist into addresses)
|
|
94
|
+
(count addresses))))
|
|
95
|
+
|
|
96
|
+
(defn delete-freed-addresses!
|
|
97
|
+
"Delete freed addresses from store and cache.
|
|
98
|
+
Used as fallback when freelist is not available.
|
|
99
|
+
|
|
100
|
+
Arguments:
|
|
101
|
+
store - The datahike store
|
|
102
|
+
addresses - Vector of addresses to delete
|
|
103
|
+
max-batch - Chunk size for multi-dissoc calls
|
|
104
|
+
sync? - Whether to perform synchronous deletion
|
|
105
|
+
|
|
106
|
+
Returns: Number of addresses deleted (async channel or sync value)"
|
|
107
|
+
[store addresses max-batch sync?]
|
|
108
|
+
(if (empty? addresses)
|
|
109
|
+
(if sync? 0 (go-try- 0))
|
|
110
|
+
(let [cache (-> store :storage :cache)]
|
|
111
|
+
(trace "Deleting" (count addresses) "freed addresses")
|
|
112
|
+
;; Evict from cache first
|
|
113
|
+
(doseq [addr addresses]
|
|
114
|
+
(wrapped/evict cache addr))
|
|
115
|
+
;; Batch delete from store, chunked by max-batch
|
|
116
|
+
(if (multi-key-capable? store)
|
|
117
|
+
(if sync?
|
|
118
|
+
(do
|
|
119
|
+
(doseq [chunk (partition-all max-batch addresses)]
|
|
120
|
+
(k/multi-dissoc store (vec chunk) {:sync? true}))
|
|
121
|
+
(count addresses))
|
|
122
|
+
(go-try-
|
|
123
|
+
(doseq [chunk (partition-all max-batch addresses)]
|
|
124
|
+
(<?- (k/multi-dissoc store (vec chunk) {:sync? false})))
|
|
125
|
+
(count addresses)))
|
|
126
|
+
;; Fallback to individual deletes for stores without multi-key support
|
|
127
|
+
(if sync?
|
|
128
|
+
(do
|
|
129
|
+
(doseq [addr addresses]
|
|
130
|
+
(k/dissoc store addr {:sync? true}))
|
|
131
|
+
(count addresses))
|
|
132
|
+
(go-try-
|
|
133
|
+
(doseq [addr addresses]
|
|
134
|
+
(<?- (k/dissoc store addr {:sync? false})))
|
|
135
|
+
(count addresses)))))))
|
|
136
|
+
|
|
137
|
+
(defn online-gc!
|
|
138
|
+
"Perform online GC during commit.
|
|
139
|
+
Clears ALL eligible freed addresses each cycle.
|
|
140
|
+
|
|
141
|
+
CRITICAL SAFETY WARNINGS:
|
|
142
|
+
- Online GC is ONLY safe for single-branch databases
|
|
143
|
+
- Multi-branch databases SKIP online GC entirely (returns 0)
|
|
144
|
+
Reason: Freed nodes from one branch may still be referenced by other branches
|
|
145
|
+
through structural sharing. Only offline GC can safely handle multi-branch
|
|
146
|
+
cleanup via full reachability analysis.
|
|
147
|
+
- Long-lived readers can be corrupted if grace period is too short
|
|
148
|
+
- For bulk imports with no readers, set :grace-period-ms 0
|
|
149
|
+
|
|
150
|
+
When freelist recycling is available (non-crypto-hash mode, single branch),
|
|
151
|
+
addresses are added to a freelist for reuse instead of being deleted from
|
|
152
|
+
the store. This eliminates LMDB delete operations, converting O(freed_nodes)
|
|
153
|
+
deletes to O(1) append.
|
|
154
|
+
|
|
155
|
+
MULTI-BRANCH LIMITATION:
|
|
156
|
+
Online GC is completely disabled when multiple branches exist. Freed addresses
|
|
157
|
+
remain in the freed-addresses tracking for offline GC (d/gc-storage) to handle
|
|
158
|
+
safely through full reachability analysis.
|
|
159
|
+
|
|
160
|
+
Options:
|
|
161
|
+
:grace-period-ms - Minimum age before deletion (default 60000 = 1 minute)
|
|
162
|
+
:max-batch - Chunk size for store deletion batches (default 10000, only for delete mode)
|
|
163
|
+
:enabled? - Enable online GC (default false for safety)
|
|
164
|
+
:sync? - Synchronous deletion (default true, only for delete mode)
|
|
165
|
+
|
|
166
|
+
Returns: Number of addresses recycled/deleted (or async channel with count)"
|
|
167
|
+
[store {:keys [grace-period-ms max-batch enabled? sync?]
|
|
168
|
+
:or {grace-period-ms 60000
|
|
169
|
+
max-batch 10000
|
|
170
|
+
enabled? false
|
|
171
|
+
sync? true}}]
|
|
172
|
+
(if-not enabled?
|
|
173
|
+
(if sync? 0 (go-try- 0))
|
|
174
|
+
(if sync?
|
|
175
|
+
;; Synchronous mode
|
|
176
|
+
(let [branches (k/get store :branches nil {:sync? true})
|
|
177
|
+
multi-branch? (> (count branches) 1)]
|
|
178
|
+
(if multi-branch?
|
|
179
|
+
(do
|
|
180
|
+
(debug "Online GC: skipped (multi-branch detected - use offline GC instead)")
|
|
181
|
+
0)
|
|
182
|
+
(let [[to-recycle _remaining] (get-and-clear-eligible-freed! store grace-period-ms)]
|
|
183
|
+
(if (seq to-recycle)
|
|
184
|
+
(let [freelist (-> store :storage :freelist)
|
|
185
|
+
crypto-hash? (-> store :storage :config :crypto-hash?)]
|
|
186
|
+
(if (and freelist (not crypto-hash?))
|
|
187
|
+
(do
|
|
188
|
+
(debug "Online GC: recycling" (count to-recycle) "addresses to freelist")
|
|
189
|
+
(recycle-freed-addresses! store to-recycle))
|
|
190
|
+
(do
|
|
191
|
+
(debug "Online GC: deleting" (count to-recycle) "addresses")
|
|
192
|
+
(delete-freed-addresses! store to-recycle max-batch true))))
|
|
193
|
+
0))))
|
|
194
|
+
;; Asynchronous mode
|
|
195
|
+
(go-try-
|
|
196
|
+
(let [branches (<?- (k/get store :branches nil {:sync? false}))
|
|
197
|
+
multi-branch? (> (count branches) 1)]
|
|
198
|
+
(if multi-branch?
|
|
199
|
+
(do
|
|
200
|
+
(debug "Online GC: skipped (multi-branch detected - use offline GC instead)")
|
|
201
|
+
0)
|
|
202
|
+
(let [[to-recycle _remaining] (get-and-clear-eligible-freed! store grace-period-ms)]
|
|
203
|
+
(if (seq to-recycle)
|
|
204
|
+
(let [freelist (-> store :storage :freelist)
|
|
205
|
+
crypto-hash? (-> store :storage :config :crypto-hash?)]
|
|
206
|
+
(if (and freelist (not crypto-hash?))
|
|
207
|
+
(do
|
|
208
|
+
(debug "Online GC: recycling" (count to-recycle) "addresses to freelist")
|
|
209
|
+
(recycle-freed-addresses! store to-recycle))
|
|
210
|
+
(do
|
|
211
|
+
(debug "Online GC: deleting" (count to-recycle) "addresses")
|
|
212
|
+
(<?- (delete-freed-addresses! store to-recycle max-batch false)))))
|
|
213
|
+
0))))))))
|
|
214
|
+
|
|
215
|
+
(defn start-background-gc!
|
|
216
|
+
"Start a background process that periodically runs online GC.
|
|
217
|
+
|
|
218
|
+
Arguments:
|
|
219
|
+
store - The datahike store
|
|
220
|
+
options - GC configuration map
|
|
221
|
+
:grace-period-ms - Minimum age before deletion (default 60000 = 1 minute)
|
|
222
|
+
:interval-ms - How often to run GC (default 10000 = 10 seconds)
|
|
223
|
+
:max-batch - Maximum addresses to delete per run (default 1000)
|
|
224
|
+
|
|
225
|
+
Returns: A channel that can be closed to stop the background GC
|
|
226
|
+
|
|
227
|
+
Example:
|
|
228
|
+
(def stop-ch (start-background-gc! store {:grace-period-ms 60000
|
|
229
|
+
:interval-ms 10000
|
|
230
|
+
:max-batch 1000}))
|
|
231
|
+
;; Later, to stop:
|
|
232
|
+
(async/close! stop-ch)"
|
|
233
|
+
[store {:keys [grace-period-ms interval-ms max-batch]
|
|
234
|
+
:or {grace-period-ms 60000
|
|
235
|
+
interval-ms 10000
|
|
236
|
+
max-batch 1000}}]
|
|
237
|
+
(let [stop-ch (async/chan)]
|
|
238
|
+
(async/go-loop []
|
|
239
|
+
(let [[_ ch] (async/alts! [stop-ch (async/timeout interval-ms)])]
|
|
240
|
+
(when-not (= ch stop-ch)
|
|
241
|
+
;; Run GC asynchronously
|
|
242
|
+
(try
|
|
243
|
+
(let [deleted (async/<! (online-gc! store {:grace-period-ms grace-period-ms
|
|
244
|
+
:max-batch max-batch
|
|
245
|
+
:enabled? true
|
|
246
|
+
:sync? false}))]
|
|
247
|
+
(when (and deleted (pos? deleted))
|
|
248
|
+
(debug "Background GC deleted" deleted "addresses")))
|
|
249
|
+
(catch #?(:clj Exception :cljs js/Error) e
|
|
250
|
+
(warn "Background GC error:" e)))
|
|
251
|
+
(recur))))
|
|
252
|
+
stop-ch))
|