datahike-browser-tests 1.0.0

package/doc/assets/temporal_index.mm

@@ -0,0 +1,74 @@
+<map version="freeplane 1.6.0">
+<!--To view this file, download free mind mapping software Freeplane from http://freeplane.sourceforge.net -->
+<node TEXT="Temporal Index" FOLDED="false" ID="ID_1420672071" CREATED="1562593263004" MODIFIED="1562593936990" STYLE="oval">
+<font SIZE="18"/>
+<hook NAME="MapStyle">
+    <properties edgeColorConfiguration="#808080ff,#ff0000ff,#0000ffff,#00ff00ff,#ff00ffff,#00ffffff,#7c0000ff,#00007cff,#007c00ff,#7c007cff,#007c7cff,#7c7c00ff" fit_to_viewport="false"/>
+
+<map_styles>
+<stylenode LOCALIZED_TEXT="styles.root_node" STYLE="oval" UNIFORM_SHAPE="true" VGAP_QUANTITY="24.0 pt">
+<font SIZE="24"/>
+<stylenode LOCALIZED_TEXT="styles.predefined" POSITION="right" STYLE="bubble">
+<stylenode LOCALIZED_TEXT="default" ICON_SIZE="12.0 pt" COLOR="#000000" STYLE="fork">
+<font NAME="SansSerif" SIZE="10" BOLD="false" ITALIC="false"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="defaultstyle.details"/>
+<stylenode LOCALIZED_TEXT="defaultstyle.attributes">
+<font SIZE="9"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="defaultstyle.note" COLOR="#000000" BACKGROUND_COLOR="#ffffff" TEXT_ALIGN="LEFT"/>
+<stylenode LOCALIZED_TEXT="defaultstyle.floating">
+<edge STYLE="hide_edge"/>
+<cloud COLOR="#f0f0f0" SHAPE="ROUND_RECT"/>
+</stylenode>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.user-defined" POSITION="right" STYLE="bubble">
+<stylenode LOCALIZED_TEXT="styles.topic" COLOR="#18898b" STYLE="fork">
+<font NAME="Liberation Sans" SIZE="10" BOLD="true"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.subtopic" COLOR="#cc3300" STYLE="fork">
+<font NAME="Liberation Sans" SIZE="10" BOLD="true"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.subsubtopic" COLOR="#669900">
+<font NAME="Liberation Sans" SIZE="10" BOLD="true"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.important">
+<icon BUILTIN="yes"/>
+</stylenode>
+</stylenode>
+<stylenode LOCALIZED_TEXT="styles.AutomaticLayout" POSITION="right" STYLE="bubble">
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level.root" COLOR="#000000" STYLE="oval" SHAPE_HORIZONTAL_MARGIN="10.0 pt" SHAPE_VERTICAL_MARGIN="10.0 pt">
+<font SIZE="18"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,1" COLOR="#0033ff">
+<font SIZE="16"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,2" COLOR="#00b439">
+<font SIZE="14"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,3" COLOR="#990000">
+<font SIZE="12"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,4" COLOR="#111111">
+<font SIZE="10"/>
+</stylenode>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,5"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,6"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,7"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,8"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,9"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,10"/>
+<stylenode LOCALIZED_TEXT="AutomaticLayout.level,11"/>
+</stylenode>
+</stylenode>
+</map_styles>
+</hook>
+<hook NAME="AutomaticEdgeColor" COUNTER="7" RULE="ON_BRANCH_CREATION"/>
+<node TEXT="Query Engine" POSITION="right" ID="ID_1863830559" CREATED="1562593273134" MODIFIED="1562593288017">
+<edge COLOR="#ff0000"/>
+</node>
+<node TEXT="Transactor" POSITION="left" ID="ID_1330328397" CREATED="1562593304683" MODIFIED="1562593311397">
+<edge COLOR="#00ff00"/>
+</node>
+</node>
+</map>

package/doc/backend-development.md

@@ -0,0 +1,78 @@
+# Backend Development
+
+Datahike storage is built on top of [konserve](https://github.com/replikativ/konserve), a universal key-value store abstraction. To add a new storage backend to Datahike, you simply implement a konserve backend. Once your konserve backend is required and registered, Datahike will transparently work on top of it.
+
+## Implementation Steps
+
+1. **Implement a konserve backend** following the [konserve documentation](https://github.com/replikativ/konserve)
+2. **Register your backend** by requiring it in your project
+3. **Use it with Datahike** by specifying the backend keyword in your configuration
+
+That's it! Datahike will automatically work with any konserve backend.
+
+## Konserve Backend Documentation
+
+See the [konserve documentation](https://github.com/replikativ/konserve) for:
+- How to implement a new backend
+- The konserve protocol specification
+- Examples of existing backends
+
+## Optional: Additional Datahike-Specific Support
+
+While konserve backends work out of the box with Datahike, you can optionally add Datahike-specific optimizations:
+
+### Serialization Handlers
+
+For performance optimization, you can add custom serialization handlers for Datahike's data structures. See [datahike-lmdb](https://github.com/replikativ/datahike-lmdb) as an example, which installs LMDB-specific Datahike serialization handlers.
+
+## Existing Backends
+
+Datahike ships with built-in support for:
+- `:memory` - In-memory storage (via konserve)
+- `:file` - File-based storage (via konserve)
+
+Additional backends available through konserve:
+- [datahike-lmdb](https://github.com/replikativ/datahike-lmdb) - LMDB backend with optimized serialization
+- [IndexedDB](https://github.com/replikativ/konserve-indexeddb) - Browser storage (via konserve)
+- [PostgreSQL](https://github.com/replikativ/konserve-pg) - PostgreSQL backend (via konserve)
+- Many more available in the [konserve ecosystem](https://github.com/replikativ/konserve)
+
+## Configuration
+
+Once your konserve backend is available, use it with Datahike by specifying the backend keyword:
+
+```clojure
+(require '[datahike.api :as d])
+
+;; Example: using a custom backend
+(def cfg {:store {:backend :your-backend-keyword
+                  :id #uuid "550e8400-e29b-41d4-a716-446655440000"
+                  ;; ... backend-specific configuration
+                  }})
+
+(d/create-database cfg)
+(def conn (d/connect cfg))
+```
+
+## Advanced: Custom Index Implementations
+
+While less common than implementing backends, you can also implement custom index data structures for Datahike. This allows exploring different index algorithms optimized for specific use cases.
+
+Datahike uses **persistent-set** as the default index. The **hitchhiker-tree** index (`:datahike.index/hitchhiker-tree`) is available for backwards compatibility and can be useful for certain workloads.
+
+To implement a custom index:
+
+1. Implement the index protocol (see existing implementations in `src/datahike/index/`)
+2. Register your index implementation
+3. Use it by specifying `:index :your-index-keyword` in the database configuration
+
+Example configuration with custom index:
+
+```clojure
+(def cfg {:store {:backend :memory
+                  :id #uuid "550e8400-e29b-41d4-a716-446655440000"}
+          :index :datahike.index/hitchhiker-tree  ; or your custom index
+          :schema-flexibility :write})
+```
+
+This is an advanced topic primarily useful for research or highly specialized performance requirements.

package/doc/bb-pod.md

@@ -0,0 +1,89 @@
+# Datahike babashka Pod
+
+With the help of [the Datahike cli](doc/cli.md) we provide the possibility to run Datahike as a [babashka pod](https://book.babashka.org/#pods).
+
+## Why Datahike as a babashka pod?
+
+A babashka pod is a way to use Datahike in babashka without Datahike being built into the babashka binary. When you
+don't want to run a whole JVM for a little script you can use babashka and write your script in your preferred language.
+Babashka already has ways to persist data to other databases but Datahike adds the possibility to write to a durable
+datalog database.
+
+## babashka pod-registry
+
+You can use the [official babashka pod-registry](https://github.com/babashka/pod-registry/tree/master/manifests/replikativ/datahike) to download the latest version of Datahike as a pod and run it with babashka.
+
+## Compilation
+
+Please run `bb ni-cli` inside the datahike repository. You'll need [GraalVM-JDK](https://www.graalvm.org/latest/getting-started/) installed with `native-compile` on your path, [babashka](https://babashka.org/) and [Clojure](https://clojure.org/guides/install_clojure).
+
+## Maturity
+
+This feature is not used in production so far. Please try it and [open issues on GitHub](https://github.com/replikativ/datahike/issues/new/choose) if you find a problem.
+
+### Supported functionality
+
+- as-of
+- connect
+- create-database
+- database-exists?
+- datoms (returns a seq of datoms as maps)
+- db
+- db-with
+- delete-database
+- entity (returns a simple map without further functionality)
+- history
+- metrics
+- pull
+- pull-many
+- q
+- release-db (release the DB-object to free memory)
+- since
+- schema
+- transact (returns the transaction-report as map)
+- with-db (a macro that avoids storing the DB-object)
+
+## Example usage
+
+```clojure
+(ns pod
+  (:require [babashka.pods :as pods]))
+
+(pods/load-pod 'replikativ/datahike "CURRENT") ;; Check https://github.com/babashka/pod-registry for latest version
+
+(require '[datahike.pod :as d])
+
+(def config {:store  {:backend :file
+                      :path "/tmp/bb-datahike-pod"}
+             :keep-history? true
+             :schema-flexibility :read})
+
+(d/delete-database config)
+
+(d/create-database config)
+
+(def conn (d/connect config))
+
+(d/transact conn [{:name  "Alice", :age   20}
+                  {:name  "Bob", :age   30}
+                  {:name  "Charlie", :age   40}
+                  {:age 15}])
+
+(def db (d/db conn))
+
+(d/q '[:find ?e ?n ?a
+       :where
+       [?e :name ?n]
+       [?e :age ?a]]
+     db)
+
+(release-db db)
+
+(d/pull (d/db conn) '[*] 3)
+
+(with-db [db (db conn)]
+  (q {:query '{:find [?e ?a ?v]
+      :where
+      [[?e ?a ?v]]}}
+     db))
+```

package/doc/benchmarking.md

@@ -0,0 +1,360 @@
+# Benchmarking Datahike
+
+There is a small command line utility integrated in this project to measure the performance of our *in-memory* and our *file* backend. It is also capable of comparing benchmarking results.
+
+To run the benchmarks, navigate to the project folder in your console and run 
+```bash
+clj -M:benchmark CMD [OPTIONS] [FILEPATHS] 
+```
+
+The command can either be `run` or `compare`.
+
+
+## Running Benchmarks
+
+```bash
+clj -M:benchmark run [OPTIONS] [OUTPUTFILEPATH]+
+```
+
+Options:
+
+| Short | Long                      | Description                                                            | Default       |
+|:-----:|---------------------------|------------------------------------------------------------------------|---------------|
+| -u    | --db-server-url URL       | Base URL for datahike server for benchmark output.                     |               |
+| -n    | --db-name DBNAME          | Database name for datahike server for benchmark output.                |               |
+| -g    | --db-token TOKEN          | Token for datahike server for benchmark output.                        |               | 
+| -t    | --tag TAG                 | Add tag to measurements; multiple tags possible.                       | `#{}`         | 
+| -o    | --output-format FORMAT    | Short form of output format to use.                                    | `edn`         | 
+| -c    | --config-name CONFIGNAME  | Name of database configuration to use.                                 | (all)         | 
+| -d    | --db-entity-counts VECTOR | Numbers of entities in database for which benchmarks should be run.    | `[0 1000]`    | 
+| -x    | --tx-entity-counts VECTOR | Numbers of entities in transaction for which benchmarks should be run. | `[0 1000]`    | 
+| -y    | --data-types TYPEVECTOR   | Vector of data types to test queries on.                               | `[:int :str]` | 
+| -z    | --data-found-opts OPTS    | Run query for existent or nonexistent values in the database.          | `:all`        | 
+| -i    | --iterations ITERATIONS   | Number of iterations of each measurement.                              | `10`          | 
+| -f    | --function FUNCTIONNAME   | Name of function to test.                                              | (all)         |
+| -q    | --query QUERYNAME         | Name of query to test.                                                 | (all)         | 
+| -h    | --help                    | Show help screen for tool usage.                                       |               |
+
+### Examples
+
+1) Benchmark *connection* time for databases with  *1000 entities* (= 4000 datoms) and *file* backend with tag *feature* and output as *edn*
+
+```bash
+TIMBRE_LEVEL=':warn' clj -M:benchmark run -f :connection -d '[1000]' -c file -t feature -o edn feature.edn
+```
+
+2) Benchmark *transaction* time for *integer* datoms and 10 entities (= 40 datoms) per transaction using *memory* backend and output as csv
+
+```bash
+TIMBRE_LEVEL=':warn' clj -M:benchmark run -f :transaction -y '[:int]' -x '[10]' -c mem-set -o csv feature.csv
+```
+
+3) Benchmark *query* time for a *simple query* for all backends, for every configuration run the query *10 times* and take the average time 
+
+```bash
+TIMBRE_LEVEL=':warn'  clj -M:benchmark run -f :query -q :simple-query -i 10
+```
+
+### Possible Options
+
+#### Database Configurations  (-c)
+
+Options for `-c`:
+- `mem-set` for in-memory database with persistent-set index
+- `mem-hht` for in-memory database with hitchhiker-tree index
+- `file-set` for database with file store backend and persistent-set index
+- `file-hht` for database with file store backend and hitchhiker-tree index
+
+Implementations:
+
+```clojure
+(def db-configs
+  [{:config-name "mem-set"
+    :config {:store {:backend :memory :id #uuid "550e8400-e29b-41d4-a716-446655440000"}
+             :schema-flexibility :write
+             :keep-history? false
+             :index :datahike.index/persistent-set}}
+   {:config-name "mem-hht"
+    :config {:store {:backend :memory :id #uuid "550e8400-e29b-41d4-a716-446655440001"}
+             :schema-flexibility :write
+             :keep-history? false
+             :index :datahike.index/hitchhiker-tree}}
+   {:config-name "file-set"
+    :config {:store {:backend :file
+                     :path "/tmp/performance-set"
+                     :id #uuid "550e8400-e29b-41d4-a716-446655440002"}
+             :schema-flexibility :write
+             :keep-history? false
+             :index :datahike.index/persistent-set}}
+   {:config-name "file-hht"
+    :config {:store {:backend :file
+                     :path "/tmp/performance-hht"
+                     :id #uuid "550e8400-e29b-41d4-a716-446655440003"}
+             :schema-flexibility :write
+             :keep-history? false
+             :index :datahike.index/hitchhiker-tree}}])
+```
+
+#### Tested Functions (-f)
+
+Options for `-f`:
+
+- `:connection`: Testing `datahike/connect`. Run can be configured via options `-c`, `-d`, `-i`
+- `:transaction`: Testing `datahike/transact`. Run can be configured via options `-c`, `-d`, `-x`, `-i`
+- `:query`: Testing `datahike/q`. Run can be configured via options `-c`, `-d`, `-y`, `-z`, `-i`, `-q`
+ 
+#### Data Types (-z)
+
+Used for query functions
+
+Options for `-z`:
+- `:int` for datatype `long`
+- `:str` for data type `String`
+
+#### Queries (-q)
+
+Options for `-q`: 
+
+- Simple query: `:simple-query`
+- Join queries: `:e-join-query` `:e-join-query-first-fixed` `:e-join-query-second-fixed` `:a-join-query` `:v-join-query`
+- Predicate queries: `:equals-query` `:equals-query-1-fixed` `:less-than-query` `:less-than-query-1-fixed`
+- Queries using arguments from bindings: `:scalar-arg-query` `:scalar-arg-query-with-join` `:vector-arg-query`
+- Aggregate queries: `:stddev-query` `:variance-query` `:max-query` `:median-query` `:avg-query`
+- Cache check queries: `:simple-query-first-run` `:simple-query-second-run`
+
+If applicable, the queries are run for each different implemented data type, for data in the database and data not in the database.  
+
+##### Simple Query 
+
+Implementation:
+
+```clojure
+(defn simple-query [db attr val]
+  {:query (conj '[:find ?e :where]
+                (conj '[?e] attr val))
+   :args [db]})
+```
+
+##### Join Queries
+
+Implementation:
+
+```clojure
+(defn e-join-query [db attr1 attr2]
+  {:query (conj '[:find ?e :where]
+                (conj '[?e] attr1 '?v1)
+                (conj '[?e] attr2 '?v2))
+   :args [db]})
+
+(defn a-join-query [db attr]
+  {:query (conj '[:find ?v1 ?v2 :where]
+                (conj '[?e1] attr '?v1)
+                (conj '[?e2] attr '?v2))
+   :args [db]})
+
+(defn v-join-query [db attr1 attr2]
+  {:query (conj '[:find ?e1 ?e2 :where]
+                (conj '[?e1] attr1 '?v)
+                (conj '[?e2] attr2 '?v))
+   :args [db]})
+
+(defn e-join-query-first-fixed [db attr1 val1 attr2]
+  {:query (conj '[:find ?v2 :where]
+                (conj '[?e] attr1 val1)
+                (conj '[?e] attr2 '?v2))
+   :args [db]})
+
+(defn e-join-query-second-fixed [db attr1 attr2 val2]
+  {:query (conj '[:find ?v1 :where]
+                (conj '[?e] attr1 '?v1)
+                (conj '[?e] attr2 val2))
+   :args [db]})
+```
+
+##### Predicate Queries
+
+Implementation:
+
+```clojure
+(defn less-than-query [db attr]
+  {:query (conj '[:find ?e1 ?e2 :where]
+                (conj '[?e1] attr '?v1)
+                (conj '[?e2] attr '?v2)
+                '[(< ?v1 ?v2)])
+   :args [db]})
+
+(defn equals-query [db attr]
+  {:query (conj '[:find ?e1 ?e2 :where]
+                (conj '[?e1] attr '?v1)
+                (conj '[?e2] attr '?v2)
+                '[(= ?v1 ?v2)])
+   :args [db]})
+
+(defn less-than-query-1-fixed [db attr comp-val]
+  {:query (conj '[:find ?e :where]
+                (conj '[?e] attr '?v)
+                (conj '[]
+                      (sequence (conj '[= ?v] comp-val))))
+   :args [db]})
+
+(defn equals-query-1-fixed [db attr comp-val]
+  {:query (conj '[:find ?e :where]
+                (conj '[?e] attr '?v)
+                (conj '[]
+                      (sequence (conj '[= ?v] comp-val))))
+   :args [db]})
+```
+
+##### Binding Queries 
+
+Implementation:
+
+```clojure
+(defn scalar-arg-query [db attr val]
+  {:query (conj '[:find ?e
+                  :in $ ?v
+                  :where]
+                (conj '[?e] attr '?v))
+   :args [db val]})
+
+(defn scalar-arg-query-with-join [db attr val]
+  {:query (conj '[:find ?e1 ?e2 ?v2
+                  :in $ ?v1
+                  :where]
+                (conj '[?e1] attr '?v1)
+                (conj '[?e2] attr '?v2))
+   :args [db val]})
+
+(defn vector-arg-query [db attr vals]
+  {:query (conj '[:find ?e
+                  :in $ ?v
+                  :where]
+                (conj '[?e] attr '?v))
+   :args [db vals]})
+```
+
+##### Aggregate Queries
+Only run for data type `:int`.
+
+```clojure
+[{:function :sum-query
+  :query {:query '[:find (sum ?x)
+                   :in [?x ...]]
+          :args [(repeatedly (count entities) #(rand-int 100))]}}
+
+ {:function :avg-query
+    :query {:query '[:find (avg ?x)
+                     :in [?x ...]]
+            :args [(repeatedly (count entities) #(rand-int 100))]}}
+
+ {:function :median-query
+  :query {:query '[:find (median ?x)
+                     :in [?x ...]]
+          :args [(repeatedly (count entities) #(rand-int 100))]}}
+ {:function :variance-query
+  :query {:query '[:find (variance ?x)
+                   :in [?x ...]]
+          :args [(repeatedly (count entities) #(rand-int 100))]}}
+
+ {:function :stddev-query
+  :query {:query '[:find (stddev ?x)
+                   :in [?x ...]]
+          :args [(repeatedly (count entities) #(rand-int 100))]}}
+
+ {:function :max-query
+  :query {:query '[:find (max ?x)
+                   :in [?x ...]]
+          :args [(repeatedly (count entities) #(rand-int 100))]}}]
+```
+
+##### Cache check queries
+
+Simple query with exact same configuration run twice. Identifiers are `:simple-query-first-run` for the first time it is run and `:simple-query-second-run` for the second run.
+
+### Output Configuration
+
+*Formats:*
+- `remote-db`; using an instance of datahike-server to upload the results. The server configuration iscontrolled by options -u -n and -g
+- `edn`
+- `csv`
+
+If an output filename is given the result is saved in a file instead of printed to stdout.
+
+The edn output will look as follows:
+
+```clojure
+[ ;; ...
+ {:context {:dh-config {:schema-flexibility :write,
+                        :keep-history? false,
+                        :index :datahike.index/persistent-set,
+                        :name "mem-set",
+                        :backend :memory}, 
+            :function :vector-arg-query, 
+            :db-entities 2500, 
+            :db-datoms 10000, 
+            :execution {:data-type :int, 
+                        :data-in-db? true}}, 
+  :time {:mean 0.17954399999999998, 
+         :median 0.172268,               
+         :std 0.02388124449855995, 
+         :count 10, 
+         :observations [0.173015 0.168094 0.174449 0.250349 0.169847 0.168364 0.168926 0.169352 0.172268 0.180776]}, 
+  :tag "bind-collection-bounds-opt"}
+  ;; ...
+]
+```
+
+
+## Comparing Benchmarks
+
+Usage:
+```bash
+clj -A:benchmark compare [-p] [FILEPATHS]* 
+```
+
+The comparison tool gives the option to *compare any number of benchmarking results* using
+a) textual *table* format (default) or
+b) *plots* (if command line option `-p` has been given).
+
+Please note: 
+1) At the moment the comparison tool can only *handle edn files* as input, so be aware of that and run the benchmarks with `--output edn` when you are planning to use this tool later. 
+2) Use *different tags* for the benchmarks you want to compare since the comparison tool assumes measurements with the same tag to belong to the same group of measurements.
+
+Example for comparison table:
+
+```bash
+clj -A:benchmark compare benchmarks1.edn benchmarks2.edn
+```
+
+Example for comparison plots:
+
+```bash
+clj -A:benchmark compare -p benchmarks1.edn benchmarks2.edn
+```
+
+The plots produced are scatter plots of the results combined with line plots using the median of the values for a measurement point.
+
+If you want to see plots for a single benchmarking result, nothing keeps you from using the comparison tool on a single file. 
+
+### Measuring Performance Changes During Development
+
+The comparison tool has proven valuable to our team for comparing different branches of the datahike project to detect performance regressions or improvements.
+
+Workflow:
+
+1) Run the benchmarks on the branch you are forking from with the option `--output edn` giving it an expressive tag, e.g. the name of the branch:
+```bash
+git checkout development
+clj -A:benchmark run -t development -o edn development.edn
+```
+
+2) Run the benchmarks on your feature branch with the option `--output edn` giving it a tag, e.g. the name of your new feature:
+```bash
+git checkout feature
+clj -A:benchmark run -t feature -o edn feature.edn
+```
+
+3) Run the comparison tool on any branch giving both of the previous output files as input:
+```bash
+clj -A:benchmark compare -p development.edn feature.edn
+```