datahike-browser-tests 1.0.0

package/bb.edn

@@ -0,0 +1,269 @@
+{:min-bb-version "0.8.0"
+ :pods {org.babashka/tools-deps-native {:version "0.1.7"}
+        clj-kondo/clj-kondo {:version "2025.07.28"}}
+ :deps {datahike/bb {:local/root "bb"}
+        org.babashka/http-client {:mvn/version "0.4.22"}}
+ :tasks {:requires [[babashka.fs :as fs]
+                    [babashka.process :as process]
+                    [clojure.edn :as edn]
+                    [clojure.string :as str]
+                    [pod.borkdude.clj-kondo :as clj-kondo]
+                    [tools.build :as build]
+                    [tools.clj-kondo :as clj-kondo-tools]
+                    [tools.deploy :as deploy]
+                    [tools.examples :as examples]
+                    [tools.npm :as npm]
+                    [tools.python :as python]
+                    [tools.release :as release]
+                    [tools.test :as test]
+                    [tools.version :as version]]
+         :init (do (def config-file "config.edn")
+                   (def config (edn/read-string (slurp config-file))))
+
+         ; tools
+
+         bench {:doc "Run benchmarks"
+                :depends [prep]
+                :task (clojure "-M:benchmark" "measure")}
+
+         ffix {:doc "Format source files"
+               :depends [prep]
+               :task (clojure "-M:ffix")}
+
+         ; checks
+
+         format {:doc "Test formatting"
+                 :depends [prep]
+                 :task (clojure "-M:format")}
+
+         lint {:doc "Run clj-kondo linter"
+               :require [[pod.borkdude.clj-kondo :as clj-kondo]]
+               :task (clj-kondo/print! (clj-kondo/run! {:lint "."}))}
+
+         test {:doc "Run all tests or restrict to 'native-image', 'back-compat' or a kaocha test id (see tests.edn)"
+               :depends [jcompile]
+               :task (apply test/-main config *command-line-args*)}
+
+         kaocha {:doc "Run kaocha with arbitrary arguments"
+                 :task (apply test/kaocha *command-line-args*)}
+
+
+         outdated {:doc "Find outdated libraries"
+                   :depends [prep]
+                   :task (clojure "-M:outdated")}
+
+         ; build and release
+
+         inc {:doc "Increment the project version: [major|minor] <version>"
+              :task (apply version/inc config-file *command-line-args*)}
+
+         tag {:doc "Return current version as a tag"
+              :task (println (version/as-tag config))}
+
+         clean {:doc "Remove build files"
+                :task (build/clean (-> config :build :clj))}
+
+         prep {:doc "Prepare git dependencies (compile Java sources)"
+               :task (clojure "-X:deps prep")}
+
+         codegen-java {:doc "Generate Java API bindings from specification"
+                       :depends [prep]
+                       :task (let [cp (str/trim (:out (shell {:out :string} "clojure" "-Spath")))]
+                               ;; First compile Java dependencies (Util, IEntity) so Clojure can load them
+                               (println "Compiling Java dependencies...")
+                               (shell "javac" "-cp" cp "-d" "target/classes"
+                                      "java/src/datahike/java/IEntity.java"
+                                      "java/src/datahike/java/Util.java")
+                               ;; Now generate the Java API
+                               (println "Generating Java API from specification...")
+                               (clojure "-M" "-m" "datahike.codegen.java" "java/src-generated"))}
+
+         codegen-ts {:doc "Generate TypeScript type definitions from specification"
+                     :task (npm/generate-typescript-definitions! "npm-package/index.d.ts")}
+
+         codegen-native {:doc "Generate native C entry points from specification"
+                         :depends [prep]
+                         :task (clojure "-M" "-m" "datahike.codegen.native" "libdatahike/src")}
+
+         python-version {:doc "Update Python package version from config.edn"
+                         :task (python/update-python-version! config "pydatahike")}
+
+         codegen-python {:doc "Generate Python bindings from specification"
+                         :depends [prep python-version]
+                         :task (clojure "-M" "-m" "datahike.codegen.python" "pydatahike/src/datahike")}
+
+         java-examples-version {:doc "Update Java examples pom.xml version to match current version"
+                                :task (examples/update-java-examples-version! config)}
+
+         codegen-clj-kondo {:doc "Generate clj-kondo config from API specification"
+                            :depends [prep]
+                            :task (clj-kondo-tools/generate-clj-kondo-config!
+                                    "resources/clj-kondo.exports/io.replikativ/datahike/config.edn"
+                                    ".clj-kondo/datahike/datahike/config.edn"
+                                    ".clj-kondo/config.edn")}
+
+         codegen-all {:doc "Generate all language bindings (TypeScript, Java, Native, Python)"
+                      :depends [codegen-ts codegen-java codegen-native codegen-python]}
+
+         codegen-report {:doc "Show code generation coverage across all bindings"
+                         :depends [prep]
+                         :task (clojure "-M" "-m" "datahike.codegen.report")}
+
+         jcompile {:doc "Compile java classes"
+                   :depends [clean codegen-java]
+                   :task (build/compile-java (-> config :build :clj))}
+
+         javadoc {:doc "Generate Javadoc for Java API (published to javadoc.io)"
+                  :task (build/javadoc {})}
+
+         ccompile {:doc "Compile clojure namespaces"
+                   :task (build/compile-clojure (-> config :build :clj))}
+
+         version-resource {:doc "Generate version resource file for embedding in builds"
+                           :task (build/write-version-resource config)}
+
+         pom {:doc "Create pom file"
+              :task (build/pom config (-> config :build :clj))}
+
+         jar {:doc "Build jar"
+              :depends [jcompile pom version-resource]
+              :task (build/jar config (-> config :build :clj))}
+
+         install {:doc "Install jar locally"
+                  :task (deploy/local config (-> config :build :clj))}
+
+         clojars {:doc "Install jar to clojars"
+                  :task (deploy/remote config (-> config :build :clj))}
+
+         release {:doc "Build and release jar to GitHub"
+                  :task (release/-main config *command-line-args*)}
+
+
+         ;; http server build and release
+
+         http-server-clean {:doc  "Remove build files"
+                            :task (build/clean (-> config :build :http-server-clj))}
+
+         http-server-jcompile {:doc     "Compile java classes"
+                               :depends [http-server-clean]
+                               :task    (build/compile-java (-> config :build :http-server-clj))}
+
+         http-server-ccompile {:doc  "Compile clojure namespaces for http-server"
+                               :task (build/compile-clojure (-> config :build :http-server-clj))}
+
+         http-server-pom {:doc  "Create pom file"
+                          :task (build/pom config (-> config :build :http-server-clj))}
+
+         http-server-uber {:doc     "Build jar"
+                           :depends [http-server-jcompile http-server-pom http-server-ccompile version-resource]
+                           :task    (build/uber config (-> config :build :http-server-clj))}
+
+         http-server-install {:doc  "Install uber jar locally"
+                              :task (deploy/local config (-> config :build :http-server-clj))}
+
+         http-server-release {:doc     "Build and release jar to GitHub"
+                              :depends [http-server-uber]
+                              :task    (let [jar (build/jar-path config (-> config :build :http-server-clj))]
+                                         (release/gh-release jar config))}
+
+         ;; native image
+
+         ni-check {:doc "Check for 'native-image' program"
+                   :task (try (shell "which" "native-image")
+                              (println "Program native-image found!")
+                              (catch Exception _
+                                (println "PATH does not contain native-image! Make sure to add your GraalVM to it.")
+                                (System/exit 1)))}
+
+         ni-cli {:doc "Build native image cli"
+                 :depends [jcompile ni-check version-resource]
+                 :task (clojure "-M:native-cli")}
+
+         ni-ccompile {:doc "Compile clojure namespaces for native-image"
+                      :depends [jcompile]
+                      :task (build/compile-clojure (-> config :build :native))}
+
+         ni-uber {:doc "Build native image uber jar"
+                  :depends [jcompile ni-ccompile version-resource]
+                  :task (build/uber config (-> config :build :native))}
+
+         ni-compile {:doc "Create native cpp library"
+                     :depends [codegen-native ni-uber]
+                     :task (build/native-compile config (-> config :build :native))}
+
+        ;; npm package
+
+         npm-version {:doc "Update npm package.json version from config.edn"
+                      :task (npm/update-package-json-version! config "npm-package")}
+
+         npm-clean {:doc "Clean compiled files from npm package directory"
+                    :task (npm/clean-npm-package! "npm-package")}
+
+         npm-build {:doc "Build npm package (clean, update version, generate types, compile ClojureScript)"
+                    :depends [prep]
+                    :task (npm/build-npm-package! config "npm-package")}
+
+         npm-test {:doc "Run npm package JavaScript tests"
+                   :task (do
+                           (println "Running npm package tests...")
+                           (let [result (shell {:dir "npm-package"
+                                                :out :inherit
+                                                :err :inherit}
+                                               "node test.js")]
+                             (when-not (zero? (:exit result))
+                               (System/exit 1))))}
+
+         node-cljs-test {:doc "Compile and run ClojureScript tests on Node.js"
+                         :depends [prep]
+                         :task (do
+                                 (println "Compiling CLJS node tests...")
+                                 (shell "npx shadow-cljs compile node-test")
+                                 (println "Running CLJS node tests...")
+                                 (shell "node target/out/node-test.js"))}
+
+         browser-test {:doc "Run browser integration tests with Karma (headless Chrome)"
+                       :depends [prep]
+                       :task (let [;; Auto-detect Chrome binary
+                                   chrome-bin (or (System/getenv "CHROME_BIN")
+                                                  (when (fs/exists? "/usr/bin/chromium-browser") "/usr/bin/chromium-browser")
+                                                  (when (fs/exists? "/usr/bin/chromium") "/usr/bin/chromium")
+                                                  (when (fs/exists? "/usr/bin/google-chrome") "/usr/bin/google-chrome"))]
+                               (when-not chrome-bin
+                                 (println "ERROR: Chrome not found. Set CHROME_BIN environment variable.")
+                                 (System/exit 1))
+
+                               ;; Start JVM test server in background
+                               (println "Starting Kabel test server on ws://localhost:47296...")
+                               (let [server-proc (process/process
+                                                  ["clojure" "-M:test" "-e"
+                                                   "(require 'datahike.kabel.browser-test-server) (datahike.kabel.browser-test-server/start-test-server!) (Thread/sleep Long/MAX_VALUE)"]
+                                                  {:out :inherit :err :inherit})]
+                                 (try
+                                   ;; Wait for server to start
+                                   (println "Waiting for server startup (10s)...")
+                                   (Thread/sleep 10000)
+
+                                   ;; Compile browser tests
+                                   (println "Compiling browser tests...")
+                                   (shell "npx shadow-cljs compile browser-ci")
+
+                                   ;; Run Karma
+                                   (println "Running tests with Karma...")
+                                   (shell {:extra-env {"CHROME_BIN" chrome-bin}} "npx karma start --single-run")
+
+                                   (finally
+                                     ;; Kill the server process
+                                     (println "Stopping test server...")
+                                     (process/destroy-tree server-proc)))))}
+
+         browser-watch {:doc "Start browser test dev server (open http://localhost:8022)"
+                        :depends [prep]
+                        :task (do
+                                (println "Starting browser test dev server...")
+                                (println "Open http://localhost:8022 in your browser")
+                                (shell "npx shadow-cljs watch browser-integration-test"))}
+
+         ;; aggregate tasks
+
+         ci {:doc "Run CI checks (Clojure/ClojureScript + npm tests)"
+             :depends [test npm-test format lint]}}}

package/benchmark/src/benchmark/cli.clj

@@ -0,0 +1,195 @@
+(ns benchmark.cli
+  (:require [clojure.tools.cli :as cli]
+            [benchmark.measure :refer [get-measurements time-statistics]]
+            [benchmark.compare :refer [compare-benchmarks]]
+            [benchmark.config :as c]
+            [benchmark.store :refer [save]]
+            [clojure.string :refer [join]]
+            [datahike.store :as ds]
+            [datahike.config :as dc]
+            [datahike.index :as di]))
+
+(def output-formats (set (keys (methods save))))
+(def config-names (set (map :config-name c/named-db-configs)))
+(def implemented-queries #{:simple-query
+                           :e-join-query :e-join-query-first-fixed :e-join-query-second-fixed
+                           :a-join-query
+                           :v-join-query
+                           :equals-query :equals-query-1-fixed
+                           :less-than-query :less-than-query-1-fixed
+                           :scalar-arg-query :scalar-arg-query-with-join
+                           :vector-arg-query
+                           :limit-query
+                           :stddev-query :variance-query :max-query :median-query :avg-query})
+(def implemented-functions #{:connection :transaction :query})
+(def implemented-data-types #{:int :str})
+
+(def index-names (-> (methods di/default-index-config) keys set (disj :default)))
+(def backend-names (-> (methods ds/default-config) keys set (disj :default)))
+(def time-stats (-> (time-statistics [0]) keys set))
+
+(def cli-options                                            ;; TODO: use default values from datahike.config when available
+  [;; CMD run
+   ["-u" "--db-server-url URL" "Base URL for datahike server, e.g. http://localhost:3000"
+    :default nil]
+   ["-n" "--db-name DBNAME" "Database name for datahike server" :default nil]
+   ["-g" "--db-token TOKEN" "Token for datahike server" :default nil]
+
+   ["-t" "--tag TAG" "Add tag to measurements"
+    :default #{}
+    :assoc-fn (fn [m k v] (assoc m k (conj (get m k) v)))]
+   ["-o" "--output-format FORMAT"
+    (str "Determines how the results will be processed. "
+         "Available are " output-formats " "
+         "Currently only edn format is supported for comparisons.")
+    :default "edn"
+    :validate [output-formats  #(str "Format " % " has not been implemented. "
+                                     "Available formats are " output-formats)]]
+   ["-e" "--statistics VECTOR"
+    (str "Statistic columns shown in the printed comparison summary. "
+         "Available are: " time-stats)
+    :default [:median :mean :std]
+    :parse-fn read-string
+    :validate [vector? (str "Must be a vector of cell names. Available are: " time-stats)
+               #(every? time-stats %) #(str "Must be a vector of cell names. Available are: " time-stats)]]
+
+   ["-c" "--config-name CONFIGNAME"
+    (str "Name of preset database configuration to use. Available are: " config-names)
+    :validate [config-names  #(str "A configuration named " % " has not been implemented. "
+                                   "Available configurations are: " config-names)]]
+
+   ["-a" "--index INDEXNAME"
+    (str "Name of index to use. Available are " index-names)
+    :default :all
+    :parse-fn read-string
+    :validate [index-names #(str "An index named " % " has not been implemented. "
+                                 "Available indices are: " index-names)]]
+   ["-l" "--history BOOLEAN"
+    (str "Boolean indication if measured databases use temporal indices")
+    :default false
+    :parse-fn read-string
+    :validate [#(contains? #{true false :all} %) "Must be a boolean value or keyword :all"]]
+   ["-b" "--backend BACKEND"
+    (str "Keyword for backend that is going to be used")
+    :default :all
+    :parse-fn read-string
+    :validate [(conj backend-names :all) #(str "A backend named " % " has not been implemented. "
+                                               "Available backends are: " backend-names)]]
+   ["-k" "--search-caches SIZES"
+    (str "Search cache sizes for which measurements should be done")
+    :default [dc/*default-search-cache-size*]
+    :parse-fn read-string
+    :validate [vector? "Must be a vector of non-negative integers."
+               #(every? nat-int? %) "Vector must consist of non-negative integers."]]
+   ["-m" "--store-caches SIZES"
+    (str "Store cache sizes for which measurements should be done")
+    :default [dc/*default-store-cache-size*]
+    :parse-fn read-string
+    :validate [vector? "Must be a vector of positive integers."
+               #(every? pos-int? %) "Vector must consist of non-negative integers."]]
+   ["-j" "--schema VALUE"
+    (str "Schema flexibility configuration. Available are: " #{:read :write})
+    :default :write
+    :parse-fn read-string
+    :validate [#{:read :write :all} #(str "A schema-flexibility named " % " has not been implemented. "
+                                          "Available values are: " #{:read :write})]]
+
+   ["-d" "--db-entity-counts VECTOR"
+    (str "Numbers of entities in database for which benchmarks should be run. "
+         "Must be given as a clojure vector of non-negative integers like '[0 10 100 1000]'.")
+    :default [0 1000]
+    :parse-fn read-string
+    :validate [vector? "Must be a vector of non-negative integers."
+               #(every? nat-int? %) "Vector must consist of non-negative integers."]]
+   ["-x" "--tx-entity-counts VECTOR"
+    (str  "Numbers of entities in transaction for which benchmarks should be run.  "
+          "Must be given as a clojure vector of non-negative integers like '[0 10 100 1000]'.")
+    :default [0 1000]
+    :parse-fn read-string
+    :validate [vector? "Must be a vector of non-negative integers."
+               #(every? nat-int? %) "Vector must consist of non-negative integers."]]
+   ["-y" "--data-types VECTOR"
+    (str "Vector of datatypes to test queries on. Available are: " implemented-data-types ".")
+    :default [:int :str]
+    :parse-fn read-string
+    :validate [vector? (str "Must be a vector of data type keywords. Available are " implemented-data-types)
+               #(every? implemented-data-types %) (str "Vector must consist of keywords for datatypes. "
+                                                       "Available are: " implemented-data-types)]]
+   ["-z" "--data-found-opts BOOLEAN"
+    (str "Boolean indicating if query is run for existent or nonexistent values in the database.")
+    :default true
+    :parse-fn read-string
+    :validate [#{true false :all} "Must be a boolean value or keyword :all."]]
+   ["-i" "--iterations ITERATIONS"
+    (str "Number of measurements for each setting taken on the same database.")
+    :default 10
+    :parse-fn read-string
+    :validate [nat-int? "Must be a non-negative integer."]]
+   ["-s" "--db-samples DB-SAMPLES"
+    (str "Number of database instances on which the full measurements are run.")
+    :default 1
+    :parse-fn read-string
+    :validate [nat-int? "Must be a non-negative integer."]]
+   ["-q" "--query QUERYNAME"
+    (str "Name of query to test. Available are " implemented-queries)
+    :default :all
+    :parse-fn read-string
+    :validate [(conj implemented-queries :all) (str "Must be a keyword for a implemented database query "
+                                                    "Available are: " implemented-queries)]]
+   ["-f" "--function FUNCTIONNAME"
+    (str "Name of function to test. Available are " implemented-functions)
+    :default :all
+    :parse-fn read-string
+    :validate [(conj implemented-functions :all) (str "Must be a keyword for implemented database function. "
+                                                      "Available are: " implemented-functions)]]
+
+   ;; CMD compare
+
+   ["-p" "--[no-]plots" "Output comparison as plots."]
+
+   ["-h" "--help"]])
+
+
+(defn print-usage-info [summary]
+  (println (str "Usage: clj -M:benchmark CMD [OPTIONS] [FILEPATHS] \n\n  Options for command 'run':\n" summary)))
+
+(defmulti run-command (fn [cmd _paths _options] cmd))
+
+(defmethod run-command :default [cmd _paths _options]
+  (throw (Exception. (str "Command '" cmd "' does not exist. Valid commands are 'measure' and 'compare'"))))
+
+(defmethod run-command "compare" [_cmd paths {:keys [plots statistics]}]
+  (compare-benchmarks paths plots statistics))
+
+
+(defmethod run-command "measure" [_cmd paths {:keys [tag output-format] :as options}]
+  (let [server-description (select-keys options [:db-server-url :db-token :db-name])]
+    (if (and (= "remote-db" output-format)
+             (some nil? (vals server-description)))
+      (do (println (str "Only partial information for remote connection has been given: " server-description))
+          (println "Please, define URL, database name, and token to save the data on a remote datahike server."))
+      (let [measurements (cond->> (get-measurements options)
+                                  (seq tag) (map (fn [entity] (assoc entity :tag (join " " tag))))
+                                  true      vec) ]
+    (save options paths measurements)))))
+
+(defn -main [& args]
+  (let [{:keys [arguments options errors summary]} (cli/parse-opts args cli-options)
+        [cmd & paths] arguments]
+    (cond
+      (some? errors)
+      (do (println "Errors:" errors)
+          (print-usage-info summary))
+
+      (:help options)
+      (print-usage-info summary)
+
+      :else (run-command cmd paths options)))
+
+  (shutdown-agents))
+
+(comment
+  (-main "measure" "-x" "[0 10000 5000]" "-t" "test-bench" "-o" "edn" "bench.edn")
+  )
+
+;TIMBRE_LEVEL=":info" clj -M:benchmark measure --backend :file --index :datahike.index/persistent-set -t pss -o edn pss.edn --schema :write --history false

package/benchmark/src/benchmark/compare.clj

@@ -0,0 +1,157 @@
+(ns benchmark.compare
+  (:require [clojure.edn :as edn]
+            [clojure.string :refer [join]]
+            [incanter.core :as ic]
+            [incanter.charts :as charts]
+            [benchmark.config :as c]
+            [taoensso.timbre :as log]
+            [clojure.java.io :as io])
+  (:import [java.awt Color]))
+
+(defn comparison-table [group filenames time-stats]
+  (let [col-mapping (zipmap (map :name c/csv-cols)
+                            c/csv-cols)
+        grouped (group-by :context group)
+        context-sort-fn (fn [context]
+                          (mapv #(get-in context (vec (rest (:path (% col-mapping))))) ;; rest since already in :context
+                                c/comparison-context-cell-order))
+        sorted-contexts (sort-by context-sort-fn (keys grouped))
+        unique-context-key-spaces (map (fn [k]
+                                         (let [{:keys [title path]} (k col-mapping)]
+                                           (->> sorted-contexts
+                                                (map (fn [context]
+                                                       (let [x (get-in context (vec (rest path)))]
+                                                         (count (str (if (keyword? x)
+                                                                       (name x)
+                                                                       x))))))
+                                                (cons (count title))
+                                                (apply max))))
+                                       c/comparison-context-cell-order)
+        num-space 8
+        col-width (+ (* 3 (- (count time-stats) 1))
+                     (* (count time-stats) num-space))
+        titles (concat (map #(get-in col-mapping [% :title])
+                            c/comparison-context-cell-order)
+                       filenames)
+        title-spaces (concat unique-context-key-spaces (repeat (count filenames) col-width))
+        subtitles (concat (repeat (count c/comparison-context-cell-order) "")
+                          (apply concat (repeat (count filenames) time-stats)))
+        subtitle-spaces (concat unique-context-key-spaces
+                                (repeat (* (count time-stats) (count filenames))
+                                        num-space))
+        dbl-fmt (str "%" num-space ".2f")
+        str-fmt (fn [cw] (str "%-" cw "s"))]
+    (str "  | " (join " | " (map #(format (str-fmt %2) %1)    titles title-spaces))    " |\n"
+         "  |-" (join "-+-" (map (fn [s] (apply str (repeat s "-"))) title-spaces))    "-|\n"
+         "  | " (join " | " (map #(format (str-fmt %2) %1) subtitles subtitle-spaces)) " |\n"
+         "  |-" (join "-+-" (map (fn [s] (apply str (repeat s "-"))) subtitle-spaces)) "-|\n"
+         (join "\n" (for [[context results] (sort-by #(context-sort-fn (key %)) grouped)]
+                      (let [times (map (fn [filename] (->> results (filter #(= (:source %) filename)) first :time))
+                                       filenames)
+                            context-cells (map (fn [context-key col-width]
+                                                 (let [context-val (get-in context (vec (rest (:path (context-key col-mapping)))) "")]
+                                                   (format (str-fmt col-width)
+                                                           (if (keyword? context-val)
+                                                             (name context-val)
+                                                             context-val))))
+                                               c/comparison-context-cell-order
+                                               unique-context-key-spaces)
+                            measurement-cells (map (fn [measurements] (join " | " (map (fn [stat] (format dbl-fmt (stat measurements)))
+                                                                                       time-stats)))
+                                                   times)]
+                        (str "  | " (join " | " (concat context-cells measurement-cells)) " |"))))
+         "\n")))
+
+
+(defn check-file-format [filename]
+  (when-not (.endsWith filename ".edn")
+    (throw (Exception. (str "File " filename "has an unsupported file type. Supported are only edn files.")))))
+
+(defn report [benchmarks filenames time-stats]
+  (let [grouped-benchmarks (group-by #(get-in % [:context :function]) benchmarks)
+        output (str (when-let [conn-benchmarks (:connection grouped-benchmarks)]
+                      (str "Connection Measurements (in s):\n"
+                           (comparison-table conn-benchmarks filenames time-stats)
+                           "\n"))
+                    (when-let [tx-benchmarks (:transaction grouped-benchmarks)]
+                      (str "Transaction Measurements (in s):\n"
+                           (comparison-table tx-benchmarks filenames time-stats)
+                           "\n"))
+                    (when-let [query-benchmarks (->> (dissoc grouped-benchmarks :connection :transaction)
+                                                     vals
+                                                     (apply concat))]
+                      (str "Query Measurements (in s):\n"
+                           (comparison-table query-benchmarks filenames time-stats))))]
+    (println output)))
+
+(defn create-plots [data] ;; 1 plot per function and context
+  (let [tags (join "," (distinct (map :tag data)))
+        colors [Color/red Color/blue Color/green Color/cyan Color/magenta Color/orange Color/yellow]
+        directory (str "plots-" tags)]
+    
+  (doall (for [function (distinct (map #(get-in % [:context :function]) data))
+               config (distinct (map #(get-in % [:context :dh-config]) data))
+               execution-details (distinct (map #(get-in % [:context :execution]) data))]
+           (let [_ (log/debug (str "Processing " function " " config " " execution-details))
+                 {:keys [tx-entities data-type data-in-db?]} execution-details
+                 filename (str (name function)
+                               "_config-" config
+                               (when tx-entities
+                                (str "_tx-ents-" tx-entities))
+                               (when data-type
+                                (str "_type-" (name data-type)))
+                               (when data-in-db?
+                                 (str "_found-" data-in-db?))
+                               "_tags-" tags
+                               ".png")
+
+                 plot-data (->> data
+                                (filter #(and (= (get-in % [:context :function]) function)
+                                              (= (get-in % [:context :dh-config]) config)
+                                              (= (get-in % [:context :execution]) execution-details)))
+                                (map #(assoc % :x (get-in % [:context :db-entities]))))]
+              (when (seq plot-data)
+                (let [filepath (str directory "/" filename)
+                      all-y (apply concat (map #(get-in % [:time :observations]) plot-data))
+                      ymin 0
+                      ymax (apply max all-y)
+                     ; ymax (quantile all-y :probs 0.9)
+                      plot (charts/scatter-plot nil nil
+                                                :title  (str "Execution time of function " function "\n"
+                                                             "for configuration " (name config) " and\n"
+                                                             "execution parameters " execution-details)
+                                                :y-label "Time (s)"
+                                                :x-label (str "Entities in database (1 entity = " (count c/schema) " datoms)")
+                                                :legend true
+                                                :series-label "")]
+                (charts/set-stroke-color plot (Color. 0 0 0 0) :dataset 0)
+                (charts/set-y-range plot ymin ymax)
+                (doall (map-indexed (fn [idx [tag group]]
+                        (let [sorted (sort-by :x group)
+                              entities (map :x sorted)
+                              time (map :time sorted)
+                              time-obs (apply concat (map :observations time))
+                              entities-rep (apply concat (map #(repeat (count (:observations %2)) %1) entities time))]
+                          
+                        (charts/add-points plot entities-rep time-obs :series-label tag)
+                        (charts/set-stroke-color plot (get colors idx) :dataset (+ 1 (* idx 2)))
+                        (charts/add-lines plot entities (map :median time) :series-label (str tag " median")))
+                        (charts/set-stroke-color plot (get colors idx) :dataset (+ 2 (* idx 2))))
+                          (group-by :tag plot-data)))
+                  (when-not (.exists (io/file directory))
+                    (.mkdir (io/file directory)))
+                  (ic/save plot filepath
+                           :width 800
+                           :height 600))))))))
+
+(defn compare-benchmarks [filenames plots? time-stats]
+  (run! check-file-format filenames)
+
+  (let [benchmarks (->> filenames
+                        (map (fn [filename]
+                          (map #(assoc % :source filename)
+                               (edn/read-string (slurp filename)))))
+                        (apply concat))]
+    (if plots?
+      (create-plots benchmarks)
+      (report benchmarks filenames time-stats))))