datahike-browser-tests 1.0.0

package/.circleci/config.yml

@@ -0,0 +1,405 @@
+version: 2.1
+
+setup: true
+
+orbs:
+  tools: replikativ/clj-tools@0
+  continuation: circleci/continuation@2.0
+
+jobs:
+  setup:
+    executor: tools/clojurecli
+    parameters:
+      setup_cljs:
+        type: boolean
+        default: true
+    steps:
+      - restore_cache:
+          keys:
+            - source-{{ .Branch }}-{{ .Revision }}
+            - source-{{ .Branch }}
+            - source-
+      - checkout
+      - save_cache:
+          key: source-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - .git
+      - restore_cache:
+          keys:
+            - deps-{{ checksum "deps.edn" }}
+            - deps-
+      - run:
+          name: resolve deps
+          command: clojure -A:dev:test:build -P
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+      - when:
+          condition: <<parameters.setup_cljs>>
+          steps:
+            - restore_cache:
+                keys:
+                  - npm-{{ checksum "package-lock.json" }}
+                  - npm-
+            - run: npm ci
+            - save_cache:
+                key: npm-{{ checksum "package-lock.json" }}
+                paths:
+                  - /home/circleci/.npm
+      - persist_to_workspace:
+          root: /home/circleci/
+          paths:
+            - .m2
+            - .npm
+            - replikativ
+  format:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Check format
+          command: bb format
+          no_output_timeout: 5m
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  native-image:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Add GitHub to known hosts
+          command: mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
+          no_output_timeout: 1m
+      - run:
+          name: Generate config
+          command: |
+            bb --version
+            bb .circleci/scripts/gen_ci.clj > generated_config.yml
+      - continuation/continue:
+          configuration_path: generated_config.yml
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  persistent-set-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Run unittests for persistent sorted set index
+          command: bb test clj-pss
+          no_output_timeout: 5m
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  hitchhiker-tree-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Run unittests for hitchhiker-tree index
+          command: bb test clj-hht
+          no_output_timeout: 5m
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  spec-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Run unittests for hitchhiker-tree index
+          command: bb test specs
+          no_output_timeout: 5m
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  backward-compatibility-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Retrieve ssh-fingerprints
+          command: mkdir ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
+          no_output_timeout: 1m
+      - run:
+          name: Run backward compatibility test
+          command: bb test back-compat
+          no_output_timeout: 5m
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  integration-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Run integration test
+          command: bb test integration
+          no_output_timeout: 5m
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  kabel-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Run kabel tests
+          command: bb test kabel
+          no_output_timeout: 5m
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  browser-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Prepare dependencies
+          command: clojure -X:deps prep
+      - run:
+          name: Start test server
+          command: clojure -M:test -e "(require 'datahike.kabel.browser-test-server) (datahike.kabel.browser-test-server/start-test-server!) (Thread/sleep Long/MAX_VALUE)"
+          background: true
+      - run:
+          name: Wait for server startup
+          command: sleep 20
+      - run:
+          name: Compile ClojureScript for browser tests
+          command: npx shadow-cljs compile :browser-ci
+          no_output_timeout: 5m
+      - run:
+          name: Run Karma browser tests
+          command: npx karma start --single-run
+          no_output_timeout: 5m
+  node-cljs-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Run CLJS node tests
+          command: bb node-cljs-test
+          no_output_timeout: 5m
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  npm-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Build npm package
+          command: bb npm-build
+          no_output_timeout: 5m
+      - run:
+          name: Run npm JS tests
+          command: bb npm-test
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  java-examples-test:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Update examples pom.xml to current version
+          command: bb java-examples-version
+      - run:
+          name: Verify local Datahike jar exists
+          command: |
+            echo "Checking .m2 directory structure:"
+            ls -la /home/circleci/.m2/repository/org/replikativ/ || echo ".m2/repository/org/replikativ not found"
+            echo ""
+            echo "Checking datahike versions:"
+            ls -la /home/circleci/.m2/repository/org/replikativ/datahike/ || echo "datahike directory not found"
+            echo ""
+            echo "Current version directory:"
+            CURRENT_VERSION=$(grep -oP '<datahike.version>\K[^<]+' examples/java/pom.xml)
+            echo "Looking for version: $CURRENT_VERSION"
+            ls -la /home/circleci/.m2/repository/org/replikativ/datahike/$CURRENT_VERSION/ || echo "$CURRENT_VERSION directory not found"
+      - run:
+          name: Download Maven dependencies
+          command: cd examples/java && mvn dependency:resolve dependency:resolve-plugins
+      - run:
+          name: Run QuickStart example
+          command: cd examples/java && mvn compile exec:java -Dexec.mainClass="examples.QuickStart"
+          no_output_timeout: 3m
+      - run:
+          name: Run SchemaExample
+          command: cd examples/java && mvn compile exec:java -Dexec.mainClass="examples.SchemaExample"
+          no_output_timeout: 3m
+      - run:
+          name: Run TimeTravelExample
+          command: cd examples/java && mvn compile exec:java -Dexec.mainClass="examples.TimeTravelExample"
+          no_output_timeout: 3m
+      - save_cache:
+          key: deps-{{ checksum "deps.edn" }}
+          paths:
+            - /home/circleci/.m2
+            - /home/circleci/.deps.clj
+  build:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Build jar
+          command: bb jar
+      - run:
+          name: Install jar to local Maven repository
+          command: bb install
+      - persist_to_workspace:
+          root: /home/circleci/
+          paths:
+            - .m2
+            - .npm
+            - replikativ
+  deploy:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Deploy to clojars
+          command: bb clojars
+  release-jar:
+    executor: tools/clojurecli
+    steps:
+      - attach_workspace:
+          at: /home/circleci
+      - run:
+          name: Release jar on GitHub
+          command: bb release jar
+
+workflows:
+  build-test-and-deploy:
+    jobs:
+      - setup:
+          context: ghcr-read-package
+          setup_cljs: true
+      - build:
+          context: ghcr-read-package
+          requires:
+            - setup
+      - native-image:
+          context:
+            - ghcr-read-package
+            - github-token
+          filters:
+            branches:
+              only: main
+          requires:
+            - setup
+      - format:
+          context: ghcr-read-package
+          requires:
+            - setup
+      - persistent-set-test:
+          context: ghcr-read-package
+          requires:
+            - build
+      - hitchhiker-tree-test:
+          context: ghcr-read-package
+          requires:
+            - build
+      - backward-compatibility-test:
+          context: ghcr-read-package
+          requires:
+            - build
+      - integration-test:
+          context: ghcr-read-package
+          requires:
+            - build
+      - kabel-test:
+          context: ghcr-read-package
+          requires:
+            - build
+      - browser-test:
+          context: ghcr-read-package
+          requires:
+            - build
+      - node-cljs-test:
+          context: ghcr-read-package
+          requires:
+            - build
+      - npm-test:
+          context: ghcr-read-package
+          requires:
+            - build
+      - java-examples-test:
+          context: ghcr-read-package
+          requires:
+            - build
+      - deploy:
+          context:
+            - ghcr-read-package
+            - clojars-deploy
+            - github-token
+          filters:
+            branches:
+              only: main
+          requires:
+            - format
+            - persistent-set-test
+            - hitchhiker-tree-test
+            - backward-compatibility-test
+            - integration-test
+            - kabel-test
+            - browser-test
+            - node-cljs-test
+            - npm-test
+            - java-examples-test
+            - native-image
+      - release-jar:
+          context:
+            - ghcr-read-package
+            - github-token
+          filters:
+            branches:
+              only: main
+          requires:
+            - format
+            - persistent-set-test
+            - hitchhiker-tree-test
+            - backward-compatibility-test
+            - integration-test
+            - kabel-test
+            - browser-test
+            - node-cljs-test
+            - npm-test
+            - java-examples-test
+            - native-image

package/.circleci/scripts/gen_ci.clj

@@ -0,0 +1,194 @@
+(ns gen-ci
+  (:require
+   [babashka.tasks :as tasks]
+   [clj-yaml.core :as yaml]
+   [clojure.string :as str]
+   [flatland.ordered.map :refer [ordered-map]]))
+
+(def graalvm-version "25.0.0")
+
+(defn run
+  ([cmd-name cmd]
+   (run cmd-name cmd nil))
+  ([cmd-name cmd no-output-timeout]
+   (let [base {:run {:name    cmd-name
+                     :command cmd}}]
+     (if no-output-timeout
+       (assoc-in base [:run :no_output_timeout] no-output-timeout)
+       base))))
+
+(defn make-graalvm-url [arch]
+  (let [myarch (case arch
+                 "amd64" "x64"
+                 "aarch64" "aarch64")]
+    (str "https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-"
+         graalvm-version
+         "/graalvm-community-jdk-"
+         graalvm-version
+         "_linux-"
+         myarch
+         "_bin.tar.gz")))
+
+(defn build-native-image
+  [arch resource-class]
+  (let [cache-key (str arch "-deps-linux-{{ checksum \"deps.edn\" }}")
+        graalvm-url (make-graalvm-url arch)]
+    (ordered-map
+     :machine
+     {:image "ubuntu-2204:2023.10.1"
+      :resource_class resource-class}
+     :working_directory "/home/circleci/replikativ"
+     :environment {:GRAALVM_VERSION graalvm-version
+                   :GRAALVM_HOME "/home/circleci/graalvm"
+                   :DTHK_PLATFORM "linux"
+                   :DTHK_ARCH arch
+                   :PATH "/bin:/home/circleci/graalvm/bin:/home/circleci/clojure/bin:/home/circleci/bin"
+                   :JAVA_HOME "/home/circleci/graalvm/bin/java"}
+     :steps
+     [:checkout
+      {:restore_cache {:keys [cache-key]}}
+      (run "Install GraalVM"
+           (format "cd /home/circleci
+/bin/curl -sL %s -o graalvm.tar.gz
+/bin/mkdir graalvm || true
+/bin/tar -xzf graalvm.tar.gz --directory graalvm --strip-components 1
+sudo update-alternatives --install /usr/bin/java java /home/circleci/graalvm/bin/java 0
+sudo update-alternatives --install /usr/bin/javac javac /home/circleci/graalvm/bin/javac 0
+sudo update-alternatives --set java /home/circleci/graalvm/bin/java
+sudo update-alternatives --set javac /home/circleci/graalvm/bin/javac"
+                   graalvm-url))
+      (run "Install Clojure"
+           "cd /home/circleci
+/bin/curl -sL https://download.clojure.org/install/linux-install-1.12.2.1565.sh -o clojure-install.sh
+/bin/chmod +x clojure-install.sh
+/bin/bash -x clojure-install.sh --prefix /home/circleci/clojure")
+      (run "Install Babashka"
+           "cd /home/circleci
+/bin/curl -sL https://raw.githubusercontent.com/babashka/babashka/master/install -o babashka-install.sh
+/bin/chmod +x babashka-install.sh
+/bin/bash -x babashka-install.sh --dir /home/circleci/bin")
+      (run "Build native image"
+           "cd /home/circleci/replikativ
+bb ni-cli")
+      (run "Build libdatahike"
+           "cd /home/circleci/replikativ
+bb ni-compile")
+      (run "Test native image"
+           "cd /home/circleci/replikativ
+bb test native-image")
+      (run "Test babashka pod"
+           "cd /home/circleci/replikativ
+bb test bb-pod")
+      (run "Test libdatahike"
+           "cd /home/circleci/replikativ
+bb test libdatahike")
+      (run "Install Python dependencies"
+           "pip install --user pytest")
+      (run "Test Python bindings"
+           "cd /home/circleci/replikativ
+bb test python")
+      {:persist_to_workspace
+       {:root "/home/circleci/"
+        :paths ["replikativ/dthk" "replikativ/libdatahike/target" "replikativ/graalvm"]}}
+      {:save_cache
+       {:paths ["~/.m2" ".deps.clj"]
+        :key cache-key}}])))
+
+(defn release-artifacts
+  [arch]
+  (let [cache-key (str arch "-deps-linux-{{ checksum \"deps.edn\" }}")]
+    (ordered-map
+     :executor "tools/clojurecli"
+     :working_directory "/home/circleci/replikativ"
+     :environment {:DTHK_PLATFORM "linux"
+                   :DTHK_ARCH arch}
+     :steps
+     [:checkout
+      {:restore_cache {:keys [cache-key]}}
+      {:attach_workspace {:at "/home/circleci"}}
+      (run "Release native image"
+           "cd /home/circleci/replikativ
+bb release native-image")
+      (run "Release libdatahike"
+           "cd /home/circleci/replikativ
+bb release libdatahike")
+      {:persist_to_workspace
+       {:root "/home/circleci/"
+        :paths ["replikativ/dthk" "replikativ/graalvm"]}}
+      {:save_cache
+       {:paths ["~/.m2" ".deps.clj"]
+        :key cache-key}}])))
+
+(defn make-config []
+  (ordered-map
+   :version 2.1
+   :orbs
+   {:tools "replikativ/clj-tools@0"}
+   :commands
+   {:setup-docker-buildx
+    {:steps
+     [{:run
+       {:name "Create multi-platform capable buildx builder"
+        :command
+        "docker run --privileged --rm tonistiigi/binfmt --install all\ndocker buildx create --name ci-builder --use"}}]}}
+   :jobs (ordered-map
+          :build-linux-amd64 (build-native-image "amd64" "large")
+          :build-linux-aarch64 (build-native-image "aarch64" "arm.large")
+          :release-linux-amd64 (release-artifacts "amd64")
+          :release-linux-aarch64 (release-artifacts "aarch64"))
+   :workflows (ordered-map
+               :version 2
+               :native-images
+               {:jobs ["build-linux-amd64"
+                       "build-linux-aarch64"
+                       {"release-linux-amd64"
+                        {:context ["dockerhub-deploy"
+                                   "github-token"]
+                         :filters {:branches {:only "main"}}
+                         :requires ["build-linux-amd64"]}}
+                       {"release-linux-aarch64"
+                        {:context ["dockerhub-deploy"
+                                   "github-token"]
+                         :filters {:branches {:only "main"}}
+                         :requires ["build-linux-aarch64"]}}]})))
+
+(def skip-config
+  {:skip-if-only [#"^doc\/.*"
+                  #"^bb\/.*"
+                  #"^dev\/.*"
+                  #"^examples\/.*"
+                  #"^test\/.*"
+                  #".*.md$"]})
+
+(defn get-changes
+  []
+  (-> (tasks/shell {:out :string} "git diff --name-only HEAD~1")
+      (:out)
+      (str/split-lines)))
+
+(defn irrelevant-change?
+  [change regexes]
+  (some? (some #(re-matches % change) regexes)))
+
+(defn anything-relevant?
+  [change-set regexes]
+  (some? (some #(not (irrelevant-change? % regexes)) change-set)))
+
+(defn main
+  []
+  (let [{:keys [skip-if-only]} skip-config
+        changed-files          (get-changes)]
+    (when (anything-relevant? changed-files skip-if-only)
+      (-> (make-config)
+          (yaml/generate-string :dumper-options {:flow-style :block})
+          println))))
+
+(when (= *file* (System/getProperty "babashka.file"))
+  (main))
+
+(comment
+  (def changed-files (get-changes))
+  (anything-relevant? changed-files (:skip-if-only skip-config))
+  (-> (make-config) 
+      (yaml/generate-string :dumper-options {:flow-style :block})
+      println))

package/.cirrus.yml

@@ -0,0 +1,60 @@
+macos_instance:
+  image: ghcr.io/cirruslabs/macos-runner:sequoia
+
+build-task:
+  only_if: $CIRRUS_BRANCH == 'main'
+  maven_cache:
+    folder: ~/.m2
+  env:
+    DTHK_PLATFORM: macos
+    DTHK_ARCH: aarch64
+    INSTALL_DIR: ${HOME}
+    GRAALVM_VERSION: 25.0.0
+    GRAALVM_HOME: ${HOME}/graalvm/Contents/Home/
+    GITHUB_TOKEN: ENCRYPTED[!86a3791406be6ffc9490f464846ec5385908ae8d3d2a9eeb1457d529060816e14e9d7b88ecd190970add6a5d5d4e7b59!]
+  script: |
+    set -euo pipefail
+
+    # install babashka
+    curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
+    chmod +x install
+    ./install --dir ${HOME}/.local/bin
+    export PATH=${HOME}/.local/bin:$PATH
+
+    # install clojure
+    curl -L -O https://github.com/clojure/brew-install/releases/latest/download/posix-install.sh
+    chmod +x posix-install.sh
+    sudo ./posix-install.sh
+
+    # install graalvm
+    pushd "$INSTALL_DIR" >/dev/null
+    curl -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-${GRAALVM_VERSION}/graalvm-community-jdk-${GRAALVM_VERSION}_${DTHK_PLATFORM}-${DTHK_ARCH}_bin.tar.gz -o graalvm.tar.gz
+    sudo xattr -r -d com.apple.quarantine graalvm.tar.gz
+    mkdir graalvm
+    tar -xzf graalvm.tar.gz --strip-components=1 -C graalvm
+    popd >/dev/null
+
+    # prepare
+    export PATH=${GRAALVM_HOME}/bin:${PATH}
+    export JAVA_HOME=${GRAALVM_HOME}
+    sudo /usr/sbin/softwareupdate --install-rosetta --agree-to-license
+    java -version
+    ${GRAALVM_HOME}bin/native-image --version
+    echo $PATH
+    native-image --version
+
+    # compile native artifacts
+    bb ni-cli
+    bb ni-compile
+
+    # test native artifacts
+    bb test native-image
+    bb test bb-pod
+    bb test libdatahike
+
+    # release native artifacts
+    bb release native-image
+    bb release libdatahike
+
+  binaries_artifacts:
+    path: "dist/*"

package/.clj-kondo/babashka/sci/config.edn

@@ -0,0 +1 @@
+{:hooks {:macroexpand {sci.core/copy-ns sci.core/copy-ns}}}

package/.clj-kondo/babashka/sci/sci/core.clj

@@ -0,0 +1,9 @@
+(ns sci.core)
+
+(defmacro copy-ns
+  ([ns-sym sci-ns]
+   `(copy-ns ~ns-sym ~sci-ns nil))
+  ([ns-sym sci-ns opts]
+   `[(quote ~ns-sym)
+     ~sci-ns
+     (quote ~opts)]))