datahike-browser-tests 1.0.0

package/test/datahike/test/unstructured_test.cljc

@@ -0,0 +1,291 @@
+(ns datahike.test.unstructured-test
+  (:require
+   #?(:cljs [cljs.test :as t :refer-macros [is deftest testing]]
+      :clj  [clojure.test :as t :refer [is deftest testing]])
+   [datahike.api :as d]
+   [datahike.experimental.unstructured :as du]))
+
+(deftest test-value-type-inference
+  (testing "Basic type inference"
+    (is (= :db.type/long (du/value->type 42)))
+    (is (= :db.type/float (du/value->type (float 42.0))))
+    (is (= :db.type/string (du/value->type "hello")))
+    (is (= :db.type/boolean (du/value->type true)))
+    (is (= :db.type/keyword (du/value->type :keyword)))
+    (is (= :db.type/ref (du/value->type {:a 1})))))
+
+(deftest test-schema-inference
+  (testing "Schema inference for basic values"
+    (is (= {:db/ident :name
+            :db/valueType :db.type/string
+            :db/cardinality :db.cardinality/one}
+           (du/infer-value-schema :name "Alice"))))
+
+  (testing "Schema inference for collections"
+    (is (= {:db/ident :tags
+            :db/valueType :db.type/keyword
+            :db/cardinality :db.cardinality/many}
+           (du/infer-value-schema :tags [:tag1 :tag2])))
+
+    (is (= {:db/ident :address
+            :db/valueType :db.type/ref
+            :db/cardinality :db.cardinality/one}
+           (du/infer-value-schema :address {:street "123 Main St"}))))
+
+  (testing "Empty vector handling"
+    (is (nil? (du/infer-value-schema :empty-tags [])))))
+
+(deftest test-process-unstructured-data
+  (testing "Processing simple data"
+    (let [data {:name "Alice" :age 30}
+          result (du/process-unstructured-data data)]
+      (is (= 2 (count (:schema result))))
+      (is (= 1 (count (:tx-data result))))
+      (is (= -1 (:db/id (first (:tx-data result)))))))
+
+  (testing "Processing nested data"
+    (let [data {:name "Bob"
+                :age 25
+                :address {:street "123 Main St"
+                          :city "Anytown"}}
+          result (du/process-unstructured-data data)]
+      ;; There should be 5 schema entries: name, age, address, street, city
+      (is (= 5 (count (:schema result))))
+      (is (= 2 (count (:tx-data result))))
+      (is (some #(= -1 (:db/id %)) (:tx-data result)))))
+
+  (testing "Processing empty vectors"
+    (let [data {:name "Charlie"
+                :tags []}
+          result (du/process-unstructured-data data)]
+      ;; The schema should only include the name attribute, not tags
+      (is (= 1 (count (:schema result))))
+      (is (= :name
+             (-> result :schema first :db/ident)))
+      ;; The transaction data should still include the empty tags vector
+      (let [tx-entity (first (:tx-data result))]
+        (is (= "Charlie" (:name tx-entity)))
+        (is (= [] (:tags tx-entity)))))))
+
+(deftest test-basic-schema-on-read
+  (testing "Basic operations with schema-on-read"
+    (let [cfg {:store {:backend :memory
+                       :id #uuid "10000000-0000-0000-0000-000000000001"}
+               :schema-flexibility :read}
+          _ (d/delete-database cfg)
+          _ (d/create-database cfg)
+          conn (d/connect cfg)]
+
+      ;; Just add simple data directly
+      (d/transact conn [{:name "Alice" :age 30}])
+
+      ;; Simple query
+      (is (= #{["Alice"]}
+             (d/q '[:find ?n :where [?e :name ?n]] (d/db conn))))
+
+      (d/release conn))))
+
+(deftest test-unstructured-schema-on-read
+  (testing "Using process-unstructured-data with schema-on-read"
+    (let [cfg {:store {:backend :memory
+                       :id #uuid "20000000-0000-0000-0000-000000000002"}
+               :schema-flexibility :read}
+          _ (d/delete-database cfg)
+          _ (d/create-database cfg)
+          conn (d/connect cfg)
+          data {:name "Bob"
+                :age 25
+                :address {:street "123 Main St"
+                          :city "Anytown"}}]
+
+      ;; Use process-unstructured-data to prepare data
+      (let [processed (du/process-unstructured-data data)
+            tx-report (d/transact conn (:tx-data processed))]
+
+        ;; Verify simple attribute
+        (is (= #{["Bob"]}
+               (d/q '[:find ?n :where [?e :name ?n]] (d/db conn))))
+
+        ;; We can't directly query nested data with the temp IDs in the schema-on-read mode
+        ;; This requires a different approach
+        ;; Let's verify we can at least retrieve the address entity id
+        (let [address-id (d/q '[:find ?a :where [?e :address ?a]] (d/db conn))]
+          (is (not (empty? address-id)))))
+
+      (d/release conn))))
+
+(deftest test-transact-unstructured-api
+  (testing "Using the transact-unstructured API with schema-on-read"
+    (let [cfg {:store {:backend :memory
+                       :id #uuid "30000000-0000-0000-0000-000000000003"}
+               :schema-flexibility :read}
+          _ (d/delete-database cfg)
+          _ (d/create-database cfg)
+          conn (d/connect cfg)
+          data {:name "Charlie"
+                :age 35
+                :skills ["programming" "design" "management"]
+                :contact {:email "charlie@example.com"
+                          :phone "555-1234"}}]
+
+      ;; Use the high-level API
+      (du/transact-unstructured conn data)
+
+      ;; Verify basic data
+      (is (= #{["Charlie"]}
+             (d/q '[:find ?n :where [?e :name ?n]] (d/db conn))))
+      (is (= #{[35]}
+             (d/q '[:find ?a :where [?e :age ?a]] (d/db conn))))
+
+      ;; Verify array data - just check if any skills were added
+      (let [skills (d/q '[:find ?s :where [_ :skills ?s]] (d/db conn))]
+        (is (not (empty? skills))))
+
+      ;; Verify that nested object was stored properly by checking contact
+      (let [contacts (d/q '[:find ?e :where [?e :email "charlie@example.com"]] (d/db conn))]
+        (is (not (empty? contacts))))
+
+      (d/release conn))))
+
+(deftest test-schema-on-write
+  (testing "Using transact-unstructured with schema-on-write"
+    (let [cfg {:store {:backend :memory
+                       :id #uuid "40000000-0000-0000-0000-000000000004"}}  ;; Default is schema-on-write
+          _ (d/delete-database cfg)
+          _ (d/create-database cfg)
+          conn (d/connect cfg)
+          data {:name "Diana"
+                :age 28
+                :active true}]
+
+      ;; First, verify that we can't insert data directly (requires schema)
+      (is (thrown? Exception
+                   (d/transact conn [data])))
+
+      ;; Now use transact-unstructured, which should infer and apply schema first
+      (du/transact-unstructured conn data)
+
+      ;; Verify the schema was created
+      (let [schema-query '[:find ?a ?t ?c
+                           :where
+                           [?e :db/ident ?a]
+                           [?e :db/valueType ?t]
+                           [?e :db/cardinality ?c]]]
+        (is (= #{[:name :db.type/string :db.cardinality/one]
+                 [:age :db.type/long :db.cardinality/one]
+                 [:active :db.type/boolean :db.cardinality/one]}
+               (d/q schema-query (d/db conn)))))
+
+      ;; Verify the data was inserted
+      (is (= #{["Diana"]}
+             (d/q '[:find ?n :where [?e :name ?n]] (d/db conn))))
+      (is (= #{[28]}
+             (d/q '[:find ?a :where [?e :age ?a]] (d/db conn))))
+      (is (= #{[true]}
+             (d/q '[:find ?a :where [?e :active ?a]] (d/db conn))))
+
+      ;; Test schema evolution with new attributes
+      (let [new-data {:name "Erik"
+                      :age 42
+                      :email "erik@example.com"  ;; New attribute
+                      :active false}]
+
+        ;; This should work and auto-add the email schema
+        (du/transact-unstructured conn new-data)
+
+        ;; Verify new schema was added
+        (let [schema-query '[:find ?a ?t ?c
+                             :where
+                             [?e :db/ident ?a]
+                             [?e :db/valueType ?t]
+                             [?e :db/cardinality ?c]]]
+          (is (= #{[:name :db.type/string :db.cardinality/one]
+                   [:age :db.type/long :db.cardinality/one]
+                   [:active :db.type/boolean :db.cardinality/one]
+                   [:email :db.type/string :db.cardinality/one]}
+                 (d/q schema-query (d/db conn)))))
+
+        ;; Verify both records exist
+        (is (= #{["Diana"] ["Erik"]}
+               (d/q '[:find ?n :where [?e :name ?n]] (d/db conn))))
+        (is (= #{["erik@example.com"]}
+               (d/q '[:find ?e :where [_ :email ?e]] (d/db conn)))))
+
+      (d/release conn))))
+
+(deftest test-cardinality-many-inference
+  (testing "Inferring cardinality/many from vector values"
+    (let [cfg {:store {:backend :memory
+                       :id #uuid "50000000-0000-0000-0000-000000000005"}}
+          _ (d/delete-database cfg)
+          _ (d/create-database cfg)
+          conn (d/connect cfg)
+          ;; First set up the schema for name with unique identity
+          _ (d/transact conn [{:db/ident :name
+                               :db/valueType :db.type/string
+                               :db/cardinality :db.cardinality/one
+                               :db/unique :db.unique/identity}])
+
+          data {:name "Fiona"
+                :tags ["clojure" "datalog" "databases"]} ;; Vector = cardinality many
+          ]
+
+      ;; This should automatically set up the tags attribute as cardinality/many
+      (du/transact-unstructured conn data)
+
+      ;; Verify the schema for tags with cardinality/many
+      (let [schema-query '[:find ?t ?c
+                           :where
+                           [?e :db/ident :tags]
+                           [?e :db/valueType ?t]
+                           [?e :db/cardinality ?c]]]
+        (is (= #{[:db.type/string :db.cardinality/many]} ;; Check cardinality/many
+               (d/q schema-query (d/db conn)))))
+
+      ;; Verify all array values were stored
+      (let [tags (d/q '[:find ?t :where [_ :tags ?t]] (d/db conn))]
+        (is (= #{"clojure" "datalog" "databases"}
+               (set (map first tags)))))
+
+      ;; Test adding more values to the array for the same entity
+      (d/transact conn [{:db/id [:name "Fiona"]
+                         :tags ["datahike"]}]) ;; Add another tag 
+
+      ;; Verify all tags are preserved (cardinality/many behavior)
+      (let [tags (d/q '[:find ?t :where [_ :tags ?t]] (d/db conn))]
+        (is (= #{"clojure" "datalog" "databases" "datahike"}
+               (set (map first tags)))))
+
+      (d/release conn))))
+
+(deftest test-schema-conflict
+  (testing "Detecting schema conflicts"
+    (let [cfg {:store {:backend :memory
+                       :id #uuid "60000000-0000-0000-0000-000000000006"}}
+          _ (d/delete-database cfg)
+          _ (d/create-database cfg)
+          conn (d/connect cfg)]
+
+      ;; First add schema for 'score' as a long
+      (d/transact conn [{:db/ident :score
+                         :db/valueType :db.type/long
+                         :db/cardinality :db.cardinality/one}])
+
+      ;; Add a data point with correct type
+      (d/transact conn [{:score 100}])
+
+      ;; Now try to add data with incompatible type (string instead of long)
+      (let [data {:name "Grace"
+                  :score "High"}] ;; Score is a string but schema says long
+
+        ;; This should detect the conflict and throw an exception
+        (is (thrown-with-msg? Exception #"Schema conflict detected"
+                              (du/transact-unstructured conn data))))
+
+      ;; Verify the data was not inserted
+      (is (empty? (d/q '[:find ?n :where [_ :name ?n]] (d/db conn))))
+
+      ;; Verify the original score still works
+      (is (= #{[100]} (d/q '[:find ?s :where [_ :score ?s]] (d/db conn))))
+
+      (d/release conn))))

package/test/datahike/test/upsert_impl_test.cljc

@@ -0,0 +1,205 @@
+(ns datahike.test.upsert-impl-test
+  (:require
+   #?(:cljs [cljs.test    :as t :refer-macros [is deftest testing]]
+      :clj  [clojure.test :as t :refer        [is deftest testing]])
+   [hitchhiker.tree.utils.async :as ha]
+   [hitchhiker.tree :as tree]
+   [hitchhiker.tree.messaging :as msg]
+   [datahike.index.hitchhiker-tree.upsert :as htu]
+   [datahike.test.utils :refer [setup-db conn-id]]
+   [datahike.constants :as const]
+   [datahike.db :as db]
+   [datahike.api :as d]))
+
+(defn upsert-helper
+  [t k]
+  (ha/<?? (msg/enqueue t [(htu/new-UpsertOp k 0 [0 1])])))
+
+(deftest hh-tree-upsert
+  (let [new-tree (tree/b-tree (tree/->Config 3 3 2))
+        projected-vec [4 :name "Marcel" 1]
+        tree (reduce upsert-helper (ha/<?? new-tree)
+                     (into (sorted-set)
+                           #{[1 :at "home" 1]
+                             [1 :age 44 1]
+                             [1 :name "Petr" 1]
+                             [2 :age 25 1]
+                             [2 :name "Ivan" 1]
+                             [3 :age 11 1]
+                             projected-vec}))]
+    (testing "Against an entry in the projection area,"
+      (testing "we are in a projection"
+        (is (= (:key (first (:op-buf tree))) projected-vec)))
+      (testing "basic lookup works"
+        (is (= [[1 :age 44 1] nil] (first (msg/lookup-fwd-iter tree [1 :age 44 1])))))
+      (testing "a totally new entry is persisted"
+        (let [new [5 :name "Jo" 3]
+              tree-after (upsert-helper tree new)]
+          ;; Is new in?
+          (is (= [new nil] (first (msg/lookup-fwd-iter tree-after new))))))
+      (testing "new = old"
+        (let [new [4 :name "Marcel" 2]
+              tree-after (upsert-helper tree new)]
+          (is (= [new nil] (first (msg/lookup-fwd-iter tree-after new))))))
+      (testing "old v <> new v,"
+        (testing "not keeping history"
+          (let [new [4 :name "New-Name" 2]
+                tree-after (upsert-helper tree new)]
+            ;; Is new in?
+            (is (= [new nil] (first (msg/lookup-fwd-iter tree-after new))))
+            ;; Is old deleted?
+            (is (= nil (msg/lookup tree-after projected-vec)))))))
+
+    (testing "Against an entry located at leaf level,"
+      (testing "basic lookup works"
+        (is (= [[1 :age 44 1] nil] (first (msg/lookup-fwd-iter tree [1 :age 44 1])))))
+      (testing "re-inserting works"
+        (let [new [1 :age 44 1]
+              tree-after (upsert-helper tree new)]
+          (is (= [new nil] (first (msg/lookup-fwd-iter tree-after new))))))
+      (testing "old v <> new v"
+        (testing "not keeping history"
+          (let [new [1 :age 40 5]
+                tree-after (upsert-helper tree new)]
+            ;; Is new in?
+            (is (= [new nil] (first (msg/lookup-fwd-iter tree-after new))))
+            ;; Is old deleted?
+            (is (= nil (msg/lookup tree-after [1 :age 44 1]))))))))
+
+  (testing "when it overflows"
+    ;; I.e. testing the projection of a deferred-op sitting on an index-node.
+    (let [new-tree (tree/b-tree (tree/->Config 3 3 2))
+          tree (reduce upsert-helper (ha/<?? new-tree)
+                       (into (sorted-set)
+                             #{[1 :age 44 1]
+                               [1 :name "Petr" 1]
+                               [2 :age 25 1]
+                               [2 :name "Ivan" 1]
+                               [3 :age 11 1]
+                               [4 :age 12 1]
+                               [4 :age 20 1]
+                               [4 :name "Paulo" 1]
+                               [4 :age 40 1]}))] ;; triggers the overflow
+
+      (is (= nil (msg/lookup tree [4 :age 12 1])))
+      (is (= nil (msg/lookup tree [4 :age 20 1])))
+      (is (= [[1 :age 44 1] nil] (first (msg/lookup-fwd-iter tree [1 :age 44 1])))))))
+
+(defn connect []
+  (let [cfg  {:schema-flexibility :read
+              :store {:backend :memory :id (random-uuid)}
+              :initial-tx         []}
+        _    (d/delete-database cfg)
+        _    (d/create-database cfg)]
+    (d/connect cfg)))
+
+(deftest datahike-upsert
+  (testing "IndexNode"
+    (let [txs  (vec (for [i (range 1000)]
+                      {:name (str "Peter" i)
+                       :age  i}))
+          conn (connect)]
+      (is (d/transact conn txs))
+      (d/release conn)))
+
+  (testing "simple upsert and history"
+    (let [txs  [[:db/add 199 :name "Peter"]
+                [:db/add 199 :name "Ptr"]]
+          conn (connect)]
+      (is (d/transact conn txs))
+      (is (not (d/datoms @conn :eavt 199 :name "Peter"))) ;; no history
+      (is (d/datoms (d/history @conn) :eavt 199 :name "Peter"))
+      (d/release conn)))
+
+  (testing "transacting the same datoms twice should work with :avet"
+    (let [dvec #(vector (:e %) (:a %) (:v %))
+          txs [[:db/add 1 :age 44]
+               [:db/add 2 :age 25]
+               [:db/add 3 :age 11]]
+          db (-> (db/empty-db {:age {:db/index true}})
+                 (d/db-with txs)
+                 ;; transacting again
+                 (d/db-with txs))]
+      (is (= [[3 :age 11]
+              [2 :age 25]
+              [1 :age 44]]
+             (map dvec (d/datoms db :avet)))))))
+
+(deftest temporal-upsert
+  (let [schema [{:db/ident       :name
+                 :db/valueType   :db.type/string
+                 :db/unique      :db.unique/identity
+                 :db/index       true
+                 :db/cardinality :db.cardinality/one}
+                {:db/ident       :age
+                 :db/valueType   :db.type/long
+                 :db/cardinality :db.cardinality/one}]
+        initial-tx (conj schema
+                         {:name "Alice"
+                          :age  25}
+                         {:name "Bob"
+                          :age  35})
+        cfg {:store {:backend :memory
+                     :id #uuid "70000000-0000-0000-0000-000000000007"}
+             :schema-flexibility :read
+             :initial-tx initial-tx}
+        conn (setup-db cfg)
+        query '[:find ?a ?t ?op
+                :where
+                [?e :name "Alice"]
+                [?e :age ?a ?t ?op]]]
+    (testing "when multiple transactions"
+      (d/transact conn [[:db/add [:name "Alice"] :age 20]
+                        [:db/add [:name "Bob"] :age 15]])
+      (d/transact conn [[:db/add [:name "Alice"] :age 10]])
+      (d/transact conn [[:db/add [:name "Alice"] :age 1]])
+      (is (= #{[25 (+ const/tx0 1) true]
+               [25 (+ const/tx0 2) false]
+               [20 (+ const/tx0 2) true]
+               [20 (+ const/tx0 3) false]
+               [10 (+ const/tx0 3) true]
+               [10 (+ const/tx0 4) false]
+               [1 (+ const/tx0 4) true]}
+             (d/q query (d/history @conn))))
+      (d/release conn))
+    (testing "when only one transaction"
+      (let [conn (setup-db cfg)]
+        (d/transact conn [[:db/add [:name "Alice"] :age 20]
+                          [:db/add [:name "Bob"] :age 15]
+                          [:db/add [:name "Alice"] :age 10]
+                          [:db/add [:name "Alice"] :age 1]])
+        (is (= #{[25 (+ const/tx0 1) true]
+                 [25 (+ const/tx0 2) false]
+                 [20 (+ const/tx0 2) true]
+                 [20 (+ const/tx0 2) false]
+                 [10 (+ const/tx0 2) true]
+                 [10 (+ const/tx0 2) false]
+                 [1 (+ const/tx0 2) true]}
+               (d/q query (d/history @conn))))
+        (d/release conn)))))
+
+(deftest upsert-read-handlers
+  (let [config {:store {:backend :file :path "/tmp/upsert-read-handlers" :id #uuid "ab1d0000-0000-0000-0000-000000000001"}
+                :schema-flexibility :write
+                :keep-history? false}
+        schema [{:db/ident       :block/string
+                 :db/valueType   :db.type/string
+                 :db/cardinality :db.cardinality/one}
+                {:db/ident       :block/children
+                 :db/valueType   :db.type/ref
+                 :db/index       true
+                 :db/cardinality :db.cardinality/one}]
+        _      (d/delete-database config)
+        _      (d/create-database config)
+        conn   (d/connect config)]
+    (d/transact conn schema)
+    (d/transact conn (vec (for [i (range 1000)]
+                            {:db/id (inc i) :block/children (inc i)})))
+    (d/release conn)
+
+    (let [conn (d/connect config)]
+      ;; Would fail if upsert read handlers are not present
+      (is (d/datoms @conn :eavt))
+      (d/release conn))
+
+    (d/delete-database config)))