datahike-browser-tests 1.0.0

package/doc/bindings/edn-conversion.md

@@ -0,0 +1,383 @@
+# EDN Conversion Rules for Language Bindings
+
+This document describes the universal rules for converting native data structures (Python dicts, JavaScript objects, Java Maps) to EDN (Extensible Data Notation) across all Datahike language bindings.
+
+## Design Goal
+
+Provide a **consistent, explicit, and predictable** mapping between native language data structures and EDN that works the same way across Python, JavaScript, and Java bindings.
+
+## The Universal Rule
+
+> **Keys are always keywordized. Values starting with `:` become keywords, everything else remains literal.**
+
+### Key Conversion
+
+All map/dict/object keys are automatically converted to EDN keywords:
+
+| Input | EDN Output | Notes |
+|-------|------------|-------|
+| `"name"` | `:name` | Simple key |
+| `"person/name"` | `:person/name` | Namespaced key |
+| `":db/id"` | `:db/id` | Leading `:` is stripped (convenience) |
+| `"keep-history?"` | `:keep-history?` | Question mark preserved |
+
+### Value Conversion
+
+Values are converted based on their type and content:
+
+| Input Type | Input Value | EDN Output | Rule |
+|------------|-------------|------------|------|
+| String starting with `:` | `":active"` | `:active` | Keyword |
+| String without `:` | `"Alice"` | `"Alice"` | String |
+| String with `\:` prefix | `"\\:literal"` | `":literal"` | Escaped literal |
+| Integer | `42` | `42` | Number |
+| Float | `3.14` | `3.14` | Number |
+| Boolean | `true` / `false` | `true` / `false` | Boolean |
+| Null/None | `null` / `None` | `nil` | Nil |
+| List/Array | `[1, 2, 3]` | `[1 2 3]` | Vector |
+| Nested Map/Dict | `{"a": 1}` | `{:a 1}` | Recursive |
+
+## Examples by Language
+
+### Python
+
+```python
+# Configuration
+import uuid
+
+config = {
+    "store": {
+        "backend": ":memory",              # Keyword
+        "id": str(uuid.uuid4())         # String (memory backend requires UUID)
+    },
+    "schema-flexibility": ":read",      # Keyword
+    "keep-history?": True               # Boolean
+}
+# → {:store {:backend :memory :id "<uuid-string>"}
+#    :schema-flexibility :read
+#    :keep-history? true}
+
+# Transaction data
+data = [
+    {"name": "Alice", "status": ":active"},
+    {"name": "Bob", "status": ":inactive"}
+]
+# → [{:name "Alice" :status :active}
+#    {:name "Bob" :status :inactive}]
+
+# Schema transaction
+schema = [
+    {
+        "db/ident": ":person/name",
+        "db/valueType": ":db.type/string",
+        "db/cardinality": ":db.cardinality/one"
+    }
+]
+# → [{:db/ident :person/name
+#     :db/valueType :db.type/string
+#     :db/cardinality :db.cardinality/one}]
+```
+
+### JavaScript
+
+```javascript
+// Configuration
+const crypto = require('crypto');
+const config = {
+    store: {
+        backend: ':memory',                // Keyword
+        id: crypto.randomUUID()         // String (memory backend requires UUID)
+    },
+    'schema-flexibility': ':read',      // Keyword
+    'keep-history?': true               // Boolean
+};
+// → {:store {:backend :memory :id "<uuid-string>"}
+//    :schema-flexibility :read
+//    :keep-history? true}
+
+// Transaction data
+const data = [
+    {name: 'Alice', status: ':active'},
+    {name: 'Bob', status: ':inactive'}
+];
+// → [{:name "Alice" :status :active}
+//    {:name "Bob" :status :inactive}]
+```
+
+### Java
+
+```java
+// Configuration
+import java.util.UUID;
+
+Map<String, Object> config = Map.of(
+    "store", Map.of(
+        "backend", ":memory",              // Keyword
+        "id", UUID.randomUUID().toString() // String (memory backend requires UUID)
+    ),
+    "schema-flexibility", ":read"       // Keyword
+);
+// → {:store {:backend :memory :id "<uuid-string>"}
+//    :schema-flexibility :read}
+
+// Or use builders (convenience)
+// Note: memory backend requires UUID identifier
+Database db = Database.memory(UUID.randomUUID().toString())
+    .schemaFlexibility(SchemaFlexibility.READ)
+    .build();
+```
+
+## Edge Cases and Escape Hatches
+
+### Literal Colon Strings
+
+To include a string that starts with `:` without it becoming a keyword, use backslash escape:
+
+```python
+# Python
+{"literal": "\\:starts-with-colon"}
+# → {:literal ":starts-with-colon"}
+
+# JavaScript
+{literal: '\\:starts-with-colon'}
+# → {:literal ":starts-with-colon"}
+
+# Java
+Map.of("literal", "\\:starts-with-colon")
+// → {:literal ":starts-with-colon"}
+```
+
+**Note:** In Python/Java string literals, you need to escape the backslash itself: `"\\:"` → literal `\:` → EDN `":value"`
+
+### Helper Modules for Complex Types
+
+All languages provide helper modules for explicit type control:
+
+```python
+# Python
+from datahike import edn
+
+{
+    "uuid": edn.uuid("550e8400-e29b-41d4-a716-446655440000"),
+    "inst": edn.inst("2024-01-01T00:00:00Z"),
+    "literal": edn.string(":force-string"),
+    "keyword": edn.keyword("name", "person")  # :person/name
+}
+```
+
+```javascript
+// JavaScript
+import {edn} from 'datahike';
+
+{
+    uuid: edn.uuid('550e8400-e29b-41d4-a716-446655440000'),
+    inst: edn.inst('2024-01-01T00:00:00Z'),
+    literal: edn.string(':force-string'),
+    keyword: edn.keyword('person', 'name')  // :person/name
+}
+```
+
+```java
+// Java
+import static datahike.edn.EDN.*;
+
+Map.of(
+    "uuid", uuid("550e8400-e29b-41d4-a716-446655440000"),
+    "inst", inst("2024-01-01T00:00:00Z"),
+    "literal", string(":force-string"),
+    "keyword", keyword("person", "name")  // :person/name
+)
+```
+
+### EDN String Escape Hatch
+
+For truly complex cases (database functions, unusual syntax), all bindings accept raw EDN strings:
+
+```python
+# Python
+db.transact('''
+[{:db/id #db/id[:db.part/user]
+  :db/fn (fn [db] (inc 1))}]
+''', input_format='edn')
+```
+
+```javascript
+// JavaScript
+await d.transact(conn, `
+[{:db/id #db/id[:db.part/user]
+  :db/fn (fn [db] (inc 1))}]
+`);
+```
+
+```java
+// Java
+Datahike.transact(conn,
+    "[{:db/id #db/id[:db.part/user] " +
+    "  :db/fn (fn [db] (inc 1))}]");
+```
+
+## Custom Backend Support
+
+Custom storage backends may require arbitrary configuration keys that are not known in advance. The map/dict/object-based API handles these naturally:
+
+```python
+# Python - custom S3 backend
+db = Database({
+    "store": {
+        "backend": ":my-s3",
+        "bucket": "my-bucket",
+        "region": "us-west-2",
+        "encryption": {
+            "type": ":aes256",
+            "key-id": "secret-key"
+        }
+    }
+})
+```
+
+```javascript
+// JavaScript - custom S3 backend
+const db = new Database({
+    store: {
+        backend: ':my-s3',
+        bucket: 'my-bucket',
+        region: 'us-west-2',
+        encryption: {
+            type: ':aes256',
+            'key-id': 'secret-key'
+        }
+    }
+});
+```
+
+```java
+// Java - custom S3 backend (Map API)
+Map<String, Object> config = Map.of(
+    "store", Map.of(
+        "backend", ":my-s3",
+        "bucket", "my-bucket",
+        "region", "us-west-2",
+        "encryption", Map.of(
+            "type", ":aes256",
+            "key-id", "secret-key"
+        )
+    )
+);
+Database db = new Database(config);
+```
+
+## Common Keyword Constants
+
+All language bindings provide pre-defined constants for frequently-used Datahike keywords:
+
+```python
+# Python
+from datahike import kw
+
+kw.DB_ID                    # :db/id
+kw.DB_IDENT                 # :db/ident
+kw.DB_VALUE_TYPE            # :db/valueType
+kw.DB_CARDINALITY           # :db/cardinality
+kw.STRING                   # :db.type/string
+kw.LONG                     # :db.type/long
+kw.REF                      # :db.type/ref
+kw.ONE                      # :db.cardinality/one
+kw.MANY                     # :db.cardinality/many
+```
+
+```javascript
+// JavaScript
+import {kw} from 'datahike';
+
+kw.DB_ID                    // :db/id
+kw.DB_IDENT                 // :db/ident
+// ... etc
+```
+
+```java
+// Java
+import static datahike.edn.Keywords.*;
+
+DB_ID                       // :db/id
+DB_IDENT                    // :db/ident
+// ... etc
+```
+
+## Design Rationale
+
+### Why `:` prefix for keyword values?
+
+**Explicit is better than implicit.** Previous versions used heuristics (e.g., `"backend": "mem"` → `:memory`), but this created ambiguity:
+- Is `"person/name"` a keyword or a string with a slash?
+- Is `"active"` a keyword or a string?
+
+The `:` prefix makes the distinction unambiguous and works consistently across all contexts.
+
+### Why not separate config rules from data rules?
+
+**Consistency reduces cognitive load.** Having one universal rule means:
+- No need to remember context-specific rules
+- Easier to teach and document
+- Predictable behavior everywhere
+
+### Why backslash escaping?
+
+**Familiar to programmers.** Backslash escaping is used in JSON, regex, shell scripts, and most programming languages. It's a well-understood convention.
+
+### Why allow EDN string fallback?
+
+**Pragmatic escape hatch.** While the conversion rules handle 99% of cases, edge cases exist (database functions, unusual Clojure syntax). Allowing raw EDN strings ensures users are never blocked.
+
+## Migration from Previous Versions
+
+### JavaScript (Breaking Change)
+
+**Old Syntax (v0.6 and earlier):**
+```javascript
+const crypto = require('crypto');
+const config = {
+    store: {
+        backend: 'memory',      // String → keyword (implicit)
+        id: crypto.randomUUID() // Memory backend requires UUID
+    }
+};
+```
+
+**New Syntax (v0.7+):**
+```javascript
+const crypto = require('crypto');
+const config = {
+    store: {
+        backend: ':memory',     // Explicit : required
+        id: crypto.randomUUID() // Memory backend requires UUID
+    }
+};
+```
+
+**Migration:** Add `:` prefix to all values that should be keywords. Note that memory backend requires UUID identifiers.
+
+### Java (Enhancement)
+
+**Syntax:**
+```java
+import java.util.UUID;
+
+Map<String, Object> config = Map.of(
+    "store", Map.of(
+        "backend", ":memory",      // Explicit : for keyword
+        "id", UUID.randomUUID().toString() // Memory backend requires UUID
+    )
+);
+
+// Or use builders (memory backend requires UUID)
+Database.memory(UUID.randomUUID().toString()).build();
+```
+
+**Migration:** Add `:` prefix to all values that should be keywords for consistency with other languages.
+
+## See Also
+
+- [Python Binding Documentation](python.md)
+- [Java Binding Documentation](java.md)
+- [JavaScript Binding Documentation](javascript.md)
+- [Custom Backends Guide](custom-backends.md)

package/doc/cli.md

@@ -0,0 +1,162 @@
+# Command line interface
+
+**Status: Beta** - The CLI is functional and tested, but the command structure may change as we refine the interface.
+
+We provide the `dthk` native executable to access Datahike databases from
+the command line.
+
+## Installation
+
+1. Download the precompiled binary for your platform from the [Datahike releases page](https://github.com/replikativ/datahike/releases)
+2. Unzip the archive
+3. Add the `dthk` executable to your PATH
+
+Supported platforms:
+- Linux (amd64, aarch64)
+- macOS (aarch64/Apple Silicon)
+
+For other platforms, build from source using GraalVM native-image (see project documentation).
+
+## Commands
+
+The CLI provides flat commands that mirror the Clojure API functions. To see all available commands, run:
+
+```bash
+$ dthk --help
+```
+
+Common commands by category:
+- **Database Operations**: `create-database`, `delete-database`, `database-exists`
+- **Transaction Operations**: `transact`, `load-entities`, `db-with`, `with`
+- **Query Operations**: `query`, `pull`, `pull-many`, `entity`, `datoms`, `seek-datoms`, `index-range`, `is-filtered`, `query-stats`
+- **Schema Operations**: `schema`, `reverse-schema`
+- **Diagnostics**: `metrics`
+- **Maintenance**: `gc-storage`
+
+Prefix syntax for database access:
+- `db:config.edn` - Dereferences a connection to get the current database value
+- `conn:config.edn` - Creates a connection for transacting
+- `edn:file.edn` - Reads EDN data from a file
+- `json:file.json` - Reads JSON data from a file
+- `asof:timestamp:config.edn` - Creates a database snapshot as-of timestamp
+- `since:timestamp:config.edn` - Creates a database view since timestamp
+- `history:config.edn` - Returns the history database
+
+## Example usage
+
+To access a database you need to provide the usual configuration for Datahike.
+Put this into a file `myconfig.edn`.
+
+```clojure
+{:store  {:backend :file
+          :path "/home/USERNAME/dh-shared-db"
+          :id #uuid "550e8400-e29b-41d4-a716-446655440000"
+          :config {:in-place? true}}
+ :keep-history? true
+ :schema-flexibility :read}
+```
+
+Now you can invoke some of our core API functions on the database. Let us add a
+fact to the database (be careful to use single ' if you do not want your shell
+to substitute parts of your Datalog ;) ):
+
+```bash
+$ dthk transact conn:myconfig.edn '[[:db/add -1 :name "Linus"]]'
+ ```
+
+And retrieve it:
+
+```bash
+$ dthk query '[:find ?n . :where [?e :name ?n]]' db:myconfig.edn
+"Linus" # prints the name
+```
+
+Note that the `conn:<file>` argument to `transact` comes before the transaction
+value(s), whereas the `db:<file>` argument to `q` comes after the query
+value, mirroring the Clojure API. As an added benefit, this also allows passing
+multiple db configuration files prefixed with `db:` for joining over arbitrary
+many databases or data files with "edn:" or "json:". Everything non-prefixed is
+read in as `edn` and passed to the query engine as well:
+
+```bash
+$ dthk query '[:find ?e . :in $ ?name :where [?e :name ?name]]' db:myconfig.edn '"Linus"'
+123
+```
+
+When passing strings as EDN, make sure to enclose double quotes as part of the
+command-line arg value. Otherwise it will be parsed as a symbol.
+
+Provided the filestore is configured with `{:in-place? true}` you can even write
+to the same database without a dedicated daemon from different shells:
+
+```bash
+# In the first shell
+$ dthk transact conn:myconfig.edn '[[:db/add -1 :name "Alice"]]'
+
+# In a second shell simultaneously
+$ dthk transact conn:myconfig.edn '[[:db/add -2 :name "Bob"]]'
+```
+
+To check that everything has been added and no write operations have overwritten
+each other:
+
+```bash
+$ dthk query '[:find (count ?e) . :in $ :where [?e :name ?n]]' db:myconfig.edn
+2 # check :)
+```
+
+# Memory model
+
+The persistent semantics of Datahike work more like `git` and less like similar
+mutable databases such as SQLite or Datalevin. In particular you can always read
+and retain snapshots (copies) of the database for free, no matter what else is
+happening in the system. The current version is tested with memory and file
+storage, but hopefully many other backends will also work with the
+`native-image`.
+
+In principle this shared memory access should even work while having a JVM
+server, e.g. datahike-server, serving the same database. Note that all reads can
+happen in parallel, only the writers experience congestion around exclusive file
+locks here. This access pattern does not provide highest throughput, but is
+extremely flexible and easy to start with.
+
+## Forking and pulling
+
+Forking is easy, it is enough to copy the folder of the store (even if the
+database is currently being written to). The only thing you need to take care of
+is to copy the DB root first and place it into the target directory last. The root
+file is a konserve internal storage file with a UUID name like `0594e3b6-9635-5c99-8142-412accf3023b.ksv`
+(the actual UUID will match your database's `:id` configuration). Then you can use e.g.
+`rsync` (or `git`) to copy all other (immutable) files into your new folder. In
+the end you copy the root file in there as well, making sure that all files it
+is referencing are reachable. Note that this will ensure that you only copy new
+data each time.
+
+## Merging
+
+Now here comes the cool part. You do not need anything more for merging than
+Datalog itself. You can use a query like this to extract all new facts that are
+in `db1` but not in `db2` like this:
+
+```bash
+dthk query '[:find ?e ?a ?v ?t :in $ $2 :where [$ ?e ?a ?v ?t] (not [$2 ?e ?a ?v ?t])]' db:config1.edn db:config2.edn
+```
+
+Since we cannot update transaction metadata, we should filter out
+`:db/txInstant`s. We can also use a trick to add `:db/add` to each element in
+the results, yielding valid transactions that we can then feed into `db2`.
+
+
+```bash
+dthk query '[:find ?db-add ?e ?a ?v ?t :in $ $2 ?db-add :where [$ ?e ?a ?v ?t] [(not= :db/txInstant ?a)] (not [$2 ?e ?a ?v ?t])]' db:config1.edn db:config2.edn ":db/add" | dthk transact db:config2.edn
+```
+
+Note that this very simple strategy assumes that the entity ids that have been
+added to `db1` do not overlap with potentially new ones added to `db2`. You can
+encode conflict resolution strategies and id mappings with Datalog as well and
+we are exploring several such strategies at the moment. This strategy is fairly
+universal, as [CRDTs can be expressed in pure
+Datalog](https://speakerdeck.com/ept/data-structures-as-queries-expressing-crdts-using-datalog).
+While it is not the most efficient way to merge, we plan to provide fast paths
+for common patterns in Datalog. Feel free to contact us if you are interested in
+complex merging strategies or have related cool ideas.

package/doc/cljdoc.edn

@@ -0,0 +1,27 @@
+{:cljdoc.doc/tree [["Readme" {:file "README.md"}]
+                   ["Getting Started" {}
+                    ["Why Datalog?" {:file "doc/datalog-vs-sql.md"}]
+                    ["Configuration" {:file "doc/config.md"}]
+                    ["Storage Backends" {:file "doc/storage-backends.md"}]
+                    ["Schema" {:file "doc/schema.md"}]]
+                   ["Core Features" {}
+                    ["Time Variance" {:file "doc/time_variance.md"}]
+                    ["Versioning (Beta)" {:file "doc/versioning.md"}]
+                    ["Distributed Architecture" {:file "doc/distributed.md"}]
+                    ["Garbage Collection" {:file "doc/gc.md"}]
+                    ["Norms (Database Migrations)" {:file "doc/norms.md"}]]
+                   ["Platform Support" {}
+                    ["ClojureScript Support" {:file "doc/cljs-support.md"}]
+                    ["JavaScript API" {:file "doc/javascript-api.md"}]
+                    ["Babashka Pod" {:file "doc/bb-pod.md"}]
+                    ["Command Line Interface" {:file "doc/cli.md"}]
+                    ["libdatahike (C/C++)" {:file "doc/libdatahike.md"}]]
+                   ["Advanced" {}
+                    ["Entity Specs" {:file "doc/entity_spec.md"}]
+                    ["Logging and Error Handling" {:file "doc/logging_and_error_handling.md"}]
+                    ["Unstructured Input (Experimental)" {:file "doc/unstructured.md"}]
+                    ["Backend Development" {:file "doc/backend-development.md"}]
+                    ["Benchmarking" {:file "doc/benchmarking.md"}]]
+                   ["Reference" {}
+                    ["Differences to Datomic" {:file "doc/datomic_differences.md"}]
+                    ["Contributing" {:file "doc/contributing.md"}]]]}

package/doc/cljs-support.md

@@ -0,0 +1,133 @@
+# ClojureScript Support
+
+**This feature is in beta. Please try it out and provide feedback.**
+
+Datahike runs in ClojureScript on both Node.js and browser environments. The
+core query engine, pull API, and entity API work identically to Clojure. The
+main differences are in storage backends and async operation.
+
+## Async Storage Model
+
+ClojureScript environments often require async I/O (IndexedDB, network). To
+support this, Datahike's persistent-sorted-set indices use *partial
+continuation-passing style* (partial-cps): storage operations return channels
+instead of values, while in-memory operations remain synchronous.
+
+This is transparent when using `datahike.api` - async backends return channels
+from `create-database`, `connect`, `transact!`, etc.
+
+## Node.js
+
+Node.js supports the file backend via
+[konserve-node-filestore](https://github.com/replikativ/konserve):
+
+```clojure
+(ns my-app.core
+  (:require [datahike.api :as d]
+            [datahike.nodejs]  ;; Registers :file backend
+            [cljs.core.async :refer [<!] :refer-macros [go]]))
+
+(go
+  (let [config {:store {:backend :file
+                        :path "./my-database"
+                        :id #uuid "550e8400-e29b-41d4-a716-446655440000"}
+                :schema-flexibility :write}]
+    (<! (d/create-database config))
+    (let [conn (<! (d/connect config))]
+      (<! (d/transact! conn [{:name "Alice"}]))
+      (println (d/q '[:find ?n :where [?e :name ?n]] (d/db conn))))))
+```
+
+## Browser
+
+Browsers cannot use file storage directly. Datahike provides two browser-compatible backends:
+
+### Memory backend
+
+Fast but not persistent across page reloads:
+
+```clojure
+{:store {:backend :memory :id #uuid "550e8400-e29b-41d4-a716-446655440000"}}
+```
+
+### IndexedDB backend
+
+Persistent browser storage via
+[konserve-indexeddb](https://github.com/replikativ/konserve):
+
+```clojure
+{:store {:backend :indexeddb :id #uuid "550e8400-e29b-41d4-a716-446655440000"}}
+```
+
+IndexedDB is async, so all operations return channels.
+
+### TieredStore
+
+For optimal performance, use TieredStore which combines a fast memory frontend
+with a persistent IndexedDB backend:
+
+```clojure
+{:store {:backend :tiered
+         :id #uuid "550e8400-e29b-41d4-a716-446655440000"
+         :frontend-config {:backend :memory
+                          :id #uuid "550e8400-e29b-41d4-a716-446655440000"}
+         :backend-config {:backend :indexeddb
+                         :id #uuid "550e8400-e29b-41d4-a716-446655440000"}}}
+```
+
+Writes go to both stores (write-through). Reads come from memory. On
+reconnection, TieredStore syncs cached data from IndexedDB into memory before
+the sync handshake, so only keys newer than cached timestamps are transferred.
+
+## Distributed Setup with KabelWriter
+
+For browser applications that need a persistent backend, use the KabelWriter to
+connect to a JVM server. The server owns the database (using file storage) and
+streams updates to browser clients via WebSockets. See [Streaming writer
+(Kabel)](distributed.md#streaming-writer-kabel) for setup instructions.
+
+This architecture means:
+- Transactions are sent to the server via RPC
+- The server writes to its file store
+- Updates are replicated to the client's TieredStore via konserve-sync
+- Queries run locally against the synchronized in-memory store
+
+## JavaScript API
+
+For JavaScript/TypeScript applications, Datahike provides a Promise-based API.
+See [JavaScript API](javascript-api.md) for documentation.
+
+**Installation:**
+```bash
+npm install datahike@next
+```
+
+**Example:**
+```javascript
+const d = require('datahike');
+const crypto = require('crypto');
+
+const config = {
+  store: {
+    backend: ':memory',
+    id: crypto.randomUUID()
+  },
+  'schema-flexibility': ':read'  // Allow schemaless data (use kebab-case)
+};
+
+await d.createDatabase(config);
+const conn = await d.connect(config);
+await d.transact(conn, [{ name: 'Alice' }]);
+const db = await d.db(conn);  // db() is async for async backends
+const results = await d.q('[:find ?n :where [?e :name ?n]]', db);
+console.log(results);
+// => [['Alice']]
+```
+
+## Limitations
+
+- **History queries**: Work but may be slower due to async index traversal
+- **Large datasets**: Browser memory constraints apply; consider server-side
+  queries for large result sets
+- **Concurrent tabs**: Each tab has its own connection; use KabelWriter for
+  cross-tab consistency