@bullet./paraql 0.1.2 → 0.3.0

package/index.js

@@ -2,6 +2,7 @@ const { Buffer } = require("buffer")
 const ReadyResource = require("ready-resource")
 
 const binding = require("./binding")
+const constants = require("./lib/constants")
 const errors = require("./lib/errors")
 const ParaQLStatement = require("./lib/statement")
 const ParaVFS = require("./lib/vfs")
@@ -13,14 +14,15 @@ module.exports = exports = class ParaQL extends ReadyResource {
       key = null
     }
 
-    const { cacheSize = 1024, ...opts } = options
+    const { authorize = null, ...opts } = options
 
     super()
 
     this._handle = null
     this._vfs = new ParaVFS(this, store, key, opts)
-    this._cacheSize = cacheSize
-    this._cache = new Map()
+
+    this._authorize = authorize
+    this._initiator = null
   }
 
   get name() {
@@ -55,7 +57,11 @@ module.exports = exports = class ParaQL extends ReadyResource {
     await this._vfs.ready()
 
     try {
-      this._handle = await binding.open("paraql.db")
+      this._handle = await binding.open(
+        this,
+        this.name,
+        this._authorize ? this._on_authorize : null,
+      )
     } catch (err) {
       throw errors.from(err)
     }
@@ -64,11 +70,6 @@ module.exports = exports = class ParaQL extends ReadyResource {
   async _close() {
     if (this.opened) {
       try {
-        for (const stmt of this._cache.values()) {
-          await stmt.finalize()
-        }
-
-        this._cache.clear()
         await binding.close(this._handle)
       } catch (err) {
         throw errors.from(err)
@@ -78,6 +79,34 @@ module.exports = exports = class ParaQL extends ReadyResource {
     await this._vfs.close()
   }
 
+  async _on_authorize(action, param1, param2, param3, param4, callback) {
+    if (!this._initiator || this.key.equals(this._initiator)) return callback(true)
+
+    try {
+      const result = await this._authorize(
+        this._initiator,
+        action,
+        param1,
+        param2,
+        param3,
+        param4,
+      )
+      callback(!!result)
+    } catch {
+      callback(false)
+    }
+  }
+
+  static async deserialize(store, data, options) {
+    const db = new ParaQL(store, null, options)
+
+    await db.ready()
+
+    await db._vfs.deserialize(data)
+
+    return db
+  }
+
   async addWriter(key) {
     await this._vfs.addWriter(key)
   }
@@ -98,11 +127,28 @@ module.exports = exports = class ParaQL extends ReadyResource {
     return this._vfs.info()
   }
 
-  async _exec(sql) {
+  async serialize() {
+    if (!this.opened) await this.ready()
+
+    if (this.closed) throw errors.ALREADY_CLOSED()
+
+    try {
+      const data = await binding.serialize(this._handle)
+
+      return Buffer.from(data)
+    } catch (err) {
+      throw errors.from(err)
+    }
+  }
+
+  async _exec(sql, initiator) {
     try {
+      this._initiator = initiator
       await binding.exec(this._handle, sql)
     } catch (err) {
       throw errors.from(err)
+    } finally {
+      this._initiator = null
     }
   }
 
@@ -114,35 +160,27 @@ module.exports = exports = class ParaQL extends ReadyResource {
     await this._vfs.exec(sql)
   }
 
-  async prepare(sql) {
+  async prepare(sql, initiator = this.local) {
     if (!this.opened) await this.ready()
 
     if (this.closed) throw errors.ALREADY_CLOSED()
 
-    let stmt = this._cache.get(sql)
-
-    if (stmt) {
-      this._cache.delete(sql)
-      this._cache.set(sql, stmt)
-      return stmt
-    }
-
-    stmt = new ParaQLStatement(this, sql)
-
-    await stmt.ready()
+    this._initiator = initiator
 
-    this._cache.set(sql, stmt)
+    try {
+      const stmt = new ParaQLStatement(this, sql)
 
-    if (this._cache.size > this._cacheSize) {
-      const oldest = this._cache.values().next().value
+      await stmt.ready()
 
-      await oldest.finalize()
+      return stmt
+    } finally {
+      this._initiator = null
     }
-
-    return stmt
   }
 }
 
+exports.AUTHORIZE_ACTION = constants.AUTHORIZE_ACTION
+
 function isObject(o) {
   return typeof o === "object" && o && !Buffer.isBuffer(o)
 }

package/lib/codecs.js

@@ -1,6 +1,5 @@
 const c = require("compact-encoding")
 const IndexEncoder = require("index-encoder")
-const zlib = require("zlib")
 
 const { OPERATION, PAGE_SIZE } = require("./constants")
 
@@ -9,6 +8,10 @@ const op = {
     c.uint8.preencode(state, operation.type)
 
     switch (operation.type) {
+      case OPERATION.DESERIALIZE: {
+        c.buffer.preencode(state, operation.data)
+        break
+      }
       case OPERATION.WRITER_ADD:
       case OPERATION.WRITER_DEL: {
         c.fixed32.preencode(state, operation.key)
@@ -24,22 +27,16 @@ const op = {
         c.any.preencode(state, operation.positional)
         break
       }
-      case OPERATION.STMT: {
-        c.string.preencode(state, operation.sql)
-      }
-      case OPERATION.BATCH: {
-        c.any.preencode(state, operation.named)
-        c.any.preencode(state, operation.positional)
-      }
-      case OPERATION.FLUSH: {
-        break
-      }
     }
   },
   encode(state, operation) {
     c.uint8.encode(state, operation.type)
 
     switch (operation.type) {
+      case OPERATION.DESERIALIZE: {
+        c.buffer.encode(state, operation.data)
+        break
+      }
       case OPERATION.WRITER_ADD:
       case OPERATION.WRITER_DEL: {
         c.fixed32.encode(state, operation.key)
@@ -55,16 +52,6 @@ const op = {
         c.any.encode(state, operation.positional)
         break
       }
-      case OPERATION.STMT: {
-        c.string.encode(state, operation.sql)
-      }
-      case OPERATION.BATCH: {
-        c.any.encode(state, operation.named)
-        c.any.encode(state, operation.positional)
-      }
-      case OPERATION.FLUSH: {
-        break
-      }
     }
   },
   decode(state) {
@@ -73,6 +60,10 @@ const op = {
     const operation = { type }
 
     switch (type) {
+      case OPERATION.DESERIALIZE: {
+        operation.data = c.buffer.decode(state)
+        break
+      }
       case OPERATION.WRITER_ADD:
       case OPERATION.WRITER_DEL: {
         operation.key = c.fixed32.decode(state)
@@ -88,16 +79,6 @@ const op = {
         operation.positional = c.any.decode(state)
         break
       }
-      case OPERATION.STMT: {
-        operation.sql = c.string.decode(state)
-      }
-      case OPERATION.BATCH: {
-        operation.named = c.any.decode(state)
-        operation.positional = c.any.decode(state)
-      }
-      case OPERATION.FLUSH: {
-        break
-      }
     }
 
     return operation
@@ -114,25 +95,3 @@ module.exports.Operation = {
     return c.decode(op, operation)
   },
 }
-
-module.exports.deflate = async function (buffer, options) {
-  return new Promise((resolve, reject) =>
-    zlib.deflate(buffer, { chunkSize: PAGE_SIZE, ...options }, (err, result) => {
-      if (err) {
-        return reject(result)
-      }
-      resolve(result)
-    }),
-  )
-}
-
-module.exports.inflate = async function (buffer, options) {
-  return new Promise((resolve, reject) =>
-    zlib.inflate(buffer, { chunkSize: PAGE_SIZE, ...options }, (err, result) => {
-      if (err) {
-        return reject(result)
-      }
-      resolve(result)
-    }),
-  )
-}

package/lib/constants.js

@@ -1,10 +1,43 @@
 module.exports.PAGE_SIZE = 4096
 module.exports.OPERATION = {
+  DESERIALIZE: 0,
   WRITER_ADD: 1,
   WRITER_DEL: 2,
   EXEC: 8,
   RUN: 9,
-  STMT: 10,
-  BATCH: 11,
-  FLUSH: 12,
+}
+module.exports.AUTHORIZE_ACTION = {
+  CREATE_INDEX: 1,
+  CREATE_TABLE: 2,
+  CREATE_TEMP_INDEX: 3,
+  CREATE_TEMP_TABLE: 4,
+  CREATE_TEMP_TRIGGER: 5,
+  CREATE_TEMP_VIEW: 6,
+  CREATE_TRIGGER: 7,
+  CREATE_VIEW: 8,
+  DELETE: 9,
+  DROP_INDEX: 10,
+  DROP_TABLE: 11,
+  DROP_TEMP_INDEX: 12,
+  DROP_TEMP_TABLE: 13,
+  DROP_TEMP_TRIGGER: 14,
+  DROP_TEMP_VIEW: 15,
+  DROP_TRIGGER: 16,
+  DROP_VIEW: 17,
+  INSERT: 18,
+  PRAGMA: 19,
+  READ: 20,
+  SELECT: 21,
+  TRANSACTION: 22,
+  UPDATE: 23,
+  ATTACH: 24,
+  DETACH: 25,
+  ALTER_TABLE: 26,
+  REINDEX: 27,
+  ANALYZE: 28,
+  CREATE_VTABLE: 29,
+  DROP_VTABLE: 30,
+  FUNCTION: 31,
+  SAVEPOINT: 32,
+  RECURSIVE: 33,
 }

package/lib/statement.d.ts

@@ -3,11 +3,6 @@ import Buffer from "bare-buffer"
 
 interface ParaQLStatement extends ReadyResource {
   readonly sourceSQL: string
-  readonly batching: boolean
-
-  batch(on: boolean): void
-
-  flush(): Promise<ParaQLStatement.FlushResult>
 
   finalize(): Promise<void>
 
@@ -44,12 +39,6 @@ declare namespace ParaQLStatement {
     changes: number
     lastInsertRowid: number
   }
-
-  export interface FlushResult {
-    changes: number
-    lastInsertRowid: number
-    errors: number
-  }
 }
 
 export = ParaQLStatement

package/lib/statement.js

@@ -11,7 +11,6 @@ module.exports = class ParaQLStatement extends ReadyResource {
     this._db = db
     this._sourceSQL = sql
     this._handle = null
-    this._queue = null
   }
 
   async _open() {
@@ -27,8 +26,6 @@ module.exports = class ParaQLStatement extends ReadyResource {
       try {
         await binding.finalize(this._handle)
         this._handle = null
-        this._queue = null
-        this._db._cache.delete(this._sourceSQL)
       } catch (err) {
         throw errors.from(err)
       }
@@ -39,28 +36,10 @@ module.exports = class ParaQLStatement extends ReadyResource {
     return this._sourceSQL
   }
 
-  get batching() {
-    return !!this._queue
-  }
-
-  batch(on) {
-    if (on === false) {
-      this._queue = null
-    } else {
-      this._queue = this._queue ?? []
-    }
-  }
-
   async finalize() {
     await this.close()
   }
 
-  async flush() {
-    const queue = this._queue
-    this._queue = []
-    return this._db._vfs.flush(this._sourceSQL, queue)
-  }
-
   async _bind(named, positional) {
     for (const key in named) {
       if (Array.isArray(named[key])) named[key] = JSON.stringify(named[key])
@@ -117,12 +96,7 @@ module.exports = class ParaQLStatement extends ReadyResource {
 
     const [named, positional] = splitParameters(params)
 
-    if (this._queue) {
-      this._queue.push({ named, positional })
-      return null
-    } else {
-      return this._db._vfs.run(this._sourceSQL, named, positional)
-    }
+    return this._db._vfs.run(this._sourceSQL, named, positional)
   }
 
   async *iterate(...params) {

package/lib/vfs.js

@@ -6,7 +6,7 @@ const ReadyResource = require("ready-resource")
 const RocksDB = require("rocksdb-native")
 
 const binding = require("../binding")
-const { PageKey, Operation, deflate, inflate } = require("./codecs")
+const { PageKey, Operation } = require("./codecs")
 const { OPERATION, PAGE_SIZE } = require("./constants")
 const Deferred = require("./deferred")
 const Encryption = require("./encryption")
@@ -19,8 +19,6 @@ module.exports = class ParaVFS extends ReadyResource {
       keyPair = null,
       encrypted = false,
       encryptionKey = null,
-      compressed = false,
-      compressionLevel = 6,
     } = options
 
     super()
@@ -30,8 +28,6 @@ module.exports = class ParaVFS extends ReadyResource {
     this._name = name
 
     this._encryption = null
-    this._compressed = compressed
-    this._compressionLevel = compressionLevel
 
     this.store = store.namespace(name)
     this.bee = new Autobee(this.store, key, {
@@ -76,10 +72,6 @@ module.exports = class ParaVFS extends ReadyResource {
     return !!this._encryption
   }
 
-  get compressed() {
-    return !!this._compressed
-  }
-
   async _open() {
     await this.bee.ready()
 
@@ -113,15 +105,22 @@ module.exports = class ParaVFS extends ReadyResource {
   }
 
   async _apply(nodes, view, host) {
-    let stmt = null
-    let changes = 0
-    let lastInsertRowid = 0
-    let errors = 0
-
     for (const node of nodes) {
-      const op = Operation.decode(this.compressed ? await inflate(node.value) : node.value)
+      const op = Operation.decode(node.value)
 
       switch (op.type) {
+        case OPERATION.DESERIALIZE: {
+          this._write(view)
+          try {
+            await this._xWrite(this.name, op.data, 0, () => {})
+            await this._flush(this.name)
+            this._interactive?.resolve()
+          } catch (err) {
+            this._interactive?.reject(err)
+          }
+          this._write(null)
+          break
+        }
         case OPERATION.WRITER_ADD: {
           const batch = view.write()
           host.addWriter(op.key)
@@ -137,7 +136,7 @@ module.exports = class ParaVFS extends ReadyResource {
         case OPERATION.EXEC: {
           this._write(view)
           try {
-            await this._db._exec(op.sql)
+            await this._db._exec(op.sql, node.key)
             this._interactive?.resolve()
           } catch (err) {
             this._interactive?.reject(err)
@@ -147,39 +146,19 @@ module.exports = class ParaVFS extends ReadyResource {
         }
         case OPERATION.RUN: {
           this._write(view)
+          let stmt = null
           try {
-            stmt = await this._db.prepare(op.sql)
+            stmt = await this._db.prepare(op.sql, node.key)
             const result = await stmt._run(op.named, op.positional)
             this._interactive?.resolve(result)
           } catch (err) {
             this._interactive?.reject(err)
+          } finally {
+            await stmt?.finalize()
           }
           this._write(null)
           break
         }
-        case OPERATION.STMT: {
-          try {
-            stmt = await this._db.prepare(op.sql)
-          } catch (err) {
-            throw errors.INTERNAL(`Received invalid statement: ${op.sql}`)
-          }
-        }
-        case OPERATION.BATCH: {
-          this._write(view)
-          try {
-            const result = await stmt._run(op.named, op.positional)
-            changes += result.changes
-            lastInsertRowid = result.lastInsertRowid
-          } catch (err) {
-            errors++
-          }
-          this._write(null)
-          break
-        }
-        case OPERATION.FLUSH: {
-          this._interactive?.resolve({ changes, lastInsertRowid, errors })
-          break
-        }
       }
     }
   }
@@ -193,23 +172,11 @@ module.exports = class ParaVFS extends ReadyResource {
   }
 
   async addWriter(key) {
-    await this.bee.append(
-      this.compressed
-        ? await deflate(Operation.encode({ type: OPERATION.WRITER_ADD, key }), {
-          level: this._compressionLevel,
-        })
-        : Operation.encode({ type: OPERATION.WRITER_ADD, key }),
-    )
+    await this.bee.append(Operation.encode({ type: OPERATION.WRITER_ADD, key }))
   }
 
   async removeWriter(key) {
-    await this.bee.append(
-      this.compressed
-        ? await deflate(Operation.encode({ type: OPERATION.WRITER_DEL, key }), {
-          level: this._compressionLevel,
-        })
-        : Operation.encode({ type: OPERATION.WRITER_DEL, key }),
-    )
+    await this.bee.append(Operation.encode({ type: OPERATION.WRITER_DEL, key }))
   }
 
   replicate(isInitiator) {
@@ -249,17 +216,13 @@ module.exports = class ParaVFS extends ReadyResource {
     return { database, temporary, total: database + temporary }
   }
 
-  async exec(sql) {
+  async deserialize(data) {
     this._interactive = new Deferred()
     try {
-      const promise = this.bee.append(
-        this.compressed
-          ? await deflate(Operation.encode({ type: OPERATION.EXEC, sql }), {
-            level: this._compressionLevel,
-          })
-          : Operation.encode({ type: OPERATION.EXEC, sql }),
+      const promise = await this.bee.append(
+        Operation.encode({ type: OPERATION.DESERIALIZE, data }),
       )
-      await this._interactive.promise
+      return await this._interactive.promise
         .then(() => promise)
         .catch(async (err) => {
           await promise
@@ -270,21 +233,12 @@ module.exports = class ParaVFS extends ReadyResource {
     }
   }
 
-  async run(sql, named, positional) {
+  async exec(sql) {
     this._interactive = new Deferred()
     try {
-      const promise = this.bee.append(
-        this.compressed
-          ? await deflate(Operation.encode({ type: OPERATION.RUN, sql, named, positional }), {
-            level: this._compressionLevel,
-          })
-          : Operation.encode({ type: OPERATION.RUN, sql, named, positional }),
-      )
-      return await this._interactive.promise
-        .then(async (result) => {
-          await promise
-          return result
-        })
+      const promise = this.bee.append(Operation.encode({ type: OPERATION.EXEC, sql }))
+      await this._interactive.promise
+        .then(() => promise)
         .catch(async (err) => {
           await promise
           throw err
@@ -294,23 +248,11 @@ module.exports = class ParaVFS extends ReadyResource {
     }
   }
 
-  async flush(sql, operations) {
+  async run(sql, named, positional) {
     this._interactive = new Deferred()
     try {
       const promise = this.bee.append(
-        this.compressed
-          ? await Promise.all(
-            [
-              { type: OPERATION.STMT, sql },
-              ...operations.map((op) => ({ type: OPERATION.BATCH, ...op })),
-              { type: OPERATION.FLUSH },
-            ].map((op) => deflate(Operation.encode(op), { level: this._compressionLevel })),
-          )
-          : [
-            { type: OPERATION.STMT, sql },
-            ...operations.map((op) => ({ type: OPERATION.BATCH, ...op })),
-            { type: OPERATION.FLUSH },
-          ].map(Operation.encode),
+        Operation.encode({ type: OPERATION.RUN, sql, named, positional }),
       )
       return await this._interactive.promise
         .then(async (result) => {
@@ -343,10 +285,6 @@ module.exports = class ParaVFS extends ReadyResource {
       value = entry?.value ?? null
     }
 
-    if (value && this.compressed) {
-      value = await inflate(value)
-    }
-
     return value
   }
 
@@ -374,10 +312,6 @@ module.exports = class ParaVFS extends ReadyResource {
 
     this._files.set(name, batch)
 
-    if (this.compressed) {
-      value = await deflate(value, { level: this._compressionLevel })
-    }
-
     if (tmp && this.encrypted) {
       value = this._encryption.encrypt(value, name, index)
     }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@bullet./paraql",
-  "version": "0.1.2",
-  "description": "Parallel multi-writer relational database with SQL syntax. Native support for encryption, compression, and vector search.",
+  "version": "0.3.0",
+  "description": "Fast, parallel, multi-writer relational database with SQL syntax. Native support for encryption, compression, and vector search.",
   "author": {
     "name": "Tomas Ravinskas",
     "email": "tomas@tomasrav.me",
@@ -49,10 +49,6 @@
     "path": {
       "bare": "bare-path",
       "default": "path"
-    },
-    "zlib": {
-      "bare": "bare-zlib",
-      "default": "zlib"
     }
   },
   "homepage": "https://github.com/getbullet-app/paraql",
@@ -72,11 +68,10 @@
     "vector-search"
   ],
   "dependencies": {
-    "autobee": "^1.0.10",
+    "autobee": "^2.7.2",
     "bare-buffer": "^3.6.2",
     "bare-fs": "^4.7.4",
     "bare-path": "^3.1.1",
-    "bare-zlib": "^1.4.1",
     "compact-encoding": "^3.3.0",
     "index-encoder": "^3.5.0",
     "ready-resource": "^1.2.0",