@bullet./paraql 0.1.2 → 0.2.0

package/index.js

@@ -2,6 +2,7 @@ const { Buffer } = require("buffer")
 const ReadyResource = require("ready-resource")
 
 const binding = require("./binding")
+const constants = require("./lib/constants")
 const errors = require("./lib/errors")
 const ParaQLStatement = require("./lib/statement")
 const ParaVFS = require("./lib/vfs")
@@ -13,14 +14,15 @@ module.exports = exports = class ParaQL extends ReadyResource {
       key = null
     }
 
-    const { cacheSize = 1024, ...opts } = options
+    const { authorize = null, ...opts } = options
 
     super()
 
     this._handle = null
     this._vfs = new ParaVFS(this, store, key, opts)
-    this._cacheSize = cacheSize
-    this._cache = new Map()
+
+    this._authorize = authorize
+    this._initiator = null
   }
 
   get name() {
@@ -55,7 +57,11 @@ module.exports = exports = class ParaQL extends ReadyResource {
     await this._vfs.ready()
 
     try {
-      this._handle = await binding.open("paraql.db")
+      this._handle = await binding.open(
+        this,
+        this.name,
+        this._authorize ? this._on_authorize : null,
+      )
     } catch (err) {
       throw errors.from(err)
     }
@@ -64,11 +70,6 @@ module.exports = exports = class ParaQL extends ReadyResource {
   async _close() {
     if (this.opened) {
       try {
-        for (const stmt of this._cache.values()) {
-          await stmt.finalize()
-        }
-
-        this._cache.clear()
         await binding.close(this._handle)
       } catch (err) {
         throw errors.from(err)
@@ -78,6 +79,34 @@ module.exports = exports = class ParaQL extends ReadyResource {
     await this._vfs.close()
   }
 
+  async _on_authorize(action, param1, param2, param3, param4, callback) {
+    if (!this._initiator || this.key.equals(this._initiator)) return callback(true)
+
+    try {
+      const result = await this._authorize(
+        this._initiator,
+        action,
+        param1,
+        param2,
+        param3,
+        param4,
+      )
+      callback(!!result)
+    } catch {
+      callback(false)
+    }
+  }
+
+  static async deserialize(store, data, options) {
+    const db = new ParaQL(store, null, options)
+
+    await db.ready()
+
+    await db._vfs.deserialize(data)
+
+    return db
+  }
+
   async addWriter(key) {
     await this._vfs.addWriter(key)
   }
@@ -98,11 +127,28 @@ module.exports = exports = class ParaQL extends ReadyResource {
     return this._vfs.info()
   }
 
-  async _exec(sql) {
+  async serialize() {
+    if (!this.opened) await this.ready()
+
+    if (this.closed) throw errors.ALREADY_CLOSED()
+
+    try {
+      const data = await binding.serialize(this._handle)
+
+      return Buffer.from(data)
+    } catch (err) {
+      throw errors.from(err)
+    }
+  }
+
+  async _exec(sql, initiator) {
     try {
+      this._initiator = initiator
       await binding.exec(this._handle, sql)
     } catch (err) {
       throw errors.from(err)
+    } finally {
+      this._initiator = null
     }
   }
 
@@ -114,35 +160,27 @@ module.exports = exports = class ParaQL extends ReadyResource {
     await this._vfs.exec(sql)
   }
 
-  async prepare(sql) {
+  async prepare(sql, initiator = this.local) {
     if (!this.opened) await this.ready()
 
     if (this.closed) throw errors.ALREADY_CLOSED()
 
-    let stmt = this._cache.get(sql)
-
-    if (stmt) {
-      this._cache.delete(sql)
-      this._cache.set(sql, stmt)
-      return stmt
-    }
-
-    stmt = new ParaQLStatement(this, sql)
-
-    await stmt.ready()
+    this._initiator = initiator
 
-    this._cache.set(sql, stmt)
+    try {
+      const stmt = new ParaQLStatement(this, sql)
 
-    if (this._cache.size > this._cacheSize) {
-      const oldest = this._cache.values().next().value
+      await stmt.ready()
 
-      await oldest.finalize()
+      return stmt
+    } finally {
+      this._initiator = null
     }
-
-    return stmt
   }
 }
 
+exports.AUTHORIZE_ACTION = constants.AUTHORIZE_ACTION
+
 function isObject(o) {
   return typeof o === "object" && o && !Buffer.isBuffer(o)
 }

package/lib/codecs.js

@@ -9,6 +9,10 @@ const op = {
     c.uint8.preencode(state, operation.type)
 
     switch (operation.type) {
+      case OPERATION.DESERIALIZE: {
+        c.buffer.preencode(state, operation.data)
+        break
+      }
       case OPERATION.WRITER_ADD:
       case OPERATION.WRITER_DEL: {
         c.fixed32.preencode(state, operation.key)
@@ -24,22 +28,16 @@ const op = {
         c.any.preencode(state, operation.positional)
         break
       }
-      case OPERATION.STMT: {
-        c.string.preencode(state, operation.sql)
-      }
-      case OPERATION.BATCH: {
-        c.any.preencode(state, operation.named)
-        c.any.preencode(state, operation.positional)
-      }
-      case OPERATION.FLUSH: {
-        break
-      }
     }
   },
   encode(state, operation) {
     c.uint8.encode(state, operation.type)
 
     switch (operation.type) {
+      case OPERATION.DESERIALIZE: {
+        c.buffer.encode(state, operation.data)
+        break
+      }
       case OPERATION.WRITER_ADD:
       case OPERATION.WRITER_DEL: {
         c.fixed32.encode(state, operation.key)
@@ -55,16 +53,6 @@ const op = {
         c.any.encode(state, operation.positional)
         break
       }
-      case OPERATION.STMT: {
-        c.string.encode(state, operation.sql)
-      }
-      case OPERATION.BATCH: {
-        c.any.encode(state, operation.named)
-        c.any.encode(state, operation.positional)
-      }
-      case OPERATION.FLUSH: {
-        break
-      }
     }
   },
   decode(state) {
@@ -73,6 +61,10 @@ const op = {
     const operation = { type }
 
     switch (type) {
+      case OPERATION.DESERIALIZE: {
+        operation.data = c.buffer.decode(state)
+        break
+      }
       case OPERATION.WRITER_ADD:
       case OPERATION.WRITER_DEL: {
         operation.key = c.fixed32.decode(state)
@@ -88,16 +80,6 @@ const op = {
         operation.positional = c.any.decode(state)
         break
       }
-      case OPERATION.STMT: {
-        operation.sql = c.string.decode(state)
-      }
-      case OPERATION.BATCH: {
-        operation.named = c.any.decode(state)
-        operation.positional = c.any.decode(state)
-      }
-      case OPERATION.FLUSH: {
-        break
-      }
     }
 
     return operation

package/lib/constants.js

@@ -1,10 +1,43 @@
 module.exports.PAGE_SIZE = 4096
 module.exports.OPERATION = {
+  DESERIALIZE: 0,
   WRITER_ADD: 1,
   WRITER_DEL: 2,
   EXEC: 8,
   RUN: 9,
-  STMT: 10,
-  BATCH: 11,
-  FLUSH: 12,
+}
+module.exports.AUTHORIZE_ACTION = {
+  CREATE_INDEX: 1,
+  CREATE_TABLE: 2,
+  CREATE_TEMP_INDEX: 3,
+  CREATE_TEMP_TABLE: 4,
+  CREATE_TEMP_TRIGGER: 5,
+  CREATE_TEMP_VIEW: 6,
+  CREATE_TRIGGER: 7,
+  CREATE_VIEW: 8,
+  DELETE: 9,
+  DROP_INDEX: 10,
+  DROP_TABLE: 11,
+  DROP_TEMP_INDEX: 12,
+  DROP_TEMP_TABLE: 13,
+  DROP_TEMP_TRIGGER: 14,
+  DROP_TEMP_VIEW: 15,
+  DROP_TRIGGER: 16,
+  DROP_VIEW: 17,
+  INSERT: 18,
+  PRAGMA: 19,
+  READ: 20,
+  SELECT: 21,
+  TRANSACTION: 22,
+  UPDATE: 23,
+  ATTACH: 24,
+  DETACH: 25,
+  ALTER_TABLE: 26,
+  REINDEX: 27,
+  ANALYZE: 28,
+  CREATE_VTABLE: 29,
+  DROP_VTABLE: 30,
+  FUNCTION: 31,
+  SAVEPOINT: 32,
+  RECURSIVE: 33,
 }

package/lib/statement.d.ts

@@ -5,10 +5,6 @@ interface ParaQLStatement extends ReadyResource {
   readonly sourceSQL: string
   readonly batching: boolean
 
-  batch(on: boolean): void
-
-  flush(): Promise<ParaQLStatement.FlushResult>
-
   finalize(): Promise<void>
 
   all<T extends ParaQLStatement.Row = ParaQLStatement.Row>(
@@ -21,6 +17,8 @@ interface ParaQLStatement extends ReadyResource {
 
   run(...params: ParaQLStatement.Parameters): Promise<ParaQLStatement.RunResult | undefined>
 
+  batch(...params: ParaQLStatement.Parameters[]): Promise<void>
+
   iterate<T extends ParaQLStatement.Row = ParaQLStatement.Row>(
     ...params: ParaQLStatement.Parameters
   ): AsyncIterableIterator<T>
@@ -44,12 +42,6 @@ declare namespace ParaQLStatement {
     changes: number
     lastInsertRowid: number
   }
-
-  export interface FlushResult {
-    changes: number
-    lastInsertRowid: number
-    errors: number
-  }
 }
 
 export = ParaQLStatement

package/lib/statement.js

@@ -11,7 +11,6 @@ module.exports = class ParaQLStatement extends ReadyResource {
     this._db = db
     this._sourceSQL = sql
     this._handle = null
-    this._queue = null
   }
 
   async _open() {
@@ -27,8 +26,6 @@ module.exports = class ParaQLStatement extends ReadyResource {
       try {
         await binding.finalize(this._handle)
         this._handle = null
-        this._queue = null
-        this._db._cache.delete(this._sourceSQL)
       } catch (err) {
         throw errors.from(err)
       }
@@ -39,28 +36,10 @@ module.exports = class ParaQLStatement extends ReadyResource {
     return this._sourceSQL
   }
 
-  get batching() {
-    return !!this._queue
-  }
-
-  batch(on) {
-    if (on === false) {
-      this._queue = null
-    } else {
-      this._queue = this._queue ?? []
-    }
-  }
-
   async finalize() {
     await this.close()
   }
 
-  async flush() {
-    const queue = this._queue
-    this._queue = []
-    return this._db._vfs.flush(this._sourceSQL, queue)
-  }
-
   async _bind(named, positional) {
     for (const key in named) {
       if (Array.isArray(named[key])) named[key] = JSON.stringify(named[key])
@@ -117,12 +96,19 @@ module.exports = class ParaQLStatement extends ReadyResource {
 
     const [named, positional] = splitParameters(params)
 
-    if (this._queue) {
-      this._queue.push({ named, positional })
-      return null
-    } else {
-      return this._db._vfs.run(this._sourceSQL, named, positional)
-    }
+    return this._db._vfs.run(this._sourceSQL, named, positional)
+  }
+
+  async batch(...params) {
+    if (this.closed) throw errors.ALREADY_CLOSED()
+
+    const batch = params.map((p) => {
+      const [named, positional] = splitParameters(p)
+
+      return { sql: this._sourceSQL, named, positional }
+    })
+
+    return this._db._vfs.batch(batch)
   }
 
   async *iterate(...params) {

package/lib/vfs.js

@@ -113,15 +113,22 @@ module.exports = class ParaVFS extends ReadyResource {
   }
 
   async _apply(nodes, view, host) {
-    let stmt = null
-    let changes = 0
-    let lastInsertRowid = 0
-    let errors = 0
-
     for (const node of nodes) {
       const op = Operation.decode(this.compressed ? await inflate(node.value) : node.value)
 
       switch (op.type) {
+        case OPERATION.DESERIALIZE: {
+          this._write(view)
+          try {
+            await this._xWrite(this.name, op.data, 0, () => {})
+            await this._flush(this.name)
+            this._interactive?.resolve()
+          } catch (err) {
+            this._interactive?.reject(err)
+          }
+          this._write(null)
+          break
+        }
         case OPERATION.WRITER_ADD: {
           const batch = view.write()
           host.addWriter(op.key)
@@ -137,7 +144,7 @@ module.exports = class ParaVFS extends ReadyResource {
         case OPERATION.EXEC: {
           this._write(view)
           try {
-            await this._db._exec(op.sql)
+            await this._db._exec(op.sql, node.key)
             this._interactive?.resolve()
           } catch (err) {
             this._interactive?.reject(err)
@@ -147,39 +154,19 @@ module.exports = class ParaVFS extends ReadyResource {
         }
         case OPERATION.RUN: {
           this._write(view)
+          let stmt = null
           try {
-            stmt = await this._db.prepare(op.sql)
+            stmt = await this._db.prepare(op.sql, node.key)
             const result = await stmt._run(op.named, op.positional)
             this._interactive?.resolve(result)
           } catch (err) {
             this._interactive?.reject(err)
+          } finally {
+            await stmt?.finalize()
           }
           this._write(null)
           break
         }
-        case OPERATION.STMT: {
-          try {
-            stmt = await this._db.prepare(op.sql)
-          } catch (err) {
-            throw errors.INTERNAL(`Received invalid statement: ${op.sql}`)
-          }
-        }
-        case OPERATION.BATCH: {
-          this._write(view)
-          try {
-            const result = await stmt._run(op.named, op.positional)
-            changes += result.changes
-            lastInsertRowid = result.lastInsertRowid
-          } catch (err) {
-            errors++
-          }
-          this._write(null)
-          break
-        }
-        case OPERATION.FLUSH: {
-          this._interactive?.resolve({ changes, lastInsertRowid, errors })
-          break
-        }
       }
     }
   }
@@ -196,8 +183,8 @@ module.exports = class ParaVFS extends ReadyResource {
     await this.bee.append(
       this.compressed
         ? await deflate(Operation.encode({ type: OPERATION.WRITER_ADD, key }), {
-          level: this._compressionLevel,
-        })
+            level: this._compressionLevel,
+          })
         : Operation.encode({ type: OPERATION.WRITER_ADD, key }),
     )
   }
@@ -206,8 +193,8 @@ module.exports = class ParaVFS extends ReadyResource {
     await this.bee.append(
       this.compressed
         ? await deflate(Operation.encode({ type: OPERATION.WRITER_DEL, key }), {
-          level: this._compressionLevel,
-        })
+            level: this._compressionLevel,
+          })
         : Operation.encode({ type: OPERATION.WRITER_DEL, key }),
     )
   }
@@ -249,17 +236,17 @@ module.exports = class ParaVFS extends ReadyResource {
     return { database, temporary, total: database + temporary }
   }
 
-  async exec(sql) {
+  async deserialize(data) {
     this._interactive = new Deferred()
     try {
-      const promise = this.bee.append(
+      const promise = await this.bee.append(
         this.compressed
-          ? await deflate(Operation.encode({ type: OPERATION.EXEC, sql }), {
-            level: this._compressionLevel,
-          })
-          : Operation.encode({ type: OPERATION.EXEC, sql }),
+          ? await deflate(Operation.encode({ type: OPERATION.DESERIALIZE, data }), {
+              level: this._compressionLevel,
+            })
+          : Operation.encode({ type: OPERATION.DESERIALIZE, data }),
       )
-      await this._interactive.promise
+      return await this._interactive.promise
         .then(() => promise)
         .catch(async (err) => {
           await promise
@@ -270,21 +257,18 @@ module.exports = class ParaVFS extends ReadyResource {
     }
   }
 
-  async run(sql, named, positional) {
+  async exec(sql) {
     this._interactive = new Deferred()
     try {
       const promise = this.bee.append(
         this.compressed
-          ? await deflate(Operation.encode({ type: OPERATION.RUN, sql, named, positional }), {
-            level: this._compressionLevel,
-          })
-          : Operation.encode({ type: OPERATION.RUN, sql, named, positional }),
+          ? await deflate(Operation.encode({ type: OPERATION.EXEC, sql }), {
+              level: this._compressionLevel,
+            })
+          : Operation.encode({ type: OPERATION.EXEC, sql }),
       )
-      return await this._interactive.promise
-        .then(async (result) => {
-          await promise
-          return result
-        })
+      await this._interactive.promise
+        .then(() => promise)
         .catch(async (err) => {
           await promise
           throw err
@@ -294,23 +278,15 @@ module.exports = class ParaVFS extends ReadyResource {
     }
   }
 
-  async flush(sql, operations) {
+  async run(sql, named, positional) {
     this._interactive = new Deferred()
     try {
       const promise = this.bee.append(
         this.compressed
-          ? await Promise.all(
-            [
-              { type: OPERATION.STMT, sql },
-              ...operations.map((op) => ({ type: OPERATION.BATCH, ...op })),
-              { type: OPERATION.FLUSH },
-            ].map((op) => deflate(Operation.encode(op), { level: this._compressionLevel })),
-          )
-          : [
-            { type: OPERATION.STMT, sql },
-            ...operations.map((op) => ({ type: OPERATION.BATCH, ...op })),
-            { type: OPERATION.FLUSH },
-          ].map(Operation.encode),
+          ? await deflate(Operation.encode({ type: OPERATION.RUN, sql, named, positional }), {
+              level: this._compressionLevel,
+            })
+          : Operation.encode({ type: OPERATION.RUN, sql, named, positional }),
       )
       return await this._interactive.promise
         .then(async (result) => {
@@ -326,6 +302,20 @@ module.exports = class ParaVFS extends ReadyResource {
     }
   }
 
+  async batch(batch) {
+    await this.bee.append(
+      this.compressed
+        ? await Promise.all(
+            batch.map((op) =>
+              deflate(Operation.encode({ type: OPERATION.RUN, ...op }), {
+                level: this._compressionLevel,
+              }),
+            ),
+          )
+        : batch.map((op) => Operation.encode({ type: OPERATION.RUN, ...op })),
+    )
+  }
+
   async _get(name, index) {
     const tmp = name !== this.name
     const key = PageKey.encode([name, index])

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bullet./paraql",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Parallel multi-writer relational database with SQL syntax. Native support for encryption, compression, and vector search.",
   "author": {
     "name": "Tomas Ravinskas",