@basictech/react 0.2.0-beta.1 → 0.2.0-beta.10

package/src/index.ts

@@ -1,6 +1,25 @@
 import { useBasic, BasicProvider } from "./AuthContext";
-import { useLiveQuery as useQuery } from "dexie-react-hooks";
+import { useLiveQuery } from "dexie-react-hooks";
+import { validateSchema, validateData, generateEmptySchema } from "./schema";
+import hello from "@basictech/schema"
+
+
+function useQuery(queryable: any) {
+    return useLiveQuery(() => {
+        if (typeof queryable === 'function') {
+            return queryable();
+        }
+        return queryable;
+    }, [queryable], []);
+}
+
+
+const sc = { 
+    validateSchema: validateSchema,
+    validateData: validateData,
+    generateEmptySchema: generateEmptySchema
+}
 
 export {
-    useBasic, BasicProvider, useQuery
-}
+    useBasic, BasicProvider, useQuery, sc, hello
+}

package/src/schema.ts

@@ -0,0 +1,159 @@
+// Basic Schema Library
+// utils for validating and interacting with Basic schemas
+import Ajv, { ErrorObject } from 'ajv'
+
+const basicJsonSchema = {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "properties": {
+        "project_id": {
+            "type": "string"
+        },
+        "namespace": {
+            "type": "string",
+        },
+        "version": {
+            "type": "integer",
+            "minimum": 0
+        },
+        "tables": {
+            "type": "object",
+            "patternProperties": {
+                "^[a-zA-Z0-9_]+$": {
+                    "type": "object",
+                    "properties": {
+                        "name": {
+                            "type": "string"
+                        },
+                        "type": {
+                            "type": "string",
+                            "enum": ["collection"]
+                        },
+                        "fields": {
+                            "type": "object",
+                            "patternProperties": {
+                                "^[a-zA-Z0-9_]+$": {
+                                    "type": "object",
+                                    "properties": {
+                                        "type": {
+                                            "type": "string",
+                                            "enum": ["string", "boolean", "number", "json"]
+                                        },
+                                        "indexed": {
+                                            "type": "boolean"
+                                        }, 
+                                        "required": {
+                                            "type": "boolean"
+                                        }
+                                    },
+                                    "required": ["type"]
+                                }
+                            },
+                            "additionalProperties": true
+                        }
+                    },
+                    "required": ["fields"]
+                }
+            },
+            "additionalProperties": true
+        }
+    },
+    "required": ["project_id", "version", "tables"]
+  }
+const ajv = new Ajv()
+const validator = ajv.compile(basicJsonSchema)
+
+type Schema = typeof basicJsonSchema
+
+function generateEmptySchema() {
+   
+}
+
+
+/**
+ * Validate a schema
+ * only checks if the schema is formatted correctly, not if can be published
+ * @param schema - The schema to validate
+ * @returns {valid: boolean, errors: any[]} - The validation result
+ */
+function validateSchema(schema: Schema) : {valid: boolean, errors: ErrorObject[]} {
+    const v = validator(schema)
+    return { 
+        valid: v,
+        errors: validator.errors || []
+    }
+}
+
+// type ErrorObject = {
+//     keyword: string;
+//     instancePath: string; 
+//     schemaPath: string;
+//     params: Record<string, any>;
+//     propertyName?: string;
+//     message?: string;
+//     schema?: any;
+//     parentSchema?: any;
+//     data?: any;
+// }
+
+
+function validateData(schema: any, table: string, data: Record<string, any>, checkRequired: boolean = true) {
+    const valid = validateSchema(schema)
+    if (!valid.valid) {
+        return { valid: false, errors: valid.errors, message: "Schema is invalid" }
+    }
+
+    const tableSchema = schema.tables[table]
+
+    if (!tableSchema) {
+        return { valid: false, errors: [{ message: `Table ${table} not found in schema` }], message: "Table not found" }
+    }
+
+    for (const [fieldName, fieldValue] of Object.entries(data)) {
+        const fieldSchema = tableSchema.fields[fieldName]
+        
+        if (!fieldSchema) {
+            return { 
+                valid: false, 
+                errors: [{ message: `Field ${fieldName} not found in schema` }],
+                message: "Invalid field"
+            }
+        }
+
+        const schemaType = fieldSchema.type
+        const valueType = typeof fieldValue
+
+        if (
+            (schemaType === 'string' && valueType !== 'string') ||
+            (schemaType === 'number' && valueType !== 'number') ||
+            (schemaType === 'boolean' && valueType !== 'boolean') ||
+            (schemaType === 'json' && valueType !== 'object')
+        ) {
+            return {
+                valid: false,
+                errors: [{ 
+                    message: `Field ${fieldName} should be type ${schemaType}, got ${valueType}` 
+                }],
+                message: "invalid type"
+            }
+        }
+    }
+
+    if (checkRequired) {
+        for (const [fieldName, fieldSchema] of Object.entries(tableSchema.fields)) {
+            if ((fieldSchema as { required?: boolean }).required && !data[fieldName]) {
+                return { valid: false, errors: [{ message: `Field ${fieldName} is required` }], message: "Required field missing" }
+            }
+        }
+    }
+
+    return { valid: true, errors: [] }
+}
+
+
+export {
+    validateSchema,
+    validateData,
+    generateEmptySchema
+}
+

package/src/sync/index.ts

@@ -0,0 +1,223 @@
+"use client"
+
+import { v7 as uuidv7 } from 'uuid';
+import { Dexie, PromiseExtended } from 'dexie';
+import 'dexie-syncable';
+import 'dexie-observable';
+
+import { syncProtocol } from './syncProtocol'
+import { SERVER_URL, log } from '../config'
+
+import { validateSchema, validateData } from '../schema'
+syncProtocol()
+
+
+// const DexieSyncStatus = {
+//   "-1": "ERROR",
+//   "0": "OFFLINE",
+//   "1": "CONNECTING",
+//   "2": "ONLINE",
+//   "3": "SYNCING",
+//   "4": "ERROR_WILL_RETRY"
+// }
+
+
+
+
+
+export class BasicSync extends Dexie {
+  basic_schema: any
+
+  constructor(name: string, options: any) {
+    super(name, options);
+
+    // --- INIT SCHEMA --- // 
+
+    //todo: handle versions?
+
+    // TODO: validate schema
+    this.basic_schema = options.schema
+    this.version(1).stores(this._convertSchemaToDxSchema(this.basic_schema))
+
+    this.version(2).stores({})
+    // this.verssion
+
+
+    // create an alias for toArray
+    // @ts-ignore
+    this.Collection.prototype.get = this.Collection.prototype.toArray
+
+
+    // --- SYNC --- // 
+
+    // this.syncable.on("statusChanged", (status, url) => { 
+    //   console.log("statusChanged", status, url)
+    // })
+
+  }
+
+  async connect({ access_token }: { access_token: string }) {
+    // const WS_URL = "ws://localhost:3003/ws"
+    const WS_URL = `${SERVER_URL}/ws`
+
+    
+    // Update sync nodes
+    await this.updateSyncNodes();
+    
+    // Proceed with the WebSocket connection
+    
+    log('Starting connection...')
+    return this.syncable.connect("websocket", WS_URL, { authToken: access_token });
+  }
+
+  async disconnect() {
+    const WS_URL = `${SERVER_URL}/ws`
+
+    return this.syncable.disconnect(WS_URL) 
+  }
+
+  private async updateSyncNodes() {
+    try {
+      const syncNodes = await this.table('_syncNodes').toArray();
+      const localSyncNodes = syncNodes.filter(node => node.type === 'local');
+      log('Local sync nodes:', localSyncNodes);
+
+      if (localSyncNodes.length > 1) {
+
+        
+        const largestNodeId = Math.max(...localSyncNodes.map(node => node.id));
+        // Check if the largest node is already the master
+        const largestNode = localSyncNodes.find(node => node.id === largestNodeId);
+        if (largestNode && largestNode.isMaster === 1) {
+          log('Largest node is already the master. No changes needed.');
+          return; // Exit the function early as no changes are needed
+        }
+
+
+        log('Largest node id:', largestNodeId);
+        log('HEISENBUG: More than one local sync node found.')
+
+        for (const node of localSyncNodes) {
+          log(`Local sync node keys:`, node.id, node.isMaster);
+          await this.table('_syncNodes').update(node.id, { isMaster: node.id === largestNodeId ? 1 : 0 });
+
+          log(`HEISENBUG: Setting ${node.id} to ${node.id === largestNodeId ? 'master' : '0'}`);
+        }
+
+        // Add a 1 second delay before returning // i dont think this helps?
+        await new Promise(resolve => setTimeout(resolve, 2000));
+
+      }
+
+      log('Sync nodes updated');
+    } catch (error) {
+      console.error('Error updating _syncNodes table:', error);
+    }
+  }
+
+  handleStatusChange(fn: any) {
+    this.syncable.on("statusChanged", fn)
+  }
+
+
+  _convertSchemaToDxSchema(schema: any) {
+    const stores = Object.entries(schema.tables).map(([key, table]: any) => {
+
+      const indexedFields = Object.entries(table.fields).filter(([key, field]: any) => field.indexed).map(([key, field]: any) => `,${key}`).join('')
+      return {
+        [key]: 'id' + indexedFields
+      }
+    })
+
+    return Object.assign({}, ...stores)
+  }
+
+  debugeroo() {
+    // console.log("debugeroo", this.syncable)
+
+    // this.syncable.list().then(x => console.log(x))
+
+    // this.syncable
+    return this.syncable
+  }
+
+  collection(name: string) {
+    // TODO: check against schema
+
+    return {
+
+      /**
+       * Returns the underlying Dexie table
+       * @type {Dexie.Table}
+       */
+      ref: this.table(name),
+
+      // --- WRITE ---- // 
+      add: (data: any) => {
+        // log("Adding data to", name, data)
+
+        const valid = validateData(this.basic_schema, name, data)
+        if (!valid.valid) {
+          log('Invalid data', valid)
+          return Promise.reject({ ... valid })
+        }
+
+        return this.table(name).add({
+          id: uuidv7(),
+          ...data
+        })
+
+      },
+
+      put: (data: any) => {
+        const valid = validateData(this.basic_schema, name, data)
+        if (!valid.valid) {
+          log('Invalid data', valid)
+          return Promise.reject({ ... valid })
+        }
+
+        return this.table(name).put({
+          id: uuidv7(),
+          ...data
+        })
+      },
+
+      update: (id: string, data: any) => {
+        const valid = validateData(this.basic_schema, name, data, false)
+        if (!valid.valid) {
+          log('Invalid data', valid)
+          return Promise.reject({ ... valid })
+        }
+
+        return this.table(name).update(id, data)
+      },
+
+      delete: (id: string) => {
+        return this.table(name).delete(id)
+      },
+
+
+      // --- READ ---- // 
+
+      get: async (id: string) => {
+        return this.table(name).get(id) 
+      },
+
+      getAll: async () => {
+        return this.table(name).toArray();
+      },
+
+      // --- QUERY ---- // 
+      // TODO: lots to do here. simplifing creating querie,  filtering/ordering/limit, and execute
+
+      query: () => this.table(name),
+
+      filter: (fn: any) => this.table(name).filter(fn).toArray(),
+
+    }
+  }
+}
+
+class QueryMethod { 
+
+}

package/src/sync/syncProtocol.js

@@ -0,0 +1,179 @@
+"use client"
+import { Dexie } from "dexie";
+import { log } from "../config";
+
+export const syncProtocol = function () {
+  log("Initializing syncProtocol");
+  // Constants:
+  var RECONNECT_DELAY = 5000; // Reconnect delay in case of errors such as network down.
+
+  Dexie.Syncable.registerSyncProtocol("websocket", {
+    sync: function (
+      context,
+      url,
+      options,
+      baseRevision,
+      syncedRevision,
+      changes,
+      partial,
+      applyRemoteChanges,
+      onChangesAccepted,
+      onSuccess,
+      onError,
+    ) {
+      // The following vars are needed because we must know which callback to ack when server sends it's ack to us.
+      var requestId = 0;
+      var acceptCallbacks = {};
+
+      // Connect the WebSocket to given url:
+      var ws = new WebSocket(url);
+
+      // console.log("ws OPTIONS", options);
+
+      // sendChanges() method:
+      function sendChanges(changes, baseRevision, partial, onChangesAccepted) {
+        log("sendChanges", changes.length, baseRevision);
+        ++requestId;
+        acceptCallbacks[requestId.toString()] = onChangesAccepted;
+
+        // In this example, the server expects the following JSON format of the request:
+        //  {
+        //      type: "changes"
+        //      baseRevision: baseRevision,
+        //      changes: changes,
+        //      partial: partial,
+        //      requestId: id
+        //  }
+        //  To make the sample simplified, we assume the server has the exact same specification of how changes are structured.
+        //  In real world, you would have to pre-process the changes array to fit the server specification.
+        //  However, this example shows how to deal with the WebSocket to fullfill the API.
+
+        ws.send(
+          JSON.stringify({
+            type: "changes",
+            changes: changes,
+            partial: partial,
+            baseRevision: baseRevision,
+            requestId: requestId,
+          }),
+        );
+      }
+
+
+
+      // When WebSocket opens, send our changes to the server.
+      ws.onopen = function (event) {
+        // Initiate this socket connection by sending our clientIdentity. If we dont have a clientIdentity yet,
+        // server will call back with a new client identity that we should use in future WebSocket connections.
+        
+        log("Opening socket - sending clientIdentity", context.clientIdentity);
+        ws.send(
+          JSON.stringify({
+            type: "clientIdentity",
+            clientIdentity: context.clientIdentity || null,
+            authToken: options.authToken
+          }),
+        );
+
+      };
+
+      // If network down or other error, tell the framework to reconnect again in some time:
+      ws.onerror = function (event) {
+        ws.close();
+        log("ws.onerror", event);
+        onError(event?.message, RECONNECT_DELAY);
+      };
+
+      // If socket is closed (network disconnected), inform framework and make it reconnect
+      ws.onclose = function (event) {
+        // console.log('🙅 ws.onclose', event)
+        onError("Socket closed: " + event.reason, RECONNECT_DELAY);
+      };
+
+      // isFirstRound: Will need to call onSuccess() only when we are in sync the first time.
+      // onSuccess() will unblock Dexie to be used by application code.
+      // If for example app code writes: db.friends.where('shoeSize').above(40).toArray(callback), the execution of that query
+      // will not run until we have called onSuccess(). This is because we want application code to get results that are as
+      // accurate as possible. Specifically when connected the first time and the entire DB is being synced down to the browser,
+      // it is important that queries starts running first when db is in sync.
+      var isFirstRound = true;
+      // When message arrive from the server, deal with the message accordingly:
+      ws.onmessage = function (event) {
+        try {
+          // Assume we have a server that should send JSON messages of the following format:
+          // {
+          //     type: "clientIdentity", "changes", "ack" or "error"
+          //     clientIdentity: unique value for our database client node to persist in the context. (Only applicable if type="clientIdentity")
+          //     message: Error message (Only applicable if type="error")
+          //     requestId: ID of change request that is acked by the server (Only applicable if type="ack" or "error")
+          //     changes: changes from server (Only applicable if type="changes")
+          //     lastRevision: last revision of changes sent (applicable if type="changes")
+          //     partial: true if server has additionalChanges to send. False if these changes were the last known. (applicable if type="changes")
+          // }
+          var requestFromServer = JSON.parse(event.data);
+          log("requestFromServer", requestFromServer, { acceptCallback, isFirstRound });
+
+          if (requestFromServer.type == "clientIdentity") {
+            context.clientIdentity = requestFromServer.clientIdentity;
+            context.save();
+
+            sendChanges(changes, baseRevision, partial, onChangesAccepted);
+
+            ws.send(
+              JSON.stringify({
+                type: "subscribe",
+                syncedRevision: syncedRevision,
+              }),
+            );
+          } else if (requestFromServer.type == "changes") {
+            applyRemoteChanges(
+              requestFromServer.changes,
+              requestFromServer.currentRevision,
+              requestFromServer.partial,
+            );
+            if (isFirstRound && !requestFromServer.partial) {
+              // Since this is the first sync round and server sais we've got all changes - now is the time to call onsuccess()
+              onSuccess({
+                // Specify a react function that will react on additional client changes
+                react: function (
+                  changes,
+                  baseRevision,
+                  partial,
+                  onChangesAccepted,
+                ) {
+                  sendChanges(
+                    changes,
+                    baseRevision,
+                    partial,
+                    onChangesAccepted,
+                  );
+                },
+                // Specify a disconnect function that will close our socket so that we dont continue to monitor changes.
+                disconnect: function () {
+                  ws.close();
+                },
+              });
+              isFirstRound = false;
+            }
+          } else if (requestFromServer.type == "ack") {
+            var requestId = requestFromServer.requestId;
+            var acceptCallback = acceptCallbacks[requestId.toString()];
+            acceptCallback(); // Tell framework that server has acknowledged the changes sent.
+            delete acceptCallbacks[requestId.toString()];
+          } else if (requestFromServer.type == "error") {
+            var requestId = requestFromServer.requestId;
+            ws.close();
+            onError(requestFromServer.message, Infinity); // Don't reconnect - an error in application level means we have done something wrong.
+          } else {
+            log("unknown message", requestFromServer);
+            ws.close();
+            onError("unknown message", Infinity);
+          }
+        } catch (e) {
+          ws.close();
+          onError(e, Infinity); // Something went crazy. Server sends invalid format or our code is buggy. Dont reconnect - it would continue failing.
+        }
+      };
+    },
+  });
+};