jason-rails 0.4.0 → 0.6.1

data/client/src/JasonProvider.tsx

@@ -1,130 +1,12 @@
-import createActions from './createActions'
-import { createConsumer } from "@rails/actioncable"
-import JasonContext from './JasonContext'
-import axios from 'axios'
-import applyCaseMiddleware from 'axios-case-converter'
+import React from 'react'
+import useJason from './useJason'
 import { Provider } from 'react-redux'
-import { createEntityAdapter, createSlice, createReducer, configureStore } from '@reduxjs/toolkit'
-import createJasonReducers from './createJasonReducers'
-import createPayloadHandler from './createPayloadHandler'
-import createOptDis from './createOptDis'
-import makeEager from './makeEager'
-import { camelizeKeys } from 'humps'
-import md5 from 'blueimp-md5'
-import _ from 'lodash'
-import React, { useState, useEffect } from 'react'
-import { validate as isUuid } from 'uuid'
+import JasonContext from './JasonContext'
 
 const JasonProvider = ({ reducers, middleware, extraActions, children }: { reducers?: any, middleware?: any, extraActions?: any, children?: React.FC }) => {
-  const [store, setStore] = useState(null)
-  const [value, setValue] = useState(null)
-  const [connected, setConnected] = useState(false)
-
-  const csrfToken = (document.querySelector("meta[name=csrf-token]") as any).content
-  axios.defaults.headers.common['X-CSRF-Token'] = csrfToken
-  const restClient = applyCaseMiddleware(axios.create(), {
-    preservedKeys: (key) => {
-      return isUuid(key)
-    }
-  })
-
-  useEffect(() => {
-    restClient.get('/jason/api/schema')
-    .then(({ data: snakey_schema }) => {
-      const schema = camelizeKeys(snakey_schema)
-
-      const serverActionQueue = function() {
-        const queue: any[] = []
-        let inFlight = false
-
-        return {
-          addItem: (item) => queue.push(item),
-          getItem: () => {
-            if (inFlight) return false
-
-            const item = queue.shift()
-            if (item) {
-              inFlight = true
-              return item
-            }
-            return false
-          },
-          itemProcessed: () => inFlight = false,
-          fullySynced: () => queue.length === 0 && !inFlight,
-          getData: () => ({ queue, inFlight })
-        }
-      }()
-
-      const consumer = createConsumer()
-      const allReducers = {
-        ...reducers,
-        ...createJasonReducers(schema)
-      }
-
-      console.log({ schema, allReducers })
-      const store = configureStore({ reducer: allReducers, middleware })
-
-      let payloadHandlers = {}
-      function handlePayload(payload) {
-        const { model, md5Hash } = payload
-        console.log({ md5Hash, fn: `${model}:${md5Hash}`, payloadHandlers, model: _.camelCase(model), payload })
-        const handler = payloadHandlers[`${_.camelCase(model)}:${md5Hash}`]
-        if (handler) {
-          handler({ ...payload, model: _.camelCase(model) })
-        }
-      }
-
-      const subscription = (consumer.subscriptions.create({
-        channel: 'Jason::Channel'
-      }, {
-        connected: () => {
-          setConnected(true)
-        },
-        received: payload => {
-          console.log("Payload received", payload)
-          handlePayload(payload)
-        },
-        disconnected: () => console.warn('Disconnected from ActionCable')
-      }));
-
-      console.log('sending message')
-      subscription.send({ message: 'test' })
-
-      function createSubscription(config) {
-        const md5Hash = md5(JSON.stringify(config))
-        console.log('Subscribe with', config, md5Hash)
-
-        _.map(config, (v, model) => {
-          payloadHandlers[`${model}:${md5Hash}`] = createPayloadHandler(store.dispatch, serverActionQueue, subscription, model, schema[model])
-        })
-        subscription.send({ createSubscription: config })
-
-        return () => removeSubscription(config)
-      }
-
-      function removeSubscription(config) {
-        subscription.send({ removeSubscription: config })
-        const md5Hash = md5(JSON.stringify(config))
-        _.map(config, (v, model) => {
-          delete payloadHandlers[`${model}:${md5Hash}`]
-        })
-      }
-      const optDis = createOptDis(schema, store.dispatch, restClient, serverActionQueue)
-      const actions = createActions(schema, store, restClient, optDis, extraActions)
-      const eager = makeEager(schema)
-
-      console.log({ actions })
-
-      setValue({
-        actions: actions,
-        subscribe: (config) => createSubscription(config),
-        eager
-      })
-      setStore(store)
-    })
-  }, [])
+  const [store, value] = useJason({ reducers, middleware, extraActions })
 
-  if(!(store && value && connected)) return <div /> // Wait for async fetch of schema to complete
+  if(!(store && value)) return <div /> // Wait for async fetch of schema to complete
 
   return <Provider store={store}>
     <JasonContext.Provider value={value}>{ children }</JasonContext.Provider>

data/client/src/createJasonReducers.ts

@@ -2,8 +2,8 @@ import { createEntityAdapter, createSlice } from '@reduxjs/toolkit'
 import pluralize from 'pluralize'
 import _ from 'lodash'
 
-function generateSlices(schema) {
-  const sliceNames = schema.map(k => pluralize(k))
+function generateSlices(models) {
+  const sliceNames = models.map(k => pluralize(k))
   const adapter = createEntityAdapter()
 
   return _.fromPairs(_.map(sliceNames, name => {
@@ -30,6 +30,59 @@ function generateSlices(schema) {
   }))
 }
 
+function generateJasonSlices(models) {
+  const initialState = _.fromPairs(_.map(models, (model_name) => {
+    return [model_name, {}]
+  }))
+
+  const modelSliceReducer = createSlice({
+    name: 'jasonModels',
+    initialState,
+    reducers: {
+      setSubscriptionIds(s,a) {
+        const { payload } = a
+        const { subscriptionId, model, ids } = payload
+        console.log({ initialState })
+        s[model][subscriptionId] = ids
+      },
+      addSubscriptionId(s,a) {
+        const { payload } = a
+        const { subscriptionId, model, id } = payload
+        s[model][subscriptionId] = _.union(s[model][subscriptionId] || [], [id])
+      },
+      removeSubscriptionId(s,a) {
+        const { payload } = a
+        const { subscriptionId, model, id } = payload
+        s[model][subscriptionId] = _.remove(s[model][subscriptionId] || [], id)
+      },
+      removeSubscription(s, a) {
+        const { payload: { subscriptionId } } = a
+        _.map(models, model => {
+          delete s[model][subscriptionId]
+        })
+      }
+    }
+  }).reducer
+
+  const jasonSliceReducer = createSlice({
+    name: 'jason',
+    initialState: {
+      connected: false,
+      queueSize: 0
+    },
+    reducers: {
+      upsert: (s,a) => ({ ...s, ...a.payload })
+    }
+  }).reducer
+
+  return { jason: jasonSliceReducer, jasonModels: modelSliceReducer }
+}
+
 export default function createJasonReducers(schema) {
-  return generateSlices(_.keys(schema))
+  const models = _.keys(schema)
+
+  return {
+    ...generateSlices(models),
+    ...generateJasonSlices(models)
+  }
 }

data/client/src/createOptDis.ts

@@ -13,7 +13,6 @@ function enrich(type, payload) {
 
 export default function createOptDis(schema, dispatch, restClient, serverActionQueue) {
   const plurals = _.keys(schema).map(k => pluralize(k))
-  let inFlight = false
 
   function enqueueServerAction (action) {
     serverActionQueue.addItem(action)
@@ -23,7 +22,6 @@ export default function createOptDis(schema, dispatch, restClient, serverActionQ
     const action = serverActionQueue.getItem()
     if (!action) return
 
-    inFlight = true
     restClient.post('/jason/api/action', action)
     .then(serverActionQueue.itemProcessed)
     .catch(e => {

data/client/src/createPayloadHandler.ts

@@ -2,18 +2,17 @@ import { apply_patch } from 'jsonpatch'
 import deepCamelizeKeys from './deepCamelizeKeys'
 import pluralize from 'pluralize'
 import _ from 'lodash'
-import { validate as isUuid } from 'uuid'
+import { validate as isUuid, v4 as uuidv4 } from 'uuid'
 
 function diffSeconds(dt2, dt1) {
   var diff =(dt2.getTime() - dt1.getTime()) / 1000
   return Math.abs(Math.round(diff))
 }
 
-export default function createPayloadHandler(dispatch, serverActionQueue, subscription, model, config) {
-  console.log({ model, config })
-  let payload = [] as any[]
-  let previousPayload = [] as any[]
-  let idx = 0
+export default function createPayloadHandler({ dispatch, serverActionQueue, transportAdapter, config }) {
+  const subscriptionId = uuidv4()
+
+  let idx = {}
   let patchQueue = {}
 
   let lastCheckAt = new Date()
@@ -21,8 +20,7 @@ export default function createPayloadHandler(dispatch, serverActionQueue, subscr
   let checkInterval
 
   function getPayload() {
-    console.log({ getPayload: model, subscription })
-    subscription.send({ getPayload: { model, config } })
+    setTimeout(() => transportAdapter.getPayload(config), 1000)
   }
 
   function camelizeKeys(item) {
@@ -31,87 +29,78 @@ export default function createPayloadHandler(dispatch, serverActionQueue, subscr
 
   const tGetPayload = _.throttle(getPayload, 10000)
 
-  function dispatchPayload() {
-    // We want to avoid updates from server overwriting changes to local state, so if there is a queue then wait.
-    if (!serverActionQueue.fullySynced()) {
-      console.log(serverActionQueue.getData())
-      setTimeout(dispatchPayload, 100)
-      return
-    }
-
-    const includeModels = (config.includeModels || []).map(m => _.camelCase(m))
-
-    console.log("Dispatching", { payload, includeModels })
-
-    includeModels.forEach(m => {
-      const subPayload = _.flatten(_.compact(camelizeKeys(payload).map(instance => instance[m])))
-      const previousSubPayload = _.flatten(_.compact(camelizeKeys(previousPayload).map(instance => instance[m])))
-
-      // Find IDs that were in the payload but are no longer
-      const idsToRemove = _.difference(previousSubPayload.map(i => i.id), subPayload.map(i => i.id))
-
-      dispatch({ type: `${pluralize(m)}/upsertMany`, payload: subPayload })
-      dispatch({ type: `${pluralize(m)}/removeMany`, payload: idsToRemove })
-    })
-
-    const idsToRemove = _.difference(previousPayload.map(i => i.id), payload.map(i => i.id))
-
-    dispatch({ type: `${pluralize(model)}/upsertMany`, payload: camelizeKeys(payload) })
-    dispatch({ type: `${pluralize(model)}/removeMany`, payload: idsToRemove })
-    previousPayload = payload
-  }
-
-  function processQueue() {
-    console.log({ idx, patchQueue })
+  function processQueue(model) {
+    console.debug("processQueue", model, idx[model], patchQueue[model])
     lastCheckAt = new Date()
-    if (patchQueue[idx]) {
-      payload = apply_patch(payload, patchQueue[idx])
-      if (patchQueue[idx]) {
-        dispatchPayload()
+    if (patchQueue[model][idx[model]]) {
+      if (!serverActionQueue.fullySynced()) {
+        console.debug(serverActionQueue.getData())
+        setTimeout(() => processQueue(model), 100)
+        return
+      }
+
+      const { payload, destroy, id, type } = patchQueue[model][idx[model]]
+
+      if (type === 'payload') {
+        dispatch({ type: `${pluralize(model)}/upsertMany`, payload })
+        const ids = payload.map(instance => instance.id)
+        dispatch({ type: `jasonModels/setSubscriptionIds`, payload: { model, subscriptionId, ids }})
+      } else if (destroy) {
+        // Middleware will determine if this model should be removed if it isn't in any other subscriptions
+        dispatch({ type: `jasonModels/removeSubscriptionId`, payload: { model, subscriptionId, id }})
+      } else {
+        dispatch({ type: `${pluralize(model)}/upsert`, payload })
+        dispatch({ type: `jasonModels/addSubscriptionId`, payload: { model, subscriptionId, id }})
       }
-      delete patchQueue[idx]
-      idx++
+
+      delete patchQueue[model][idx[model]]
+      idx[model]++
       updateDeadline = null
-      processQueue()
+      processQueue(model)
     // If there are updates in the queue that are ahead of the index, some have arrived out of order
     // Set a deadline for new updates before it declares the update missing and refetches.
-    } else if (_.keys(patchQueue).length > 0 && !updateDeadline) {
+    } else if (_.keys(patchQueue[model]).length > 0 && !updateDeadline) {
       var t = new Date()
       t.setSeconds(t.getSeconds() + 3)
       updateDeadline = t
-      setTimeout(processQueue, 3100)
+      setTimeout(() => processQueue(model), 3100)
     // If more than 10 updates in queue, or deadline has passed, restart
-    } else if (_.keys(patchQueue).length > 10 || (updateDeadline && diffSeconds(updateDeadline, new Date()) < 0)) {
+    } else if (_.keys(patchQueue[model]).length > 10 || (updateDeadline && diffSeconds(updateDeadline, new Date()) < 0)) {
       tGetPayload()
       updateDeadline = null
     }
   }
 
   function handlePayload(data) {
-    const { value, idx: newIdx, diff, latency, type } = data
-    console.log({ data })
+    const { idx: newIdx, model: snake_model, type } = data
+    const model = _.camelCase(snake_model)
 
-    if (type === 'payload') {
-      if (!value) return null;
+    idx[model] = idx[model] || 0
+    patchQueue[model] = patchQueue[model] || {}
 
-      payload = value
-      dispatchPayload()
-      idx = newIdx + 1
+    if (type === 'payload') {
+      idx[model] = newIdx
       // Clear any old changes left in the queue
-      patchQueue= _.pick(patchQueue, _.keys(patchQueue).filter(k => k > newIdx + 1))
-      return
+      patchQueue[model] = _.pick(patchQueue[model], _.keys(patchQueue[model]).filter(k => k > newIdx + 1))
     }
 
-    patchQueue[newIdx] = diff
-
-    processQueue()
+    patchQueue[model][newIdx] = camelizeKeys({ ...data, model })
+    console.debug("Added to queue", model, idx[model], camelizeKeys({ ...data, model }), serverActionQueue.getData())
+    processQueue(model)
 
     if (diffSeconds((new Date()), lastCheckAt) >= 3) {
       lastCheckAt = new Date()
-      console.log('Interval lost. Pulling from server')
+      console.debug('Interval lost. Pulling from server')
       tGetPayload()
     }
   }
 
-  return handlePayload
+  tGetPayload()
+
+  // Clean up after ourselves
+  function tearDown() {
+    dispatch({ type: `jasonModels/removeSubscription`, payload: { subscriptionId }})
+  }
+
+  return { handlePayload, tearDown }
 }

data/client/src/createServerActionQueue.test.ts

@@ -0,0 +1,42 @@
+import createServerActionQueue from './createServerActionQueue'
+
+test('Adding items', () => {
+  const serverActionQueue = createServerActionQueue()
+  serverActionQueue.addItem({ type: 'entity/add', payload: { id: 'abc', attribute: 1 } })
+  const item = serverActionQueue.getItem()
+  expect(item).toStrictEqual({ type: 'entity/add', payload: { id: 'abc', attribute: 1 } })
+})
+
+test('Deduping of items that will overwrite each other', () => {
+  const serverActionQueue = createServerActionQueue()
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 1 } })
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 2 } })
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 3 } })
+
+  const item = serverActionQueue.getItem()
+
+  expect(item).toStrictEqual({ type: 'entity/upsert', payload: { id: 'abc', attribute: 3 } })
+})
+
+test('Deduping of items with a superset', () => {
+  const serverActionQueue = createServerActionQueue()
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 1 } })
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 2, attribute2: 'test' } })
+
+  const item = serverActionQueue.getItem()
+
+  expect(item).toStrictEqual({ type: 'entity/upsert', payload: { id: 'abc', attribute: 2, attribute2: 'test' } })
+})
+
+test("doesn't dedupe items with some attributes missing", () => {
+  const serverActionQueue = createServerActionQueue()
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 1 } })
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute2: 'test' } })
+
+  const item = serverActionQueue.getItem()
+  serverActionQueue.itemProcessed()
+  const item2 = serverActionQueue.getItem()
+
+  expect(item).toStrictEqual({ type: 'entity/upsert', payload: { id: 'abc', attribute: 1 } })
+  expect(item2).toStrictEqual({ type: 'entity/upsert', payload: { id: 'abc', attribute2: 'test' } })
+})

data/client/src/createServerActionQueue.ts

@@ -0,0 +1,47 @@
+// A FIFO queue with deduping of actions whose effect will be cancelled by later actions
+
+import _ from 'lodash'
+
+export default function createServerActionQueue() {
+  const queue: any[] = []
+  let inFlight = false
+
+  function addItem(item) {
+    // Check if there are any items ahead in the queue that this item would effectively overwrite.
+    // In that case we can remove them
+    // If this is an upsert && item ID is the same && current item attributes are a superset of the earlier item attributes
+    const { type, payload } = item
+    if (type.split('/')[1] !== 'upsert') {
+      queue.push(item)
+      return
+    }
+
+    _.remove(queue, item => {
+      const { type: itemType, payload: itemPayload } = item
+      if (type !== itemType) return false
+      if (itemPayload.id !== payload.id) return false
+
+      // Check that all keys of itemPayload are in payload.
+      return _.difference(_.keys(itemPayload),_.keys(payload)).length === 0
+    })
+
+    queue.push(item)
+  }
+
+  return {
+    addItem,
+    getItem: () => {
+      if (inFlight) return false
+
+      const item = queue.shift()
+      if (item) {
+        inFlight = true
+        return item
+      }
+      return false
+    },
+    itemProcessed: () => inFlight = false,
+    fullySynced: () => queue.length === 0 && !inFlight,
+    getData: () => ({ queue, inFlight })
+  }
+}