jason-rails 0.4.1 → 0.5.0

data/client/src/createJasonReducers.ts

@@ -2,8 +2,8 @@ import { createEntityAdapter, createSlice } from '@reduxjs/toolkit'
 import pluralize from 'pluralize'
 import _ from 'lodash'
 
-function generateSlices(schema) {
-  const sliceNames = schema.map(k => pluralize(k))
+function generateSlices(models) {
+  const sliceNames = models.map(k => pluralize(k))
   const adapter = createEntityAdapter()
 
   return _.fromPairs(_.map(sliceNames, name => {
@@ -30,6 +30,52 @@ function generateSlices(schema) {
   }))
 }
 
+function generateJasonSlices(models) {
+  const initialState = _.fromPairs(_.map(models, (model_name) => {
+    return [model_name, {}]
+  }))
+
+  const modelSliceReducer = createSlice({
+    name: 'jasonModels',
+    initialState,
+    reducers: {
+      setSubscriptionIds(s,a) {
+        const { payload } = a
+        const { subscriptionId, model, ids } = payload
+        s[model][subscriptionId] = ids
+      },
+      addSubscriptionId(s,a) {
+        const { payload } = a
+        const { subscriptionId, model, id } = payload
+        s[model][subscriptionId] = _.union(s[model][subscriptionId] || [], [id])
+      },
+      removeSubscriptionId(s,a) {
+        const { payload } = a
+        const { subscriptionId, model, id } = payload
+        s[model][subscriptionId] = _.remove(s[model][subscriptionId] || [], id)
+      }
+    }
+  }).reducer
+
+  const jasonSliceReducer = createSlice({
+    name: 'jason',
+    initialState: {
+      connected: false,
+      queueSize: 0
+    },
+    reducers: {
+      upsert: (s,a) => ({ ...s, ...a.payload })
+    }
+  }).reducer
+
+  return { jason: jasonSliceReducer, jasonModels: modelSliceReducer }
+}
+
 export default function createJasonReducers(schema) {
-  return generateSlices(_.keys(schema))
+  const models = _.keys(schema)
+
+  return {
+    ...generateSlices(models),
+    ...generateJasonSlices(models)
+  }
 }

data/client/src/createOptDis.ts

@@ -13,7 +13,6 @@ function enrich(type, payload) {
 
 export default function createOptDis(schema, dispatch, restClient, serverActionQueue) {
   const plurals = _.keys(schema).map(k => pluralize(k))
-  let inFlight = false
 
   function enqueueServerAction (action) {
     serverActionQueue.addItem(action)
@@ -23,7 +22,6 @@ export default function createOptDis(schema, dispatch, restClient, serverActionQ
     const action = serverActionQueue.getItem()
     if (!action) return
 
-    inFlight = true
     restClient.post('/jason/api/action', action)
     .then(serverActionQueue.itemProcessed)
     .catch(e => {

data/client/src/createPayloadHandler.ts

@@ -2,18 +2,17 @@ import { apply_patch } from 'jsonpatch'
 import deepCamelizeKeys from './deepCamelizeKeys'
 import pluralize from 'pluralize'
 import _ from 'lodash'
-import { validate as isUuid } from 'uuid'
+import { validate as isUuid, v4 as uuidv4 } from 'uuid'
 
 function diffSeconds(dt2, dt1) {
   var diff =(dt2.getTime() - dt1.getTime()) / 1000
   return Math.abs(Math.round(diff))
 }
 
-export default function createPayloadHandler(dispatch, serverActionQueue, subscription, model, config) {
-  console.log({ model, config })
-  let payload = [] as any[]
-  let previousPayload = [] as any[]
-  let idx = 0
+export default function createPayloadHandler({ dispatch, serverActionQueue, subscription, config }) {
+  const subscriptionId = uuidv4()
+
+  let idx = {}
   let patchQueue = {}
 
   let lastCheckAt = new Date()
@@ -21,8 +20,7 @@ export default function createPayloadHandler(dispatch, serverActionQueue, subscr
   let checkInterval
 
   function getPayload() {
-    console.log({ getPayload: model, subscription })
-    subscription.send({ getPayload: { model, config } })
+    setTimeout(() => subscription.send({ getPayload: config }), 1000)
   }
 
   function camelizeKeys(item) {
@@ -31,87 +29,73 @@ export default function createPayloadHandler(dispatch, serverActionQueue, subscr
 
   const tGetPayload = _.throttle(getPayload, 10000)
 
-  function dispatchPayload() {
-    // We want to avoid updates from server overwriting changes to local state, so if there is a queue then wait.
-    if (!serverActionQueue.fullySynced()) {
-      console.log(serverActionQueue.getData())
-      setTimeout(dispatchPayload, 100)
-      return
-    }
-
-    const includeModels = (config.includeModels || []).map(m => _.camelCase(m))
-
-    console.log("Dispatching", { payload, includeModels })
-
-    includeModels.forEach(m => {
-      const subPayload = _.flatten(_.compact(camelizeKeys(payload).map(instance => instance[m])))
-      const previousSubPayload = _.flatten(_.compact(camelizeKeys(previousPayload).map(instance => instance[m])))
-
-      // Find IDs that were in the payload but are no longer
-      const idsToRemove = _.difference(previousSubPayload.map(i => i.id), subPayload.map(i => i.id))
-
-      dispatch({ type: `${pluralize(m)}/upsertMany`, payload: subPayload })
-      dispatch({ type: `${pluralize(m)}/removeMany`, payload: idsToRemove })
-    })
-
-    const idsToRemove = _.difference(previousPayload.map(i => i.id), payload.map(i => i.id))
-
-    dispatch({ type: `${pluralize(model)}/upsertMany`, payload: camelizeKeys(payload) })
-    dispatch({ type: `${pluralize(model)}/removeMany`, payload: idsToRemove })
-    previousPayload = payload
-  }
-
-  function processQueue() {
-    console.log({ idx, patchQueue })
+  function processQueue(model) {
+    console.debug("processQueue", model, idx[model], patchQueue[model])
     lastCheckAt = new Date()
-    if (patchQueue[idx]) {
-      payload = apply_patch(payload, patchQueue[idx])
-      if (patchQueue[idx]) {
-        dispatchPayload()
+    if (patchQueue[model][idx[model]]) {
+      if (!serverActionQueue.fullySynced()) {
+        console.debug(serverActionQueue.getData())
+        setTimeout(() => processQueue(model), 100)
+        return
       }
-      delete patchQueue[idx]
-      idx++
+
+      const { payload, destroy, id, type } = patchQueue[model][idx[model]]
+
+      if (type === 'payload') {
+        dispatch({ type: `${pluralize(model)}/upsertMany`, payload })
+        const ids = payload.map(instance => instance.id)
+        dispatch({ type: `jasonModels/setSubscriptionIds`, payload: { model, subscriptionId, ids }})
+      } else if (destroy) {
+        dispatch({ type: `${pluralize(model)}/remove`, payload: id })
+        dispatch({ type: `jasonModels/removeSubscriptionId`, payload: { model, subscriptionId, id }})
+      } else {
+        dispatch({ type: `${pluralize(model)}/upsert`, payload })
+        dispatch({ type: `jasonModels/addSubscriptionId`, payload: { model, subscriptionId, id }})
+      }
+
+      delete patchQueue[model][idx[model]]
+      idx[model]++
       updateDeadline = null
-      processQueue()
+      processQueue(model)
     // If there are updates in the queue that are ahead of the index, some have arrived out of order
     // Set a deadline for new updates before it declares the update missing and refetches.
-    } else if (_.keys(patchQueue).length > 0 && !updateDeadline) {
+    } else if (_.keys(patchQueue[model]).length > 0 && !updateDeadline) {
       var t = new Date()
       t.setSeconds(t.getSeconds() + 3)
       updateDeadline = t
-      setTimeout(processQueue, 3100)
+      setTimeout(() => processQueue(model), 3100)
     // If more than 10 updates in queue, or deadline has passed, restart
-    } else if (_.keys(patchQueue).length > 10 || (updateDeadline && diffSeconds(updateDeadline, new Date()) < 0)) {
+    } else if (_.keys(patchQueue[model]).length > 10 || (updateDeadline && diffSeconds(updateDeadline, new Date()) < 0)) {
       tGetPayload()
       updateDeadline = null
     }
   }
 
   function handlePayload(data) {
-    const { value, idx: newIdx, diff, latency, type } = data
-    console.log({ data })
+    const { idx: newIdx, model: snake_model, type } = data
+    const model = _.camelCase(snake_model)
 
-    if (type === 'payload') {
-      if (!value) return null;
+    idx[model] = idx[model] || 0
+    patchQueue[model] = patchQueue[model] || {}
 
-      payload = value
-      dispatchPayload()
-      idx = newIdx + 1
+    if (type === 'payload') {
+      idx[model] = newIdx
       // Clear any old changes left in the queue
-      patchQueue= _.pick(patchQueue, _.keys(patchQueue).filter(k => k > newIdx + 1))
-      return
+      patchQueue[model] = _.pick(patchQueue[model], _.keys(patchQueue[model]).filter(k => k > newIdx + 1))
     }
 
-    patchQueue[newIdx] = diff
-
-    processQueue()
+    patchQueue[model][newIdx] = camelizeKeys({ ...data, model })
+    console.debug("Added to queue", model, idx[model], camelizeKeys({ ...data, model }), serverActionQueue.getData())
+    processQueue(model)
 
     if (diffSeconds((new Date()), lastCheckAt) >= 3) {
       lastCheckAt = new Date()
-      console.log('Interval lost. Pulling from server')
+      console.debug('Interval lost. Pulling from server')
       tGetPayload()
     }
   }
 
+  tGetPayload()
+
   return handlePayload
 }

data/client/src/createServerActionQueue.test.ts

@@ -0,0 +1,42 @@
+import createServerActionQueue from './createServerActionQueue'
+
+test('Adding items', () => {
+  const serverActionQueue = createServerActionQueue()
+  serverActionQueue.addItem({ type: 'entity/add', payload: { id: 'abc', attribute: 1 } })
+  const item = serverActionQueue.getItem()
+  expect(item).toStrictEqual({ type: 'entity/add', payload: { id: 'abc', attribute: 1 } })
+})
+
+test('Deduping of items that will overwrite each other', () => {
+  const serverActionQueue = createServerActionQueue()
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 1 } })
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 2 } })
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 3 } })
+
+  const item = serverActionQueue.getItem()
+
+  expect(item).toStrictEqual({ type: 'entity/upsert', payload: { id: 'abc', attribute: 3 } })
+})
+
+test('Deduping of items with a superset', () => {
+  const serverActionQueue = createServerActionQueue()
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 1 } })
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 2, attribute2: 'test' } })
+
+  const item = serverActionQueue.getItem()
+
+  expect(item).toStrictEqual({ type: 'entity/upsert', payload: { id: 'abc', attribute: 2, attribute2: 'test' } })
+})
+
+test("doesn't dedupe items with some attributes missing", () => {
+  const serverActionQueue = createServerActionQueue()
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute: 1 } })
+  serverActionQueue.addItem({ type: 'entity/upsert', payload: { id: 'abc', attribute2: 'test' } })
+
+  const item = serverActionQueue.getItem()
+  serverActionQueue.itemProcessed()
+  const item2 = serverActionQueue.getItem()
+
+  expect(item).toStrictEqual({ type: 'entity/upsert', payload: { id: 'abc', attribute: 1 } })
+  expect(item2).toStrictEqual({ type: 'entity/upsert', payload: { id: 'abc', attribute2: 'test' } })
+})

data/client/src/createServerActionQueue.ts

@@ -0,0 +1,47 @@
+// A FIFO queue with deduping of actions whose effect will be cancelled by later actions
+
+import _ from 'lodash'
+
+export default function createServerActionQueue() {
+  const queue: any[] = []
+  let inFlight = false
+
+  function addItem(item) {
+    // Check if there are any items ahead in the queue that this item would effectively overwrite.
+    // In that case we can remove them
+    // If this is an upsert && item ID is the same && current item attributes are a superset of the earlier item attributes
+    const { type, payload } = item
+    if (type.split('/')[1] !== 'upsert') {
+      queue.push(item)
+      return
+    }
+
+    _.remove(queue, item => {
+      const { type: itemType, payload: itemPayload } = item
+      if (type !== itemType) return false
+      if (itemPayload.id !== payload.id) return false
+
+      // Check that all keys of itemPayload are in payload.
+      return _.difference(_.keys(itemPayload),_.keys(payload)).length === 0
+    })
+
+    queue.push(item)
+  }
+
+  return {
+    addItem,
+    getItem: () => {
+      if (inFlight) return false
+
+      const item = queue.shift()
+      if (item) {
+        inFlight = true
+        return item
+      }
+      return false
+    },
+    itemProcessed: () => inFlight = false,
+    fullySynced: () => queue.length === 0 && !inFlight,
+    getData: () => ({ queue, inFlight })
+  }
+}

data/client/src/pruneIdsMiddleware.ts

@@ -0,0 +1,24 @@
+import _ from 'lodash'
+import pluralize from 'pluralize'
+
+const pruneIdsMiddleware = schema => store => next => action => {
+  const { type } = action
+  const result = next(action)
+  const state = store.getState()
+  if (type === 'jasonModels/setSubscriptionIds') {
+    // Check every model
+    _.map(_.keys(schema), model => {
+      let ids = []
+      _.map(state.jasonModels[model], (subscribedIds, k) => {
+        ids = _.union(ids, subscribedIds)
+      })
+      // Find IDs currently in Redux that aren't in any subscription
+      const idsToRemove = _.difference(state[pluralize(model)].ids, ids)
+      store.dispatch({ type: `${pluralize(model)}/removeMany`, payload: idsToRemove })
+    })
+  }
+
+  return result
+}
+
+export default pruneIdsMiddleware

data/client/src/restClient.ts

@@ -0,0 +1,13 @@
+import axios from 'axios'
+import applyCaseMiddleware from 'axios-case-converter'
+import { validate as isUuid } from 'uuid'
+
+const csrfToken = (document?.querySelector("meta[name=csrf-token]") as any)?.content
+axios.defaults.headers.common['X-CSRF-Token'] = csrfToken
+const restClient = applyCaseMiddleware(axios.create() as any, {
+  preservedKeys: (key) => {
+    return isUuid(key)
+  }
+})
+
+export default restClient

data/client/src/useJason.test.ts

@@ -0,0 +1,81 @@
+import { renderHook, act } from '@testing-library/react-hooks'
+import useJason from './useJason'
+import restClient from './restClient'
+
+jest.mock('./restClient')
+
+test('it works', async () => {
+  const resp = { data: { post: {} } };
+  // @ts-ignore
+  restClient.get.mockResolvedValue(resp);
+
+  const { result, waitForNextUpdate } = renderHook(() => useJason({ reducers: {
+    test: (s,a) => s || {}
+  }}));
+
+  await waitForNextUpdate()
+  const [store, value, connected] = result.current
+  const { handlePayload, subscribe } = value
+
+  const subscription = subscribe({ post: {} })
+
+  handlePayload({
+    type: 'payload',
+    model: 'post',
+    payload: [{ id: 4, name: 'test' }],
+    md5Hash: subscription.md5Hash,
+    idx: 1
+  })
+
+  handlePayload({
+    id: 4,
+    model: 'post',
+    destroy: true,
+    md5Hash: subscription.md5Hash,
+    idx: 2
+  })
+
+  handlePayload({
+    id: 5,
+    model: 'post',
+    payload: { id: 5, name: 'test2' },
+    md5Hash: subscription.md5Hash,
+    idx: 3
+  })
+})
+
+test('pruning IDs', async () => {
+  const resp = { data: { post: {} } };
+
+  // @ts-ignore
+  restClient.get.mockResolvedValue(resp);
+
+  const { result, waitForNextUpdate } = renderHook(() => useJason({ reducers: {
+    test: (s,a) => s || {}
+  }}));
+
+  await waitForNextUpdate()
+  const [store, value, connected] = result.current
+  const { handlePayload, subscribe } = value
+
+  const subscription = subscribe({ post: {} })
+
+  handlePayload({
+    type: 'payload',
+    model: 'post',
+    payload: [{ id: 4, name: 'test' }],
+    md5Hash: subscription.md5Hash,
+    idx: 1
+  })
+
+  handlePayload({
+    type: 'payload',
+    model: 'post',
+    payload: [{ id: 5, name: 'test it out' }],
+    md5Hash: subscription.md5Hash,
+    idx: 2
+  })
+
+  // The ID 4 should have been pruned
+  expect(store.getState().posts.ids).toStrictEqual([5])
+})