@knowlearning/agents 0.0.1

package/agents/browser/embedded.js

@@ -0,0 +1,164 @@
+import { v1 as uuid } from 'npm/unscoped/uuid/9.0.0'
+import MutableProxy from '../../persistence/json.js'
+import Experience from './experience.js'
+
+export default EmbeddedAgent()
+
+function EmbeddedAgent() {
+  let messageIndex = 0
+  let resolveSession
+  const session = new Promise(r => resolveSession = r)
+  const responses = {}
+  const watchers = {}
+
+  async function send(message) {
+    const requestId = message.requestId || uuid()
+
+    messageIndex += 1
+    //  default to window opener if present
+    const upstream = window.opener ? window.opener : window.parent
+    try {
+      upstream
+        .postMessage({
+          ...message,
+          session: await session,
+          requestId,
+          index: messageIndex
+        }, '*')
+      return new Promise((resolve, reject) => {
+        responses[requestId] = { resolve, reject }
+      })
+    }
+    catch (error) {
+      console.log('ERROR POSTING MESSAGE UP', message, error)
+    }
+  }
+
+  addEventListener('message', async ({ data }) => {
+    if (data.type === 'setup') resolveSession(data.session)
+    else if (responses[data.requestId]) {
+      const { resolve, reject } = responses[data.requestId]
+      if (data.error) reject(data.error)
+      else resolve(data.response)
+    }
+    else if (data.ii !== undefined) {
+      const { domain, user, scope } = data
+      const key = `${domain}/${user}/${scope}`
+      if (watchers[key]) watchers[key].forEach(fn => fn(data))
+    }
+  })
+
+  function environment() {
+    return send({ type: 'environment' })
+  }
+
+  function state(scope, user, domain) {
+    let watchFn
+    let resolveKey
+    const key = new Promise(r => resolveKey = r)
+
+    const promise = new Promise(async resolve => {
+      const { domain: d, auth: { user: u }, scope: s } = await environment()
+      domain = domain || d
+      user = user || u
+      scope = scope || s
+      resolveKey(`${domain}/${user}/${scope}`)
+      resolve(send({ type: 'state', scope, user, domain }))
+    })
+
+    promise.watch = fn => {
+      key
+        .then( async k => {
+          if (!watchers[k]) watchers[k] = []
+          watchers[k].push(fn)
+          watchFn = fn
+        })
+      return promise
+    }
+
+    promise.unwatch = () => {
+      key
+        .then( k => {
+          watchers[k] = watchers[k].filter(x => x !== watchFn)
+          if (watchers[k].length === 0) delete watchers[k]
+        })
+
+      return promise
+    }
+
+    return promise
+  }
+
+  async function patch(root, scopes) {
+    //  TODO: consider watch function added to return to receive progress
+    return send({ type: 'patch', root, scopes })
+  }
+
+  async function mutate (scope, initialize=true) {
+    const handlePatch = patch => interact(scope, patch)
+
+    if (!scope) scope = (await environment()).scope
+
+    if (initialize) {
+      //  TODO: consider setting up watcher and updating state
+      return new MutableProxy(await state(scope) || {}, handlePatch)
+    }
+    else return new MutableProxy({}, handlePatch)
+  }
+
+  function interact(scope, patch) {
+    return send({ type: 'interact', scope, patch })
+  }
+
+  async function upload(name, type, data, id=uuid()) {
+    const url = await send({ type: 'upload', name, contentType: type, id })
+
+    if (!data) return url
+    else {
+      console.log('UPLOADING TO URL!!!!!!!!!!!!!!', id, url)
+      const headers = { 'Content-Type': type }
+      const response = await fetch(url, {method: 'PUT', headers, body: data})
+      const { ok, statusText } = response
+
+      console.log(ok, statusText)
+
+      if (ok) return id
+      else throw new Error(statusText)
+    }
+  }
+
+  async function download(id, passthrough=false) {
+    const url = await send({ type: 'download', id })
+    return passthrough ? url : fetch(url)
+  }
+
+  function metadata(id) {
+    return state(id, 'metadata', 'core')
+  }
+
+  function login(provider, username, password) {
+    return send({ type: 'login', provider, username, password })
+  }
+
+  function logout() { return send({ type: 'logout' }) }
+
+  function disconnect() { return send({ type: 'disconnect' }) }
+
+  function reconnect() { return send({ type: 'reconnect' }) }
+
+  return {
+    login,
+    logout,
+    environment,
+    state,
+    mutate,
+    patch,
+    interact,
+    metadata,
+    upload,
+    download,
+    disconnect,
+    reconnect,
+    Experience
+  }
+}

package/agents/browser/experience.js

@@ -0,0 +1,126 @@
+import { v1 as uuid, validate as validateUUID } from 'npm/unscoped/uuid/9.0.0'
+
+const copy = x => JSON.parse(JSON.stringify(x))
+
+const watchers = {}
+
+export default function Experience(environment, iframe) {
+  const postMessageQueue = []
+  let frameLoaded = false
+  let embeddedAgentInitialized = false
+  const session = uuid()
+
+  const postMessage = m => new Promise((resolve, reject) => {
+    const message = { ...copy(m), session }
+    postMessageQueue.push({ message, sent: resolve })
+    if (frameLoaded) processPostMessageQueue()
+  })
+
+  const processPostMessageQueue = () => {
+    while (iframe.parentNode && postMessageQueue.length) {
+      const { message, sent } = postMessageQueue.shift()
+      iframe.contentWindow.postMessage(message, '*')
+      sent()
+    }
+  }
+
+  this.remove = () => {
+    if (iframe.parentNode) iframe.parentNode.removeChild(iframe)
+  }
+
+  const handleMessage = async message => {
+    const { requestId, type } = message
+
+    const sendDown = response => {
+      if (message.type === 'environment') Object.assign(response, environment)
+      return postMessage({ requestId, response })
+    }
+
+    if (type === 'error') {
+      console.error(message)
+      sendDown({})
+    }
+    else if (type === 'environment') {
+      sendDown(await Agent.environment())
+    }
+    else if (type === 'interact') {
+      await Agent.interact(message.scope, message.patch)
+      sendDown({}) // TODO: might want to send down the interaction index
+    }
+    else if (type === 'state') {
+      const { scope, user, domain } = message
+      const statePromise = Agent.state(scope, user, domain)
+
+      const key = `${session}/${domain}/${user}/${scope}`
+
+      //  only watch once per key per experience session
+      if (!watchers[key]) {
+        watchers[key] = statePromise.unwatch //  TODO: consider how to use this...
+        statePromise.watch(postMessage)
+      }
+
+      sendDown(await statePromise)
+    }
+    else if (type === 'patch') {
+      const { root, scopes } = message
+      sendDown(await Agent.patch(root, scopes))
+    }
+    else if (type === 'upload') {
+      const { name, contentType, id } = message
+      sendDown(await Agent.upload(name, contentType, false, id))
+    }
+    else if (type === 'download') {
+      sendDown(await Agent.download(message.id, true))
+    }
+    else if (type === 'login') {
+      const { provider, username, password } = message
+      sendDown(await Agent.login(provider, username, password))
+    }
+    else if (type === 'logout') Agent.logout()
+    else if (type === 'disconnect') sendDown(await Agent.disconnect())
+    else if (type === 'reconnect') sendDown(await Agent.reconnect())
+    else {
+      console.log('Unknown message type passed up...', message)
+      sendDown({})
+    }
+  }
+
+  window.addEventListener('message', ({ data }) => {
+    if (data.session === session) {
+      embeddedAgentInitialized = true
+      //  TODO: ensure message index ordering!!!!!!!!!!!!!!!!!!!! (no order guarantee given in postMessage protocol, so we need to make a little buffer here)
+      handleMessage(data)
+    }
+  })
+
+  //  write in a temporary loading notification while frame loads
+  const cw = iframe.contentWindow
+  if (cw) cw.document.body.innerHTML = 'Loading...'
+
+  //  TODO: make sure content security policy headers for embedded domain always restrict iframe
+  //        src to only self for embedded domain
+  Agent
+    .environment()
+    .then(async ({ embed }) => {
+      iframe.onload = () => {
+        frameLoaded = true
+        processPostMessageQueue()
+      }
+      const { protocol, pathname } = window.location
+      if (validateUUID(environment.content)) {
+        //  TODO: fix development port workararounds for embed
+        if (protocol === 'http:') embed = embed.replace(':32002', ':32020')
+        iframe.src = `${protocol}//${embed}${pathname}?s=${session}`
+      }
+      else {
+        iframe.src = environment.content
+      }
+
+      while(!embeddedAgentInitialized) {
+        postMessage({ type: 'setup', session })
+        await new Promise(r => setTimeout(r, 100))
+      }
+    })
+
+  return this
+}

package/agents/browser/firebase-auth.js

@@ -0,0 +1,60 @@
+import { initializeApp } from 'npm/unscoped/firebase/9.12.1/app'
+import {
+  getAuth,
+  signOut as firebaseSignOut,
+  GoogleAuthProvider,
+  OAuthProvider,
+  onAuthStateChanged,
+  signInWithRedirect,
+  signInWithEmailAndPassword,
+  signInAnonymously
+} from 'npm/unscoped/firebase/9.12.1/auth'
+
+initializeApp({
+  "apiKey": "AIzaSyAxjYuF-2JmXxlXnGgNu2CO4Q41EAtUgrY",
+  "authDomain": "opensourcelearningplatform.firebaseapp.com",
+  "databaseURL": "https://opensourcelearningplatform.firebaseio.com",
+  "projectId": "opensourcelearningplatform",
+  "storageBucket": "opensourcelearningplatform.appspot.com",
+  "messagingSenderId": "831020253582",
+  "appId: 1:831020253582": "web:6e1dfc6e89f5c2107164df",
+  "measurementId": "G-8QD6034RJ0"
+})
+
+let currentUser
+const authClient = getAuth()
+const onAuth = fn => onAuthStateChanged(authClient, user => {
+  if (!user) signInAnonymously(authClient)
+  else if (user.uid !== currentUser) {
+    const { uid } = user
+    currentUser = uid
+    const provider = user.isAnonymous ? 'anonymous' : user.providerData[0].providerId
+    fn({ user: uid, provider, getIdToken: () => user.getIdToken() })
+  }
+})
+
+const authProviders = {
+  google: () => new GoogleAuthProvider(),
+  microsoft: () => new OAuthProvider('microsoft.com')
+}
+
+const login = async (provider, username, password) => {
+  if (authProviders[provider]) {
+    signInWithRedirect(authClient, authProviders[provider]())
+    return { success: true }
+  }
+  else if (provider === 'email') {
+    try {
+      await signInWithEmailAndPassword(authClient, username, password)
+      location.reload()
+    }
+    catch (error) { return { success: false } }
+  }
+}
+
+const logout = async () => {
+  await firebaseSignOut(authClient)
+  location.reload()
+}
+
+export { onAuth, login, logout }

package/agents/browser/root.js

@@ -0,0 +1,21 @@
+import { v1 as uuid } from 'npm/unscoped/uuid/9.0.0'
+import { applyPatch } from 'npm/unscoped/fast-json-patch/3.1.1'
+import { onAuth, login, logout } from './firebase-auth.js'
+import GenericAgent from '../generic.js'
+import Experience from './experience.js'
+
+const Agent = new GenericAgent({
+    host: window.location.host,
+    protocol: window.location.protocol === 'https:' ? 'wss' : 'ws',
+    token: new Promise(r => onAuth(({ getIdToken: g }) => r(g()))),
+    WebSocket,
+    uuid,
+    fetch,
+    applyPatch,
+    login,
+    logout
+})
+
+Agent.Experience = Experience
+
+export default Agent

package/agents/deno.js

@@ -0,0 +1,14 @@
+import { applyPatch } from 'https://esm.sh/fast-json-patch@3.1.1'
+import Agent from './generic.js'
+
+const SERVE_HOST = Deno.env.get("SERVE_HOST")
+const SERVICE_ACCOUNT_TOKEN = Deno.env.get("SERVICE_ACCOUNT_TOKEN")
+
+export default new Agent({
+  host: SERVE_HOST,
+  token: Deno.readTextFile(SERVICE_ACCOUNT_TOKEN),
+  WebSocket,
+  uuid: () => crypto.randomUUID(),
+  fetch,
+  applyPatch
+})

package/agents/generic.js

@@ -0,0 +1,305 @@
+import MutableProxy from '../persistence/json.js'
+
+//  transform our custom path implementation to the standard JSONPatch path
+function standardJSONPatch(patch) {
+  return patch.map(p => {
+    return {...p, path: '/' + p.path.map(sanitizeJSONPatchPathSegment).join('/')}
+  })
+}
+
+function sanitizeJSONPatchPathSegment(s) {
+  if (typeof s === "string") return s.replaceAll('~', '~0').replaceAll('/', '~1')
+  else return s
+}
+
+export default function Agent({ host, token, WebSocket, protocol='ws', uuid, fetch, applyPatch, login, logout }) {
+  let ws
+  let user
+  let domain
+  const session = uuid()
+  let server
+  let si = -1
+  let authed = false
+  const states = {}
+  const responses = {}
+  const watchers = {}
+  const lastInteractionUpdateForWatchedScope = {}
+  const messageQueue = []
+  let resolveEnvironment
+  let disconnected = false
+  const environment = new Promise(r => resolveEnvironment = r)
+
+  const sessionData = new MutableProxy({}, patch => queueMessage({scope: 'sessions', patch}))
+
+  sessionData[session] = {
+    subscriptions: {},
+    uploads: {},
+    downloads: {},
+    patches: {}
+  }
+
+  function removeWatcher(key, fn) {
+    const watcherIndex = watchers[key].findIndex(x => x === fn)
+    if (watcherIndex > -1) watchers[key].splice(watcherIndex, 1)
+    else console.warn('TRIED TO REMOVE WATCHER THAT DOES NOT EXIST')
+  }
+
+  this.environment = function () { return environment }
+
+  this.state = state
+  this.upload = upload
+  this.download = download
+  this.interact = interact
+  this.patch = patch
+  this.mutate = mutate
+  this.metadata = metadata
+
+  this.login = login
+  this.logout = logout
+  this.disconnect = disconnect
+  this.reconnect = reconnect
+
+  initWS()
+
+  let lastSentSI = -1
+  function flushMessageQueue() {
+    //  TODO: probably want to make this loop async so we don't try and push more to
+    //        a closed connection
+    while (authed && ws.readyState === WebSocket.OPEN && lastSentSI+1 < messageQueue.length) {
+      lastSentSI += 1
+      ws.send(JSON.stringify(messageQueue[lastSentSI]))
+    }
+  }
+
+  function lastMessageResponse() { //  TODO: handle error responses
+    return new Promise(r => responses[si].push(r))
+  }
+
+  function queueMessage(message) {
+    si += 1
+    responses[si] = []
+    messageQueue.push({...message, si, ts: Date.now()})
+    flushMessageQueue()
+  }
+
+  function initWS() {
+    ws = new WebSocket(`${protocol}://${host}`)
+
+    ws.onopen = async () => ws.send(JSON.stringify({ token: await token, session }))
+
+    ws.onmessage = async ({ data }) => {
+      try {
+        const message = JSON.parse(data)
+        if (message.error) console.warn('ERROR RESPONSE', message)
+
+        if (!authed) {
+          authed = true
+          if (!user) { // this is the first authed websocket connection
+            user = message.auth.user
+            domain = message.domain
+            server = message.server
+            resolveEnvironment(message)
+          }
+          else {
+            lastSentSI = message.ack
+          }
+          flushMessageQueue()
+        }
+        else {
+          if (message.si !== undefined) {
+            if (responses[message.si]) {
+              //  TODO: remove "acknowledged" messages fromt queue and do accounting with si
+              responses[message.si].forEach(fn => fn(message))
+              delete responses[message.si]
+              ws.send(JSON.stringify({ack: message.si})) //  acknowledgement that we have received the response for this message
+            }
+            else {
+              //  TODO: consider what to do here... probably want to throw error if in dev env
+              console.warn('received MULTIPLE responses for message with si', message.si, message)
+            }
+          }
+          else {
+            const key = `${message.domain}/${message.user}/${message.scope}`
+            if (watchers[key]) {
+              if (sessionData[session].subscriptions[key] + 1 !== message.ii) {
+                //  TODO: this seems to be an error that happens with decent regularity (an answer with a given si was skipped/failed)
+                //        we should be wary of out-of-order ii being passed down (maybe need to wait for older ones???)
+                console.warn('UNEXPECTED UPDATE INTERACTION INDEX!!!!!!!!!!! last index in session', sessionData[session].subscriptions[key], ' passed index ', message.ii)
+              }
+              states[key] = await states[key] || {}
+
+              const lastResetPatchIndex = message.patch.findLastIndex(p => p.path.length === 0)
+              if (lastResetPatchIndex > -1) states[key] = message.patch[lastResetPatchIndex].value
+
+              applyPatch(states[key], standardJSONPatch(message.patch.slice(lastResetPatchIndex + 1)))
+              watchers[key].forEach(fn => fn({ ...message, state: states[key] }))
+              sessionData[session].subscriptions[key] = message.ii
+            }
+          }
+        }
+      }
+      catch (error) {
+        console.error('ERROR HANDLING WS MESSAGE', error)
+      }
+    }
+
+    ws.onerror = async error => {
+      //console.log('WS CONNECTION ERROR', error.message)
+    }
+
+    ws.onclose = async () => {
+      authed = false
+      console.log(`CLOSED DOMAIN ${domain} USER ${user} SESSION ${session} CONNECTION TO SERVER ${server}`)
+      if (!disconnected) initWS() // TODO: don't do this if we are purposefully unloading...
+    }
+  }
+
+  function state(scope, u, d) {
+    let watchFn
+    let resolveKey
+    const key = new Promise(r => resolveKey = r)
+
+    const promise = new Promise(async resolveState => {
+      await environment // environment not set until first connection
+
+      if (!u) u = user
+      if (!d) d = domain
+
+      const k = `${d || domain}/${u || user}/${scope}`
+      resolveKey(k)
+
+      if (states[k] === undefined) {
+        watchers[k] = []
+        states[k] = new Promise(async resolve => {
+          sessionData[session].subscriptions[k] = null
+          const { ii, state } = await lastMessageResponse()
+          sessionData[session].subscriptions[k] = ii
+          resolve(state)
+        })
+      }
+
+      //  TODO: make sure to get the state that represents the last interaction we sent
+      //        when this "state" method was called!!!!!!!!!!!!!!!!!!!!!!!
+      //        Right now this is simply returning whatever we have at the time we get here
+      //        when some interaction responses may not have been applied yet...
+      //        POSSIBLE: wait for most recent interaction update response for key k that we sent...
+      await lastInteractionUpdateForWatchedScope[k]
+      //  TODO: probably something like await updateMessageReceivedForLastInteractionWeSentForKey[k]
+      resolveState(states[k])
+    })
+
+    promise.watch = fn => {
+      key
+        .then( k => {
+          if (watchFn) throw new Error('Only one watcher allowed') //  TODO: consider allowing more watchers per state call
+          watchers[k].push(fn)
+          watchFn = fn
+        })
+      return promise
+    }
+
+    promise.unwatch = async () => {
+      const k = await key
+      removeWatcher(k, watchFn)
+      if (watchers[k].length === 0) {
+        delete sessionData[session].subscriptions[k]
+        delete watchers[k]
+      }
+    }
+
+    return promise
+  }
+
+  //  TODO: if no data, set up streaming upload
+  async function upload(name, type, data, id=uuid()) {
+    //  TODO: include data size info...
+    sessionData[session].uploads[id] = { url: null, sent: 0, name, type }
+    const { url } = await lastMessageResponse()
+    sessionData[session].uploads[id].url = url
+
+    if (!data) return url
+    else {
+      const headers = { 'Content-Type': type }
+      const response = await fetch(url, {method: 'PUT', headers, body: data})
+      const { ok, statusText } = response
+
+      if (ok) return id
+      else throw new Error(statusText)
+    }
+  }
+
+   async function download(id, passthrough=false) {
+    sessionData[session].downloads[id] = { url: null, size: null, sent: 0 }
+    const { url } = await lastMessageResponse()
+    sessionData[session].downloads[id].url = url
+
+    if (passthrough) return url
+    else {
+      const response = await fetch(url)
+      const { ok, statusText } = response
+
+      if (ok) return response
+      else throw new Error(statusText)
+    }
+  }
+
+  async function patch(root, scopes) {
+    sessionData[session].patches[uuid()] = { root, scopes }
+    const { swaps } = await lastMessageResponse()
+    return { swaps }
+  }
+
+  async function interact(scope, patch) {
+    queueMessage({scope, patch})
+    const response = lastMessageResponse()
+
+    await environment
+
+    const key = `${domain}/${user}/${scope}`
+
+    //  if we are watching this key, we want to keep track of last interaction we fired
+    if (states[key] !== undefined) {
+      let resolve
+      lastInteractionUpdateForWatchedScope[key] = new Promise(r => resolve = r)
+
+      const { ii } = await response
+
+      const resolveAndUnwatch = (update) => {
+        if (update.ii === ii) {
+          resolve(ii)
+          removeWatcher(key, resolveAndUnwatch)
+        }
+      }
+
+      watchers[key].push(resolveAndUnwatch)
+
+      return { ii }
+    }
+    else {
+      const { ii } = await response
+      return { ii }
+    }
+  }
+
+  async function mutate(scope, initialize=true) {
+    const initial = initialize ? await this.state(scope) || {} : {}
+    return new MutableProxy(initial, patch => interact(scope, patch))
+  }
+
+  function metadata(id) {
+    return state(id, 'metadata', 'core')
+  }
+
+  function disconnect() {
+    console.log('DISCONNECTED AGENT!!!!!!!!!!!!!!!')
+    disconnected = true
+    ws.close()
+  }
+
+  function reconnect() {
+    console.log('RECONNECTED AGENT!!!!!!!!!!!!!!!')
+    initWS()
+  }
+
+  return this
+}

package/agents/node.js

@@ -0,0 +1,17 @@
+import fs from 'fs'
+import WebSocket from 'ws'
+import { v1 as uuid } from 'uuid'
+import fastJSONPatch from 'fast-json-patch'
+import fetch from 'node-fetch'
+import Agent from './generic.js'
+
+const { SERVE_HOST, SERVICE_ACCOUNT_TOKEN } = process.env
+
+export default new Agent({
+  host: SERVE_HOST,
+  token: fs.promises.readFile(SERVICE_ACCOUNT_TOKEN).then(f => f.toString()),
+  WebSocket,
+  uuid,
+  fetch,
+  applyPatch: fastJSONPatch.applyPatch
+})

package/create-folder-content.js

@@ -0,0 +1,67 @@
+import { extensionToType } from './metadata-utils.js'
+
+//  these utility functions work in a web environment
+const fileFromEntry = entry => new Promise((s,e) => entry.file(s,e))
+const getTypeFromEntry = entry => fileFromEntry(entry).then(file => file.type)
+const getDataFromEntry = entry => fileFromEntry(entry).then(file => file.arrayBuffer())
+
+//  entries is of form: [{ fullPath, name }, ....]
+export default async function createFolderContent(entries, upload, uuid, getType=getTypeFromEntry, getData=getDataFromEntry) {
+  //  TODO: swap out internal references in files
+  const swaps = {}
+  const roots = {}
+  const filenames = []
+  entries.forEach(e => {
+    const id = uuid(e.fullPath)
+    swaps[e.fullPath] = id
+    roots[id] = true
+    filenames.push(e.name)
+  })
+  const metadata = []
+  const body = []
+  await Promise.all(
+    entries.map(async entry => {
+      const id = swaps[entry.fullPath]
+      const type = await getType(entry)
+      let data = await getData(entry)
+      if (!type.startsWith('image/')) {
+        //  TODO: this is probably the most fragile part and we probably want unit tests
+        //        for reasonable return values for getData
+        if (data instanceof ArrayBuffer || data instanceof Uint8Array) {
+          data = new TextDecoder("utf-8").decode(data)
+        }
+        Object
+          .entries(swaps)
+          .forEach(([from, to]) => {
+            let reference = ''
+            if (from === entry.fullPath) reference = `./${entry.name}`
+            else {
+              const e = entry.fullPath.split('/')
+              const f = from.split('/')
+              while (e[0] !== undefined && f[0] !== undefined && e[0] === f[0]) {
+                e.shift()
+                f.shift()
+              }
+              // for every entry dir past the common prefix, add a ../ for the reference
+              if (e.length > 1) e.slice(0,-1).forEach(() => reference += '../')
+              else reference = './'
+
+              reference += f.join('/')
+            }
+            if (data.includes(reference)) {
+              //  if found a reference to this swap, then
+              //  the from reference cannot be a root
+              //  TODO: fix the bug where this will delete root content that is part of a cycle
+              delete roots[to]
+              data = data.replaceAll(reference, to)
+            }
+          })
+      }
+      const segments = entry.name.split('.')
+      const contentType = extensionToType(segments[segments.length - 1])
+
+      await upload(entry.fullPath, contentType, data, id)
+    })
+  )
+  return Object.keys(roots)
+}

package/metadata-utils.js

@@ -0,0 +1,20 @@
+const tePairs = [
+    ["application/javascript;syntax=vue-template", "vue"],
+    ["application/javascript", "js"],
+    ["application/json", "json"],
+    ["text/html", "html"],
+    ["text/css", "css"],
+    ["text/plain", "txt"],
+    ["text/yaml", "yaml"],
+    ["image/png", "png"],
+    ["text/plain;syntax=Dockerfile", "Dockerfile"]
+  ]
+  
+  const typeToExtensionMap = tePairs.reduce((o,[t,e]) => (o[t]=e, o), {})
+  const extensionToTypeMap = tePairs.reduce((o,[t,e]) => (o[e]=t, o), {})
+  
+  // TODO: do better fallback matching (do a smarter look at params/subtypes)
+  const typeToExtension = t => typeToExtensionMap[t] || typeToExtensionMap[t.split(';')[0]]
+  const extensionToType = e => extensionToTypeMap[e] || "text/plain"
+  
+  export { typeToExtension, extensionToType }

package/package.js

@@ -0,0 +1,6 @@
+//  TODO: export browser, node, deno, and generic agents
+export { default as BrowserAgent } from './lib/agents/browser/embedded.js'
+export { default as NodeAgent } from './lib/agents/node.js'
+export { default as DenoAgent } from './lib/agents/deno.js'
+export { default as GenericAgent } from './lib/agents/generic.js'
+export { default as createPersistentVuexStore } from './persistence/vuex.js'

package/package.json

@@ -0,0 +1,29 @@
+{
+    "name": "@knowlearning/agents",
+    "version": "0.0.1",
+    "description": "API for embedding applications in KnowLearning systems.",
+    "main": "package.js",
+    "type": "module",
+    "directories": {
+      "example": "examples",
+      "test": "test"
+    },
+    "scripts": {
+      "test": "echo \"Error: no test specified\" && exit 1"
+    },
+    "repository": {
+      "type": "git",
+      "url": "git+https://github.com/knowlearning/core.git"
+    },
+    "author": "KnowLearning",
+    "license": "MPL-2.0",
+    "bugs": {
+      "url": "https://github.com/knowlearning/core/issues"
+    },
+    "homepage": "https://github.com/knowlearning/core#readme",
+    "dependencies": {
+      "fast-json-patch": "^3.1.1",
+      "uuid": "^8.3.2"
+    }
+  }
+  

package/persistence/json.js

@@ -0,0 +1,92 @@
+const proxies = new Set()
+
+// TODO make serialization more reliable... basically: handle if prop includes "
+function serializePath(path) {
+  return `/${path.join('/')}`
+}
+
+//  TODO: add extra argument for "ignorePrefixes" to ignore interactions at certain paths
+export default function MutableProxy(state, interact, ephemeralPaths={}, parentPath=[], rootState) {
+  if (ephemeralPaths[serializePath(parentPath)]) return state
+  if (!rootState) rootState = state
+  if (proxies.has(state)) {
+    //  TODO: simply return with no effect if redundantly setting state like a[x] = a[x]
+    throw new Error(`Cannot add mutable state to multiple mutable parents. Attempted Path: ${serializePath(parentPath)}`)
+  }
+
+  const isArray = Array.isArray(state)
+
+  const childMutableProxy = (prop, value) => MutableProxy(
+    value,
+    interact,
+    ephemeralPaths,
+    [...parentPath, prop],
+    rootState
+  )
+
+  //  recursively ensure child objects are converted to proxies (unless they are in an ephemeral path)
+  Object
+    .entries(state)
+    .filter(([, value]) => value instanceof Object)
+    .forEach(([key, value]) => state[key] = childMutableProxy(key, value))
+
+  const traps = {
+    set(target, prop, value) {
+      const path = [...parentPath, prop]
+      const serializedPath = serializePath(path)
+
+      if (ephemeralPaths[serializedPath]) {
+        target[prop] = value
+        return true
+      }
+
+      if (isArray) {
+        if (!/^\d+$/.test(prop)) {
+          target[prop] = value
+          return true
+        }
+        else prop = parseInt(prop)
+      }
+
+      if (value === undefined) {
+        console.log('EXTRA ERROR INFO. target, prop, value', target, prop, value)
+        throw new Error(`Setting properties to undefined is not supported. Please use a delete statement. Attempted to set ${serializedPath}`)
+      }
+
+      //  TODO: probably want to batch interactions that represent array mutations... and/or do
+      //        something smart around insertions/deletions
+      //  TODO: if is array and prop is last element, consider passing -1 as prop
+      interact([{
+        op: target[prop] ? 'replace' : 'add',
+        value: JSON.parse(JSON.stringify(value)), //  TODO: more efficient sanitization
+        path
+      }])
+
+      if (value instanceof Object) target[prop] = childMutableProxy(prop, value)
+      else target[prop] = value
+
+      return true
+    },
+    deleteProperty(target, prop) {
+      if (prop in target) {
+        proxies.delete(proxy)
+        delete target[prop]
+
+        if (isArray) {
+          if (!/^\d+$/.test(prop)) return true
+          else prop = parseInt(prop)
+        }
+
+        //  TODO: if is array and prop is last element, consider passing -1 as prop
+        const path = [...parentPath, prop]
+        if (!ephemeralPaths[serializePath(path)]) interact([{ op: 'remove', path }])
+      }
+      return true
+    }
+  }
+
+  const proxy = new Proxy(state, traps)
+  proxies.add(proxy)
+
+  return proxy
+}

package/persistence/vuex.js

@@ -0,0 +1,84 @@
+import { createStore } from 'vuex'
+import MutableProxy from './json.js'
+
+//  TODO: consider path serialization approach. Also consider just using a mutable proxy from agent.state...
+
+const copy = x => JSON.parse(JSON.stringify(x))
+
+export default async function (storeDefinition) {
+  //  Set up persistance
+  const scope ='whatevah' //  TODO: resolve what scope should be...
+  //  TODO: assess the function of this from the old implementation
+  const savedState = await Agent.state(scope)
+
+  const scopedPaths = getScopedPaths(storeDefinition)
+  const stateAttachedStore = await attachModuleState(savedState, storeDefinition, scopedPaths)
+  const s = stateAttachedStore.state
+  const originalState = s instanceof Function ? s() :  s
+  const handlePatch = patch => {
+    console.log('PATCHING STATE', patch)
+    Agent.interact(scope, patch)
+  }
+
+  if (savedState === null) handlePatch([{ op: 'add', path: [], value: copy(originalState) }])
+
+  // ephemeral paths for the root mutable proxy are any paths with a scope specified (including null scopes)
+  const ephemeralPaths = {}
+  Object.keys(scopedPaths).forEach(key => ephemeralPaths[key] = true)
+
+  stateAttachedStore.state = () => MutableProxy(savedState || originalState, handlePatch, ephemeralPaths)
+  return createStore(stateAttachedStore)
+}
+
+// TODO: sanitize module names for path serialization approach
+function descendantPaths(path, pathSet) {
+  return (
+    Object
+      .keys(pathSet)
+      .filter(p => p.startsWith(path + '/')) // filter for descendant paths
+      .reduce((acc, p) => {
+        const descendantPart = p.slice(path.length)
+        acc[descendantPart] = pathSet[p]
+        return acc
+      }, {})
+  )
+}
+
+async function attachModuleState(state, module, scopedPaths, path='') {
+  await Promise.all(
+    Object
+      .entries(module.modules || {})
+      .map(async ([subModuleName, subModule]) => {
+        const subModuleStartState = state ? state[subModuleName] : null
+
+        // vuex expects to initialize state from submodules
+        if (subModuleStartState) delete state[subModuleName]
+
+        module.modules[subModuleName] = await attachModuleState(subModuleStartState, subModule, scopedPaths, `${path}/${subModuleName}`)
+      })
+  )
+
+  // if our path is in scoped paths, return new Mutable proxy attached to scope
+  const scope = scopedPaths[path]
+  if (scope) {
+    const handlePatch = patch => Agent.interact(scope, patch)
+    const initState = await Agent.state(scope)
+    const ephemeralPaths = descendantPaths(path, scopedPaths)
+    state = MutableProxy(initState || {}, handlePatch, ephemeralPaths)
+    if (!initState) Object.assign(state, module.state())
+  }
+
+  return state ? { ...module, state: () => state } : module
+}
+
+function getScopedPaths(module, path="", paths={}) {
+  if (module.scope !== undefined) paths[path] = module.scope
+
+  Object
+    .entries(module.modules || {})
+    .forEach(([subModuleName, subModule]) => {
+      getScopedPaths(subModule, `${path}/${subModuleName}`, paths)
+    })
+
+  return paths
+}