@live-change/task-service 0.8.31 → 0.8.33

package/model.js

@@ -51,6 +51,10 @@ const taskProperties = {
     },
     default: []
   },
+  maxRetries: {
+    type: Number,
+    default: 5
+  },
   progress: {
     type: Object,
     properties: {
@@ -82,6 +86,56 @@ const Task = definition.model({
     },
     byState: {
       property: ['state']
+    },
+    runningRootsByName: {
+      property: ['name'],
+      function: async function(input, output, { tableName }) {
+        function mapFunction(obj) {
+          if(!obj) return null
+          if(['done', 'failed', 'canceled'].includes(obj.state)) return null
+          if(obj.causeType === tableName) return null
+          return { id: `"${obj.name}"_${obj.id}`, to: obj.id }
+        }
+        await input.table(tableName).onChange(async (obj, oldObj) => {
+          await output.change(mapFunction(obj), mapFunction(oldObj))
+        })
+      },
+      parameters: {
+        tableName: definition.name + '_Task'
+      }
+    },
+    byRoot: {
+      function: async function(input, output, { tableName }) {
+        async function findAncestors(object){
+          const result = []
+          let current = object
+          while(current) {
+            result.push(`"${current.causeType}":"${current.cause}"`)
+            current = current.causeType === tableName
+              ? await input.table(tableName).object(current.cause).get()
+              : null
+          }
+          //console.log("FOUND ANCESTORS", result, "FOR", object.id)
+          return result
+        }
+        await input.table(tableName).onChange(async (obj, oldObj) => {
+          const id = obj?.id || oldObj?.id
+          const ancestors = obj ? await findAncestors(obj) : []
+          const oldAncestors = oldObj ? await findAncestors(oldObj) : []
+          //console.log("ANCESTORS", id, oldAncestors, '=>', ancestors)
+          const addedAncestors = ancestors.filter(ancestor => !oldAncestors.includes(ancestor))
+          const removedAncestors = oldAncestors.filter(ancestor => !ancestors.includes(ancestor))
+          for(const ancestor of addedAncestors) {
+            await output.change({ id: `${ancestor}_${id}`, to: id }, null)
+          }
+          for(const ancestor of removedAncestors) {
+            await output.change(null, { id: `${ancestor}_${id}`, to: id })
+          }
+        })
+      },
+      parameters: {
+        tableName: definition.name + '_Task'
+      }
     }
   }
 })
@@ -106,11 +160,35 @@ definition.view({
       type: Task
     }
   },
-  async daoPath({ hash }) {
+  async daoPath({ causeType, cause, hash }) {
     return Task.indexRangePath('byCauseAndHash', [causeType, cause, hash], { limit: 23 })
   }
 })
 
+definition.view({
+  name: 'tasksByRoot',
+  properties: {
+    rootType: {
+      type: String,
+      validation: ['nonEmpty']
+    },
+    root: {
+      type: String,
+      validation: ['nonEmpty']
+    },
+    ...App.rangeProperties
+  },
+  returns: {
+    type: Task
+  },
+  async daoPath(props) {
+    const { rootType, root } = props
+    const range = App.extractRange(props)
+    if(!range.limit) range.limit = 256
+    return Task.indexRangePath('byRoot', [rootType, root], range)
+  }
+})
+
 definition.view({
   name: 'task',
   internal: true,
@@ -126,3 +204,25 @@ definition.view({
     return Task.path(task)
   }
 })
+
+definition.view({
+  name: 'runningTaskRootsByName',
+  internal: true,
+  global: true,
+  properties: {
+    name: {
+      type: String,
+      validation: ['nonEmpty']
+    },
+    ...App.rangeProperties
+  },
+  returns: {
+    type: Task
+  },
+  async daoPath(props) {
+    const { name } = props
+    const range = App.extractRange(props)
+    if(!range.limit) range.limit = 256
+    return Task.indexRangePath('runningRootsByName', [name], range)
+  }
+})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/task-service",
-  "version": "0.8.31",
+  "version": "0.8.33",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -22,8 +22,8 @@
   },
   "type": "module",
   "dependencies": {
-    "@live-change/framework": "^0.8.31",
-    "@live-change/relations-plugin": "^0.8.31"
+    "@live-change/framework": "^0.8.33",
+    "@live-change/relations-plugin": "^0.8.33"
   },
-  "gitHead": "9ab10c6f2854dd5472cc6becb128951db16218e3"
+  "gitHead": "98ff6f9c09e5fc1f408010df6cc8038eff571276"
 }

package/task.js

@@ -3,29 +3,32 @@ const app = App.app()
 
 import crypto from 'crypto'
 
+import PQueue from 'p-queue'
+
 function upperFirst(string) {
-  return string.charAt(0).toUpperCase() + string.slice(1)
+  return string[0].toUpperCase() + string.slice(1)
 }
 
 async function triggerOnTaskStateChange(taskObject, causeType, cause) {
-  await app.trigger({
+  if(!taskObject?.state) throw new Error('Task object state is not defined in ' + taskObject)
+  if(!taskObject?.id) throw new Error('Task object id is not defined in '+taskObject)
+  await app.trigger({ type: 'task'+upperFirst(taskObject.state), }, {
     ...taskObject,
-    type: 'task'+upperFirst(taskObject.state),
     task: taskObject.id,
     causeType,
     cause
   })
-  await app.trigger({
+  await app.trigger({ type: 'task'+upperFirst(taskObject.name)+upperFirst(taskObject.state) }, {
     ...taskObject,
-    type: 'task'+upperFirst(taskObject.name)+upperFirst(taskObject.state),
     task: taskObject.id,
     causeType,
     cause
   })
   await app.trigger({
+    type: `${taskObject.causeType}_${taskObject.cause}OwnedTask`
+           +`${upperFirst(taskObject.name)}${upperFirst(taskObject.state)}`
+  }, {
     ...taskObject,
-    type: `${taskObject.causeType}_${taksObject.cause}OwnedTask`
-      +`${upperFirst(taskObject.name)}${upperFirst(taskObject.state)}`,
     task: taskObject.id,
     causeType,
     cause
@@ -33,7 +36,6 @@ async function triggerOnTaskStateChange(taskObject, causeType, cause) {
 }
 
 async function createOrReuseTask(taskDefinition, props, causeType, cause) {
-
   const propertiesJson = JSON.stringify(props)
   const hash = crypto
     .createHash('sha256')
@@ -48,60 +50,102 @@ async function createOrReuseTask(taskDefinition, props, causeType, cause) {
   const oldTask = similarTasks.find(similarTask => similarTask.name === taskDefinition.name
     && JSON.stringify(similarTask.properties) === propertiesJson)
 
-  let taskObject = oldTask
-    ? await app.serviceViewGet('task', 'task', { task: oldTask.to })
+  const taskObject = oldTask
+    ? await app.serviceViewGet('task', 'task', { task: oldTask.to || oldTask.id })
     : {
       id: app.generateUid(),
       name: taskDefinition.name,
       properties: props,
       hash,
-      state: 'created'
+      state: 'created',
+      retries: [],
+      maxRetries: taskDefinition.maxRetries ?? 5
     }
 
   if(!oldTask) {
     /// app.emitEvents
-    await app.triggerService('task', {
+    await app.triggerService({ service: 'task', type: 'task_createCauseOwnedTask' }, {
       ...taskObject,
-      type: 'task_createCaseOwnedTask',
       causeType,
       cause,
       task: taskObject.id
     })
     await triggerOnTaskStateChange(taskObject, causeType, cause)
   }
-
+  return taskObject
 }
 
 async function startTask(taskFunction, props, causeType, cause){
-  const taskObject = createOrReuseTask(taskFunction.definition, props, causeType, cause)
+  const taskObject = await createOrReuseTask(taskFunction.definition, props, causeType, cause)
   const context = {
     causeType,
     cause,
-    taskObject
+    taskObject,
   }
   const promise = taskFunction(props, context)
-  return { task: taskObject.id, taskObject, promise }
+  return { task: taskObject.id, taskObject, promise, causeType, cause }
 }
 
-export default function task(definition) {
-  const taskFunction = async (props, context, emit) => {
+export default function task(definition, serviceDefinition) {
+  if(!definition) throw new Error('Task definition is not defined')
+  if(!serviceDefinition) throw new Error('Service definition is not defined')
+  const taskFunction = async (props, context, emit, reportProgress = () => {}) => {
     if(!emit) emit = (events) =>
       app.emitEvents(definition.name, Array.isArray(events) ? events : [events], {})
 
     let taskObject = context.taskObject
-      ?? await createOrReuseTask(taskDefinition, props, context.causeType, context.cause)
+      ?? await createOrReuseTask(definition, props, context.causeType, context.cause)
+
+    if(!taskObject?.state) throw new Error('Task object state is not defined in ' + taskObject)
+    if(!taskObject?.id) throw new Error('Task object id is not defined in '+taskObject)
 
-    const maxRetries = definition.maxRetries ?? 5
+    const updateQueue = new PQueue({ concurrency: 1 })
 
     async function updateTask(data) {
-      await app.triggerService('task', {
+      if(typeof data !== 'object') throw new Error('Task update data is not an object' + JSON.stringify(data))
+      if(!taskObject?.state) throw new Error('Task object state is not defined in ' + JSON.stringify(taskObject))
+      if(!taskObject?.id) throw new Error('Task object id is not defined in ' + JSON.stringify(taskObject))
+/*      console.log("UPDATING TASK", {
         ...data,
-        type: 'task_updateCaseOwnedTask',
+        type: 'task_updateCauseOwnedTask',
         causeType: context.causeType,
         cause: context.cause,
         task: taskObject.id
       })
-      taskObject = await app.serviceViewGet('task', 'task', { task: oldTask.to })
+      console.trace("UPDATING TASK!")*/
+      await updateQueue.add(async () => {
+        const result = await app.triggerService({ service: 'task', type: 'task_updateCauseOwnedTask' }, {
+          ...data,
+          causeType: context.causeType,
+          cause: context.cause,
+          task: taskObject.id
+        })
+
+        taskObject = await app.serviceViewGet('task', 'task', { task: taskObject.id })
+        //console.log("UPDATED TASK", taskObject, result)
+      })
+    }
+
+    let selfProgress = { current: 0, total: 0 }
+    const subtasksProgress = []
+    let progressUpdateTimer, lastProgressUpdate = 0
+    const progressThrottleTime = 400
+    function updateProgress() {
+      if(progressUpdateTimer) clearTimeout(progressUpdateTimer)
+      const current = selfProgress.current + subtasksProgress.reduce(
+        (sum, progress) => sum + progress.current * (progress.factor ?? 1), 0)
+      const total = selfProgress.total + subtasksProgress.reduce(
+        (sum, progress) => sum + progress.total * (progress.factor ?? 1), 0)
+      reportProgress(current, total, selfProgress.action)
+
+      if(lastProgressUpdate + progressThrottleTime > Date.now()) { // ignore this update, do it later
+        setTimeout(updateProgress, progressThrottleTime - lastProgressUpdate - Date.now())
+        return
+      }
+      console.log("UPDATE", definition.name, "PROGRESS", current, total, selfProgress, subtasksProgress)
+      updateTask({
+        progress: { ...selfProgress, current, total }
+      })
     }
 
     const runTask = async () => {
@@ -114,18 +158,39 @@ export default function task(definition) {
         const result = await definition.execute(props, {
           ...context,
           task: {
-            async run(taskFunction, props) {
-              return await taskFunction(props, {
-                ...context,
-                causeType: definition.name + '_Task',
-                cause: taskObject.id
-              }, (events) => app.emitEvents(definition.name,
-                Array.isArray(events) ? events : [events], {}))
+            id: taskObject.id,
+            async run(taskFunction, props, progressFactor = 1) {
+              //console.log("SUBTASK RUN", taskFunction.definition.name, props)
+              const subtaskProgress = { current: 0, total: 1, factor: progressFactor }
+              subtasksProgress.push(subtaskProgress)
+              const result = await taskFunction(
+                props,
+                {
+                  ...context,
+                  taskObject: undefined,
+                  task: taskObject.id,
+                  causeType: 'task_Task',
+                  cause: taskObject.id
+                },
+                (events) => app.emitEvents(definition.name,
+                Array.isArray(events) ? events : [events], {}),
+                (current, total, action) => {
+                  subtaskProgress.current = current
+                  subtaskProgress.total = total
+                  updateProgress()
+                }
+              )
+              //console.log("SUBTASK DONE", taskFunction.definition.name, props, '=>', result)
+              subtaskProgress.current = subtaskProgress.total
+              updateProgress()
+              return result
             },
-            async progress(current, total) {
-              await updateTask({
-                progress: { current, total }
-              })
+            async progress(current, total, action, opts) {
+              selfProgress = {
+                ...opts,
+                current, total, action
+              }
+              updateProgress()
             }
           }
         })
@@ -136,34 +201,70 @@ export default function task(definition) {
         })
         await triggerOnTaskStateChange(taskObject, context.causeType, context.cause)
       } catch(error) {
-        if(taskObject.retries.length >= maxRetries) {
+        console.log("TASK ERROR", error.message, error.stack)
+        /*console.log("RETRIES", taskObject.retries?.length, maxRetries)*/
+        if((taskObject.retries?.length || 0) >= taskObject.maxRetries) {
           await updateTask({
             state: 'failed',
             doneAt: new Date(),
-            error: error.message
+            error: error.stack ?? error.message ?? error
+          })
+        } else {
+          const retriesCount = (taskObject.retries || []).length
+          await updateTask({
+            state: 'retrying',
+            retries: [...(taskObject.retries || []), {
+              startedAt: taskObject.startedAt,
+              failedAt: new Date(),
+              error: error.stack ?? error.message ?? error
+            }]
           })
-          await triggerOnTaskStateChange(taskObject, context.causeType, context.cause)
+          await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(2, retriesCount)))
         }
-        await updateTask(taskObject.id, {
-          state: 'retrying',
-          retries: [...taskObject.retries, {
-            startedAt: taskObject.startedAt,
-            failedAt: new Date(),
-            error: error.message
-          }]
-        })
         await triggerOnTaskStateChange(taskObject, context.causeType, context.cause)
       }
     }
 
-    /// TODO: implement task queues
     while(taskObject.state !== 'done' && taskObject.state !== 'failed') {
       await runTask()
+      console.log("TASK", definition.name, "AFTER RUNTASK", taskObject)
     }
 
     return taskObject.result
   }
 
+  serviceDefinition.beforeStart(async () => {
+    setTimeout(async () => {
+      let gt = ""
+      let tasksToRestart = await app.viewGet('runningTaskRootsByName', {
+        name: definition.name,
+        gt,
+        limit: 25
+      })
+      while(tasksToRestart.length > 0) {
+        console.log("FOUND", tasksToRestart.length, "TASKS", definition.name, "TO RESTART")
+        for(const task of tasksToRestart) {
+          console.log("RESTARTING TASK", task)
+          const taskObject = { ...task, id: task.to ?? task.id }
+          const context = {
+            causeType: task.causeType,
+            cause: task.cause,
+            taskObject,
+          }
+          const promise = taskFunction(taskObject.properties, context)
+          /// run async = ignore promise
+          await new Promise(resolve => setTimeout(resolve, 1000)) // wait a second
+        }
+        gt = tasksToRestart[tasksToRestart.length - 1].id
+        tasksToRestart = await app.viewGet('runningTaskRootsByName', {
+          name: definition.name,
+          gt,
+          limit: 25
+        })
+      }
+    }, 500)
+  })
+
   taskFunction.definition = definition
   taskFunction.start = async (props, causeType, cause) => {
     return await startTask(taskFunction, props, causeType, cause)