fleetmap-reports 2.0.363 → 2.0.365

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fleetmap-reports",
-  "version": "2.0.363",
+  "version": "2.0.365",
   "description": "",
   "main": "src/index.js",
   "scripts": {

package/src/partnerReports/driverranking-report.js

@@ -1,8 +1,9 @@
+const axios = require('axios')
 const traccarHelper = require('../util/traccar')
 const { getDriverData, devicesByDriver } = require('../util/driver')
 const { getCanDriverStyleMessages, parseCanDriverStyleMessage } = require('../util/xpert')
 const { calculateRPMSections } = require('./performance-report')
-const { getTranslations, isClientSide, convertToLocaleString, convertMS, executeServerSide } = require('../util/utils')
+const { getTranslations, isClientSide, convertToLocaleString, convertMS, getServerHost } = require('../util/utils')
 const jsPDF = require('jspdf')
 const { headerFromUser, addTable } = require('../util/pdfDocument')
 const { getStyle } = require('../reportStyle')
@@ -57,6 +58,20 @@ function toGrade (score) {
   return grades.find(g => score >= g.min).grade
 }
 
+// Splits [from, to] into consecutive sub-ranges of at most chunkDays each
+function splitDateRange (from, to, chunkDays) {
+  const chunkMs = chunkDays * 24 * 60 * 60 * 1000
+  const end = new Date(to).getTime()
+  const chunks = []
+  let chunkStart = new Date(from).getTime()
+  while (chunkStart < end) {
+    const chunkEnd = Math.min(chunkStart + chunkMs, end)
+    chunks.push({ from: new Date(chunkStart), to: new Date(chunkEnd) })
+    chunkStart = chunkEnd
+  }
+  return chunks.length ? chunks : [{ from: new Date(from), to: new Date(to) }]
+}
+
 async function create (from, to, userData, traccar) {
   const reportData = {
     drivers: []
@@ -73,25 +88,8 @@ async function create (from, to, userData, traccar) {
   }
 
   if (isClientSide() && (userData.drivers.length > 50 || ((new Date(to).getTime() - new Date(from).getTime()) > (1000 * 60 * 60 * 24 * 15)))) {
-    const sliced = automaticReports.sliceArray(userData.drivers, 5)
-    userData.requestToRunServerSide = true
-    userData.byDriver = true
-    // Narrow the devices sent per-slice to just the ones those drivers actually used, so a single
-    // server invocation doesn't have to hold the whole fleet's route data in memory at once (was
-    // causing Runtime.OutOfMemory on large fleets).
-    const allDriversData = await executeServerSide('driverranking-report', sliced, from, to, userData, userData.drivers.length, traccar, 5, (driversSlice) => {
-      const deviceIds = new Set()
-      driversSlice.forEach(d => {
-        const ids = deviceIdsByDriver.get(d.uniqueId)
-        if (ids) {
-          ids.forEach(id => deviceIds.add(id))
-        }
-      })
-      return { devices: userData.devices.filter(device => deviceIds.has(device.id)) }
-    })
-
-    allDriversData.sort((a, b) => b.overallScore - a.overallScore)
-    allDriversData.forEach((d, index) => { d.rank = index + 1 })
+    const rawRows = await runServerSideGrid(from, to, userData, deviceIdsByDriver, traccar)
+    const allDriversData = scoreAndRankDrivers(mergeRawRows(rawRows), userData)
 
     reportData.drivers = allDriversData
     reportData.target = buildTargets(userData)
@@ -100,44 +98,70 @@ async function create (from, to, userData, traccar) {
   }
 
   const devices = userData.devices
-  let deviceCount = 0
   const driversData = new Map()
-  const overspeedEvents = await getOverspeedEvents(from, to, userData, devices, userData, traccar)
 
-  const sliced = automaticReports.sliceArray(devices, 5)
+  // Process the period in chunks so we never hold more than ~a week of raw GPS positions for the
+  // whole device set in memory at once - a single busy device can have tens of thousands of
+  // positions over a month, which was causing Runtime.OutOfMemory even after narrowing devices/drivers.
+  const dateChunks = splitDateRange(from, to, 7)
+  const totalDeviceOperations = devices.length * dateChunks.length
+  let deviceCount = 0
 
-  for (const slice of sliced) {
-    const allInOne = await traccarHelper.getAllInOne(traccar, from, to, slice, true, true, false, false, deviceCount, devices.length)
+  for (const dateChunk of dateChunks) {
+    const overspeedEvents = await getOverspeedEvents(dateChunk.from, dateChunk.to, userData, devices, userData, traccar)
+    const sliced = automaticReports.sliceArray(devices, 5)
 
-    const result = []
-    const requests = []
-    for (const _chunk of automaticReports.sliceArray(slice.map(d => d.id), 1)) {
-      requests.push(traccar.reports.reportsEventsGet(from, to, _chunk, null, ['alarm']))
-    }
+    for (const slice of sliced) {
+      const allInOne = await traccarHelper.getAllInOne(traccar, dateChunk.from, dateChunk.to, slice, true, true, false, false, deviceCount, totalDeviceOperations)
 
-    try {
-      result.push(...(await Promise.all(requests)))
-    } catch (e) {
-      console.log(e)
-    }
-    const alarmEvents = result.flatMap(d => d.data)
+      const result = []
+      const requests = []
+      for (const _chunk of automaticReports.sliceArray(slice.map(d => d.id), 1)) {
+        requests.push(traccar.reports.reportsEventsGet(dateChunk.from, dateChunk.to, _chunk, null, ['alarm']))
+      }
+
+      try {
+        result.push(...(await Promise.all(requests)))
+      } catch (e) {
+        console.log(e)
+      }
+      const alarmEvents = result.flatMap(d => d.data)
+
+      for (const device of slice) {
+        const route = allInOne.route.filter(t => t.deviceId === device.id)
+        const trips = allInOne.trips.filter(t => t.deviceId === device.id)
 
-    for (const device of slice) {
-      const route = allInOne.route.filter(t => t.deviceId === device.id)
-      const trips = allInOne.trips.filter(t => t.deviceId === device.id)
+        const deviceOverspeedEvents = overspeedEvents.find(e => e.device.id === device.id)
+        const deviceAlarmEvents = alarmEvents.filter(e => e.deviceId === device.id)
 
-      const deviceOverspeedEvents = overspeedEvents.find(e => e.device.id === device.id)
-      const deviceAlarmEvents = alarmEvents.filter(e => e.deviceId === device.id)
+        await processDevice(driversData, device, userData, { route, trips }, deviceAlarmEvents, deviceOverspeedEvents)
+        deviceCount++
+      }
 
-      await processDevice(driversData, device, userData, { route, trips }, deviceAlarmEvents, deviceOverspeedEvents)
-      deviceCount++
+      console.log('LOADING_MESSAGE:' + slice[0].name)
+      console.log(`PROGRESS_PERC:${deviceCount / totalDeviceOperations * 100}`)
     }
+  }
+
+  const rawRows = Array.from(driversData.values())
 
-    console.log('LOADING_MESSAGE:' + slice[0].name)
-    console.log(`PROGRESS_PERC:${deviceCount / devices.length * 100}`)
+  // When invoked as one cell of the client's (driverSlice x dateChunk) grid, return the raw
+  // accumulated totals for just this chunk - the client sums them across chunks for the same
+  // driver before computing scores, since scores are relative to the whole requested period.
+  if (userData.rawMode) {
+    reportData.drivers = rawRows
+    return reportData
   }
 
-  const allDriversData = (Array.from(driversData.values())).filter(d => d.distance > 0)
+  reportData.drivers = scoreAndRankDrivers(rawRows, userData)
+  reportData.target = buildTargets(userData)
+  reportData.coefficient = buildCoefficients()
+
+  return reportData
+}
+
+function scoreAndRankDrivers (rawRows, userData) {
+  const allDriversData = rawRows.filter(d => d.distance > 0)
   allDriversData.forEach(d => {
     d.avgFuelConsumption = getCanAvgConsumption(d.distance, d.spentFuel).byKms
 
@@ -163,12 +187,86 @@ async function create (from, to, userData, traccar) {
   })
   allDriversData.sort((a, b) => b.overallScore - a.overallScore)
   allDriversData.forEach((d, index) => { d.rank = index + 1 })
+  return allDriversData
+}
 
-  reportData.drivers.push(...allDriversData)
-  reportData.target = buildTargets(userData)
-  reportData.coefficient = buildCoefficients()
+const mergedFields = ['distance', 'highEngineRPM', 'hardBraking', 'hardAcceleration', 'hardCornering',
+  'overspeed', 'spentFuel', 'geofenceAlarm', 'continuesDrivingAlarm', 'reverseAlarm', 'otherAlarm']
+
+// Sums each driver's raw totals across every (driverSlice x dateChunk) grid cell they appeared in
+function mergeRawRows (rawRows) {
+  const merged = new Map()
+  rawRows.forEach(row => {
+    let acc = merged.get(row.driverId)
+    if (!acc) {
+      acc = { driverId: row.driverId, name: row.name }
+      mergedFields.forEach(field => { acc[field] = 0 })
+      merged.set(row.driverId, acc)
+    }
+    mergedFields.forEach(field => { acc[field] += row[field] || 0 })
+  })
+  return Array.from(merged.values())
+}
 
-  return reportData
+// Fans the report out across a grid of (driverSlice x dateChunk) requests to the backend, so no
+// single Lambda invocation has to hold more than a handful of drivers' worth of one week of route
+// data in memory at once, and progress feedback arrives once per grid cell rather than once per
+// driver-slice (which could take minutes on its own with a long date range).
+async function runServerSideGrid (from, to, userData, deviceIdsByDriver, traccar) {
+  const driverSlices = automaticReports.sliceArray(userData.drivers, 5)
+  const dateChunks = splitDateRange(from, to, 7)
+
+  const cells = []
+  driverSlices.forEach(driverSlice => {
+    dateChunks.forEach(dateChunk => cells.push({ driverSlice, dateChunk }))
+  })
+
+  const cookie = await axios.get('/pinmeapi/cookie/get', { withCredentials: true }).then(d => d.data)
+  const url = `https://${getServerHost()}/pinmeapi/reports/driverranking-report`
+  const rawRows = []
+  let completed = 0
+  const maxConcurrent = 10
+
+  for (let i = 0; i < cells.length; i += maxConcurrent) {
+    const batch = cells.slice(i, i + maxConcurrent)
+    await Promise.all(batch.map(async ({ driverSlice, dateChunk }) => {
+      const deviceIds = new Set()
+      driverSlice.forEach(d => {
+        const ids = deviceIdsByDriver.get(d.uniqueId)
+        if (ids) {
+          ids.forEach(id => deviceIds.add(id))
+        }
+      })
+      const slicedDevices = userData.devices.filter(device => deviceIds.has(device.id))
+
+      try {
+        const response = await axios.post(url, {
+          from: dateChunk.from,
+          to: dateChunk.to,
+          userData: {
+            ...userData,
+            drivers: driverSlice,
+            devices: slicedDevices,
+            geofences: [],
+            requestToRunServerSide: true,
+            rawMode: true
+          },
+          cookie
+        }, { withCredentials: true })
+        const result = response.data[0]
+        if (result) {
+          rawRows.push(...(result.drivers || []))
+        }
+      } catch (e) {
+        console.error(e)
+      }
+      completed++
+      console.log('LOADING_MESSAGE:' + driverSlice[0].name)
+      console.log(`PROGRESS_PERC:${(completed / cells.length) * 100}`)
+    }))
+  }
+
+  return rawRows
 }
 
 function buildTargets (userData) {
@@ -219,6 +317,7 @@ async function processDevice (driversData, device, userData, allInOne, alarmEven
     let driverData = driversData.get(d.id)
     if (!driverData) {
       driverData = {
+        driverId: d.id,
         name: d.name,
         distance: 0,
         highEngineRPM: 0,