fleetmap-reports 2.0.199 → 2.0.200

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fleetmap-reports",
-  "version": "2.0.199",
+  "version": "2.0.200",
   "description": "",
   "main": "src/index.js",
   "scripts": {
@@ -23,6 +23,8 @@
     "jspdf": "^2.5.1",
     "jspdf-autotable": "3.8.1",
     "moment": "2.30.1",
+    "stream-chain": "^3.4.0",
+    "stream-json": "^1.9.1",
     "traccar-api": "^1.0.5",
     "wkt": "^0.1.1"
   },

package/src/util/parallel.js

@@ -37,7 +37,7 @@ exports.parallel = (report, method, toSlice, ...args) => {
               resolve(_result.flat())
             }
           })
-          worker.on('error', e => {console.error(e); reject(e)})
+          worker.on('error', e => { console.error(e); reject(e) })
           worker.on('exit', (code, signal) => {
             if (code !== 0) {
               const error = `Worker exited with code ${code} and signal ${signal}`

package/src/util/traccar.js

@@ -1,5 +1,9 @@
 const automaticReports = require('../automaticReports')
 const { convertFromUTC, isClientSide, convertFromLocal } = require('./utils')
+const { parser } = require('stream-json')
+const { streamArray } = require('stream-json/streamers/StreamArray')
+const { chain } = require('stream-chain')
+const { pick } = require('stream-json/filters/Pick')
 
 async function getRoute (traccar, from, to, devices) {
   const devicesToSlice = devices.slice()
@@ -66,6 +70,58 @@ async function getSummary (traccar, from, to, devices) {
   return result.map(r => r.summary).flat()
 }
 
+async function getItems (traccar, u, _chunk, counter, devicesPerRequest, ignorePercentage,
+  totalDevices, currentDeviceCount, routeOnly) {
+  const x = routeOnly
+    ? await streamJson(traccar, u)
+    : await traccar.axios.get(u, {
+      jar: traccar.cookieJar,
+      withCredentials: true,
+      timeout: 900000
+    }).then(r => r.data)
+
+  console.log('LOADING_MESSAGE:' + _chunk[0].name)
+  if (counter) {
+    counter.count += devicesPerRequest
+    ignorePercentage || console.log(`PROGRESS_PERC:${Math.round(counter.count / totalDevices * 100)}`)
+  } else {
+    currentDeviceCount += devicesPerRequest
+    ignorePercentage || console.log(`PROGRESS_PERC:${Math.round(currentDeviceCount / totalDevices * 100)}`)
+  }
+  console.log('done devices', _chunk.map(d => d.id), 'position count', x.route && x.route.length)
+  return x
+}
+
+async function streamJson (traccar, u) {
+  console.log('streaming', u)
+  const response = await traccar.axios.get(u, {
+    responseType: 'stream',
+    jar: traccar.cookieJar,
+    withCredentials: true,
+    timeout: 900000
+  })
+
+  const result = {
+    route: []
+  }
+
+  return new Promise((resolve, reject) => {
+    const pipeline = chain([
+      response.data, // Axios stream (Node readable)
+      parser(), // Tokenize
+      pick({ filter: 'route' }), // Pick 'route', 'summary', etc.
+      streamArray() // Stream each item in the array
+    ])
+
+    pipeline.on('data', ({ value }) => {
+      result.route.push(value) // Or stream to a DB/file/processor
+      console.log('pushed to', result.route.length)
+    })
+    pipeline.on('end', () => resolve(result))
+    pipeline.on('error', reject)
+  })
+}
+
 async function getAllInOne (
   traccar,
   from,
@@ -95,36 +151,15 @@ async function getAllInOne (
       const u = url + '&' + _chunk.map(d => `deviceId=${d.id}`).join('&')
       console.log(u)
       requests.push(
-        traccar.axios.get(u, {
-          jar: traccar.cookieJar,
-          withCredentials: true,
-          timeout: 900000
-        }).catch(e => {
-          console.log(e.message, 'try again', u)
-          return traccar.axios.get(u, {
-            jar: traccar.cookieJar,
-            withCredentials: true,
-            timeout: 900000
-          })
-        }).then(r => r.data).then(x => {
-          console.log('LOADING_MESSAGE:' + _chunk[0].name)
-          if (counter) {
-            counter.count += devicesPerRequest
-            ignorePercentage || console.log(`PROGRESS_PERC:${Math.round(counter.count / totalDevices * 100)}`)
-          } else {
-            currentDeviceCount += devicesPerRequest
-            ignorePercentage || console.log(`PROGRESS_PERC:${Math.round(currentDeviceCount / totalDevices * 100)}`)
-          }
-          console.log('done devices', _chunk.map(d => d.id), 'position count:', x.route && x.route.length)
-          return x
-        }))
+        getItems(traccar, u, _chunk, counter, devicesPerRequest, ignorePercentage, totalDevices, currentDeviceCount,
+          !getTrips && !getStops && !getSummary))
     }
     const now = new Date()
     console.log('parallel', requests.length)
     try {
       result.push(...(await Promise.all(requests)))
     } catch (e) {
-      console.log('parallel', e.message, e.config)
+      console.error('parallel', e.message, e.config)
     }
     console.log('took', new Date() - now, 'ms')
   }

package/src/zone-report.js

@@ -23,10 +23,11 @@ const { checkGeofenceIn } = require('./util/geofence')
 const { parallel } = require('./util/parallel')
 const { getDataByDay } = require('./util/trips')
 const axios = require('axios').default
-const sliceSize = 3
+const sliceSize = 4
 const fileName = 'ZoneReport'
 
 async function createZoneReport (from, to, userData, traccar) {
+  console.log('createZoneReport 200', from, to)
   const reportData = []
   const devices = devicesToProcess(userData)
   const allData = {
@@ -482,7 +483,7 @@ function analyseAlerts (alerts, deviceRoute, userData, from, to, device) {
 async function getInAndOutEvents (devices, route, userData) {
   const events = []
   const geofencesFeatures = userData.geofences.map(g => convertToFeature(g))
-  console.log('getInAndOutEvents', devices)
+  console.log('getInAndOutEvents', devices.length)
   devices.forEach(d => {
     const deviceRoute = route.filter(p => p.deviceId === d.id)
     const routePoints = deviceRoute.sort(sortPositions).map(p => convertPositionToFeature(p))