fleetmap-reports 2.0.196 → 2.0.198

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fleetmap-reports",
-  "version": "2.0.196",
+  "version": "2.0.198",
   "description": "",
   "main": "src/index.js",
   "scripts": {
@@ -14,20 +14,22 @@
     "@turf/distance": "^6.5.0",
     "@turf/helpers": "^6.5.0",
     "@turf/point-to-line-distance": "^6.5.0",
-    "axios": "1.8.2",
+    "axios": "1.6.7",
     "country-reverse-geocoding": "^0.2.2",
     "docx": "^7.3.0",
     "file-saver": "^2.0.5",
     "fleetmap-partners": "1.0.185",
     "json-as-xlsx": "2.5.6",
-    "JSONStream": "^1.3.5",
     "jspdf": "^2.5.1",
     "jspdf-autotable": "3.8.1",
     "moment": "2.30.1",
+    "stream-chain": "^3.4.0",
+    "stream-json": "^1.9.1",
     "traccar-api": "^1.0.5",
     "wkt": "^0.1.1"
   },
   "devDependencies": {
+    "axios-debug-log": "^0.8.4",
     "eslint": "^8.15.0",
     "eslint-config-standard": "^17.0.0",
     "eslint-plugin-import": "^2.26.0",

package/src/util/traccar.js

@@ -1,6 +1,9 @@
 const automaticReports = require('../automaticReports')
 const { convertFromUTC, isClientSide, convertFromLocal } = require('./utils')
-const { parse } = require('JSONStream')
+const { parser } = require('stream-json')
+const { streamArray } = require('stream-json/streamers/StreamArray')
+const { chain } = require('stream-chain')
+const { pick } = require('stream-json/filters/Pick')
 
 async function getRoute (traccar, from, to, devices) {
   const devicesToSlice = devices.slice()
@@ -67,6 +70,56 @@ async function getSummary (traccar, from, to, devices) {
   return result.map(r => r.summary).flat()
 }
 
+async function getItems (traccar, u, _chunk, counter, devicesPerRequest, ignorePercentage,
+  totalDevices, currentDeviceCount, routeOnly) {
+  const x = routeOnly
+    ? await streamJson(traccar, u)
+    : await traccar.axios.get(u, {
+      jar: traccar.cookieJar,
+      withCredentials: true,
+      timeout: 900000
+    }).then(r => r.data)
+
+  console.log('LOADING_MESSAGE:' + _chunk[0].name)
+  if (counter) {
+    counter.count += devicesPerRequest
+    ignorePercentage || console.log(`PROGRESS_PERC:${Math.round(counter.count / totalDevices * 100)}`)
+  } else {
+    currentDeviceCount += devicesPerRequest
+    ignorePercentage || console.log(`PROGRESS_PERC:${Math.round(currentDeviceCount / totalDevices * 100)}`)
+  }
+  console.log('done devices', _chunk.map(d => d.id), 'position count', x.route && x.route.length)
+  return x
+}
+
+async function streamJson (traccar, u) {
+  const response = await traccar.axios.get(u, {
+    responseType: 'stream',
+    jar: traccar.cookieJar,
+    withCredentials: true,
+    timeout: 900000
+  })
+
+  const result = {
+    route: []
+  }
+
+  return new Promise((resolve, reject) => {
+    const pipeline = chain([
+      response.data, // Axios stream (Node readable)
+      parser(), // Tokenize
+      pick({ filter: 'route' }), // Pick 'route', 'summary', etc.
+      streamArray() // Stream each item in the array
+    ])
+
+    pipeline.on('data', ({ value }) => {
+      result.route.push(value) // Or stream to a DB/file/processor
+    })
+    pipeline.on('end', () => resolve(result))
+    pipeline.on('error', reject)
+  })
+}
+
 async function getAllInOne (
   traccar,
   from,
@@ -96,39 +149,8 @@ async function getAllInOne (
       const u = url + '&' + _chunk.map(d => `deviceId=${d.id}`).join('&')
       console.log(u)
       requests.push(
-        traccar.axios.get(u, {
-          jar: traccar.cookieJar,
-          withCredentials: true,
-          timeout: 900000,
-          responseType: 'stream'
-        }).then(r => {
-          const stream = r.data
-          const jsonStream = parse()
-          stream.pipe(jsonStream)
-          let responseData = []
-          jsonStream.on('data', data => {
-            responseData = data
-          })
-          return new Promise((resolve, reject) => {
-            jsonStream.on('close', data => {
-              resolve(responseData)
-            })
-            jsonStream.on('error', err => {
-              reject(err)
-            })
-          })
-        }).then(x => {
-          console.log('LOADING_MESSAGE:' + _chunk[0].name)
-          if (counter) {
-            counter.count += devicesPerRequest
-            ignorePercentage || console.log(`PROGRESS_PERC:${Math.round(counter.count / totalDevices * 100)}`)
-          } else {
-            currentDeviceCount += devicesPerRequest
-            ignorePercentage || console.log(`PROGRESS_PERC:${Math.round(currentDeviceCount / totalDevices * 100)}`)
-          }
-          console.log('done devices', _chunk.map(d => d.id), 'position count:', x.route && x.route.length)
-          return x
-        }))
+        getItems(traccar, u, _chunk, counter, devicesPerRequest, ignorePercentage, totalDevices, currentDeviceCount,
+          !getTrips && !getStops && !getSummary))
     }
     const now = new Date()
     console.log('parallel', requests.length)