@transportme/vline-nsp-reader 1.0.2 → 1.0.4

package/lib/{table-reader.mjs → nsp/table-reader.mjs}

@@ -1,4 +1,4 @@
-import PDFParser from 'pdf2json'
+import PDFParser from '@transportme/pdf2json'
 import fs from 'fs/promises'
 
 export default class TableReader {

package/lib/{vline-nsp.mjs → nsp/vline-nsp.mjs}

@@ -92,5 +92,5 @@ export async function getNSPVersion() {
     nspVersion.addFile(new NSPFile(name.replace(/ NSP.+/, '').trim(), $(button).attr('href'), nspVersion))
   })
 
-  return Object.values(nspVersions).sort((a, b) => b.effective - a.effective)
+  return Object.values(nspVersions).sort((a, b) => b.effective - a.effective).filter(version => version.files.length > 5)
 }

package/lib/pass/pass-pdf-reader.mjs

@@ -0,0 +1,88 @@
+import PassTableReader from './pass-table-reader.mjs'
+
+export default class PassPDFReader {
+
+  #filePath
+  #pages
+
+  constructor(filePath) {
+    this.#filePath = filePath
+  }
+
+  getStations(table) {
+    return table.map(row => row[0].replace(/a ?r ?r$/, '').replace(/d ?e ?p$/, '').trim())
+  }
+
+  getBody(table) {
+    return table.map(row => row.slice(1))
+  }
+
+  async readRuns() {
+    const tableReader = new PassTableReader(this.#filePath)
+    const tables = await tableReader.read()
+
+    const runs = []
+
+    for (const table of tables) {
+      const stations = this.getStations(table)
+      const body = this.getBody(table)
+
+      for (let columnIndex = 0; columnIndex < body[0].length; columnIndex++) {
+        let currentRun = {
+          type: '',
+          stops: []
+        }
+
+        let lastStation
+        for (let stationIndex = 0; stationIndex < stations.length; stationIndex++) {
+          const stationName = stations[stationIndex]
+          const stopData = (body[stationIndex][columnIndex] || '').replace('.', ':')
+          if (stationName === 'Service') {
+            currentRun.type = stopData[0] + stopData.slice(1).toLowerCase()
+            continue
+          }
+          if (stationName === 'Service Information') continue
+          if (!stopData || stopData.length === 1) continue
+          if (stationName === 'Change Service' && stopData.length) {
+            currentRun.stops.push(lastStation)
+            runs.push(currentRun)
+            lastStation = null
+            currentRun = {
+              type: stopData[0] + stopData.slice(1).toLowerCase(),
+              stops: []
+            }
+
+            continue
+          }
+
+          if (lastStation && lastStation.name === stationName) {
+            lastStation.dep = stopData
+          } else {
+            if (lastStation) currentRun.stops.push(lastStation)
+            lastStation = {
+              name: stationName,
+              arr: stopData,
+              dep: stopData
+            }
+          }
+        }
+
+        currentRun.stops.push(lastStation)
+        runs.push(currentRun)
+      }
+    }
+
+    return runs.map(run => this.setRunData(run))
+  }
+
+  setRunData(run) {
+    return {
+      ...run,
+      origin: run.stops[0].name,
+      destination: run.stops[run.stops.length - 1].name,
+      departureTime: run.stops[0].dep,
+      destinationArrivalTime: run.stops[run.stops.length - 1].arr,
+    }
+  }
+
+}

package/lib/pass/pass-table-reader.mjs

@@ -0,0 +1,131 @@
+import PDFParser from '@transportme/pdf2json'
+import fs from 'fs/promises'
+
+export default class PassTableReader {
+
+  #file
+
+  constructor(file) {
+    this.#file = file
+  }
+
+  parserCallback(data) {
+    return data.Pages.flatMap(page => {
+      const { HLines, VLines } = page
+
+      const heightFrequency = VLines.reduce((acc, line) => {
+        if (!acc[line.l]) acc[line.l] = 0
+        acc[line.l]++
+        return acc
+      }, {})
+      const commonHeight = parseFloat(Object
+        .keys(heightFrequency)
+        .map(height => ({ height, freq: heightFrequency[height] }))
+        .sort((a, b) => b.freq - a.freq)[0].height)
+
+      const rowStarts = VLines
+        .filter(fill => Math.abs(fill.l - commonHeight) < 0.1)
+        .map(fill => fill.y)
+        .filter((e, i, a) => a.indexOf(e) === i)
+        .sort((a, b) => a - b)
+
+      const tables = rowStarts.slice(1).reduce((acc, row) => {
+        const currTable = acc[acc.length - 1]
+        const tableEnd = currTable[currTable.length - 1]
+        if (row - tableEnd > commonHeight * 1.5) {
+          // New table
+          acc.push([row])
+        } else {
+          // Existing table
+          currTable.push(row)
+        }
+
+        return acc
+      }, [[rowStarts[0]]])
+
+      return tables.map(rowStarts => {
+        const tableStart = rowStarts[0]
+        const tableEnd = rowStarts[rowStarts.length - 1]
+
+        const colStarts = HLines
+          .filter(fill => tableStart - 0.1 < fill.y && fill.y < tableEnd + commonHeight * 1.5)
+          .map(fill => fill.x)
+          .filter((e, i, a) => a.indexOf(e) === i)
+          .sort((a, b) => a - b)
+
+        const tableData = []
+
+        page.Texts.forEach(text => {
+          let textContent = decodeURIComponent(text.R[0].T)
+
+          let firstYGreater = rowStarts.find(r => r > text.y + 0.3)
+          let currentRow = rowStarts.indexOf(firstYGreater) - 1
+          if (currentRow < 0) return
+
+          let currentCol = colStarts.findLastIndex(c => c < text.x + 0.4)
+
+          if (!tableData[currentRow]) tableData[currentRow] = []
+
+          if (!tableData[currentRow][currentCol]) tableData[currentRow][currentCol] = textContent
+          else tableData[currentRow][currentCol] += ` ${textContent}`
+        })
+
+        for (let y = 0; y < tableData.length; y++) {
+          if (!tableData[y]) tableData[y] = []
+          for (let x = 0; x < tableData[y].length; x++) {
+            if (!tableData[y][x]) tableData[y][x] = ''
+          }
+        }
+
+        return tableData
+      })
+
+      console.log(tables)
+
+      console.log(heightFrequency)
+      console.log(VLines)
+      console.log(rowStarts)
+
+      let pageData = []
+      let smallTable = rowStarts.length === 2
+
+      return pageData
+    })
+
+    return pages[0]
+
+    // return pages.map(page => {
+    //   let maxSize = Math.max(...page.map(row => row.length))
+    //   let blankCells = Array(maxSize).fill('')
+
+    //   return page.map(row => row.map(g => g.replace(/  +/g, ' ').trim()).concat(blankCells).slice(0, maxSize))
+    // })
+  }
+
+  read() {
+    return new Promise(async (resolve, reject) => {
+      let pdfParser = new PDFParser()
+
+      pdfParser.on("pdfParser_dataReady", data => {
+        try {
+          resolve(this.parserCallback(data))
+        } catch (err) {
+          reject(err)
+        }
+      })
+
+      pdfParser.on("pdfParser_dataError", err => {
+        reject(err)
+      })
+
+      try {
+        let pdfBuffer = await fs.readFile(this.#file)
+        pdfParser.parseBuffer(pdfBuffer)
+      } catch (err) {
+        reject(err)
+      }
+    })
+  
+  }
+
+}

package/lib.mjs

@@ -1,7 +1,9 @@
-import { getNSPVersion, NSPFile, NSPVersion } from './lib/vline-nsp.mjs'
+import { getNSPVersion, NSPFile, NSPVersion } from './lib/nsp/vline-nsp.mjs'
+import PassPDFReader from './lib/pass/pass-pdf-reader.mjs'
 
 export {
   getNSPVersion,
   NSPFile,
-  NSPVersion
+  NSPVersion,
+  PassPDFReader
 }

package/package.json

@@ -1,18 +1,18 @@
 {
   "name": "@transportme/vline-nsp-reader",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "main": "lib.mjs",
   "scripts": {
-    "test": "mocha"
+    "test": "mocha './{,!(node_modules)/**}/*.test.mjs'"
   },
   "author": "",
   "license": "ISC",
   "description": "",
   "dependencies": {
+    "@transportme/pdf2json": "^4.0.10000004",
     "async": "^3.2.6",
     "cheerio": "^1.0.0",
-    "node-fetch": "^3.3.2",
-    "pdf2json": "github:eyeballcode/pdf2json"
+    "node-fetch": "^3.3.2"
   },
   "devDependencies": {
     "chai": "^5.1.2",

package/read-pdf.mjs

@@ -1,12 +1,10 @@
-import util from 'util'
-import NSPPDFReader from './lib/nsp-pdf-reader.mjs'
-import TableReader from './lib/table-reader.mjs'
+import PassTableReader from './lib/pass/pass-table-reader.mjs'
 
 // let nspReader = new NSPPDFReader(process.argv[2])
 // await nspReader.read()
 // console.log(util.inspect(nspReader.getAllRuns(), { depth: null, colors: true, maxArrayLength: null }))
 
-let tableReader = new TableReader(process.argv[2])
-let pages = await tableReader.read()
+let tableReader = new PassTableReader(process.argv[2])
+let tables = await tableReader.read()
 
-for (let page of pages) console.table(page)
+for (let table of tables) console.table(table)