@transportme/vline-nsp-reader 1.0.3 → 1.0.5

package/lib/heat/constants.mjs

@@ -0,0 +1,4 @@
+export default {
+  VLINE_HOST: 'https://vline.com.au',
+  HEAT_PAGE: '/Timetables/heat'
+}

package/lib/heat/vline-heat.mjs

@@ -0,0 +1,93 @@
+import fetch from 'node-fetch';
+import constants from './constants.mjs'
+import { load as parseHTML } from 'cheerio'
+import async from 'async'
+import fs from 'fs/promises'
+import path from 'path'
+import { pipeline } from 'stream/promises'
+import { createWriteStream } from 'fs'
+import PassPDFReader from '../pass/pass-pdf-reader.mjs'
+
+export class HeatTimetable {
+
+  files = []
+
+  constructor() {
+  }
+
+  addFile(file) {
+    this.files.push(file)
+  }
+
+  async saveFiles(outputDir) {
+    try {
+      await fs.mkdir(outputDir)
+    } catch (e) {}
+    await async.forEach(this.files, async file => {
+      await file.download(outputDir)
+    })
+  }
+
+}
+
+export class HeatTimetableFile {
+
+  line
+  type
+  href
+
+  #filePath 
+
+  constructor(line, type, href) {
+    this.line = line
+    this.type = type
+    this.href = href
+  }
+
+  async download(outputDir) {
+    this.#filePath = path.join(outputDir, `${this.line} - ${this.type}.pdf`)
+
+    let response = await fetch(constants.VLINE_HOST + this.href)
+    let outputStream = createWriteStream(this.#filePath)
+
+    await pipeline(response.body, outputStream)
+  }
+
+  setFilePath(filePath) {
+    this.#filePath = filePath
+  }
+
+  async extractRuns() {
+    let reader = new PassPDFReader(this.#filePath)
+    return await reader.readRuns()
+  }
+
+  static fromFile(pathname) {
+    const filename = path.basename(pathname).replace('.pdf', '')
+    const [line, type] = filename.split(' - ')
+    const file = new HeatTimetableFile(line, type, '')
+    file.setFilePath(pathname)
+
+    return file
+  }
+
+}
+
+export async function getHeatTimetables() {
+  let body = await (await fetch(constants.VLINE_HOST + constants.HEAT_PAGE)).text()
+  let $ = parseHTML(body)
+
+  let buttons = Array.from($('div.TimeTableHeaderMainContainer > a.button-file-link-caption'))
+  const timetable = new HeatTimetable()
+
+  buttons.forEach(button => {
+    let text = $(button).text().replace(/PDF.+/, '').replace(' extreme heat timetable', '').replace(/via .+\(/, '(').replace(/ \/ \w+/, '')
+    let data = text.match(/([\w ]+) \((.+)\)/)
+    if (!data) return null
+
+    let [_, line, type] = data
+    timetable.addFile(new HeatTimetableFile(line, type, $(button).attr('href')))
+  })
+
+  return timetable.files.length ? timetable : null
+}

package/lib/pass/pass-pdf-reader.mjs

@@ -0,0 +1,89 @@
+import PassTableReader from './pass-table-reader.mjs'
+
+export default class PassPDFReader {
+
+  #filePath
+  #pages
+
+  constructor(filePath) {
+    this.#filePath = filePath
+  }
+
+  getStations(table) {
+    return table.map(row => row[0].replace(/a ?r ?r$/, '').replace(/d ?e ?p$/, '').trim())
+  }
+
+  getBody(table) {
+    return table.map(row => row.slice(1))
+  }
+
+  async readRuns() {
+    const tableReader = new PassTableReader(this.#filePath)
+    const tables = await tableReader.read()
+
+    const runs = []
+
+    for (const table of tables) {
+      const stations = this.getStations(table)
+      const body = this.getBody(table)
+
+      for (let columnIndex = 0; columnIndex < body[0].length; columnIndex++) {
+        let currentRun = {
+          type: '',
+          stops: []
+        }
+
+        let lastStation
+        for (let stationIndex = 0; stationIndex < stations.length; stationIndex++) {
+          const stationName = stations[stationIndex]
+          const stopData = (body[stationIndex][columnIndex] || '').replace('.', ':')
+          if (stationName === 'Service') {
+            currentRun.type = stopData[0] + stopData.slice(1).toLowerCase()
+            continue
+          }
+          if (stationName === 'Service Information') continue
+          if (!stopData || stopData.length === 1) continue
+          if (stationName === 'Change Service' && !currentRun.stops.length) continue
+          if (stationName === 'Change Service' && stopData.length) {
+            currentRun.stops.push(lastStation)
+            runs.push(currentRun)
+            lastStation = null
+            currentRun = {
+              type: stopData[0] + stopData.slice(1).toLowerCase(),
+              stops: []
+            }
+
+            continue
+          }
+
+          if (lastStation && lastStation.name === stationName) {
+            lastStation.dep = stopData
+          } else {
+            if (lastStation) currentRun.stops.push(lastStation)
+            lastStation = {
+              name: stationName,
+              arr: stopData,
+              dep: stopData
+            }
+          }
+        }
+
+        if (lastStation) currentRun.stops.push(lastStation)
+        if (currentRun.stops.length) runs.push(currentRun)
+      }
+    }
+
+    return runs.map(run => this.setRunData(run))
+  }
+
+  setRunData(run) {
+    return {
+      ...run,
+      origin: run.stops[0].name,
+      destination: run.stops[run.stops.length - 1].name,
+      departureTime: run.stops[0].dep,
+      destinationArrivalTime: run.stops[run.stops.length - 1].arr,
+    }
+  }
+
+}

package/lib/pass/pass-table-reader.mjs

@@ -0,0 +1,132 @@
+import PDFParser from '@transportme/pdf2json'
+import fs from 'fs/promises'
+
+export default class PassTableReader {
+
+  #file
+
+  constructor(file) {
+    this.#file = file
+  }
+
+  parserCallback(data) {
+    return data.Pages.flatMap(page => {
+      const { HLines, VLines } = page
+
+      const heightFrequency = VLines.reduce((acc, line) => {
+        if (!acc[line.l]) acc[line.l] = 0
+        acc[line.l]++
+        return acc
+      }, {})
+      const commonHeight = parseFloat(Object
+        .keys(heightFrequency)
+        .map(height => ({ height, freq: heightFrequency[height] }))
+        .sort((a, b) => b.freq - a.freq)[0].height)
+
+      const rowStarts = VLines
+        .filter(fill => Math.abs(fill.l - commonHeight) < 0.1)
+        .map(fill => fill.y)
+        .filter((e, i, a) => a.indexOf(e) === i)
+        .sort((a, b) => a - b)
+
+      const tables = rowStarts.slice(1).reduce((acc, row) => {
+        const currTable = acc[acc.length - 1]
+        const tableEnd = currTable[currTable.length - 1]
+        if (row - tableEnd > commonHeight * 1.5) {
+          // New table
+          acc.push([row])
+        } else {
+          // Existing table
+          currTable.push(row)
+        }
+
+        return acc
+      }, [[rowStarts[0]]])
+
+      return tables.map(rowStarts => {
+        const tableStart = rowStarts[0]
+        const tableEnd = rowStarts[rowStarts.length - 1]
+
+        const colStarts = HLines
+          .filter(fill => tableStart - 0.1 < fill.y && fill.y < tableEnd + commonHeight * 1.5)
+          .map(fill => fill.x)
+          .filter((e, i, a) => a.indexOf(e) === i)
+          .sort((a, b) => a - b)
+
+        const tableData = []
+
+        page.Texts.forEach(text => {
+          if (text.y < tableStart - commonHeight || text.y > tableEnd + commonHeight * 1.5) return
+
+          let textContent = decodeURIComponent(text.R[0].T)
+
+          let currentRow = rowStarts.findIndex(r => r > text.y + 0.3) - 1
+          if (currentRow < 0) currentRow = rowStarts.length - 1
+
+          let currentCol = colStarts.findLastIndex(c => c < text.x + 0.4)
+
+          if (!tableData[currentRow]) tableData[currentRow] = []
+
+          if (!tableData[currentRow][currentCol]) tableData[currentRow][currentCol] = textContent
+          else tableData[currentRow][currentCol] += ` ${textContent}`
+        })
+
+        for (let y = 0; y < tableData.length; y++) {
+          if (!tableData[y]) tableData[y] = []
+          for (let x = 0; x < tableData[y].length; x++) {
+            if (!tableData[y][x]) tableData[y][x] = ''
+          }
+        }
+
+        return tableData
+      })
+
+      console.log(tables)
+
+      console.log(heightFrequency)
+      console.log(VLines)
+      console.log(rowStarts)
+
+      let pageData = []
+      let smallTable = rowStarts.length === 2
+
+      return pageData
+    })
+
+    return pages[0]
+
+    // return pages.map(page => {
+    //   let maxSize = Math.max(...page.map(row => row.length))
+    //   let blankCells = Array(maxSize).fill('')
+
+    //   return page.map(row => row.map(g => g.replace(/  +/g, ' ').trim()).concat(blankCells).slice(0, maxSize))
+    // })
+  }
+
+  read() {
+    return new Promise(async (resolve, reject) => {
+      let pdfParser = new PDFParser()
+
+      pdfParser.on("pdfParser_dataReady", data => {
+        try {
+          resolve(this.parserCallback(data))
+        } catch (err) {
+          reject(err)
+        }
+      })
+
+      pdfParser.on("pdfParser_dataError", err => {
+        reject(err)
+      })
+
+      try {
+        let pdfBuffer = await fs.readFile(this.#file)
+        pdfParser.parseBuffer(pdfBuffer)
+      } catch (err) {
+        reject(err)
+      }
+    })
+  
+  }
+
+}

package/lib.mjs

@@ -1,7 +1,13 @@
-import { getNSPVersion, NSPFile, NSPVersion } from './lib/vline-nsp.mjs'
+import { getHeatTimetables, HeatTimetable, HeatTimetableFile } from './lib/heat/vline-heat.mjs'
+import { getNSPVersion, NSPFile, NSPVersion } from './lib/nsp/vline-nsp.mjs'
+import PassPDFReader from './lib/pass/pass-pdf-reader.mjs'
 
 export {
   getNSPVersion,
+  getHeatTimetables,
   NSPFile,
-  NSPVersion
+  NSPVersion,
+  PassPDFReader,
+  HeatTimetable,
+  HeatTimetableFile
 }

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@transportme/vline-nsp-reader",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "main": "lib.mjs",
   "scripts": {
-    "test": "mocha"
+    "test": "mocha './{,!(node_modules)/**}/*.test.mjs'"
   },
   "author": "",
   "license": "ISC",
   "description": "",
   "dependencies": {
-    "@transportme/pdf2json": "^4.0.10000002",
+    "@transportme/pdf2json": "^4.0.10000004",
     "async": "^3.2.6",
     "cheerio": "^1.0.0",
     "node-fetch": "^3.3.2"

package/read-pdf.mjs

@@ -1,12 +1,10 @@
-import util from 'util'
-import NSPPDFReader from './lib/nsp-pdf-reader.mjs'
-import TableReader from './lib/table-reader.mjs'
+import PassTableReader from './lib/pass/pass-table-reader.mjs'
 
 // let nspReader = new NSPPDFReader(process.argv[2])
 // await nspReader.read()
 // console.log(util.inspect(nspReader.getAllRuns(), { depth: null, colors: true, maxArrayLength: null }))
 
-let tableReader = new TableReader(process.argv[2])
-let pages = await tableReader.read()
+let tableReader = new PassTableReader(process.argv[2])
+let tables = await tableReader.read()
 
-for (let page of pages) console.table(page)
+for (let table of tables) console.table(table)