@transportme/vline-nsp-reader 1.0.0

package/lib/constants.mjs

@@ -0,0 +1,4 @@
+export default {
+  VLINE_CORPORATE_HOST: 'https://corporate.vline.com.au',
+  NSP_PAGE: '/Network-Access/Network-service-plan'
+}

package/lib/nsp-pdf-reader.mjs

@@ -0,0 +1,153 @@
+import { getOperationDays } from './nsp-utils.mjs'
+import TableReader from './table-reader.mjs'
+
+export default class NSPPDFReader {
+
+  #filePath
+  #pages
+
+  #stations = []
+
+  constructor(filePath) {
+    this.#filePath = filePath
+  }
+
+  getHeaderIndex(pageNum) {
+    return this.#pages[pageNum].findIndex(row => row[0] === 'Train Movement Type')
+  }
+
+  getHeader(pageNum) {
+    let headerIndex = this.getHeaderIndex(pageNum)
+    return this.#pages[pageNum].slice(0, headerIndex + 1)
+  }
+
+  getBody(pageNum) {
+    let headerIndex = this.getHeaderIndex(pageNum)
+    return this.#pages[pageNum].slice(headerIndex + 2)
+  }
+
+  getStations(pageNum) {
+    if (this.#stations[pageNum]) return this.#stations[pageNum]
+    let body = this.getBody(pageNum)
+    let stationList = body.map(row => row[0])
+
+    let output = []
+    let lastStation = stationList[0]
+    for (let i = 0; i < stationList.length; i++) {
+      if (stationList[i]) {
+        lastStation = stationList[i]
+        output.push(stationList[i])
+      } else {
+        output.push(lastStation)
+      }
+    }
+
+    this.#stations[pageNum] = output
+    return output
+  }
+
+  async read() {
+    let tableReader = new TableReader(this.#filePath)
+    let rawPages = await tableReader.read()
+
+    let pages = []
+    for (let page of rawPages) {
+      let header = page.slice(0, 4)
+      if (header.some(row => row[0] === 'Business ID')) pages.push(page)
+      else {
+        let colDiff = pages[pages.length - 1][0].length - page[0].length
+        if (colDiff > 0) for (let row of page) row.push(...Array(colDiff).fill(''))
+        pages[pages.length - 1].push(...page)
+      }
+    }
+
+    this.#pages = pages
+  }
+
+  __setPageData(data) { this.#pages = data }
+
+  processStation(station) {
+    if (station.depTime.endsWith('*')) {
+      station.express = true
+      station.depTime = station.depTime.slice(0, -1)
+    }
+
+    if (station.depTime && !station.arrTime) {
+      if (station.depTime.includes('/')) {
+        let [ arrTime, depTime ] = station.depTime.split('/')
+        station.arrTime = arrTime, station.depTime = depTime
+      } else {
+        station.arrTime = station.depTime
+      }
+    }
+
+    return station
+  }
+
+  getRuns(pageNum) {
+    let header = this.getHeader(pageNum)
+    let body = this.getBody(pageNum)
+    let stations = this.getStations(pageNum)
+
+    let runs = []
+    for (let runIndex = 2; runIndex < header[0].length; runIndex++) {
+      let runData = {
+        tdn: header[0][runIndex].slice(0, 4),
+        conditional: header[0][runIndex].length > 4,
+        daysRunCode: header[1][runIndex],
+        daysRun: getOperationDays(header[1][runIndex]),
+        operator: null, movementType: null,
+        formedBy: null, forming: null,
+        vehicleType: null, stations: []
+      }
+
+      for (let headerRow = 2; headerRow < header.length; headerRow++) {
+        let rowName = header[headerRow][0]
+        let rowData = header[headerRow][runIndex]
+        if (rowName === 'Operator') runData.operator = rowData
+        else if (rowName === 'Train Movement Type') runData.movementType = rowData
+        else if (rowName === 'Master Vehicle Formation') runData.vehicleType = rowData
+        else if (rowName === 'Formed By On Arrival') runData.formedBy = rowData
+      }
+
+      let lastStation = null
+      let lastStationName = null
+      for (let stationIndex = 0; stationIndex < stations.length - 1; stationIndex++) {
+        let currentStation = stations[stationIndex]
+        let stationData = body[stationIndex][runIndex]
+        if (!stationData) continue
+        if (lastStationName === currentStation) {
+          let amendmentType = body[stationIndex][1]
+          if (amendmentType === 'Plat') lastStation.plat = stationData
+          else if (amendmentType === 'Arr') lastStation.arrTime = stationData
+          else lastStation.track = stationData
+        } else {
+          if (lastStation) runData.stations.push(lastStation)
+          lastStation = {
+            name: currentStation,
+            arrTime: null,
+            depTime: stationData,
+            plat: null, track: null, express: false
+          }
+        }
+
+        lastStationName = currentStation
+      }
+
+      runData.stations.push(lastStation)
+      runData.stations = runData.stations.map(station => this.processStation(station))
+      let forming = body[stations.length - 1][runIndex].toUpperCase()
+      runData.forming = forming.length ? forming : null
+      runs.push(runData)
+    }
+
+    return runs
+  }
+
+  getAllRuns() {
+    let runs = []
+    for (let i = 0; i < this.#pages.length; i++) runs.push(...this.getRuns(i))
+    return runs
+  }
+
+}

package/lib/nsp-utils.mjs

@@ -0,0 +1,33 @@
+const daysOfWeek = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
+const dayMapping = {
+  'M': 'Mon',
+  'Tu': 'Tue',
+  'W': 'Wed',
+  'Th': 'Thu',
+  'F': 'Fri',
+  'Sat': 'Sat',
+  'Sun': 'Sun',
+  'Su': 'Sun'
+}
+
+export function getOperationDayLetter(letters) {
+  return letters.match(/([A-Z][a-z]*)/g)
+}
+
+export function getOperationDays(days) {
+  let parts = days.split('+')
+
+  if (parts.length === 1) {
+    if (days.endsWith('O')) {
+      return getOperationDayLetter(days.slice(0, -1)).map(code => dayMapping[code])
+    } else if (days.endsWith('E')) {
+      let excludedDays = getOperationDayLetter(days.slice(0, -1)).map(code => dayMapping[code])
+      return daysOfWeek.slice(0, -2).filter(day => !excludedDays.includes(day))
+    } else if (days === 'MF') return daysOfWeek.slice(0, -2)
+    else if (days === 'Daily') return daysOfWeek.slice(0)
+    else if (daysOfWeek.includes(days)) return [ days ]
+    return []
+  }
+  
+  return parts.map(part => getOperationDays(part)).reduce((acc, e) => acc.concat(e), [])
+}

package/lib/table-reader.mjs

@@ -0,0 +1,98 @@
+import PDFParser from 'pdf2json'
+import fs from 'fs/promises'
+
+export default class TableReader {
+
+  #file
+
+  constructor(file) {
+    this.#file = file
+  }
+
+  parserCallback(data) {
+    // PDF's contain pages and each page contains Texts. These texts have an x and y value.
+    // So finding Texts with equal y values seems like the solution.
+    // However, some y values are off by 0.010 pixels/points so let's first find what the smallest y value could be.
+
+    // Let's find Texts with the same x value and look for the smallest y distance of these Texts (on the same page of course)
+    // Then use those smallest y values (per page) to find Texts that seem to be on the same row
+    // If no smallest y value (per page) can be found, use 0 as smallest distance.
+
+
+    // now lets find Texts with 'the same' y-values, Actually y-values in the range of y-smallestYValue and y+smallestYValue:
+
+    let pages = data.Pages.map(page => {
+      let fills = page.Fills
+      let colStarts = fills.filter(fill => {
+        return fill.h > fill.w
+      }).map(fill => fill.x).filter((e, i, a) => a.indexOf(e) === i).sort((a, b) => a - b)
+
+      let rowStarts = fills.filter(fill => {
+        return fill.h < fill.w
+      }).map(fill => fill.y).filter((e, i, a) => a.indexOf(e) === i).sort((a, b) => a - b)
+
+      let pageData = []
+      let smallTable = rowStarts.length === 2
+
+      page.Texts.forEach(text => {
+        let textContent = decodeURIComponent(text.R[0].T)
+        if (textContent.match(/^(Effective|As at|Replaces)/)) return
+
+        let firstYGreater = rowStarts.find(r => r > text.y + 0.3)
+        let difference = firstYGreater - text.y
+        let currentRow = rowStarts.indexOf(firstYGreater) - 1
+        if (!smallTable && difference > 0.6) currentRow--
+        if (currentRow < 0) return
+
+        let currentCol = colStarts.findIndex(c => c > text.x + 0.4) - 1
+
+        if (!pageData[currentRow]) pageData[currentRow] = []
+
+        if (!pageData[currentRow][currentCol]) pageData[currentRow][currentCol] = textContent
+        else pageData[currentRow][currentCol] += ` ${textContent}`
+      })
+
+      for (let y = 0; y < pageData.length; y++) {
+        if (!pageData[y]) pageData[y] = []
+        for (let x = 0; x < pageData[y].length; x++) {
+          if (!pageData[y][x]) pageData[y][x] = ''
+        }
+      }
+      return pageData
+    })
+
+    return pages.map(page => {
+      let maxSize = Math.max(...page.map(row => row.length))
+      let blankCells = Array(maxSize).fill('')
+
+      return page.map(row => row.map(g => g.replace(/  +/g, ' ').trim()).concat(blankCells).slice(0, maxSize))
+    })
+  }
+
+  read() {
+    return new Promise(async (resolve, reject) => {
+      let pdfParser = new PDFParser()
+
+      pdfParser.on("pdfParser_dataReady", data => {
+        try {
+          resolve(this.parserCallback(data))
+        } catch (err) {
+          reject(err)
+        }
+      })
+
+      pdfParser.on("pdfParser_dataError", err => {
+        reject(err)
+      })
+
+      try {
+        let pdfBuffer = await fs.readFile(this.#file)
+        pdfParser.parseBuffer(pdfBuffer)
+      } catch (err) {
+        reject(err)
+      }
+    })
+  
+  }
+
+}

package/lib/vline-nsp.mjs

@@ -0,0 +1,96 @@
+import fetch from 'node-fetch';
+import constants from './constants.mjs'
+import { load as parseHTML } from 'cheerio'
+import async from 'async'
+import fs from 'fs/promises'
+import path from 'path'
+import { pipeline } from 'stream/promises'
+import { createWriteStream } from 'fs'
+import NSPPDFReader from './nsp-pdf-reader.mjs'
+
+export class NSPVersion {
+
+  version
+  effective
+
+  files = []
+
+  constructor(version, effective) {
+    this.version = version
+    this.effective = effective
+  }
+
+  addFile(file) {
+    this.files.push(file)
+  }
+
+  async saveFiles(outputDir) {
+    try {
+      await fs.mkdir(outputDir)
+    } catch (e) {}
+    await async.forEach(this.files, async file => {
+      await file.download(outputDir)
+    })
+  }
+
+}
+
+export class NSPFile {
+
+  name
+  href
+
+  #nspVersion
+  #filePath 
+
+  constructor(name, href, nspVersion) {
+    this.name = name
+    this.href = href
+
+    this.#nspVersion = nspVersion
+  }
+
+  async download(outputDir) {
+    this.#filePath = path.join(outputDir, `${this.#nspVersion.version} ${this.name}.pdf`)
+
+    let response = await fetch(constants.VLINE_CORPORATE_HOST + this.href)
+    let outputStream = createWriteStream(this.#filePath)
+
+    await pipeline(response.body, outputStream)
+  }
+
+  setFilePath(filePath) {
+    this.#filePath = filePath
+  }
+
+  async extractRuns() {
+    let reader = new NSPPDFReader(this.#filePath)
+    await reader.read()
+    return reader.getAllRuns()
+  }
+
+}
+
+export async function getNSPVersion() {
+  let body = await (await fetch(constants.VLINE_CORPORATE_HOST + constants.NSP_PAGE)).text()
+  let $ = parseHTML(body)
+
+  let buttons = Array.from($('div#publication-list > a.btn.button-file-link-caption'))
+
+  let nspVersions = {}
+
+  buttons.filter(button => $(button).text().trim().startsWith('FP')).forEach(button => {
+    let text = $(button).text()
+    let data = text.match(/(FP\w+) (\d+)-(\d+)-(\d+) (.+)/)
+    if (!data) return null
+
+    let [_, version, day, month, year, name] = data
+
+    if (!nspVersions[version]) nspVersions[version] = new NSPVersion(version, new Date(`${`20${year}`.slice(-4)}-${month}-${day}`))
+
+    let nspVersion = nspVersions[version]
+    nspVersion.addFile(new NSPFile(name.replace(/ NSP.+/, '').trim(), $(button).attr('href'), nspVersion))
+  })
+
+  return Object.values(nspVersions).sort((a, b) => b.effective - a.effective)
+}

package/lib.mjs

@@ -0,0 +1,7 @@
+import { getNSPVersion, NSPFile, NSPVersion } from './lib/vline-nsp.mjs'
+
+export {
+  getNSPVersion,
+  NSPFile,
+  NSPVersion
+}

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@transportme/vline-nsp-reader",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "mocha"
+  },
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "dependencies": {
+    "async": "^3.2.6",
+    "cheerio": "^1.0.0",
+    "node-fetch": "^3.3.2",
+    "pdf2json": "github:eyeballcode/pdf2json"
+  },
+  "devDependencies": {
+    "chai": "^5.1.2",
+    "mocha": "^10.8.2",
+    "nock": "^13.5.6",
+    "tmp-promise": "^3.0.3"
+  }
+}

package/read-pdf.mjs

@@ -0,0 +1,12 @@
+import util from 'util'
+import NSPPDFReader from './lib/nsp-pdf-reader.mjs'
+import TableReader from './lib/table-reader.mjs'
+
+// let nspReader = new NSPPDFReader(process.argv[2])
+// await nspReader.read()
+// console.log(util.inspect(nspReader.getAllRuns(), { depth: null, colors: true, maxArrayLength: null }))
+
+let tableReader = new TableReader(process.argv[2])
+let pages = await tableReader.read()
+
+for (let page of pages) console.table(page)