@mountainpass/addressr 2.0.3 → 2.1.0

package/lib/service/address-service.js

@@ -8,9 +8,13 @@ exports.dropIndex = dropIndex;
 exports.fetchGnafFile = fetchGnafFile;
 exports.getAddress = getAddress;
 exports.getAddresses = getAddresses;
+exports.getLocality = getLocality;
 exports.loadGnaf = loadGnaf;
 exports.mapAddressDetails = mapAddressDetails;
 exports.searchForAddress = searchForAddress;
+exports.searchForLocality = searchForLocality;
+exports.searchForPostcode = searchForPostcode;
+exports.searchForState = searchForState;
 exports.setAddresses = setAddresses;
 exports.unzipFile = unzipFile;
 var _debug = _interopRequireDefault(require("debug"));
@@ -23,7 +27,7 @@ var _nodeStream = _interopRequireDefault(require("node:stream"));
 var _unzipStream = _interopRequireDefault(require("unzip-stream"));
 var _elasticsearch = require("../client/elasticsearch");
 var _streamDown = _interopRequireDefault(require("../utils/stream-down"));
-var _setLinkOptions = require("./setLinkOptions");
+var _setLinkOptions = require("./set-link-options");
 var _keyv = require("keyv");
 var _keyvFile = require("keyv-file");
 var _nodeCrypto = _interopRequireDefault(require("node:crypto"));
@@ -54,10 +58,10 @@ const ONE_DAY_MS = 1000 * ONE_DAY_S;
 const THIRTY_DAYS_MS = ONE_DAY_MS * 30;
 const ES_INDEX_NAME = process.env.ES_INDEX_NAME || 'addressr';
 async function dropIndex() {
-  await (0, _elasticsearch.dropIndex)(global.esClient);
+  await (0, _elasticsearch.dropIndex)(globalThis.esClient);
 }
 async function clearAddresses() {
-  await (0, _elasticsearch.initIndex)(global.esClient, true);
+  await (0, _elasticsearch.initIndex)(globalThis.esClient, true);
 }
 async function setAddresses(addr) {
   await clearAddresses();
@@ -458,7 +462,7 @@ function mapGeo(geoSite, context, geoDefault) {
       })
     };
   }) : [];
-  const def = geoDefault && !foundDefault ? geoDefault.map(geo => {
+  const defaults = geoDefault && !foundDefault ? geoDefault.map(geo => {
     return {
       default: true,
       ...(geo.GEOCODE_TYPE_CODE !== '' && {
@@ -475,7 +479,7 @@ function mapGeo(geoSite, context, geoDefault) {
       })
     };
   }) : [];
-  return sites.concat(def);
+  return [...sites, ...defaults];
 }
 function mapToSla(fla) {
   return fla.join(', ');
@@ -706,6 +710,13 @@ async function loadAddressDetails(file, expectedCount, context, {
         }
         const indexingBody = [];
         for (const row of chunk.data) {
+          // Accumulate postcodes per locality for the locality index
+          if (context.postcodesByLocality && row.LOCALITY_PID && row.POSTCODE) {
+            if (!context.postcodesByLocality[row.LOCALITY_PID]) {
+              context.postcodesByLocality[row.LOCALITY_PID] = new Set();
+            }
+            context.postcodesByLocality[row.LOCALITY_PID].add(row.POSTCODE);
+          }
           const item = mapAddressDetails(row, context, actualCount, expectedCount);
           items.push(item);
           actualCount += 1;
@@ -819,7 +830,7 @@ async function loadAddressDetails(file, expectedCount, context, {
 }
 async function searchForAddress(searchString, p, pageSize = PAGE_SIZE) {
   //  const searchString = '657 The Entrance Road'; //'2/25 TOTTERDE'; // 'UNT 2, BELCONNEN';
-  const searchResp = await global.esClient.search({
+  const searchResp = await globalThis.esClient.search({
     index: ES_INDEX_NAME,
     body: {
       from: (p - 1 || 0) * pageSize,
@@ -875,6 +886,132 @@ async function searchForAddress(searchString, p, pageSize = PAGE_SIZE) {
   logger('hits', JSON.stringify(searchResp.body.hits, undefined, 2));
   return searchResp;
 }
+async function searchForLocality(searchString, p, pageSize = PAGE_SIZE) {
+  const searchResp = await globalThis.esClient.search({
+    index: _elasticsearch.ES_LOCALITY_INDEX_NAME,
+    body: {
+      from: (p - 1 || 0) * pageSize,
+      size: pageSize,
+      query: {
+        bool: {
+          ...(searchString && {
+            should: [{
+              multi_match: {
+                fields: ['locality_name'],
+                query: searchString,
+                fuzziness: 'AUTO',
+                type: 'bool_prefix',
+                lenient: true,
+                operator: 'AND'
+              }
+            }, {
+              multi_match: {
+                fields: ['locality_name'],
+                query: searchString,
+                type: 'phrase_prefix',
+                lenient: true,
+                operator: 'AND'
+              }
+            }]
+          })
+        }
+      },
+      sort: ['_score', {
+        'locality_name.raw': {
+          order: 'asc'
+        }
+      }]
+    }
+  });
+  logger('locality hits', JSON.stringify(searchResp.body.hits, undefined, 2));
+  return searchResp;
+}
+async function getLocality(pid) {
+  const resp = await globalThis.esClient.get({
+    index: _elasticsearch.ES_LOCALITY_INDEX_NAME,
+    id: `/localities/${pid}`
+  });
+  return resp;
+}
+async function searchForPostcode(searchString) {
+  const searchResp = await globalThis.esClient.search({
+    index: _elasticsearch.ES_LOCALITY_INDEX_NAME,
+    body: {
+      from: 0,
+      size: 0,
+      query: {
+        bool: {
+          filter: [{
+            prefix: {
+              postcodes: searchString
+            }
+          }]
+        }
+      },
+      aggs: {
+        postcodes: {
+          terms: {
+            field: 'postcodes',
+            size: 20
+          },
+          aggs: {
+            localities: {
+              terms: {
+                field: 'locality_name.raw',
+                size: 100
+              }
+            }
+          }
+        }
+      }
+    }
+  });
+  logger('postcode hits', JSON.stringify(searchResp.body.aggregations, undefined, 2));
+  return searchResp;
+}
+async function searchForState(searchString) {
+  const query = searchString ? {
+    bool: {
+      should: [{
+        prefix: {
+          state_abbreviation: searchString.toUpperCase()
+        }
+      }, {
+        wildcard: {
+          state_name: `*${searchString.toUpperCase()}*`
+        }
+      }]
+    }
+  } : {
+    match_all: {}
+  };
+  const searchResp = await globalThis.esClient.search({
+    index: _elasticsearch.ES_LOCALITY_INDEX_NAME,
+    body: {
+      from: 0,
+      size: 0,
+      query,
+      aggs: {
+        states: {
+          terms: {
+            field: 'state_abbreviation',
+            size: 20
+          },
+          aggs: {
+            state_name: {
+              terms: {
+                field: 'state_name',
+                size: 1
+              }
+            }
+          }
+        }
+      }
+    }
+  });
+  logger('state hits', JSON.stringify(searchResp.body.aggregations, undefined, 2));
+  return searchResp;
+}
 async function sendIndexRequest(indexingBody, initialBackoff = Number.parseInt(process.env.ADDRESSR_INDEX_BACKOFF || '30000'), {
   refresh = false
 } = {}) {
@@ -882,7 +1019,7 @@ async function sendIndexRequest(indexingBody, initialBackoff = Number.parseInt(p
   // eslint-disable-next-line no-constant-condition
   for (let count = 0; true; count++) {
     try {
-      const resp = await global.esClient.bulk({
+      const resp = await globalThis.esClient.bulk({
         refresh,
         body: indexingBody,
         timeout: process.env.ADDRESSR_INDEX_TIMEOUT || '300s'
@@ -962,17 +1099,17 @@ function buildSynonyms(context) {
 const {
   readdir
 } = require('node:fs').promises;
-async function getFiles(currentDir, baseDir) {
-  const dir = _nodePath.default.resolve(baseDir, currentDir);
-  logger(`reading ${dir} (${currentDir} in ${baseDir})`);
-  const dirents = await readdir(dir, {
+async function getFiles(currentDirectory, baseDirectory) {
+  const directory = _nodePath.default.resolve(baseDirectory, currentDirectory);
+  logger(`reading ${directory} (${currentDirectory} in ${baseDirectory})`);
+  const dirents = await readdir(directory, {
     withFileTypes: true
   });
   const files = await Promise.all(dirents.map(dirent => {
-    const res = `${currentDir}/${dirent.name}`;
-    return dirent.isDirectory() ? getFiles(res, baseDir) : res;
+    const result = `${currentDirectory}/${dirent.name}`;
+    return dirent.isDirectory() ? getFiles(result, baseDirectory) : result;
   }));
-  return Array.prototype.concat(...files);
+  return files.flat();
 }
 function countFileLines(filePath) {
   return new Promise((resolve, reject) => {
@@ -1024,11 +1161,13 @@ async function loadGnafData(directory, {
     //   throw new Error(`Cannot file '${countsFile}' or '${contentsFile}'`)
     // }
   }
-  const loadContext = {};
+  const loadContext = {
+    postcodesByLocality: {}
+  };
   await loadAuthFiles(files, directory, loadContext, filesCounts);
   // loadContext now contains all the auth files, so we can build the synonyms
   const synonyms = buildSynonyms(loadContext);
-  await (0, _elasticsearch.initIndex)(global.esClient, process.env.ES_CLEAR_INDEX || false, synonyms);
+  await (0, _elasticsearch.initIndex)(globalThis.esClient, process.env.ES_CLEAR_INDEX || false, synonyms);
   const addressDetailFiles = files.filter(f => f.match(/ADDRESS_DETAIL/) && f.match(/\/Standard\//));
   logger('addressDetailFiles', addressDetailFiles);
   for (const detailFile of addressDetailFiles) {
@@ -1074,6 +1213,51 @@ async function loadGnafData(directory, {
       });
     }
   }
+
+  // Phase 2: Index localities (separate post-load phase)
+  // Failures here must not affect address loading above
+  try {
+    await (0, _elasticsearch.initLocalityIndex)(globalThis.esClient, process.env.ES_CLEAR_INDEX || false, synonyms);
+    const localityIndexingBody = [];
+    for (const detailFile of addressDetailFiles) {
+      const state = _nodePath.default.basename(detailFile, _nodePath.default.extname(detailFile)).replace(/_.*/, '');
+      if (COVERED_STATES.length === 0 || COVERED_STATES.includes(state)) {
+        const stateName = await loadState(files, directory, state);
+        const localities = await loadLocality(files, directory, state);
+        for (const l of localities) {
+          if (l.LOCALITY_NAME === '') continue;
+          localityIndexingBody.push({
+            index: {
+              _index: _elasticsearch.ES_LOCALITY_INDEX_NAME,
+              _id: `/localities/${l.LOCALITY_PID}`
+            }
+          });
+          // Derive postcodes: prefer accumulated from ADDRESS_DETAIL,
+          // fall back to PRIMARY_POSTCODE from LOCALITY table
+          const accumulatedPostcodes = loadContext.postcodesByLocality[l.LOCALITY_PID];
+          const postcodes = accumulatedPostcodes ? [...accumulatedPostcodes] : l.PRIMARY_POSTCODE ? [l.PRIMARY_POSTCODE] : [];
+          localityIndexingBody.push({
+            locality_name: l.LOCALITY_NAME,
+            locality_class_code: l.LOCALITY_CLASS_CODE,
+            locality_class_name: localityClassCodeToName(l.LOCALITY_CLASS_CODE, loadContext),
+            primary_postcode: postcodes[0] || '',
+            postcodes,
+            state_abbreviation: state,
+            state_name: stateName,
+            locality_pid: l.LOCALITY_PID
+          });
+        }
+      }
+    }
+    if (localityIndexingBody.length > 0) {
+      await sendIndexRequest(localityIndexingBody, undefined, {
+        refresh: true
+      });
+    }
+    logger('Locality indexing complete');
+  } catch (error_) {
+    error('Locality indexing failed (address loading unaffected):', error_);
+  }
 }
 async function fileExists(countsFile) {
   try {
@@ -1306,14 +1490,14 @@ async function loadGnaf({
     if (contents.length === 0) {
       throw new Error(`Data dir '${unzipped}' is empty`);
     }
-    const gnafDir = await (0, _glob.glob)('**/G-NAF/', {
+    const gnafDirectory = await (0, _glob.glob)('**/G-NAF/', {
       cwd: unzipped
     });
-    console.log(gnafDir);
-    if (gnafDir.length === 0) {
+    console.log(gnafDirectory);
+    if (gnafDirectory.length === 0) {
       throw new Error(`Cannot find 'G-NAF' directory in Data dir '${unzipped}'`);
     }
-    mainDirectory = _nodePath.default.dirname(`${unzipped}/${gnafDir[0].slice(0, -1)}`);
+    mainDirectory = _nodePath.default.dirname(`${unzipped}/${gnafDirectory[0].slice(0, -1)}`);
   }
   logger('Main Data dir', mainDirectory);
   await loadGnafData(mainDirectory, {
@@ -1330,7 +1514,7 @@ async function loadGnaf({
  **/
 async function getAddress(addressId) {
   try {
-    const jsonX = await global.esClient.get({
+    const jsonX = await globalThis.esClient.get({
       index: ES_INDEX_NAME,
       id: `/addresses/${addressId}`
     });

package/lib/src/server2.js

@@ -2,20 +2,20 @@
 
 var _debug = _interopRequireDefault(require("debug"));
 var _elasticsearch = require("../client/elasticsearch");
-var _printVersion = require("../service/printVersion");
-var _waycharterServer = require("./waycharterServer");
+var _printVersion = require("../service/print-version");
+var _waycharterServer = require("./waycharter-server");
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 const logger = (0, _debug.default)('api');
-(0, _waycharterServer.startRest2Server)().then(() => {
+(0, _waycharterServer.startRest2Server)().then(async () => {
   logger('connecting es client');
-  const p1 = (0, _elasticsearch.esConnect)().then(esClient => {
-    global.esClient = esClient;
-    logger('es client connected');
-  });
-  p1.then(() => {
-    console.log('=======================');
-    console.log('Addressr - API Server 2');
-    console.log('=======================');
-    (0, _printVersion.printVersion)();
-  });
+  const esClient = await (0, _elasticsearch.esConnect)();
+  globalThis.esClient = esClient;
+  logger('es client connected');
+  console.log('=======================');
+  console.log('Addressr - API Server 2');
+  console.log('=======================');
+  (0, _printVersion.printVersion)();
+}).catch(error => {
+  console.error('Failed to start server:', error);
+  throw error;
 });

package/lib/src/{waycharterServer.js → waycharter-server.js}

@@ -107,6 +107,140 @@ function startRest2Server() {
       parameters: ['q']
     }]
   });
+  const localitiesType = waycharter.registerCollection({
+    itemPath: '/:pid',
+    itemLoader: async ({
+      pid
+    }) => {
+      const resp = await (0, _addressService.getLocality)(pid);
+      const source = resp.body._source;
+      const hash = _nodeCrypto.default.createHash('md5').update(JSON.stringify(source)).digest('hex');
+      return {
+        body: source,
+        headers: {
+          etag: `"${_version.version}-${hash}"`,
+          'cache-control': `public, max-age=${ONE_WEEK}`
+        },
+        status: 200
+      };
+    },
+    collectionPath: '/localities',
+    collectionLoader: async ({
+      page,
+      q
+    }) => {
+      if (q && q.length > 1) {
+        const foundLocalities = await (0, _addressService.searchForLocality)(q, page + 1, PAGE_SIZE);
+        const body = foundLocalities.body.hits.hits.map(h => {
+          return {
+            name: h._source.locality_name,
+            state: {
+              name: h._source.state_name,
+              abbreviation: h._source.state_abbreviation
+            },
+            ...(h._source.locality_class_code && {
+              class: {
+                code: h._source.locality_class_code,
+                name: h._source.locality_class_name
+              }
+            }),
+            ...(h._source.primary_postcode && {
+              postcode: h._source.primary_postcode
+            }),
+            score: h._score,
+            pid: h._id.replace('/localities/', '')
+          };
+        });
+        const responseHash = _nodeCrypto.default.createHash('md5').update(JSON.stringify(body)).digest('hex');
+        return {
+          body,
+          hasMore: page < foundLocalities.body.hits.total.value / PAGE_SIZE - 1,
+          headers: {
+            etag: `"${_version.version}-${responseHash}"`,
+            'cache-control': `public, max-age=${ONE_WEEK}`
+          }
+        };
+      } else {
+        return {
+          body: [],
+          hasMore: false,
+          headers: {
+            etag: `"${_version.version}"`,
+            'cache-control': `public, max-age=${ONE_WEEK}`
+          }
+        };
+      }
+    },
+    filters: [{
+      rel: 'https://addressr.io/rels/locality-search',
+      parameters: ['q']
+    }]
+  });
+  const postcodesType = waycharter.registerCollection({
+    collectionPath: '/postcodes',
+    collectionLoader: async ({
+      q
+    }) => {
+      if (q && q.length > 2) {
+        const result = await (0, _addressService.searchForPostcode)(q);
+        const buckets = result.body.aggregations.postcodes.buckets;
+        const body = buckets.map(bucket => ({
+          postcode: bucket.key,
+          localities: bucket.localities.buckets.map(l => ({
+            name: l.key
+          }))
+        }));
+        const responseHash = _nodeCrypto.default.createHash('md5').update(JSON.stringify(body)).digest('hex');
+        return {
+          body,
+          hasMore: false,
+          headers: {
+            etag: `"${_version.version}-${responseHash}"`,
+            'cache-control': `public, max-age=${ONE_WEEK}`
+          }
+        };
+      } else {
+        return {
+          body: [],
+          hasMore: false,
+          headers: {
+            etag: `"${_version.version}"`,
+            'cache-control': `public, max-age=${ONE_WEEK}`
+          }
+        };
+      }
+    },
+    filters: [{
+      rel: 'https://addressr.io/rels/postcode-search',
+      parameters: ['q']
+    }]
+  });
+  const statesType = waycharter.registerCollection({
+    collectionPath: '/states',
+    collectionLoader: async ({
+      q
+    }) => {
+      const result = await (0, _addressService.searchForState)(q && q.length > 1 ? q : undefined);
+      const buckets = result.body.aggregations.states.buckets;
+      const body = buckets.map(bucket => ({
+        abbreviation: bucket.key,
+        name: bucket.state_name.buckets[0]?.key || bucket.key
+      }));
+      const responseHash = _nodeCrypto.default.createHash('md5').update(JSON.stringify(body)).digest('hex');
+      return {
+        body,
+        hasMore: false,
+        headers: {
+          etag: `"${_version.version}-${responseHash}"`,
+          'cache-control': `public, max-age=${ONE_WEEK}`
+        }
+      };
+    },
+    filters: [{
+      rel: 'https://addressr.io/rels/state-search',
+      parameters: ['q']
+    }]
+  });
   waycharter.registerResourceType({
     path: '/health',
     loader: async () => {
@@ -127,7 +261,7 @@ function startRest2Server() {
     loader: async () => {
       return {
         body: {},
-        links: [...addressesType.additionalPaths, {
+        links: [...addressesType.additionalPaths, ...localitiesType.additionalPaths, ...postcodesType.additionalPaths, ...statesType.additionalPaths, {
           rel: 'https://addressr.io/rels/health',
           uri: '/health'
         }],

package/lib/swagger.js

@@ -5,7 +5,7 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.startServer = startServer;
 exports.stopServer = stopServer;
-exports.swaggerDoc = void 0;
+exports.swaggerDocument = void 0;
 exports.swaggerInit = swaggerInit;
 var _debug = _interopRequireDefault(require("debug"));
 var _express = _interopRequireDefault(require("express"));
@@ -32,12 +32,12 @@ var options = {
 
 // The Swagger document (require it, build it programmatically, fetch it from a URL, ...)
 var spec = (0, _nodeFs.readFileSync)(_nodePath.default.join(__dirname, 'api/swagger.yaml'), 'utf8');
-var swaggerDoc = exports.swaggerDoc = (0, _jsYaml.load)(spec);
-global.swaggerDoc = swaggerDoc;
+var swaggerDocument = exports.swaggerDocument = (0, _jsYaml.load)(spec);
+globalThis.swaggerDocument = swaggerDocument;
 function swaggerInit() {
   // Initialize the Swagger middleware
   return new Promise(resolve => {
-    (0, _swaggerTools.initializeMiddleware)(swaggerDoc, function (middleware) {
+    (0, _swaggerTools.initializeMiddleware)(swaggerDocument, function (middleware) {
       // Interpret Swagger resources and attach metadata to request - must be first in swagger-tools middleware chain
       const metaData = middleware.swaggerMetadata();
       app.use(metaData);
@@ -55,7 +55,7 @@ function swaggerInit() {
         // apiDocs: '/api-docs',
         // swaggerUi: '/docs',
       }));
-      app.use(function (error_, request, res, next) {
+      app.use(function (error_, request, response, next) {
         if (error_.failedValidation) {
           // handle validation errror
           const rehydratedError = Object.assign({}, error_);
@@ -70,13 +70,13 @@ function swaggerInit() {
             delete rehydratedError.results;
           }
           error('error!!!', error_.message, JSON.stringify(rehydratedError, undefined, 2));
-          res.status(error_.code === 'SCHEMA_VALIDATION_FAILED' ? '500' : '400').json(rehydratedError);
+          response.status(error_.code === 'SCHEMA_VALIDATION_FAILED' ? '500' : '400').json(rehydratedError);
         } else {
           next();
         }
       });
-      global.swaggerApp = app;
-      global.swaggerMiddleware = middleware;
+      globalThis.swaggerApp = app;
+      globalThis.swaggerMiddleware = middleware;
       resolve({
         app,
         middleware

package/lib/utils/stream-down.js

@@ -2,21 +2,19 @@
 
 var _progress = _interopRequireDefault(require("progress"));
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
-const {
-  parse
-} = require('node:url');
 const http = require('node:https');
 const fs = require('node:fs');
 const pathUtil = require('node:path');
-module.exports = function (url, path, size) {
-  const uri = parse(url);
+module.exports = function streamDown(url, path, size) {
+  const uri = new URL(url);
   if (!path) {
-    path = pathUtil.basename(uri.path);
+    path = pathUtil.basename(uri.pathname);
   }
-  const file = fs.createWriteStream(path);
+  const file = fs.createWriteStream(path); // eslint-disable-line security/detect-non-literal-fs-filename -- path is internal
+
   return new Promise(function (resolve, reject) {
-    http.get(uri.href).on('response', function (res) {
-      const length = res.headers['content-length'] ? Number.parseInt(res.headers['content-length'], 10) : size;
+    http.get(uri.toString()).on('response', function (response) {
+      const length = response.headers['content-length'] ? Number.parseInt(response.headers['content-length'], 10) : size;
       //   let downloaded = 0;
       //   let percent = 0;
       var bar = new _progress.default('  downloading [:bar] :rate/bps :percent :etas', {
@@ -25,7 +23,7 @@ module.exports = function (url, path, size) {
         width: 20,
         total: length
       });
-      res.on('data', function (chunk) {
+      response.on('data', function (chunk) {
         file.write(chunk);
         //   downloaded += chunk.length;
         //percent = ((100.0 * downloaded) / len).toFixed(2);
@@ -39,8 +37,8 @@ module.exports = function (url, path, size) {
         //   );
       }).on('end', function () {
         file.end();
-        console.log(`\n${uri.path} downloaded to: ${path}`);
-        resolve(res);
+        console.log(`\n${uri.pathname} downloaded to: ${path}`);
+        resolve(response);
       }).on('error', function (error) {
         reject(error);
       });

package/lib/version.js

@@ -5,4 +5,4 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.version = void 0;
 // Generated by genversion.
-const version = exports.version = '2.0.3';
+const version = exports.version = '2.1.0';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mountainpass/addressr",
-  "version": "2.0.3",
+  "version": "2.1.0",
   "description": "Australian Address Validation, Search and Autocomplete",
   "author": {
     "name": "Mountain Pass",

package/scripts/check-version.js

@@ -7,5 +7,5 @@ if (!satisfies(process.version, version)) {
   console.log(
     `Required node version ${version} not satisfied with current version ${process.version}.`,
   );
-  process.exit(1);
+  process.exit(1); // eslint-disable-line no-process-exit, n/no-process-exit -- postinstall version check
 }