@mountainpass/addressr 2.4.2 → 2.4.4

package/lib/service/address-service.js

@@ -31,8 +31,7 @@ var _elasticsearch = require("../client/elasticsearch");
 var _streamDown = _interopRequireDefault(require("../utils/stream-down"));
 var _setLinkOptions = require("./set-link-options");
 var _rangeExpansion = require("./range-expansion");
-var _keyv = require("keyv");
-var _keyvFile = require("keyv-file");
+var _gnafPackageFetch = require("./gnaf-package-fetch");
 var _nodeCrypto = _interopRequireDefault(require("node:crypto"));
 var _glob = require("glob");
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
@@ -44,21 +43,17 @@ function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e
 const fsp = _nodeFs.default.promises;
 var logger = (0, _debug.default)('api');
 var error = (0, _debug.default)('error');
-const cache = new _keyv.Keyv({
-  store: new _keyvFile.KeyvFile({
-    filename: 'target/keyv-file.msgpack'
-  })
-});
+
+// `cache`, `ONE_DAY_MS`, and `THIRTY_DAYS_MS` previously lived here for
+// fetchPackageData. Moved to ./gnaf-package-fetch.js along with that
+// function — see the comment above its `import` and P033.
+
 const PAGE_SIZE = process.env.PAGE_SIZE || 8;
 function getCoveredStates() {
   const covered = process.env.COVERED_STATES || '';
   return covered == '' ? [] : covered.split(',');
 }
 const COVERED_STATES = getCoveredStates();
-const ONE_DAY_S = 60 /*sec*/ * 60 /*min*/ * 24; /*hours*/
-
-const ONE_DAY_MS = 1000 * ONE_DAY_S;
-const THIRTY_DAYS_MS = ONE_DAY_MS * 30;
 const ES_INDEX_NAME = process.env.ES_INDEX_NAME || 'addressr';
 async function dropIndex() {
   await (0, _elasticsearch.dropIndex)(globalThis.esClient);
@@ -98,56 +93,17 @@ async function setAddresses(addr) {
   //logger(await searchForAddress('657 The Entrance Road')); //'2/25 TOTTERDE'; // 'UNT 2, BELCONNEN';);
 }
 
-// need to try proxying this to modify the headers if we want to use got's cache implementation
+// fetchPackageData and the GNAF_PACKAGE_URL constant moved to
+// ./gnaf-package-fetch.js so the data.gov.au CKAN fetch is testable in
+// raw Node ESM (this file uses babel-only bare imports). See P033 for the
+// source-inspection anti-pattern that motivated the extraction. The new
+// module also adds a Mozilla-prefixed compatible-mode User-Agent header
+// required by data.gov.au's CloudFront WAF (without it, fetch returns
+// HTTP 403 — surfaced by run 25032179791).
 
-// SEE https://data.gov.au/data/dataset/19432f89-dc3a-4ef3-b943-5326ef1dbecc
-const GNAF_PACKAGE_URL = process.env.GNAF_PACKAGE_URL || 'https://data.gov.au/data/api/3/action/package_show?id=19432f89-dc3a-4ef3-b943-5326ef1dbecc';
-async function fetchPackageData() {
-  const packageUrl = GNAF_PACKAGE_URL;
-  // See if we have the value in cache
-  const cachedResponse = await cache.get(packageUrl);
-  logger('cached gnaf package data', cachedResponse);
-  let age = 0;
-  if (cachedResponse !== undefined) {
-    cachedResponse.headers['x-cache'] = 'HIT';
-    const created = new Date(cachedResponse.headers.date);
-    logger('created', created);
-    age = Date.now() - created;
-    if (age <= ONE_DAY_MS) {
-      return cachedResponse;
-    }
-  }
-  // cached value was older than one day, so go fetch
-  try {
-    const fetchResponse = await fetch(packageUrl);
-    const body = await fetchResponse.text();
-    const headers = Object.fromEntries(fetchResponse.headers.entries());
-    logger('fresh gnaf package data', {
-      body,
-      headers
-    });
-    await cache.set(packageUrl, {
-      body,
-      headers
-    });
-    headers['x-cache'] = 'MISS';
-    return {
-      body,
-      headers
-    };
-  } catch (error_) {
-    // we were unable to fetch. if we have cached value that isn't stale, return in
-    if (cachedResponse !== undefined && age < THIRTY_DAYS_MS) {
-      cachedResponse.headers['warning'] = '110	custom/1.0 "Response is Stale"';
-      return cachedResponse;
-    }
-    // otherwise, throw the original network error
-    throw error_;
-  }
-}
 const GNAF_DIR = process.env.GNAF_DIR || `target/gnaf`;
 async function fetchGnafFile() {
-  const response = await fetchPackageData();
+  const response = await (0, _gnafPackageFetch.fetchPackageData)();
   const pack = JSON.parse(response.body);
   // id as of 16/07 for zip is 4b084096-65e4-4c8e-abbe-5e54ff85f42f
   const dataResource = pack.result.resources.find(r => r.state === 'active' && r.mimetype === 'application/zip');

package/lib/service/gnaf-package-fetch.js

@@ -0,0 +1,115 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.LOADER_USER_AGENT = exports.GNAF_PACKAGE_URL = void 0;
+exports.fetchPackageData = fetchPackageData;
+var _keyv = require("keyv");
+var _keyvFile = require("keyv-file");
+var _debug = _interopRequireDefault(require("debug"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+// @jtbd JTBD-400 (Ship Releases Reliably From Trunk)
+//
+// Extracted from service/address-service.js so the loader's data.gov.au CKAN
+// fetch is testable behaviourally. service/address-service.js uses babel-only
+// bare imports that raw Node ESM cannot resolve; this module uses clean ESM
+// with `.js` extensions throughout, so behavioural tests can import it
+// directly. See P033 for the source-inspection anti-pattern this avoids.
+//
+// fetchPackageData accepts `fetch` and `cache` via dependency injection, with
+// defaults to globalThis.fetch and a Keyv-file cache backed by
+// target/keyv-file.msgpack (cwd-relative, NOT module-relative — must match
+// the address-service.js call site's process.cwd()).
+
+const logger = (0, _debug.default)('api');
+const ONE_DAY_MS = 24 * 60 * 60 * 1000;
+const THIRTY_DAYS_MS = 30 * ONE_DAY_MS;
+
+// Dataset registration on data.gov.au:
+// https://data.gov.au/data/dataset/19432f89-dc3a-4ef3-b943-5326ef1dbecc
+const GNAF_PACKAGE_URL = exports.GNAF_PACKAGE_URL = process.env.GNAF_PACKAGE_URL || 'https://data.gov.au/data/api/3/action/package_show?id=19432f89-dc3a-4ef3-b943-5326ef1dbecc';
+
+// data.gov.au's CloudFront WAF returns HTTP 403 on Node-default User-Agent
+// (the CKAN package_show response is HTML "<!DOCTYPE…" instead of JSON).
+// A Mozilla-prefixed compatible-mode UA that identifies the tool gets
+// through. Verified 2026-04-28 via curl: same URL with no UA → 403; with
+// this UA → 200. Surfaced by the failed ADR 029 Phase 1 step 5 v2 populate
+// (run 25032179791).
+const LOADER_USER_AGENT = exports.LOADER_USER_AGENT = 'Mozilla/5.0 (compatible; addressr-loader; +https://github.com/mountain-pass/addressr)';
+
+// Default cache instance — same Keyv-file backend service/address-service.js
+// originally created. Path is cwd-relative by design (resolves against
+// process.cwd() at runtime, matching the addressr-loader CLI's working dir).
+const defaultCache = new _keyv.Keyv({
+  store: new _keyvFile.KeyvFile({
+    filename: 'target/keyv-file.msgpack'
+  })
+});
+
+/**
+ * Fetch the data.gov.au CKAN package_show response for the G-NAF dataset.
+ *
+ * On cache hit (entry < 1 day old): returns the cached response with
+ * `x-cache: HIT` annotation, no network call.
+ *
+ * On cache miss: fetches with the LOADER_USER_AGENT header, persists the
+ * response to cache, returns with `x-cache: MISS`.
+ *
+ * On fetch failure: returns the cached response with a stale `warning`
+ * header if one exists and is younger than 30 days; otherwise rethrows.
+ *
+ * @param {object} [deps] dependency injection for testing
+ * @param {typeof globalThis.fetch} [deps.fetch] fetch implementation (default: globalThis.fetch)
+ * @param {{ get: (k: string) => Promise<any>, set: (k: string, v: any) => Promise<any> }} [deps.cache] Keyv-shaped cache (default: target/keyv-file.msgpack)
+ * @returns {Promise<{ body: string, headers: Record<string, string> }>}
+ */
+async function fetchPackageData({
+  fetch: fetchFunction = globalThis.fetch,
+  cache = defaultCache
+} = {}) {
+  const packageUrl = GNAF_PACKAGE_URL;
+  const cachedResponse = await cache.get(packageUrl);
+  logger('cached gnaf package data', cachedResponse);
+  let age = 0;
+  if (cachedResponse !== undefined) {
+    cachedResponse.headers['x-cache'] = 'HIT';
+    const created = new Date(cachedResponse.headers.date);
+    logger('created', created);
+    age = Date.now() - created;
+    if (age <= ONE_DAY_MS) {
+      return cachedResponse;
+    }
+  }
+  // Cached value was older than one day, so go fetch.
+  try {
+    const fetchResponse = await fetchFunction(packageUrl, {
+      headers: {
+        'User-Agent': LOADER_USER_AGENT
+      }
+    });
+    const body = await fetchResponse.text();
+    const headers = Object.fromEntries(fetchResponse.headers.entries());
+    logger('fresh gnaf package data', {
+      body,
+      headers
+    });
+    await cache.set(packageUrl, {
+      body,
+      headers
+    });
+    headers['x-cache'] = 'MISS';
+    return {
+      body,
+      headers
+    };
+  } catch (error_) {
+    // Network unreachable: serve stale-up-to-30-days from cache rather
+    // than fail the load entirely.
+    if (cachedResponse !== undefined && age < THIRTY_DAYS_MS) {
+      cachedResponse.headers['warning'] = '110\tcustom/1.0 "Response is Stale"';
+      return cachedResponse;
+    }
+    throw error_;
+  }
+}

package/lib/version.js

@@ -5,4 +5,4 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.version = void 0;
 // Generated by genversion.
-const version = exports.version = '2.4.2';
+const version = exports.version = '2.4.4';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mountainpass/addressr",
-  "version": "2.4.2",
+  "version": "2.4.4",
   "description": "Australian Address Validation, Search and Autocomplete",
   "author": {
     "name": "Mountain Pass",