@mountainpass/addressr 1.1.8 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/example.html +1 -1
- package/lib/.claude/skills/c4/scripts/c4-generate.js +18 -20
- package/lib/.claude/skills/c4/scripts/c4-lib.js +23 -25
- package/lib/.claude/skills/c4-check/scripts/c4-check.js +8 -10
- package/lib/ci/pipeline.js +4 -6
- package/lib/client/elasticsearch.js +7 -14
- package/lib/controllers/Addresses.js +2 -3
- package/lib/dagger-version.js +1 -0
- package/lib/eslint.config.js +92 -0
- package/lib/loader.js +4 -5
- package/lib/server.js +2 -3
- package/lib/service/DefaultService.js +5 -7
- package/lib/service/address-service.js +96 -97
- package/lib/service/printVersion.js +2 -3
- package/lib/src/server2.js +2 -3
- package/lib/src/waycharterServer.js +11 -15
- package/lib/swagger.js +12 -21
- package/lib/utils/stream-down.js +2 -3
- package/lib/version.js +2 -1
- package/package.json +31 -32
|
@@ -3,55 +3,41 @@
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", {
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
|
-
exports.dropIndex = dropIndex;
|
|
7
6
|
exports.clearAddresses = clearAddresses;
|
|
8
|
-
exports.
|
|
7
|
+
exports.dropIndex = dropIndex;
|
|
9
8
|
exports.fetchGnafFile = fetchGnafFile;
|
|
10
|
-
exports.unzipFile = unzipFile;
|
|
11
|
-
exports.mapAddressDetails = mapAddressDetails;
|
|
12
|
-
exports.searchForAddress = searchForAddress;
|
|
13
|
-
exports.loadGnaf = loadGnaf;
|
|
14
9
|
exports.getAddress = getAddress;
|
|
15
10
|
exports.getAddresses = getAddresses;
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
var
|
|
22
|
-
var
|
|
23
|
-
var
|
|
24
|
-
var _httpLinkHeader = require("http-link-header");
|
|
25
|
-
var
|
|
26
|
-
var
|
|
27
|
-
var
|
|
28
|
-
var
|
|
29
|
-
var _nodePath2 = _interopRequireDefault(_nodePath);
|
|
30
|
-
var _nodeStream = require("node:stream");
|
|
31
|
-
var _nodeStream2 = _interopRequireDefault(_nodeStream);
|
|
32
|
-
var _unzipStream = require("unzip-stream");
|
|
33
|
-
var _unzipStream2 = _interopRequireDefault(_unzipStream);
|
|
11
|
+
exports.loadGnaf = loadGnaf;
|
|
12
|
+
exports.mapAddressDetails = mapAddressDetails;
|
|
13
|
+
exports.searchForAddress = searchForAddress;
|
|
14
|
+
exports.setAddresses = setAddresses;
|
|
15
|
+
exports.unzipFile = unzipFile;
|
|
16
|
+
var _debug = _interopRequireDefault(require("debug"));
|
|
17
|
+
var _directoryExists = _interopRequireDefault(require("directory-exists"));
|
|
18
|
+
var _nodeFs = _interopRequireDefault(require("node:fs"));
|
|
19
|
+
var _httpLinkHeader = _interopRequireDefault(require("http-link-header"));
|
|
20
|
+
var _papaparse = _interopRequireDefault(require("papaparse"));
|
|
21
|
+
var _nodePath = _interopRequireDefault(require("node:path"));
|
|
22
|
+
var _nodeStream = _interopRequireDefault(require("node:stream"));
|
|
23
|
+
var _unzipStream = _interopRequireDefault(require("unzip-stream"));
|
|
34
24
|
var _elasticsearch = require("../client/elasticsearch");
|
|
35
|
-
var _streamDown = require("../utils/stream-down");
|
|
36
|
-
var _streamDown2 = _interopRequireDefault(_streamDown);
|
|
25
|
+
var _streamDown = _interopRequireDefault(require("../utils/stream-down"));
|
|
37
26
|
var _setLinkOptions = require("./setLinkOptions");
|
|
38
|
-
var _keyv = require("keyv");
|
|
39
|
-
var _keyv2 = _interopRequireDefault(_keyv);
|
|
27
|
+
var _keyv = _interopRequireDefault(require("keyv"));
|
|
40
28
|
var _keyvFile = require("keyv-file");
|
|
41
|
-
var _nodeCrypto = require("node:crypto");
|
|
42
|
-
var
|
|
43
|
-
var _globPromise = require("glob-promise");
|
|
44
|
-
var _globPromise2 = _interopRequireDefault(_globPromise);
|
|
29
|
+
var _nodeCrypto = _interopRequireDefault(require("node:crypto"));
|
|
30
|
+
var _glob = require("glob");
|
|
45
31
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
46
|
-
/* eslint-disable eslint-comments/disable-enable-pair */
|
|
32
|
+
/* eslint-disable @eslint-community/eslint-comments/disable-enable-pair */
|
|
47
33
|
/* eslint-disable security/detect-non-literal-regexp */
|
|
48
34
|
/* eslint-disable security/detect-object-injection */
|
|
49
35
|
/* eslint-disable security/detect-non-literal-fs-filename */
|
|
50
36
|
|
|
51
|
-
const fsp =
|
|
52
|
-
var logger = (0,
|
|
53
|
-
var error = (0,
|
|
54
|
-
const cache = new
|
|
37
|
+
const fsp = _nodeFs.default.promises;
|
|
38
|
+
var logger = (0, _debug.default)('api');
|
|
39
|
+
var error = (0, _debug.default)('error');
|
|
40
|
+
const cache = new _keyv.default({
|
|
55
41
|
store: new _keyvFile.KeyvFile({
|
|
56
42
|
filename: 'target/keyv-file.msgpack'
|
|
57
43
|
})
|
|
@@ -108,7 +94,7 @@ async function setAddresses(addr) {
|
|
|
108
94
|
// need to try proxying this to modify the headers if we want to use got's cache implementation
|
|
109
95
|
|
|
110
96
|
// SEE https://data.gov.au/data/dataset/19432f89-dc3a-4ef3-b943-5326ef1dbecc
|
|
111
|
-
const GNAF_PACKAGE_URL = process.env.GNAF_PACKAGE_URL || 'https://data.gov.au/api/3/action/package_show?id=19432f89-dc3a-4ef3-b943-5326ef1dbecc';
|
|
97
|
+
const GNAF_PACKAGE_URL = process.env.GNAF_PACKAGE_URL || 'https://data.gov.au/data/api/3/action/package_show?id=19432f89-dc3a-4ef3-b943-5326ef1dbecc';
|
|
112
98
|
async function fetchPackageData() {
|
|
113
99
|
const packageUrl = GNAF_PACKAGE_URL;
|
|
114
100
|
// See if we have the value in cache
|
|
@@ -126,18 +112,22 @@ async function fetchPackageData() {
|
|
|
126
112
|
}
|
|
127
113
|
// cached value was older than one day, so go fetch
|
|
128
114
|
try {
|
|
129
|
-
const
|
|
130
|
-
|
|
115
|
+
const fetchResponse = await fetch(packageUrl);
|
|
116
|
+
const body = await fetchResponse.text();
|
|
117
|
+
const headers = Object.fromEntries(fetchResponse.headers.entries());
|
|
131
118
|
logger('fresh gnaf package data', {
|
|
132
|
-
body
|
|
133
|
-
headers
|
|
119
|
+
body,
|
|
120
|
+
headers
|
|
134
121
|
});
|
|
135
122
|
await cache.set(packageUrl, {
|
|
136
|
-
body
|
|
137
|
-
headers
|
|
123
|
+
body,
|
|
124
|
+
headers
|
|
138
125
|
});
|
|
139
|
-
|
|
140
|
-
return
|
|
126
|
+
headers['x-cache'] = 'MISS';
|
|
127
|
+
return {
|
|
128
|
+
body,
|
|
129
|
+
headers
|
|
130
|
+
};
|
|
141
131
|
} catch (error_) {
|
|
142
132
|
// we were unable to fetch. if we have cached value that isn't stale, return in
|
|
143
133
|
if (cachedResponse !== undefined && age < THIRTY_DAYS_MS) {
|
|
@@ -159,12 +149,12 @@ async function fetchGnafFile() {
|
|
|
159
149
|
logger('dataResource', JSON.stringify(dataResource, undefined, 2));
|
|
160
150
|
logger('url', dataResource.url);
|
|
161
151
|
logger('headers', JSON.stringify(response.headers, undefined, 2));
|
|
162
|
-
const basename =
|
|
152
|
+
const basename = _nodePath.default.basename(dataResource.url);
|
|
163
153
|
logger('basename', basename);
|
|
164
154
|
const complete_path = GNAF_DIR;
|
|
165
155
|
const incomplete_path = `${complete_path}/incomplete`;
|
|
166
156
|
await new Promise((resolve, reject) => {
|
|
167
|
-
|
|
157
|
+
_nodeFs.default.mkdir(incomplete_path, {
|
|
168
158
|
recursive: true
|
|
169
159
|
}, error_ => {
|
|
170
160
|
if (error_) reject(error_);else resolve();
|
|
@@ -172,7 +162,7 @@ async function fetchGnafFile() {
|
|
|
172
162
|
});
|
|
173
163
|
const destination = `${complete_path}/${basename}`;
|
|
174
164
|
await new Promise((resolve, reject) => {
|
|
175
|
-
|
|
165
|
+
_nodeFs.default.mkdir(incomplete_path, {
|
|
176
166
|
recursive: true
|
|
177
167
|
}, error_ => {
|
|
178
168
|
if (error_) reject(error_);else resolve();
|
|
@@ -181,7 +171,7 @@ async function fetchGnafFile() {
|
|
|
181
171
|
// see if the file exists already
|
|
182
172
|
try {
|
|
183
173
|
await new Promise((resolve, reject) => {
|
|
184
|
-
|
|
174
|
+
_nodeFs.default.access(destination, _nodeFs.default.constants.R_OK, error_ => {
|
|
185
175
|
if (error_) reject(error_);else resolve();
|
|
186
176
|
});
|
|
187
177
|
});
|
|
@@ -191,7 +181,7 @@ async function fetchGnafFile() {
|
|
|
191
181
|
// file doesn't exist, so we need to download it.
|
|
192
182
|
logger('Starting G-NAF download');
|
|
193
183
|
try {
|
|
194
|
-
await (0,
|
|
184
|
+
await (0, _streamDown.default)(dataResource.url, `${incomplete_path}/${basename}`, dataResource.size);
|
|
195
185
|
await fsp.rename(`${incomplete_path}/${basename}`, destination);
|
|
196
186
|
logger('Finished downloading G-NAF', destination);
|
|
197
187
|
return destination;
|
|
@@ -202,32 +192,32 @@ async function fetchGnafFile() {
|
|
|
202
192
|
}
|
|
203
193
|
}
|
|
204
194
|
async function unzipFile(file) {
|
|
205
|
-
const extname =
|
|
206
|
-
const basenameWithoutExtention =
|
|
195
|
+
const extname = _nodePath.default.extname(file);
|
|
196
|
+
const basenameWithoutExtention = _nodePath.default.basename(file, extname);
|
|
207
197
|
const incomplete_path = `${GNAF_DIR}/incomplete/${basenameWithoutExtention}`;
|
|
208
198
|
const complete_path = `${GNAF_DIR}/${basenameWithoutExtention}`;
|
|
209
|
-
const exists = await (0,
|
|
199
|
+
const exists = await (0, _directoryExists.default)(complete_path);
|
|
210
200
|
if (exists) {
|
|
211
201
|
logger('directory exits. Skipping extract', complete_path);
|
|
212
202
|
// already extracted. Move along.
|
|
213
203
|
return complete_path;
|
|
214
204
|
} else {
|
|
215
205
|
await new Promise((resolve, reject) => {
|
|
216
|
-
|
|
206
|
+
_nodeFs.default.mkdir(incomplete_path, {
|
|
217
207
|
recursive: true
|
|
218
208
|
}, error_ => {
|
|
219
209
|
if (error_) reject(error_);else resolve();
|
|
220
210
|
});
|
|
221
211
|
});
|
|
222
|
-
const readStream =
|
|
212
|
+
const readStream = _nodeFs.default.createReadStream(file);
|
|
223
213
|
logger('before pipe');
|
|
224
214
|
let prom = new Promise((resolve, reject) => {
|
|
225
|
-
readStream.pipe(
|
|
215
|
+
readStream.pipe(_unzipStream.default.Parse()).pipe(_nodeStream.default.Transform({
|
|
226
216
|
objectMode: true,
|
|
227
217
|
transform: function (entry, encoding, callback) {
|
|
228
218
|
const entryPath = `${incomplete_path}/${entry.path}`;
|
|
229
219
|
if (entry.isDirectory) {
|
|
230
|
-
|
|
220
|
+
_nodeFs.default.mkdir(entryPath, {
|
|
231
221
|
recursive: true
|
|
232
222
|
}, error_ => {
|
|
233
223
|
if (error_) {
|
|
@@ -239,15 +229,15 @@ async function unzipFile(file) {
|
|
|
239
229
|
}
|
|
240
230
|
});
|
|
241
231
|
} else {
|
|
242
|
-
const dirname =
|
|
243
|
-
|
|
232
|
+
const dirname = _nodePath.default.dirname(entryPath);
|
|
233
|
+
_nodeFs.default.mkdir(dirname, {
|
|
244
234
|
recursive: true
|
|
245
235
|
}, error_ => {
|
|
246
236
|
if (error_) {
|
|
247
237
|
entry.autodrain();
|
|
248
238
|
callback(error_);
|
|
249
239
|
} else {
|
|
250
|
-
|
|
240
|
+
_nodeFs.default.stat(entryPath, (error_, stats) => {
|
|
251
241
|
if (error_ && error_.code !== 'ENOENT') {
|
|
252
242
|
logger('error statting file', error_);
|
|
253
243
|
entry.autodrain();
|
|
@@ -262,7 +252,7 @@ async function unzipFile(file) {
|
|
|
262
252
|
} else {
|
|
263
253
|
// size is different, so extract the file
|
|
264
254
|
logger('extracting', entryPath);
|
|
265
|
-
entry.pipe(
|
|
255
|
+
entry.pipe(_nodeFs.default.createWriteStream(entryPath)).on('finish', () => {
|
|
266
256
|
logger('finished extracting', entryPath);
|
|
267
257
|
callback();
|
|
268
258
|
}).on('error', error => {
|
|
@@ -285,7 +275,7 @@ async function unzipFile(file) {
|
|
|
285
275
|
});
|
|
286
276
|
await prom;
|
|
287
277
|
return await new Promise((resolve, reject) => {
|
|
288
|
-
|
|
278
|
+
_nodeFs.default.rename(incomplete_path, complete_path, error_ => {
|
|
289
279
|
if (error_) reject(error_);else resolve(complete_path);
|
|
290
280
|
});
|
|
291
281
|
});
|
|
@@ -701,7 +691,7 @@ async function loadAddressDetails(file, expectedCount, context, {
|
|
|
701
691
|
} = {}) {
|
|
702
692
|
let actualCount = 0;
|
|
703
693
|
await new Promise((resolve, reject) => {
|
|
704
|
-
|
|
694
|
+
_papaparse.default.parse(_nodeFs.default.createReadStream(file), {
|
|
705
695
|
header: true,
|
|
706
696
|
skipEmptyLines: true,
|
|
707
697
|
chunkSize: Number.parseInt(process.env.ADDRESSR_LOADING_CHUNK_SIZE || '10') * 1024 * 1024,
|
|
@@ -924,7 +914,6 @@ async function sendIndexRequest(indexingBody, initialBackoff = Number.parseInt(p
|
|
|
924
914
|
// parser.pause();
|
|
925
915
|
// paused = true;
|
|
926
916
|
await new Promise(resolve => {
|
|
927
|
-
// eslint-disable-next-line no-undef
|
|
928
917
|
setTimeout(() => {
|
|
929
918
|
resolve();
|
|
930
919
|
}, backoff);
|
|
@@ -938,7 +927,7 @@ async function sendIndexRequest(indexingBody, initialBackoff = Number.parseInt(p
|
|
|
938
927
|
}
|
|
939
928
|
async function getStateName(abbr, file) {
|
|
940
929
|
return await new Promise((resolve, reject) => {
|
|
941
|
-
|
|
930
|
+
_papaparse.default.parse(_nodeFs.default.createReadStream(file), {
|
|
942
931
|
header: true,
|
|
943
932
|
delimiter: '|',
|
|
944
933
|
complete: results => {
|
|
@@ -974,7 +963,7 @@ const {
|
|
|
974
963
|
readdir
|
|
975
964
|
} = require('node:fs').promises;
|
|
976
965
|
async function getFiles(currentDir, baseDir) {
|
|
977
|
-
const dir =
|
|
966
|
+
const dir = _nodePath.default.resolve(baseDir, currentDir);
|
|
978
967
|
logger(`reading ${dir} (${currentDir} in ${baseDir})`);
|
|
979
968
|
const dirents = await readdir(dir, {
|
|
980
969
|
withFileTypes: true
|
|
@@ -987,7 +976,7 @@ async function getFiles(currentDir, baseDir) {
|
|
|
987
976
|
}
|
|
988
977
|
function countFileLines(filePath) {
|
|
989
978
|
return new Promise((resolve, reject) => {
|
|
990
|
-
const readStream =
|
|
979
|
+
const readStream = _nodeFs.default.createReadStream(filePath, 'utf-8');
|
|
991
980
|
let lines = 0;
|
|
992
981
|
let last;
|
|
993
982
|
readStream.on('data', function (chunk) {
|
|
@@ -1020,7 +1009,10 @@ async function loadGnafData(directory, {
|
|
|
1020
1009
|
} else {
|
|
1021
1010
|
// may21 was missing the counts file
|
|
1022
1011
|
files = await getFiles('.', directory);
|
|
1023
|
-
for (
|
|
1012
|
+
// Only count lines for files we'll actually load (Authority Code + covered states).
|
|
1013
|
+
// Counting all files (~5GB across all states) takes hours on CI runners.
|
|
1014
|
+
const filesToCount = COVERED_STATES.length > 0 ? files.filter(f => f.match(/Authority/) || COVERED_STATES.some(s => _nodePath.default.basename(f).startsWith(`${s}_`))) : files;
|
|
1015
|
+
for (const file of filesToCount) {
|
|
1024
1016
|
const lines = await countFileLines(`${directory}/${file}`);
|
|
1025
1017
|
filesCounts[file] = lines - 1;
|
|
1026
1018
|
}
|
|
@@ -1040,7 +1032,7 @@ async function loadGnafData(directory, {
|
|
|
1040
1032
|
const addressDetailFiles = files.filter(f => f.match(/ADDRESS_DETAIL/) && f.match(/\/Standard\//));
|
|
1041
1033
|
logger('addressDetailFiles', addressDetailFiles);
|
|
1042
1034
|
for (const detailFile of addressDetailFiles) {
|
|
1043
|
-
const state =
|
|
1035
|
+
const state = _nodePath.default.basename(detailFile, _nodePath.default.extname(detailFile)).replace(/_.*/, '');
|
|
1044
1036
|
if (COVERED_STATES.length === 0 || COVERED_STATES.includes(state)) {
|
|
1045
1037
|
loadContext.state = state;
|
|
1046
1038
|
loadContext.stateName = await loadState(files, directory, state);
|
|
@@ -1085,7 +1077,7 @@ async function loadGnafData(directory, {
|
|
|
1085
1077
|
}
|
|
1086
1078
|
async function fileExists(countsFile) {
|
|
1087
1079
|
try {
|
|
1088
|
-
await fsp.access(countsFile,
|
|
1080
|
+
await fsp.access(countsFile, _nodeFs.default.constants.F_OK);
|
|
1089
1081
|
return true;
|
|
1090
1082
|
} catch (error_) {
|
|
1091
1083
|
error(error_);
|
|
@@ -1095,7 +1087,7 @@ async function fileExists(countsFile) {
|
|
|
1095
1087
|
async function loadFileCounts(countsFile) {
|
|
1096
1088
|
const filesCounts = {};
|
|
1097
1089
|
await new Promise((resolve, reject) => {
|
|
1098
|
-
|
|
1090
|
+
_papaparse.default.parse(_nodeFs.default.createReadStream(countsFile), {
|
|
1099
1091
|
header: true,
|
|
1100
1092
|
skipEmptyLines: true,
|
|
1101
1093
|
step: function (row) {
|
|
@@ -1142,7 +1134,7 @@ async function loadStreetLocality(files, directory, state) {
|
|
|
1142
1134
|
return [];
|
|
1143
1135
|
} else {
|
|
1144
1136
|
return await new Promise((resolve, reject) => {
|
|
1145
|
-
|
|
1137
|
+
_papaparse.default.parse(_nodeFs.default.createReadStream(`${directory}/${localityFile}`), {
|
|
1146
1138
|
header: true,
|
|
1147
1139
|
delimiter: '|',
|
|
1148
1140
|
complete: results => {
|
|
@@ -1163,7 +1155,7 @@ async function loadLocality(files, directory, state) {
|
|
|
1163
1155
|
return [];
|
|
1164
1156
|
} else {
|
|
1165
1157
|
return await new Promise((resolve, reject) => {
|
|
1166
|
-
|
|
1158
|
+
_papaparse.default.parse(_nodeFs.default.createReadStream(`${directory}/${localityFile}`), {
|
|
1167
1159
|
header: true,
|
|
1168
1160
|
delimiter: '|',
|
|
1169
1161
|
complete: results => {
|
|
@@ -1187,7 +1179,7 @@ async function loadSiteGeo(files, directory, state, loadContext, filesCounts) {
|
|
|
1187
1179
|
const expectedCount = filesCounts[geoFile];
|
|
1188
1180
|
let count = 0;
|
|
1189
1181
|
return await new Promise((resolve, reject) => {
|
|
1190
|
-
|
|
1182
|
+
_papaparse.default.parse(_nodeFs.default.createReadStream(`${directory}/${geoFile}`), {
|
|
1191
1183
|
header: true,
|
|
1192
1184
|
delimiter: '|',
|
|
1193
1185
|
chunk: function (chunk, parser) {
|
|
@@ -1234,7 +1226,7 @@ async function loadDefaultGeo(files, directory, state, loadContext, filesCounts)
|
|
|
1234
1226
|
const expectedCount = filesCounts[geoFile];
|
|
1235
1227
|
let count = 0;
|
|
1236
1228
|
return await new Promise((resolve, reject) => {
|
|
1237
|
-
|
|
1229
|
+
_papaparse.default.parse(_nodeFs.default.createReadStream(`${directory}/${geoFile}`), {
|
|
1238
1230
|
header: true,
|
|
1239
1231
|
delimiter: '|',
|
|
1240
1232
|
chunk: function (chunk, parser) {
|
|
@@ -1275,9 +1267,9 @@ async function loadAuthFiles(files, directory, loadContext, filesCounts) {
|
|
|
1275
1267
|
const authCodeFiles = files.filter(f => f.match(/Authority Code/));
|
|
1276
1268
|
logger('authCodeFiles', authCodeFiles);
|
|
1277
1269
|
for (const authFile of authCodeFiles) {
|
|
1278
|
-
const contextKey =
|
|
1270
|
+
const contextKey = _nodePath.default.basename(authFile, _nodePath.default.extname(authFile));
|
|
1279
1271
|
await new Promise((resolve, reject) => {
|
|
1280
|
-
|
|
1272
|
+
_papaparse.default.parse(_nodeFs.default.createReadStream(`${directory}/${authFile}`), {
|
|
1281
1273
|
delimiter: '|',
|
|
1282
1274
|
header: true,
|
|
1283
1275
|
complete: function (results) {
|
|
@@ -1304,22 +1296,29 @@ async function loadAuthFiles(files, directory, loadContext, filesCounts) {
|
|
|
1304
1296
|
async function loadGnaf({
|
|
1305
1297
|
refresh = false
|
|
1306
1298
|
} = {}) {
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1299
|
+
let mainDirectory;
|
|
1300
|
+
if (process.env.GNAF_TEST_FIXTURE_DIR) {
|
|
1301
|
+
// Skip download, extraction, and directory scanning — use pre-prepared fixture.
|
|
1302
|
+
mainDirectory = process.env.GNAF_TEST_FIXTURE_DIR;
|
|
1303
|
+
logger('Using test fixture dir', mainDirectory);
|
|
1304
|
+
} else {
|
|
1305
|
+
const file = await fetchGnafFile();
|
|
1306
|
+
const unzipped = await unzipFile(file);
|
|
1307
|
+
logger('Data dir', unzipped);
|
|
1308
|
+
const contents = await fsp.readdir(unzipped);
|
|
1309
|
+
logger('Data dir contents', contents);
|
|
1310
|
+
if (contents.length === 0) {
|
|
1311
|
+
throw new Error(`Data dir '${unzipped}' is empty`);
|
|
1312
|
+
}
|
|
1313
|
+
const gnafDir = await (0, _glob.glob)('**/G-NAF/', {
|
|
1314
|
+
cwd: unzipped
|
|
1315
|
+
});
|
|
1316
|
+
console.log(gnafDir);
|
|
1317
|
+
if (gnafDir.length === 0) {
|
|
1318
|
+
throw new Error(`Cannot find 'G-NAF' directory in Data dir '${unzipped}'`);
|
|
1319
|
+
}
|
|
1320
|
+
mainDirectory = _nodePath.default.dirname(`${unzipped}/${gnafDir[0].slice(0, -1)}`);
|
|
1321
1321
|
}
|
|
1322
|
-
const mainDirectory = _nodePath2.default.dirname(`${unzipped}/${gnafDir[0].slice(0, -1)}`);
|
|
1323
1322
|
logger('Main Data dir', mainDirectory);
|
|
1324
1323
|
await loadGnafData(mainDirectory, {
|
|
1325
1324
|
refresh
|
|
@@ -1346,13 +1345,13 @@ async function getAddress(addressId) {
|
|
|
1346
1345
|
};
|
|
1347
1346
|
logger('json', json);
|
|
1348
1347
|
delete json._id;
|
|
1349
|
-
const link = new
|
|
1348
|
+
const link = new _httpLinkHeader.default();
|
|
1350
1349
|
link.set({
|
|
1351
1350
|
rel: 'self',
|
|
1352
1351
|
uri: `/addresses/${addressId}`
|
|
1353
1352
|
});
|
|
1354
1353
|
// TODO: store hash in address
|
|
1355
|
-
const hash =
|
|
1354
|
+
const hash = _nodeCrypto.default.createHash('md5').update(JSON.stringify(json)).digest('hex');
|
|
1356
1355
|
return {
|
|
1357
1356
|
link,
|
|
1358
1357
|
json,
|
|
@@ -1397,7 +1396,7 @@ async function getAddresses(url, swagger, q, p = 1) {
|
|
|
1397
1396
|
try {
|
|
1398
1397
|
const foundAddresses = await searchForAddress(q, p);
|
|
1399
1398
|
logger('foundAddresses', foundAddresses);
|
|
1400
|
-
const link = new
|
|
1399
|
+
const link = new _httpLinkHeader.default();
|
|
1401
1400
|
link.set({
|
|
1402
1401
|
rel: 'describedby',
|
|
1403
1402
|
uri: `/docs/#operations-${swagger.path.get['x-swagger-router-controller'].toLowerCase()}-${swagger.path.get.operationId}`,
|
|
@@ -1454,7 +1453,7 @@ async function getAddresses(url, swagger, q, p = 1) {
|
|
|
1454
1453
|
}
|
|
1455
1454
|
const responseBody = mapToSearchAddressResponse(foundAddresses);
|
|
1456
1455
|
logger('responseBody', JSON.stringify(responseBody, undefined, 2));
|
|
1457
|
-
const linkTemplate = new
|
|
1456
|
+
const linkTemplate = new _httpLinkHeader.default();
|
|
1458
1457
|
const op = swagger.path.get;
|
|
1459
1458
|
(0, _setLinkOptions.setLinkOptions)(op, url, linkTemplate);
|
|
1460
1459
|
return {
|
|
@@ -4,11 +4,10 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
6
|
exports.printVersion = printVersion;
|
|
7
|
-
var _dotenv = require("dotenv");
|
|
8
|
-
var _dotenv2 = _interopRequireDefault(_dotenv);
|
|
7
|
+
var _dotenv = _interopRequireDefault(require("dotenv"));
|
|
9
8
|
var _version = require("../version");
|
|
10
9
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
11
|
-
|
|
10
|
+
_dotenv.default.config();
|
|
12
11
|
function printVersion() {
|
|
13
12
|
let environment = process.env.NODE_ENV || 'development';
|
|
14
13
|
if (environment === 'development') {
|
package/lib/src/server2.js
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
-
var _debug = require("debug");
|
|
4
|
-
var _debug2 = _interopRequireDefault(_debug);
|
|
3
|
+
var _debug = _interopRequireDefault(require("debug"));
|
|
5
4
|
var _elasticsearch = require("../client/elasticsearch");
|
|
6
5
|
var _printVersion = require("../service/printVersion");
|
|
7
6
|
var _waycharterServer = require("./waycharterServer");
|
|
8
7
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
9
|
-
const logger = (0,
|
|
8
|
+
const logger = (0, _debug.default)('api');
|
|
10
9
|
(0, _waycharterServer.startRest2Server)().then(() => {
|
|
11
10
|
logger('connecting es client');
|
|
12
11
|
const p1 = (0, _elasticsearch.esConnect)().then(esClient => {
|
|
@@ -5,27 +5,23 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
5
5
|
});
|
|
6
6
|
exports.startRest2Server = startRest2Server;
|
|
7
7
|
exports.stopServer = stopServer;
|
|
8
|
-
var _debug = require("debug");
|
|
9
|
-
var
|
|
10
|
-
var _express = require("express");
|
|
11
|
-
var _express2 = _interopRequireDefault(_express);
|
|
8
|
+
var _debug = _interopRequireDefault(require("debug"));
|
|
9
|
+
var _express = _interopRequireDefault(require("express"));
|
|
12
10
|
var _nodeHttp = require("node:http");
|
|
13
11
|
var _waycharter = require("@mountainpass/waycharter");
|
|
14
12
|
var _addressService = require("../service/address-service");
|
|
15
13
|
var _version = require("../version");
|
|
16
|
-
var _nodeCrypto = require("node:crypto");
|
|
17
|
-
var _nodeCrypto2 = _interopRequireDefault(_nodeCrypto);
|
|
14
|
+
var _nodeCrypto = _interopRequireDefault(require("node:crypto"));
|
|
18
15
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
19
16
|
//import connect from 'connect';
|
|
20
17
|
|
|
21
|
-
var app = (0,
|
|
18
|
+
var app = (0, _express.default)();
|
|
22
19
|
const ONE_DAY = 60 * 60 * 24;
|
|
23
20
|
const ONE_WEEK = ONE_DAY * 7;
|
|
24
21
|
var serverPort = process.env.PORT || 8080;
|
|
25
|
-
var logger = (0,
|
|
26
|
-
var error = (0,
|
|
27
|
-
error.log = console.error.bind(console);
|
|
28
|
-
|
|
22
|
+
var logger = (0, _debug.default)('api');
|
|
23
|
+
var error = (0, _debug.default)('error');
|
|
24
|
+
error.log = console.error.bind(console);
|
|
29
25
|
let server;
|
|
30
26
|
const PAGE_SIZE = process.env.PAGE_SIZE || 8;
|
|
31
27
|
function startRest2Server() {
|
|
@@ -85,7 +81,7 @@ function startRest2Server() {
|
|
|
85
81
|
pid: h._id.replace('/addresses/', '')
|
|
86
82
|
};
|
|
87
83
|
});
|
|
88
|
-
const responseHash =
|
|
84
|
+
const responseHash = _nodeCrypto.default.createHash('md5').update(JSON.stringify(body)).digest('hex');
|
|
89
85
|
return {
|
|
90
86
|
body,
|
|
91
87
|
hasMore: page < foundAddresses.body.hits.total.value / PAGE_SIZE - 1,
|
|
@@ -111,7 +107,7 @@ function startRest2Server() {
|
|
|
111
107
|
parameters: ['q']
|
|
112
108
|
}]
|
|
113
109
|
});
|
|
114
|
-
|
|
110
|
+
waycharter.registerResourceType({
|
|
115
111
|
path: '/health',
|
|
116
112
|
loader: async () => {
|
|
117
113
|
return {
|
|
@@ -126,14 +122,14 @@ function startRest2Server() {
|
|
|
126
122
|
};
|
|
127
123
|
}
|
|
128
124
|
});
|
|
129
|
-
|
|
125
|
+
waycharter.registerResourceType({
|
|
130
126
|
path: '/',
|
|
131
127
|
loader: async () => {
|
|
132
128
|
return {
|
|
133
129
|
body: {},
|
|
134
130
|
links: [...addressesType.additionalPaths, {
|
|
135
131
|
rel: 'https://addressr.io/rels/health',
|
|
136
|
-
|
|
132
|
+
uri: '/health'
|
|
137
133
|
}],
|
|
138
134
|
headers: {
|
|
139
135
|
etag: `"${_version.version}"`,
|
package/lib/swagger.js
CHANGED
|
@@ -3,38 +3,35 @@
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", {
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
|
-
exports.swaggerDoc = undefined;
|
|
7
|
-
exports.swaggerInit = swaggerInit;
|
|
8
6
|
exports.startServer = startServer;
|
|
9
7
|
exports.stopServer = stopServer;
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
var
|
|
13
|
-
var
|
|
8
|
+
exports.swaggerDoc = void 0;
|
|
9
|
+
exports.swaggerInit = swaggerInit;
|
|
10
|
+
var _debug = _interopRequireDefault(require("debug"));
|
|
11
|
+
var _express = _interopRequireDefault(require("express"));
|
|
14
12
|
var _nodeFs = require("node:fs");
|
|
15
13
|
var _nodeHttp = require("node:http");
|
|
16
14
|
var _jsYaml = require("js-yaml");
|
|
17
|
-
var _nodePath = require("node:path");
|
|
18
|
-
var _nodePath2 = _interopRequireDefault(_nodePath);
|
|
15
|
+
var _nodePath = _interopRequireDefault(require("node:path"));
|
|
19
16
|
var _swaggerTools = require("swagger-tools");
|
|
20
17
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
21
18
|
//import connect from 'connect';
|
|
22
19
|
|
|
23
|
-
var app = (0,
|
|
20
|
+
var app = (0, _express.default)();
|
|
24
21
|
var serverPort = process.env.PORT || 8080;
|
|
25
|
-
var logger = (0,
|
|
26
|
-
var error = (0,
|
|
27
|
-
error.log = console.error.bind(console);
|
|
22
|
+
var logger = (0, _debug.default)('api');
|
|
23
|
+
var error = (0, _debug.default)('error');
|
|
24
|
+
error.log = console.error.bind(console);
|
|
28
25
|
|
|
29
26
|
// swaggerRouter configuration
|
|
30
27
|
var options = {
|
|
31
|
-
swaggerUi:
|
|
32
|
-
controllers:
|
|
28
|
+
swaggerUi: _nodePath.default.join(__dirname, '/swagger.json'),
|
|
29
|
+
controllers: _nodePath.default.join(__dirname, './controllers'),
|
|
33
30
|
useStubs: process.env.NODE_ENV === 'development' // Conditionally turn on stubs (mock mode)
|
|
34
31
|
};
|
|
35
32
|
|
|
36
33
|
// The Swagger document (require it, build it programmatically, fetch it from a URL, ...)
|
|
37
|
-
var spec = (0, _nodeFs.readFileSync)(
|
|
34
|
+
var spec = (0, _nodeFs.readFileSync)(_nodePath.default.join(__dirname, 'api/swagger.yaml'), 'utf8');
|
|
38
35
|
var swaggerDoc = exports.swaggerDoc = (0, _jsYaml.load)(spec);
|
|
39
36
|
global.swaggerDoc = swaggerDoc;
|
|
40
37
|
function swaggerInit() {
|
|
@@ -111,12 +108,6 @@ function startServer() {
|
|
|
111
108
|
server.listen(serverPort, function () {
|
|
112
109
|
logger('📡 Addressr is listening on port %d ( http://localhost:%d ) ', serverPort, serverPort);
|
|
113
110
|
logger('📑 Swagger-ui is available on http://localhost:%d/docs', serverPort);
|
|
114
|
-
if (process.env.NODE_ENV !== 'PRODUCTION') {
|
|
115
|
-
// ngrok.connect(serverPort).then(url => {
|
|
116
|
-
// logger('📡 Addressr is listening at %s', url);
|
|
117
|
-
// logger('📑 Swagger-ui is available on %s/docs/', url);
|
|
118
|
-
// });
|
|
119
|
-
}
|
|
120
111
|
});
|
|
121
112
|
return `http://localhost:${serverPort}`;
|
|
122
113
|
});
|
package/lib/utils/stream-down.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
-
var _progress = require("progress");
|
|
4
|
-
var _progress2 = _interopRequireDefault(_progress);
|
|
3
|
+
var _progress = _interopRequireDefault(require("progress"));
|
|
5
4
|
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
|
|
6
5
|
const {
|
|
7
6
|
parse
|
|
@@ -20,7 +19,7 @@ module.exports = function (url, path, size) {
|
|
|
20
19
|
const length = res.headers['content-length'] ? Number.parseInt(res.headers['content-length'], 10) : size;
|
|
21
20
|
// let downloaded = 0;
|
|
22
21
|
// let percent = 0;
|
|
23
|
-
var bar = new
|
|
22
|
+
var bar = new _progress.default(' downloading [:bar] :rate/bps :percent :etas', {
|
|
24
23
|
complete: '=',
|
|
25
24
|
incomplete: ' ',
|
|
26
25
|
width: 20,
|