@osm-conflation-engine/cli 3.8.0 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +26 -0
- package/dist/config.schema.json +0 -5
- package/dist/index.cjs +4 -4
- package/dist/index.d.cts +18 -8
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.mts +18 -8
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +4 -4
- package/dist/index.mjs.map +1 -1
- package/package.json +14 -13
package/LICENSE
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2021-2023 Kyℓe Hensel
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
The OpenStreetMap logo is a trademark of the OpenStreetMap Foundation, and is used with their permission.
|
|
26
|
+
This project is not endorsed by or affiliated with the OpenStreetMap Foundation.
|
package/dist/config.schema.json
CHANGED
|
@@ -77,10 +77,6 @@
|
|
|
77
77
|
"description": {
|
|
78
78
|
"type": "string"
|
|
79
79
|
},
|
|
80
|
-
"git_repository": {
|
|
81
|
-
"description": "format should be `https://github.com/a/b`",
|
|
82
|
-
"type": "string"
|
|
83
|
-
},
|
|
84
80
|
"name": {
|
|
85
81
|
"type": "string"
|
|
86
82
|
},
|
|
@@ -92,7 +88,6 @@
|
|
|
92
88
|
}
|
|
93
89
|
},
|
|
94
90
|
"required": [
|
|
95
|
-
"git_repository",
|
|
96
91
|
"name",
|
|
97
92
|
"description",
|
|
98
93
|
"wiki_page",
|
package/dist/index.cjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},s=(n,r,a)=>(a=n==null?{}:e(i(n)),o(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n));let c=require("node:path"),l=require("node:os"),u=require("node:fs"),d=require("ajv"),f=require("node:readline/promises"),p=require("h3-js"),m=require("node:timers/promises"),h=require("node:child_process"),g=require("d3-geo"),_=require("pbf2json");_=s(_,1);let v=require("node:crypto"),ee=require("@actions/core");const
|
|
2
|
-
`);n.end(),await new Promise(e=>n.on(`finish`,e))}function
|
|
1
|
+
Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},s=(n,r,a)=>(a=n==null?{}:e(i(n)),o(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n));let c=require("node:path"),l=require("node:os"),u=require("node:fs"),d=require("ajv"),f=require("node:readline/promises"),p=require("h3-js"),m=require("node:timers/promises"),h=require("node:child_process"),g=require("d3-geo"),_=require("pbf2json");_=s(_,1);let v=require("node:crypto"),ee=require("@actions/core"),y=require("tag2link");y=s(y,1);const b=process.env.NODE_ENV===`test`,x=[`download`,`match`,`conflate`],te={PBF:`o.pbf`,OVERPASS:`o.overpass.geo.json`,POSTPASS:`o.postpass.geo.json`,OSM_PROCESSED_WITH_REF:`osm-processed-with-ref.jsonl`,OSM_PROCESSED_NO_REF:`osm-processed-no-ref.jsonl`,OSM_PROCESSED_OTHER:`osm-processed-other.json`,IGNORE_LIST:`ignore-list.json`,MATCHES:`matches.json`},S=`https://osm-conflation-engine.kyle.kiwi`,C=e=>({"User-Agent":`https://github.com/osm-nz/osm-conflation-engine on behalf of ${e.metadata.wiki_page}`});let ne=function(e){return e[e.None=0]=`None`,e[e.IsLastEditedByImporter=1]=`IsLastEditedByImporter`,e[e.IsRecentlyChanged=2]=`IsRecentlyChanged`,e[e.IsChecked=4]=`IsChecked`,e[e.IsCheckedRecently=8]=`IsCheckedRecently`,e}({});function w(e){let t=e.config.osm_data?.check_date_key||`check_date`,n=new Set([t,e.config.merge.osm_key]).difference(new Set(e.config.osm_data.tags_to_keep));if(n.size)throw Error(`config.osm_data.tags_to_keep is missing some tags that are referenced in other parts of the config: ${[...n].join(`, `)}`);let r=[];for(let t of e.config.osm_data.tags_to_keep)t.startsWith(`/`)&&t.endsWith(`/`)&&r.push(new RegExp(t.slice(1,-1)));let i={};function a(t){if(t in i)return i[t];let n=e.config.osm_data.tags_to_keep.includes(t);if(!n){for(let e of r)if(e.test(t)){n=!0;break}}return i[t]=n,n}function o(e){let t={};if(e)for(let n in e){let r=e[n];a(n)&&r&&(t[n]=r)}return t}return{pickTags:o}}let re=function(e){return e[e.OneToOne=1]=`OneToOne`,e[e.OneToMany=2]=`OneToMany`,e[e.ManyToOne=3]=`ManyToOne`,e[e.ManyToMany=4]=`ManyToMany`,e[e.Delete=5]=`Delete`,e[e.Guess=6]=`Guess`,e}({});const ie=e=>e?(Date.now()-+new Date(e))/1e3/60/60/24/365<4?1:2:0;async function T(e,t){let n=(0,f.createInterface)({input:(0,u.createReadStream)(e)}),r={};for await(let e of n){let n=JSON.parse(e),i=t(n);r[i]=n}return r}async function E(e,t){let n=(0,u.createWriteStream)(e,{encoding:`utf8`});for(let e of t)n.write(JSON.stringify(e)),n.write(`
|
|
2
|
+
`);n.end(),await new Promise(e=>n.on(`finish`,e))}function D(e,t){let n=t||3;switch(e.type){case`GeometryCollection`:throw Error(`not supported`);case`Point`:return[(0,p.latLngToCell)(e.coordinates[1],e.coordinates[0],n)];case`MultiPoint`:case`LineString`:case`MultiLineString`:case`Polygon`:return(0,p.polygonToCellsExperimental)(e.coordinates,n,p.POLYGON_TO_CELLS_FLAGS.containmentOverlapping,!0).toSorted();case`MultiPolygon`:{let t=new Set;for(let r of e.coordinates){let e=(0,p.polygonToCellsExperimental)(r,n,p.POLYGON_TO_CELLS_FLAGS.containmentOverlapping,!0);for(let n of e)t.add(n)}return[...t].toSorted()}default:throw TypeError(e)}}const ae=(e=>(e.setDate(e.getDate()-90),+e))(new Date);function O(e,t,n){n.sectors=D({type:`Point`,coordinates:n.centroid},e.config.merge.sector_resolution);let r=e.config.osm_data?.check_date_key||`check_date`;+new Date(n.metadata?.timestamp??0)>ae&&(n.flags|=2);let i=n.metadata?.user&&(e.callbacks.isImportUser?e.callbacks.isImportUser?.(n.metadata.user):n.metadata?.user?.endsWith(`_import`));(n.metadata?.version===1||i)&&(n.flags|=1),n.tags[r]&&(n.flags|=4),ie(n.tags[r])===1&&(n.flags|=8);let a=n.tags[e.config.merge.osm_key];if(a)if(t.count++,t.duplicateRefs[a])t.duplicateRefs[a].push(n);else if(t.withRef[a])t.duplicateRefs[a]=[t.withRef[a],n],delete t.withRef[a];else if(a.includes(`;`)){let e=a.split(`;`);for(let r of e)t.semi[r]=n}else t.withRef[a]=n;else t.noRef[n.id]=n}async function k(e,t){let{withRef:n,noRef:r,...i}=t;await E(e.tempFileNames.osm_processed_with_ref,Object.values(n)),await E(e.tempFileNames.osm_processed_no_ref,Object.values(r)),await u.promises.writeFile(e.tempFileNames.osm_processed_other,JSON.stringify(i))}async function oe(e,t,n){let r=e.tempFileNames.overpass;if(e.use_cache&&(0,u.existsSync)(r)){console.info(`Using cached overpass response`),await k(e,JSON.parse(await u.promises.readFile(r,`utf8`)));return}let i=`
|
|
3
3
|
[out:json][timeout:480];
|
|
4
4
|
nwr["${e.config.merge.osm_key}"];
|
|
5
5
|
out meta center;
|
|
6
|
-
`,a=t?await u.promises.readFile(t,`utf8`):i;console.info(`fetching OSM data via overpass…`);let o=await fetch(`${n||`https://overpass-api.de/api/interpreter`}?data=${encodeURIComponent(a)}`,{headers:
|
|
6
|
+
`,a=t?await u.promises.readFile(t,`utf8`):i;console.info(`fetching OSM data via overpass…`);let o=await fetch(`${n||`https://overpass-api.de/api/interpreter`}?data=${encodeURIComponent(a)}`,{headers:C(e.config)});if(!o.ok){let e=await o.text()||o.statusText;throw Error(`HTTP Error ${o.status} ${e}`)}let s=await o.json(),{pickTags:c}=w(e),l={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0};for(let t of s.elements)O(e,l,{id:`${t.type[0]}${t.id}`,centroid:t.type===`node`?[t.lon,t.lat]:t.center?[t.center.lon,t.center.lat]:[0,0],tags:c(t.tags),metadata:{changeset:t.changeset,timestamp:t.timestamp,user:t.user,version:t.version},sectors:[],flags:0});await u.promises.writeFile(r,JSON.stringify(l)),await k(e,l)}const A=`https://osm-internal.download.geofabrik.de`,j=`https://www.openstreetmap.org`,se=e=>e.match(/name="csrf-token" content="([^"]+)"/)[1];async function ce(e,t){if(t.startsWith(A)||e.warnings.push(`(!) Planet PBF url is not from ${A}, so metadata may not be available.`),!process.env.OSM_AUTH)return e.warnings.push(`(!) No OSM credentials, so the planet extract will not include metadata`),{url:t.replace(A,`https://download.geofabrik.de`).replace(`-internal.osm.pbf`,`.osm.pbf`),cookies:``};let n=new URLSearchParams(process.env.OSM_AUTH),r=C(e.config),i=await fetch(`${A}/get_cookie?action=get_authorization_url`,{method:`POST`,headers:r}).then(e=>e.json()),a=await fetch(`${j}/login?cookie_test=true`,{headers:r}),o=a.headers.get(`Set-Cookie`),s=await fetch(`${j}/login`,{method:`POST`,body:new URLSearchParams({username:n.get(`username`),password:n.get(`password`),referer:`/`,commit:`Login`,authenticity_token:se(await a.text())}),headers:{...r,Cookie:o,"Content-Type":`application/x-www-form-urlencoded`}});if(s.status!==200)throw Error(`Login failed`);o+=`; ${s.headers.get(`Set-Cookie`)}`;let c=await fetch(i.authorization_url,{headers:{...r,Cookie:o}}),l=c.url.startsWith(j)?await fetch(i.authorization_url,{method:`POST`,body:new URLSearchParams({client_id:i.client_id,redirect_uri:i.redirect_uri,authenticity_token:se(await c.text()),state:i.state,response_type:`code`,scope:`read_prefs`,nonce:``,code_challenge:``,code_challenge_method:``,commit:`Authorize`}),headers:{...r,Cookie:o,"Content-Type":`application/x-www-form-urlencoded`}}).then(e=>e.text()):await c.text();if(!l.includes(`gf_download_oauth`))throw Error(`No token`);return{url:t,cookies:l}}async function le(e,t){let n=e.tempFileNames.pbf,{cookies:r,url:i}=await ce(e,t);if(await u.promises.mkdir((0,c.dirname)(n),{recursive:!0}),await u.promises.rm(n,{recursive:!0,force:!0}),/^https?:\/\//.test(i)){console.info(`downloading from`,i,`to`,n);let e=(0,h.spawn)(`curl`,[`-L`,i,`-H`,`Cookie: ${r}`,`--output`,(0,c.basename)(n)],{cwd:(0,c.dirname)(n)});e.stdout.pipe(process.stdout),e.stderr.pipe(process.stderr),await new Promise((t,n)=>{e.on(`close`,e=>e?n(e):t())}),await(0,m.setTimeout)(5e3)}else console.info(`copying file from`,i,`to`,n),await u.promises.copyFile(i,n);console.info(n,await u.promises.stat(n))}async function ue(e,t,n){let r=e.tempFileNames.postpass;if(e.use_cache&&(0,u.existsSync)(r)){console.info(`Using cached postpass response`),await k(e,JSON.parse(await u.promises.readFile(r,`utf8`)));return}let i=`
|
|
7
7
|
SELECT osm_type, osm_id, tags, ST_PointOnSurface(geom) AS geom
|
|
8
8
|
FROM postpass_pointlinepolygon
|
|
9
9
|
WHERE tags ? '${e.config.merge.osm_key}'
|
|
10
|
-
`,a=t?await u.promises.readFile(t,`utf8`):i;console.info(`fetching OSM data via postpass…`);let o=await fetch(n||`https://postpass.geofabrik.de/api/interpreter`,{method:`POST`,body:new URLSearchParams({data:a}),headers:S(e.config)});if(!o.ok){let e=await o.text()||o.statusText;throw Error(`HTTP Error ${o.status} ${e}`)}let s=await o.json(),{pickTags:c}=C(e),l={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0},d=new Set;for(let t of s.features){let{osm_type:n,osm_id:r,tags:i}=t.properties,a=`${n.toLowerCase()}${r}`;d.has(a)||(d.add(a),D(e,l,{id:a,centroid:t.geometry?.type?(0,g.geoCentroid)(t.geometry):[0,0],tags:c(i),metadata:void 0,sectors:[],flags:0}))}await u.promises.writeFile(r,JSON.stringify(l)),await O(e,l)}async function ue(e){let t=e.config.merge.osm_key,n=e.config.e2e_tests?.ignore_list_file_path;if(y&&!n)throw Error(`When running in the unit test envrironment, you must supply config.e2e_tests.ignoreListFilePath`);let r=y&&n?JSON.parse(await u.promises.readFile(n,`utf8`)):await fetch(`${x}/api/ignore_list/${encodeURIComponent(t)}?return_ids_only=true`).then(e=>e.json());await u.promises.writeFile(e.tempFileNames.ignore_list,JSON.stringify(r.result,null,2))}async function de(e,t){if(e.use_cache&&(0,u.existsSync)(e.tempFileNames.osm_processed_other)){console.info(`Using cached osm data`);return}let{pickTags:n}=C(e);console.info(`Starting preprocess of OSM data...`),await O(e,await new Promise((r,i)=>{let a={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0},o=!1,s=!1,c=0;_.default.createReadStream({file:e.tempFileNames.pbf,tags:t,leveldb:`/tmp`,metadata:!0}).on(`data`,t=>{s||=!0;let r=t.type===`node`?t:t.centroid,i=t.meta;i&&(o=!0);let l=n(t.tags);D(e,a,{id:`${t.type[0]}${t.id}`,centroid:[+r.lon,+r.lat],metadata:i?{changeset:i.changeset,timestamp:i.timestamp?new Date(1e3*i.timestamp).toISOString():void 0,user:i.user,version:i.version}:void 0,tags:l,sectors:[],flags:0}),c+=1,c%1e3||process.stdout.write(`.`)}).on(`finish`,()=>{if(o||e.warnings.push(`No metadata extracted from the planet file!`),!s)throw Error(`Nothing extracted from the pbf file, presumably it’s invalid.`);r(a)}).on(`error`,i)}))}async function fe(e){if(await ue(e),e.use_cache&&(0,u.existsSync)(e.tempFileNames.osm_processed_other)){console.info(`Using cached OSM data`);return}switch(e.config.osm_data.source.type){case`pbf`:await ce(e,e.config.osm_data.source.pbf_url),await de(e,e.config.osm_data.source.pbf_filter);break;case`overpass`:await oe(e,e.config.osm_data.source.overpass_query_file,e.config.osm_data.source.overpass_server_url);break;case`postpass`:await le(e,e.config.osm_data.source.postpass_query_file,e.config.osm_data.source.postpass_server_url);break;default:throw TypeError(e.config.osm_data.source)}}function pe(e,t,n){let r=[];for(let i in t.withRef){let a=i,o=t.withRef[a];if(!(a in n)&&!(o.flags&8)){if(e.callbacks.getAltRefs?.(a).some(e=>e in n))continue;r.push(a)}}let i={};for(let t in n){let r=t,a=n[r];for(let t of a.sectors){i[t]||={};let n=e.callbacks.getLocalKeyForSource(a);i[t][n]=r}}let a={},o=[];for(let n of r){let r=t.withRef[n],s=e.callbacks.getLocalKeyForOsm(r),c=new Set(r.sectors.map(e=>i[e]?.[s]).filter(Boolean));if(c.size>1&&console.warn(`${r.id} spans multiple sectors, and there are conflicting localKeys across the sectors. Therefore, it will be straight-up deleted`),c.size===1){let e=[...c][0];t.withRef[e]?o.push(n):a[e]=n}else o.push(n)}return{refsThatChanged:a,toDelete:o}}const me=(e,t,n=5)=>`${e.toFixed(n)},${t.toFixed(n)}`,M=6371008.8;function N(e,t){let n=Math.max(0,t.minLat-e.maxLat,e.minLat-t.maxLat),r=Math.max(0,t.minLng-e.maxLng,e.minLng-t.maxLng);return Math.hypot(n,r)}function P(e){return Math.max(Math.abs(e.maxLat-e.minLat),Math.abs(e.maxLng-e.minLng))}const F=new Map,he=(e,t,n)=>{let r=e.callbacks.getLocalKeyForSource(t);return n.filter(t=>{let n=F.get(t);return n||(n=e.callbacks.getLocalKeyForOsm(t),F.set(t,n)),n===r}).map(e=>{let n=M*(0,g.geoDistance)(e.centroid,t.centroid);return Object.assign(e,{offset:n})}).filter(e=>e.offset<200).toSorted((e,t)=>e.offset-t.offset)};async function ge(e,t,n){let r=Object.keys(t).length,i=new Set(JSON.parse(await u.promises.readFile(e.tempFileNames.ignore_list,`utf8`)));console.info(`processing deleted data...`),console.time(`matchDeletions`);let{refsThatChanged:a,toDelete:o}=pe(e,n,t);console.timeEnd(`matchDeletions`),console.info(`Processing data...`);let s=0;console.time(`matchLoop`);let c={1:[],2:[],3:{},4:[],5:o,6:[]},l=Object.values(n.noRef),d={};for(let e in n.semi){let t=e.split(`;`);for(let[n,r]of t.entries())d[r]||=[],d[r].push({value:e,isFirst:!n})}let f={};for(let t in n.semi){let r=n.semi[t],i=r.tags[e.config.merge.osm_key];i&&(f[i]=r)}for(let o in t){let u=o,p=a[u]||u;if(i.has(p)||p in a)continue;let m=n.withRef[p]||e.callbacks.getAltRefs?.(p).map(e=>n.withRef[e]).find(Boolean),h=t[u],g=n.duplicateRefs[p],_=f[p];if(d[p])if(d[p][0].isFirst)p=d[p][0].value;else continue;let v=n.semi[p];if(m&&v){let e=p.split(`;`);c[4].push({osm:[m.id,v.id],source:e})}else if(m)c[1].push({osm:m.id,source:u});else if(g)c[2].push({osm:g.map(e=>e.id),source:p});else if(_)c[1].push({osm:_.id,source:u});else if(v){let e=p.split(`;`);c[3][v.id]||=[],c[3][v.id]?.push(...e)}else{let t=he(e,h,l);t.length===1?c[1].push({source:p,osm:t[0].id}):c[6].push({source:p,osmCandidates:t.map(e=>e.id)})}s+=1,s%1e3||process.stdout.write(`${(s/r*100).toFixed(1)}% `)}return console.timeEnd(`matchLoop`),await u.promises.writeFile(e.tempFileNames.matches,JSON.stringify(c,null,void 0)),{output:c}}const I=2e-4;function L({lat:e,lng:t}){let n=[[t,e+I],[t+I,e],[t,e-I],[t-I,e]];return n.push(n[0]),[n]}function _e({lat:e,lng:t}){let n=[[t+I/2,e+I/2],[t+I/2,e-I/2],[t-I/2,e-I/2],[t-I/2,e+I/2]];return n.push(n[0]),[n]}function R(e){let t={minLat:1/0,minLng:1/0,maxLat:-1/0,maxLng:-1/0};function n([e,n]){let r=+n,i=+e;r<t.minLat&&(t.minLat=r),i<t.minLng&&(t.minLng=i),r>t.maxLat&&(t.maxLat=r),i>t.maxLng&&(t.maxLng=i)}for(let t of e)switch(t.geometry.type){case`Point`:n(t.geometry.coordinates);break;case`MultiPoint`:case`LineString`:t.geometry.coordinates.forEach(n);break;case`MultiLineString`:case`Polygon`:for(let e of t.geometry.coordinates)e.forEach(n);break;case`MultiPolygon`:for(let e of t.geometry.coordinates)for(let t of e)t.forEach(n);break;default:}return t}function z({minLng:e,minLat:t,maxLng:n,maxLat:r}){return{type:`Polygon`,coordinates:[[[e,t],[n,t],[n,r],[e,r],[e,t]]]}}function B(e,t,n,r,i=0){if(!r.length)return{};let a=R(r),o=E(z(a),e.config.merge.sector_resolution),s=i>10;if(s&&console.info(`(!) Couldn't split ${t} (${r.length} left)`),r.length<=110||s)return{[t]:{features:r,bbox:a,sectorIds:o,...n}};if(a.maxLat-a.minLat>a.maxLng-a.minLng){let o=a.minLat+(a.maxLat-a.minLat)/2,s=[[],[]];for(let e of r)s[+((0,g.geoCentroid)(e.geometry)[1]<o)].push(e);let[c,l]=s;return{...B(e,`${t}^N`,n,c,i+1),...B(e,`${t}^S`,n,l,i+1)}}let c=a.minLng+(a.maxLng-a.minLng)/2,l=[[],[]];for(let e of r)l[+((0,g.geoCentroid)(e.geometry)[0]<c)].push(e);let[u,d]=l;return{...B(e,`${t}^E`,n,u,i+1),...B(e,`${t}^W`,n,d,i+1)}}const{sin:ve,cos:ye,floor:be,sqrt:xe,PI:Se}=Math;function V(e){let t=be(xe(e)),n=(e-t**2)*Se/(2*((t+1)**2-t**2-1))||0,[r,i]=[t*ye(n),t*ve(n)];return[+r.toFixed(2),+i.toFixed(2)]}function Ce(e){for(let t in e){let n=e[t],r={};for(let e of n.features)if(e.geometry.type===`Point`){let[t,n]=e.geometry.coordinates,i=me(n,t,4);r[i]||=[],r[i].push(e)}for(let e of Object.values(r))if(e.length>1)for(let t=0;t<e.length;t+=1){let[n,r]=V(t),[i,a]=e[t].geometry.coordinates;e[t].geometry={type:`Point`,coordinates:[i+4e-6*n,a+4e-6*r]}}}return e}function H(e){return(0,v.createHash)(`sha256`).update(e,`utf8`).digest(`hex`)}function we(e){let t=0,n=0,r=0,i=0;for(let a of e)switch(a.properties.__action){case`delete`:n+=1;break;case`move`:r+=1;break;case`edit`:i+=1;break;default:t+=1}return{count:[t&&`add ${t}`,n&&`delete ${n}`,r&&`move ${r}`,i&&`edit ${i}`].filter(e=>!!e).join(`, `),totalCount:t+n+r+i}}const Te={n:`node`,w:`way`,r:`relation`},U=e=>`https://osm.org/${Te[e[0]]}/${e.slice(1)}`,W=e=>`[${e}](${U(e)})`;function G(e,t){let n=`# Matches – ${re[t]}\n\n`,r=e[t];if(Array.isArray(r))for(let e of r){if(typeof e==`string`){n+=`- ${e}\n`;continue}let t=`source`in e?Array.isArray(e.source)?e.source:[e.source]:[],r=`osm`in e?Array.isArray(e.osm)?e.osm:[e.osm]:`osmCandidates`in e?e.osmCandidates:[],i=`osmCandidates`in e?` or `:` + `;n+=`- ${t.join(` + `)||`?`} ⇄ ${r.map(W).join(i)||`?`}\n`}else for(let e in r){let t=r[e]||[];n+=`- ${t.join(` + `)||`?`} ⇄ ${W(e)}\n`}return n}function K(e,t){let n=`# Conflation – ${t}\n\n`;for(let r in e){let i=`\n## ${r}\n\n`;n+=i;for(let i in e[r]){let a=`\n### ${i}\n\n`;n+=a;for(let a of e[r][i].features){if((a.properties.__action||`create`)!==t)continue;let e=t===`create`?a.id:W(a.id),r=Object.entries(a.properties).filter(([e])=>!e.startsWith(`__`)).map(e=>e.join(`=`)).join(` + `)||`∅`;switch(t){case`create`:case`edit`:case`delete`:n+=`- ${e}\t${r}\n`;break;case`move`:{let[[e,t],[r,i]]=a.geometry.coordinates,o=M*(0,g.geoDistance)([e,t],[r,i]),s=`${U(a.id)}?mlat=${i}&mlon=${r}#map=13/${i}/${r}`;n+=`- [${a.id}](${s})\tneeds to move ${o|0}m to ${i},${r}\n`;break}default:throw TypeError(t)}}n.endsWith(a)&&(n=n.slice(0,-a.length))}n.endsWith(i)&&(n=n.slice(0,-i.length))}return n}function Ee(e,t){let n={"report-match-OneToOne.md":G(e,1),"report-match-OneToMany.md":G(e,2),"report-match-ManyToMany.md":G(e,4),"report-match-ManyToOne.md":G(e,3),"report-match-Delete.md":G(e,5),"report-match-Guess.md":G(e,6),"report-conflation-create.md":K(t,`create`),"report-conflation-edit.md":K(t,`edit`),"report-conflation-move.md":K(t,`move`),"report-conflation-delete.md":K(t,`delete`)};for(let e in n)n[e].length>5e6&&(n[e]=`_file too large_`);return n}function De(e){return`${e.replaceAll(/[^\dA-Za-zĀāĒēĪīŌōŪū]/g,``).slice(0,100)}_${H(e).slice(0,6)}`}async function Oe(e,t,n,r){let i=e.config.metadata.git_repository.match(`https://github.com/([^/]+)/([^/]+)`);if(!i)throw Error(`git_repository is not a valid github url`);let a=e.config.output?.folder||(0,c.join)(process.cwd(),`output`),o=`datasets`,s=Ee(n,r),l=Object.entries(r).flatMap(([e,t])=>Object.entries(t).map(([t,n])=>{let r=[e,t].filter(Boolean).join(` - `);return{id:De(r),category:e,group:t,title:r,bbox:n.bbox,instructions:n.instructions,...we(n.features)}})),d={fields:[],results:l.map(e=>({id:e.id,url:`https://${i[1]}.github.io/${i[2]}/${o}/${e.id}.osmPatch.geo.json`,name:e.title,title:e.title,totalCount:e.totalCount,source:``,snippet:e.count,extent:[[e.bbox.minLng,e.bbox.minLat],[e.bbox.maxLng,e.bbox.maxLat]],instructions:e.instructions,groupCategories:[e.category,`/Categories/Addresses`]})).toSorted((e,t)=>e.name.localeCompare(t.name))},f={type:`FeatureCollection`,features:l.map(({bbox:e,...t})=>({type:`Feature`,geometry:z(e),properties:t})).toSorted((e,t)=>e.properties.title.localeCompare(t.properties.title)),__reports:[...Object.keys(s),...e.config.output?.custom_report_file_names||[]]};await u.promises.mkdir((0,c.join)(a,o),{recursive:!0}),await u.promises.writeFile((0,c.join)(a,`index.json`),JSON.stringify(d,null,y?2:void 0)),await u.promises.writeFile((0,c.join)(a,`index.geo.json`),JSON.stringify(f,null,y?2:void 0)),await u.promises.writeFile((0,c.join)(a,`metrics.json`),JSON.stringify(t,null,2));for(let e in s)await u.promises.writeFile((0,c.join)(a,e),s[e]);for(let e of l){let t={type:`FeatureCollection`,...r[e.category][e.group]};await u.promises.writeFile((0,c.join)(a,o,`${e.id}.osmPatch.geo.json`),JSON.stringify(t,null,y?2:void 0))}}function q(e,t){let n={...e};for(let e in t)n[e]?n[e]!==t[e]&&(n[e]+=`;${t[e]}`):n[e]=t[e];return n}function J(e,t){return R([{type:`Feature`,geometry:z(e),properties:{}},{type:`Feature`,geometry:z(t),properties:{}}])}function ke(e){let t=!0;for(;t;){t=!1;for(let n in e){let r=e[n];if(r.features.length>40)continue;let i=Object.entries(e).filter(([e,t])=>t.sectorIds.some(e=>r.sectorIds.includes(e))&&e!==n&&r.features.length+t.features.length<110).toSorted(([,e],[,t])=>N(r.bbox,e.bbox)-N(r.bbox,t.bbox)).find(([,e])=>{let t=P(J(r.bbox,e.bbox))<=4,n=P(r.bbox)>4&&P(e.bbox)>4;return t||n});if(!i)continue;let[a,o]=i;r.bbox=J(r.bbox,o.bbox),r.sectorIds=[...new Set([...r.sectorIds,...o.sectorIds])],r.changesetTags=q(r.changesetTags||{},o.changesetTags||{}),o.instructions&&(r.instructions||=``,r.instructions+=`\n\n${o.instructions}`,r.instructions.trim()),r.features.push(...o.features),delete e[n],delete e[a];let{name:s}=q({name:n},{name:a});e[s]=r,t=!0}}return e}async function Ae(e,t,n,r,i){let a=new Set(JSON.parse(await u.promises.readFile(e.tempFileNames.ignore_list,`utf8`))),o=0,s=0,c=0,l=new Set(Object.values(n.semi));function d(e){e.flags&1&&o++,e.flags&2&&s++,e.flags&8&&c++}for(let e of Object.values(n.withRef))d(e);for(let e of Object.values(n.noRef))d(e);for(let e of l)d(e);for(let e of Object.values(n.duplicateRefs))for(let t of e)d(t);let f={config:e.config,warnings:e.warnings,countsByPhase:{init:{sourceDataset:Object.keys(t).length,ignored:a.size,osm:{withRef:Object.keys(n.withRef).length,noRef:Object.keys(n.noRef).length,semi:l.size,duplicateRefs:Object.values(n.duplicateRefs).flat().length,lastEditedByImporter:o,recentlyChanged:s,recentlyChecked:c}},matched:{1:r[1].length,2:r[2].length,3:Object.keys(r[3]).length,4:r[4].length,5:r[5].length,6:r[6].length},conflated:{create:0,edit:0,delete:0,perfect:0}}};for(let e of Object.values(i).flatMap(e=>Object.values(e)))for(let t of e.features)t.properties.__action===`edit`?f.countsByPhase.conflated.edit++:t.properties.__action===`delete`?f.countsByPhase.conflated.delete++:f.countsByPhase.conflated.create++;return f.countsByPhase.conflated.perfect=n.count-f.countsByPhase.conflated.edit-f.countsByPhase.conflated.delete,f}async function je(e){if(y)return;if(!process.env.GITHUB_ACTIONS){console.info(`Skipping run history update, because not running in GitHub Actions`);return}let t=await(0,ee.getIDToken)(`osm-conflation-engine`),n=e.config.merge.osm_key,r=await fetch(`${x}/api/run_history/${n}`,{method:`PUT`,headers:{Authorization:`Bearer ${t}`},body:JSON.stringify(e)});if(r.status!==200)throw Error(await r.text())}const Y=`invalid value returned by your callback function`;function Me(e){for(let t in e)if(!(t===`__action`&&e[t]===`edit`))return!1;return!0}function X(e){if(!e)return!1;if(typeof e!=`object`||typeof e.tags!=`object`)throw TypeError(Y);return!!e.geometry||!Me(e.tags)}function Z(e,t,n){if(!t&&!n)throw Error(`requires either osm or source`);t&&(e.tags.__action||=`edit`);let[r,i]=t?.centroid||n?.centroid;return{type:`Feature`,id:t?.id||n?.id,geometry:e.tags.__action===`delete`?{type:`Polygon`,coordinates:_e({lat:i,lng:r})}:e.geometry||(e.tags.__action===`edit`?{type:`Polygon`,coordinates:L({lat:i,lng:r})}:{type:`Point`,coordinates:[r,i]}),properties:e.tags}}async function Ne(e,t,n){let r={...n.noRef};for(let e of[`withRef`,`semi`,`duplicateRefs`])for(let t in n[e]){let i=n[e][t];if(Array.isArray(i))for(let e of i)r[e.id]=e;else r[i.id]=i}let i={};function a(t,n,r){if(!t)return;t.warnings?.length&&e.warnings.push(...t.warnings);let a=t.extraFeatures||t.createFeatures;a&&(i[n]||={},i[n][r]||=[],i[n][r].push(...a))}let o=JSON.parse(await u.promises.readFile(e.tempFileNames.matches,`utf8`));for(let{source:n,osm:s}of o[1]){let o=r[s],c=t[n];if(o.flags&8)continue;let l=await e.callbacks.mergeOneToOne({osm:o,source:c});if(!l)continue;if(typeof l!=`object`)throw TypeError(Y);let u=l.category||``,d=l.group||o.sectors[0];a(l.extra,u,d),X(l.diff)&&(i[u]||={},i[u][d]||=[],i[u][d].push(Z(l.diff,o,c)))}for(let{source:n,osm:s}of o[2]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=t[n],l=await e.callbacks.mergeOneToMany?.({osm:o,source:c});if(!l)continue;if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(Y);let u=l.category||``;a(l.extra,u,l.group||o[0]?.sectors[0]||c.sectors[0]);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(Y);if(!X(r))continue;let a=l.group||n.sectors[0];i[u]||={},i[u][a]||=[],i[u][a].push(Z(r,n,c))}}for(let n in o[3]){let s=n,c=r[s],l=o[3][s].map(e=>t[e]);if(c.flags&8)continue;let u=await e.callbacks.mergeManyToOne?.({osm:c,source:l});if(!u)continue;if(typeof u!=`object`)throw TypeError(Y);let d=u.category||``,f=u.group||c.sectors[0];a(u.extra,d,f),X(u.diff)&&(i[d]||={},i[d][f]||=[],i[d][f].push(Z(u.diff,c)))}for(let{source:n,osm:s}of o[4]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=n.map(e=>t[e]),l=await e.callbacks.mergeManyToMany?.({osm:o,source:c});if(!l)continue;if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(Y);let u=l.category||``;a(l.extra,u,l.group||o[0]?.sectors[0]||c[0].sectors[0]);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(Y);if(!X(r))continue;let a=l.group||n.sectors[0];i[u]||={},i[u][a]||=[],i[u][a].push(Z(r,n))}}for(let t of o[5]){let r=n.withRef[t];if(r.flags&8)continue;let o=await e.callbacks.deleteFeature?.({osm:r});if(!o)continue;if(typeof o!=`object`)throw TypeError(Y);let s=o.category||``,c=o.group||r.sectors[0];a(o.extra,s,c),X(o.diff)&&(i[s]||={},i[s][c]||=[],i[s][c].push(Z(o.diff,r)))}for(let r of o[6]){let o=t[r.source],s=await e.callbacks.create({source:o,osmCandidates:r.osmCandidates.map(e=>n.noRef[e])});if(!s)continue;if(typeof s!=`object`)throw TypeError(Y);let c=s.selection?n.noRef[s.selection]:void 0,l=s.category||``,u=s.group||c?.sectors[0]||`unknown`;a(s.extra,l,u),X(s.diff)&&(i[l]||={},i[l][u]||=[],i[l][u].push(Z(s.diff,c,o)))}let s=await e.callbacks.addCustomLayers?.();for(let e in s)for(let t in s[e])a(s[e][t],e,t);let c={};for(let n in i){c[n]||={};for(let a in i[n]){let{changesetTags:o={},instructions:s}=await e.callbacks.getChangesetTags?.({category:n,group:a})||{};o.created_by||=`osm-conflation-engine`,o.import||=`yes`,o.source||=e.config.metadata.wiki_page;let l=i[n][a];e.callbacks.postprocessLayer&&await e.callbacks.postprocessLayer({category:n,group:a,osmData:r,sourceData:t,features:l});let u=B(e,a,{changesetTags:o,instructions:s},l);Object.assign(c[n],u)}Ce(c[n]),ke(c[n])}let l=await Ae(e,t,n,o,c);return await Oe(e,l,o,c),await je(l),l}async function Q(e){console.info(`Reading source data into memory...`);let t=0,n={},r=(0,f.createInterface)({input:(0,u.createReadStream)(e.config.source_data.file),crlfDelay:1/0});for await(let i of r){let r=JSON.parse(i),a=r.properties[e.config.merge.dataset_column];n[a]=Object.assign(r,{sectors:E(r.geometry,e.config.merge.sector_resolution),centroid:r.geometry.type===`Point`?r.geometry.coordinates:(0,g.geoCentroid)(r.geometry)}),t++}console.info(`\tread ${t} rows.`),console.info(`Reading osm data into memory (withRef)...`);let i=await w(e.tempFileNames.osm_processed_with_ref,t=>t.tags[e.config.merge.osm_key]);console.info(`Reading osm data into memory (noRef)...`);let a=await w(e.tempFileNames.osm_processed_no_ref,e=>e.id);console.info(`Reading osm data into memory (other)...`);let o={withRef:i,noRef:a,...JSON.parse(await u.promises.readFile(e.tempFileNames.osm_processed_other,`utf8`))};return console.info(` read data.`),{sourceData:n,osmData:o}}function Pe(e){for(let t of e)if(!b.includes(t))throw Error(`‘${t}’ is not a valid step. The options are: ${b.join(`, `)}`)}async function $(e){let t=H(e.metadata.name).slice(0,12),n=y?`-mock`:``,r=(0,c.join)((0,l.tmpdir)(),`oce-${n}`,t);return await u.promises.mkdir(r,{recursive:!0}),{id:t,tempFolder:r,tempFileNames:Object.fromEntries(Object.entries(te).map(([e,t])=>[e.toLowerCase(),(0,c.join)(r,t)]))}}async function Fe(e,t,n={}){let r=JSON.parse(await u.promises.readFile(new URL(`config.schema.json`,require("url").pathToFileURL(__filename).href),`utf8`)),i=new d.Ajv().compile(r);if(!i(e))throw AggregateError(i.errors.map(e=>TypeError(e.message+JSON.stringify(e,null,2),{cause:e})),`Config file does not conform to the schema`);let{id:a,tempFolder:o,tempFileNames:s}=await $(e),c={use_cache:!!n.use_cache,id:a,tempFolder:o,tempFileNames:s,config:e,callbacks:t,warnings:[]};console.info(`temp folder:`,o);let{steps:l=b}=n;Pe(l),l.includes(`download`)&&(console.time(`download`),await fe(c),console.timeEnd(`download`));let f,p;l.includes(`match`)&&({sourceData:f,osmData:p}=await Q(c),console.time(`match`),await ge(c,f,p),console.timeEnd(`match`));let m;return l.includes(`conflate`)&&((!f||!p)&&({sourceData:f,osmData:p}=await Q(c)),console.time(`conflate`),m=await Ne(c,f,p),console.timeEnd(`conflate`)),console.info(`done.`),{conflate:m}}exports.OsmFlags=ne,exports.getTempFileNames=$,exports.readJsonL=w,exports.run=Fe,exports.writeJsonL=T;
|
|
10
|
+
`,a=t?await u.promises.readFile(t,`utf8`):i;console.info(`fetching OSM data via postpass…`);let o=await fetch(n||`https://postpass.geofabrik.de/api/interpreter`,{method:`POST`,body:new URLSearchParams({data:a}),headers:C(e.config)});if(!o.ok){let e=await o.text()||o.statusText;throw Error(`HTTP Error ${o.status} ${e}`)}let s=await o.json(),{pickTags:c}=w(e),l={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0},d=new Set;for(let t of s.features){let{osm_type:n,osm_id:r,tags:i}=t.properties,a=`${n.toLowerCase()}${r}`;d.has(a)||(d.add(a),O(e,l,{id:a,centroid:t.geometry?.type?(0,g.geoCentroid)(t.geometry):[0,0],tags:c(i),metadata:void 0,sectors:[],flags:0}))}await u.promises.writeFile(r,JSON.stringify(l)),await k(e,l)}async function de(e){let t=e.config.merge.osm_key,n=e.config.e2e_tests?.ignore_list_file_path;if(b&&!n)throw Error(`When running in the unit test envrironment, you must supply config.e2e_tests.ignoreListFilePath`);let r=b&&n?JSON.parse(await u.promises.readFile(n,`utf8`)):await fetch(`${S}/api/ignore_list/${encodeURIComponent(t)}?return_ids_only=true`).then(e=>e.json());await u.promises.writeFile(e.tempFileNames.ignore_list,JSON.stringify(r.result,null,2))}async function fe(e,t){if(e.use_cache&&(0,u.existsSync)(e.tempFileNames.osm_processed_other)){console.info(`Using cached osm data`);return}let{pickTags:n}=w(e);console.info(`Starting preprocess of OSM data...`),await k(e,await new Promise((r,i)=>{let a={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0},o=!1,s=!1,c=0;_.default.createReadStream({file:e.tempFileNames.pbf,tags:t,leveldb:`/tmp`,metadata:!0}).on(`data`,t=>{s||=!0;let r=t.type===`node`?t:t.centroid,i=t.meta;i&&(o=!0);let l=n(t.tags);O(e,a,{id:`${t.type[0]}${t.id}`,centroid:[+r.lon,+r.lat],metadata:i?{changeset:i.changeset,timestamp:i.timestamp?new Date(1e3*i.timestamp).toISOString():void 0,user:i.user,version:i.version}:void 0,tags:l,sectors:[],flags:0}),c+=1,c%1e3||process.stdout.write(`.`)}).on(`finish`,()=>{if(o||e.warnings.push(`No metadata extracted from the planet file!`),!s)throw Error(`Nothing extracted from the pbf file, presumably it’s invalid.`);r(a)}).on(`error`,i)}))}async function pe(e){if(await de(e),e.use_cache&&(0,u.existsSync)(e.tempFileNames.osm_processed_other)){console.info(`Using cached OSM data`);return}switch(e.config.osm_data.source.type){case`pbf`:await le(e,e.config.osm_data.source.pbf_url),await fe(e,e.config.osm_data.source.pbf_filter);break;case`overpass`:await oe(e,e.config.osm_data.source.overpass_query_file,e.config.osm_data.source.overpass_server_url);break;case`postpass`:await ue(e,e.config.osm_data.source.postpass_query_file,e.config.osm_data.source.postpass_server_url);break;default:throw TypeError(e.config.osm_data.source)}}function me(e,t,n){let r=[];for(let i in t.withRef){let a=i,o=t.withRef[a];if(!(a in n)&&!(o.flags&8)){if(e.callbacks.getAltRefs?.(a).some(e=>e in n))continue;r.push(a)}}let i={};for(let t in n){let r=t,a=n[r];for(let t of a.sectors){i[t]||={};let n=e.callbacks.getLocalKeyForSource(a);i[t][n]=r}}let a={},o=[];for(let n of r){let r=t.withRef[n],s=e.callbacks.getLocalKeyForOsm(r),c=new Set(r.sectors.map(e=>i[e]?.[s]).filter(Boolean));if(c.size>1&&console.warn(`${r.id} spans multiple sectors, and there are conflicting localKeys across the sectors. Therefore, it will be straight-up deleted`),c.size===1){let e=[...c][0];t.withRef[e]?o.push(n):a[e]=n}else o.push(n)}return{refsThatChanged:a,toDelete:o}}const he=(e,t,n=5)=>`${e.toFixed(n)},${t.toFixed(n)}`,M=6371008.8;function N(e,t){let n=Math.max(0,t.minLat-e.maxLat,e.minLat-t.maxLat),r=Math.max(0,t.minLng-e.maxLng,e.minLng-t.maxLng);return Math.hypot(n,r)}function P(e){return Math.max(Math.abs(e.maxLat-e.minLat),Math.abs(e.maxLng-e.minLng))}const F=new Map,ge=(e,t,n)=>{let r=e.callbacks.getLocalKeyForSource(t);return n.filter(t=>{let n=F.get(t);return n||(n=e.callbacks.getLocalKeyForOsm(t),F.set(t,n)),n===r}).map(e=>{let n=M*(0,g.geoDistance)(e.centroid,t.centroid);return Object.assign(e,{offset:n})}).filter(e=>e.offset<200).toSorted((e,t)=>e.offset-t.offset)};async function _e(e,t,n){let r=Object.keys(t).length,i=new Set(JSON.parse(await u.promises.readFile(e.tempFileNames.ignore_list,`utf8`)));console.info(`processing deleted data...`),console.time(`matchDeletions`);let{refsThatChanged:a,toDelete:o}=me(e,n,t);console.timeEnd(`matchDeletions`),console.info(`Processing data...`);let s=0;console.time(`matchLoop`);let c={1:[],2:[],3:{},4:[],5:o,6:[]},l=Object.values(n.noRef),d={};for(let e in n.semi){let t=e.split(`;`);for(let[n,r]of t.entries())d[r]||=[],d[r].push({value:e,isFirst:!n})}let f={};for(let t in n.semi){let r=n.semi[t],i=r.tags[e.config.merge.osm_key];i&&(f[i]=r)}for(let o in t){let u=o,p=a[u]||u;if(i.has(p)||p in a)continue;let m=n.withRef[p]||e.callbacks.getAltRefs?.(p).map(e=>n.withRef[e]).find(Boolean),h=t[u],g=n.duplicateRefs[p],_=f[p];if(d[p])if(d[p][0].isFirst)p=d[p][0].value;else continue;let v=n.semi[p];if(m&&v){let e=p.split(`;`);c[4].push({osm:[m.id,v.id],source:e})}else if(m)c[1].push({osm:m.id,source:u});else if(g)c[2].push({osm:g.map(e=>e.id),source:p});else if(_)c[1].push({osm:_.id,source:u});else if(v){let e=p.split(`;`);c[3][v.id]||=[],c[3][v.id]?.push(...e)}else{let t=ge(e,h,l);t.length===1?c[1].push({source:p,osm:t[0].id}):c[6].push({source:p,osmCandidates:t.map(e=>e.id)})}s+=1,s%1e3||process.stdout.write(`${(s/r*100).toFixed(1)}% `)}return console.timeEnd(`matchLoop`),await u.promises.writeFile(e.tempFileNames.matches,JSON.stringify(c,null,void 0)),{output:c}}const I=2e-4;function ve({lat:e,lng:t}){let n=[[t,e+I],[t+I,e],[t,e-I],[t-I,e]];return n.push(n[0]),[n]}function ye({lat:e,lng:t}){let n=[[t+I/2,e+I/2],[t+I/2,e-I/2],[t-I/2,e-I/2],[t-I/2,e+I/2]];return n.push(n[0]),[n]}function L(e){let t={minLat:1/0,minLng:1/0,maxLat:-1/0,maxLng:-1/0};function n([e,n]){let r=+n,i=+e;r<t.minLat&&(t.minLat=r),i<t.minLng&&(t.minLng=i),r>t.maxLat&&(t.maxLat=r),i>t.maxLng&&(t.maxLng=i)}for(let t of e)switch(t.geometry.type){case`Point`:n(t.geometry.coordinates);break;case`MultiPoint`:case`LineString`:t.geometry.coordinates.forEach(n);break;case`MultiLineString`:case`Polygon`:for(let e of t.geometry.coordinates)e.forEach(n);break;case`MultiPolygon`:for(let e of t.geometry.coordinates)for(let t of e)t.forEach(n);break;default:}return t}function R({minLng:e,minLat:t,maxLng:n,maxLat:r}){return{type:`Polygon`,coordinates:[[[e,t],[n,t],[n,r],[e,r],[e,t]]]}}function z(e,t,n,r,i=0){if(!r.length)return{};let a=L(r),o=D(R(a),e.config.merge.sector_resolution),s=i>10;if(s&&console.info(`(!) Couldn't split ${t} (${r.length} left)`),r.length<=110||s)return{[t]:{features:r,bbox:a,sectorIds:o,...n}};if(a.maxLat-a.minLat>a.maxLng-a.minLng){let o=a.minLat+(a.maxLat-a.minLat)/2,s=[[],[]];for(let e of r)s[+((0,g.geoCentroid)(e.geometry)[1]<o)].push(e);let[c,l]=s;return{...z(e,`${t}^N`,n,c,i+1),...z(e,`${t}^S`,n,l,i+1)}}let c=a.minLng+(a.maxLng-a.minLng)/2,l=[[],[]];for(let e of r)l[+((0,g.geoCentroid)(e.geometry)[0]<c)].push(e);let[u,d]=l;return{...z(e,`${t}^E`,n,u,i+1),...z(e,`${t}^W`,n,d,i+1)}}function be(e){let t={},n={};for(let r in e){let i=r.replace(/^([^^]+)(\^\w)+$/,(e,n)=>(t[n]||=0,t[n]+=1,`${n} ${t[n]}`));n[i]=e[r]}return n}const{sin:xe,cos:Se,floor:Ce,sqrt:we,PI:Te}=Math;function Ee(e){let t=Ce(we(e)),n=(e-t**2)*Te/(2*((t+1)**2-t**2-1))||0,[r,i]=[t*Se(n),t*xe(n)];return[+r.toFixed(2),+i.toFixed(2)]}function De(e){for(let t in e){let n=e[t],r={};for(let e of n.features)if(e.geometry.type===`Point`){let[t,n]=e.geometry.coordinates,i=he(n,t,4);r[i]||=[],r[i].push(e)}for(let e of Object.values(r))if(e.length>1)for(let t=0;t<e.length;t+=1){let[n,r]=Ee(t),[i,a]=e[t].geometry.coordinates;e[t].geometry={type:`Point`,coordinates:[i+4e-6*n,a+4e-6*r]}}}return e}function B(e){return(0,v.createHash)(`sha256`).update(e,`utf8`).digest(`hex`)}function Oe(e){let t=0,n=0,r=0,i=0;for(let a of e)switch(a.properties.__action){case`delete`:n+=1;break;case`move`:r+=1;break;case`edit`:i+=1;break;default:t+=1}return{count:[t&&`add ${t}`,n&&`delete ${n}`,r&&`move ${r}`,i&&`edit ${i}`].filter(e=>!!e).join(`, `),totalCount:t+n+r+i}}const ke={n:`node`,w:`way`,r:`relation`},V=e=>`https://osm.org/${ke[e[0]]}/${e.slice(1)}`,H=e=>`[${e}](${V(e)})`;function U(e,t,n){function r(t){return e.formatterUrl?`[${t}](${e.formatterUrl.replaceAll(`$1`,t)})`:t}let i=`# Matches – ${re[n]}\n\n`,a=t[n];if(Array.isArray(a))for(let e of a){if(typeof e==`string`){i+=`- ${e}\n`;continue}let t=`source`in e?Array.isArray(e.source)?e.source:[e.source]:[],n=`osm`in e?Array.isArray(e.osm)?e.osm:[e.osm]:`osmCandidates`in e?e.osmCandidates:[],a=`osmCandidates`in e?` or `:` + `;i+=`- ${t.map(r).join(` + `)||`?`} ⇄ ${n.map(H).join(a)||`?`}\n`}else for(let e in a){let t=a[e]||[];i+=`- ${t.map(r).join(` + `)||`?`} ⇄ ${H(e)}\n`}return i}function W(e,t){let n=`# Conflation – ${t}\n\n`;for(let r in e){let i=`\n## ${r}\n\n`;n+=i;for(let i in e[r]){let a=`\n### ${i}\n\n`;n+=a;for(let a of e[r][i].features){if((a.properties.__action||`create`)!==t)continue;let e=t===`create`?a.id:H(a.id),r=Object.entries(a.properties).filter(([e])=>!e.startsWith(`__`)).map(e=>e.join(`=`)).join(` + `)||`∅`;switch(t){case`create`:case`edit`:case`delete`:n+=`- ${e}\t${r}\n`;break;case`move`:{let[[e,t],[r,i]]=a.geometry.coordinates,o=M*(0,g.geoDistance)([e,t],[r,i]),s=`${V(a.id)}?mlat=${i}&mlon=${r}#map=13/${i}/${r}`;n+=`- [${a.id}](${s})\tneeds to move ${o|0}m to ${i},${r}\n`;break}default:throw TypeError(t)}}n.endsWith(a)&&(n=n.slice(0,-a.length))}n.endsWith(i)&&(n=n.slice(0,-i.length))}return n}function Ae(e,t,n){let r={"report-match-OneToOne.md":U(e,t,1),"report-match-OneToMany.md":U(e,t,2),"report-match-ManyToMany.md":U(e,t,4),"report-match-ManyToOne.md":U(e,t,3),"report-match-Delete.md":U(e,t,5),"report-match-Guess.md":U(e,t,6),"report-conflation-create.md":W(n,`create`),"report-conflation-edit.md":W(n,`edit`),"report-conflation-move.md":W(n,`move`),"report-conflation-delete.md":W(n,`delete`)};for(let e in r)r[e].length>5e6&&(r[e]=`_file too large_`);return r}const G=`metrics-history.json`,K=()=>({lastUpdated:``,rows:[]});async function je(e,t){let n=e.config.merge.osm_key,r=b?K():await fetch(`${S}/api/static/${n}/${G}`,{headers:C(e.config)}).then(async e=>{if(!e.ok){let t=await e.text()||e.statusText,n=Error(`HTTP Error ${e.status} ${t}`);if(e.status===404)return console.error(ReferenceError(`Recursive History does not exist, presumably this is the first run. History will be reset`,{cause:n})),K();throw n}return await e.json()}),i=b?`MOCK`:new Date().toISOString().split(`T`,1)[0],a={date:i,conflated:t.countsByPhase.conflated},o=r.rows.at(-1);return JSON.stringify(o)===JSON.stringify(a)?console.info(`No change in statistics, not updating table`):r.rows.push(a),r.lastUpdated=i,r}function Me(e){return`${e.replaceAll(/[^\dA-Za-zĀāĒēĪīŌōŪū]/g,``).slice(0,100)}_${B(e).slice(0,6)}`}async function Ne(e,t,n,r){let i=e.config.output?.folder||(0,c.join)(process.cwd(),`output`),a=`datasets`,o=Ae(e,n,r),s=Object.entries(r).flatMap(([e,t])=>Object.entries(t).map(([t,n])=>{let r=[e,t].filter(Boolean).join(` - `);return{id:Me(r),category:e,group:t,title:r,bbox:n.bbox,instructions:n.instructions,...Oe(n.features)}})),l={type:`FeatureCollection`,features:s.map(({bbox:e,...t})=>({type:`Feature`,geometry:R(e),properties:t})).toSorted((e,t)=>e.properties.title.localeCompare(t.properties.title)),__reports:[...Object.keys(o),...e.config.output?.custom_report_file_names||[]]},d=await je(e,t);await u.promises.mkdir((0,c.join)(i,a),{recursive:!0}),await u.promises.writeFile((0,c.join)(i,`index.geo.json`),JSON.stringify(l,null,b?2:void 0)),await u.promises.writeFile((0,c.join)(i,`metrics.json`),JSON.stringify(t,null,2)),await u.promises.writeFile((0,c.join)(i,G),JSON.stringify(d,null,2));for(let e in o)await u.promises.writeFile((0,c.join)(i,e),o[e]);for(let e of s){let t={type:`FeatureCollection`,...r[e.category][e.group]};await u.promises.writeFile((0,c.join)(i,a,`${e.id}.osmPatch.geo.json`),JSON.stringify(t,null,b?2:void 0))}}function q(e,t){let n={...e};for(let e in t)n[e]?n[e]!==t[e]&&(n[e]+=`;${t[e]}`):n[e]=t[e];return n}function J(e,t){return L([{type:`Feature`,geometry:R(e),properties:{}},{type:`Feature`,geometry:R(t),properties:{}}])}function Pe(e){let t=!0;for(;t;){t=!1;for(let n in e){let r=e[n];if(r.features.length>40)continue;let i=Object.entries(e).filter(([e,t])=>t.sectorIds.some(e=>r.sectorIds.includes(e))&&e!==n&&r.features.length+t.features.length<110).toSorted(([,e],[,t])=>N(r.bbox,e.bbox)-N(r.bbox,t.bbox)).find(([,e])=>{let t=P(J(r.bbox,e.bbox))<=4,n=P(r.bbox)>4&&P(e.bbox)>4;return t||n});if(!i)continue;let[a,o]=i;r.bbox=J(r.bbox,o.bbox),r.sectorIds=[...new Set([...r.sectorIds,...o.sectorIds])],r.changesetTags=q(r.changesetTags||{},o.changesetTags||{}),o.instructions&&(r.instructions||=``,r.instructions+=`\n\n${o.instructions}`,r.instructions.trim()),r.features.push(...o.features),delete e[n],delete e[a];let{name:s}=q({name:n},{name:a});e[s]=r,t=!0}}return e}async function Fe(e,t,n,r,i){let a=new Set(JSON.parse(await u.promises.readFile(e.tempFileNames.ignore_list,`utf8`))),o=0,s=0,c=0,l=new Set(Object.values(n.semi));function d(e){e.flags&1&&o++,e.flags&2&&s++,e.flags&8&&c++}for(let e of Object.values(n.withRef))d(e);for(let e of Object.values(n.noRef))d(e);for(let e of l)d(e);for(let e of Object.values(n.duplicateRefs))for(let t of e)d(t);let f={config:e.config,warnings:e.warnings,countsByPhase:{init:{sourceDataset:Object.keys(t).length,ignored:a.size,osm:{withRef:Object.keys(n.withRef).length,noRef:Object.keys(n.noRef).length,semi:l.size,duplicateRefs:Object.values(n.duplicateRefs).flat().length,lastEditedByImporter:o,recentlyChanged:s,recentlyChecked:c}},matched:{1:r[1].length,2:r[2].length,3:Object.keys(r[3]).length,4:r[4].length,5:r[5].length,6:r[6].length},conflated:{create:0,edit:0,delete:0,perfect:0}}};for(let e of Object.values(i).flatMap(e=>Object.values(e)))for(let t of e.features)t.properties.__action===`edit`?f.countsByPhase.conflated.edit++:t.properties.__action===`delete`?f.countsByPhase.conflated.delete++:f.countsByPhase.conflated.create++;return f.countsByPhase.conflated.perfect=n.count-f.countsByPhase.conflated.edit-f.countsByPhase.conflated.delete,f}async function Ie(e){if(b)return;if(!process.env.GITHUB_ACTIONS){console.info(`Skipping run history update, because not running in GitHub Actions`);return}let t=await(0,ee.getIDToken)(`osm-conflation-engine`),n=e.config.merge.osm_key,r=await fetch(`${S}/api/run_history/${n}`,{method:`PUT`,headers:{Authorization:`Bearer ${t}`},body:JSON.stringify(e)});if(r.status!==200)throw Error(await r.text())}const Y=`invalid value returned by your callback function`;function Le(e){for(let t in e)if(!(t===`__action`&&e[t]===`edit`))return!1;return!0}function X(e){if(!e)return!1;if(typeof e!=`object`||typeof e.tags!=`object`)throw TypeError(Y);return!!e.geometry||!Le(e.tags)}function Z(e,t,n){if(!t&&!n)throw Error(`requires either osm or source`);t&&(e.tags.__action||=`edit`);let[r,i]=t?.centroid||n?.centroid;return{type:`Feature`,id:t?.id||n?.id,geometry:e.tags.__action===`delete`?{type:`Polygon`,coordinates:ye({lat:i,lng:r})}:e.geometry||(e.tags.__action===`edit`?{type:`Polygon`,coordinates:ve({lat:i,lng:r})}:{type:`Point`,coordinates:[r,i]}),properties:e.tags}}async function Re(e,t,n){let r={...n.noRef};for(let e of[`withRef`,`semi`,`duplicateRefs`])for(let t in n[e]){let i=n[e][t];if(Array.isArray(i))for(let e of i)r[e.id]=e;else r[i.id]=i}let i={};function a(t,n,r){if(!t)return;t.warnings?.length&&e.warnings.push(...t.warnings);let a=t.extraFeatures||t.createFeatures;a&&(i[n]||={},i[n][r]||=[],i[n][r].push(...a))}let o=JSON.parse(await u.promises.readFile(e.tempFileNames.matches,`utf8`));for(let{source:n,osm:s}of o[1]){let o=r[s],c=t[n];if(o.flags&8)continue;let l=await e.callbacks.mergeOneToOne({osm:o,source:c});if(!l)continue;if(typeof l!=`object`)throw TypeError(Y);let u=l.category||``,d=l.group||o.sectors[0];a(l.extra,u,d),X(l.diff)&&(i[u]||={},i[u][d]||=[],i[u][d].push(Z(l.diff,o,c)))}for(let{source:n,osm:s}of o[2]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=t[n],l=await e.callbacks.mergeOneToMany?.({osm:o,source:c});if(!l)continue;if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(Y);let u=l.category||``;a(l.extra,u,l.group||o[0]?.sectors[0]||c.sectors[0]);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(Y);if(!X(r))continue;let a=l.group||n.sectors[0];i[u]||={},i[u][a]||=[],i[u][a].push(Z(r,n,c))}}for(let n in o[3]){let s=n,c=r[s],l=o[3][s].map(e=>t[e]);if(c.flags&8)continue;let u=await e.callbacks.mergeManyToOne?.({osm:c,source:l});if(!u)continue;if(typeof u!=`object`)throw TypeError(Y);let d=u.category||``,f=u.group||c.sectors[0];a(u.extra,d,f),X(u.diff)&&(i[d]||={},i[d][f]||=[],i[d][f].push(Z(u.diff,c)))}for(let{source:n,osm:s}of o[4]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=n.map(e=>t[e]),l=await e.callbacks.mergeManyToMany?.({osm:o,source:c});if(!l)continue;if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(Y);let u=l.category||``;a(l.extra,u,l.group||o[0]?.sectors[0]||c[0].sectors[0]);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(Y);if(!X(r))continue;let a=l.group||n.sectors[0];i[u]||={},i[u][a]||=[],i[u][a].push(Z(r,n))}}for(let t of o[5]){let r=n.withRef[t];if(r.flags&8)continue;let o=await e.callbacks.deleteFeature?.({osm:r});if(!o)continue;if(typeof o!=`object`)throw TypeError(Y);let s=o.category||``,c=o.group||r.sectors[0];a(o.extra,s,c),X(o.diff)&&(i[s]||={},i[s][c]||=[],i[s][c].push(Z(o.diff,r)))}for(let r of o[6]){let o=t[r.source],s=await e.callbacks.create({source:o,osmCandidates:r.osmCandidates.map(e=>n.noRef[e])});if(!s)continue;if(typeof s!=`object`)throw TypeError(Y);let c=s.selection?n.noRef[s.selection]:void 0,l=s.category||``,u=s.group||c?.sectors[0]||`unknown`;a(s.extra,l,u),X(s.diff)&&(i[l]||={},i[l][u]||=[],i[l][u].push(Z(s.diff,c,o)))}let s=await e.callbacks.addCustomLayers?.();for(let e in s)for(let t in s[e])a(s[e][t],e,t);let c={};for(let n in i){c[n]||={};for(let a in i[n]){let{changesetTags:o={},instructions:s}=await e.callbacks.getChangesetTags?.({category:n,group:a})||{};o.created_by||=`osm-conflation-engine`,o.import||=`yes`,o.source||=e.config.metadata.wiki_page;let l=i[n][a];e.callbacks.postprocessLayer&&await e.callbacks.postprocessLayer({category:n,group:a,osmData:r,sourceData:t,features:l});let u=z(e,a,{changesetTags:o,instructions:s},l);u=be(u),Object.assign(c[n],u)}De(c[n]),Pe(c[n])}let l=await Fe(e,t,n,o,c);return await Ne(e,l,o,c),await Ie(l),l}async function Q(e){console.info(`Reading source data into memory...`);let t=0,n={},r=(0,f.createInterface)({input:(0,u.createReadStream)(e.config.source_data.file),crlfDelay:1/0});for await(let i of r){let r=JSON.parse(i),a=r.properties[e.config.merge.dataset_column];n[a]=Object.assign(r,{sectors:D(r.geometry,e.config.merge.sector_resolution),centroid:r.geometry.type===`Point`?r.geometry.coordinates:(0,g.geoCentroid)(r.geometry)}),t++}console.info(`\tread ${t} rows.`),console.info(`Reading osm data into memory (withRef)...`);let i=await T(e.tempFileNames.osm_processed_with_ref,t=>t.tags[e.config.merge.osm_key]);console.info(`Reading osm data into memory (noRef)...`);let a=await T(e.tempFileNames.osm_processed_no_ref,e=>e.id);console.info(`Reading osm data into memory (other)...`);let o={withRef:i,noRef:a,...JSON.parse(await u.promises.readFile(e.tempFileNames.osm_processed_other,`utf8`))};return console.info(` read data.`),{sourceData:n,osmData:o}}const ze=[`deprecated`,`normal`,`preferred`];function Be(e){return y.default.filter(t=>t.key===`Key:${e}`).toSorted((e,t)=>ze.indexOf(t.rank)-ze.indexOf(e.rank))[0]?.url}function Ve(e){for(let t of e)if(!x.includes(t))throw Error(`‘${t}’ is not a valid step. The options are: ${x.join(`, `)}`)}async function $(e){let t=B(e.metadata.name).slice(0,12),n=b?`-mock`:``,r=(0,c.join)((0,l.tmpdir)(),`oce-${n}`,t);return await u.promises.mkdir(r,{recursive:!0}),{id:t,tempFolder:r,tempFileNames:Object.fromEntries(Object.entries(te).map(([e,t])=>[e.toLowerCase(),(0,c.join)(r,t)]))}}async function He(e,t,n={}){let r=JSON.parse(await u.promises.readFile(new URL(`config.schema.json`,require("url").pathToFileURL(__filename).href),`utf8`)),i=new d.Ajv().compile(r);if(!i(e))throw AggregateError(i.errors.map(e=>TypeError(e.message+JSON.stringify(e,null,2),{cause:e})),`Config file does not conform to the schema`);let{id:a,tempFolder:o,tempFileNames:s}=await $(e),c={use_cache:!!n.use_cache,id:a,tempFolder:o,tempFileNames:s,config:e,callbacks:t,formatterUrl:Be(e.merge.osm_key),warnings:[]};console.info(`temp folder:`,o);let{steps:l=x}=n;Ve(l),l.includes(`download`)&&(console.time(`download`),await pe(c),console.timeEnd(`download`));let f,p;l.includes(`match`)&&({sourceData:f,osmData:p}=await Q(c),console.time(`match`),await _e(c,f,p),console.timeEnd(`match`));let m;return l.includes(`conflate`)&&((!f||!p)&&({sourceData:f,osmData:p}=await Q(c)),console.time(`conflate`),m=await Re(c,f,p),console.timeEnd(`conflate`)),console.info(`done.`),{conflate:m}}exports.OsmFlags=ne,exports.getTempFileNames=$,exports.readJsonL=T,exports.run=He,exports.writeJsonL=E;
|
package/dist/index.d.cts
CHANGED
|
@@ -180,7 +180,6 @@ interface FeatureCollection<G extends Geometry | null = Geometry, P = GeoJsonPro
|
|
|
180
180
|
interface Config {
|
|
181
181
|
$schema: (string & {}) | 'https://unpkg.com/@osm-conflation-engine/cli/dist/config.schema.json' | 'node_modules/@osm-conflation-engine/cli/config.schema.json';
|
|
182
182
|
metadata: {
|
|
183
|
-
/** format should be `https://github.com/a/b` */git_repository: string;
|
|
184
183
|
name: string;
|
|
185
184
|
description: string;
|
|
186
185
|
wiki_page: string;
|
|
@@ -280,6 +279,7 @@ interface Ctx<G extends Geometry = Geometry, P extends GeoJsonProperties = GeoJs
|
|
|
280
279
|
tempFileNames: Record<Lowercase<FileNames>, string>;
|
|
281
280
|
config: Config;
|
|
282
281
|
callbacks: Callbacks<G, P>;
|
|
282
|
+
formatterUrl: string | undefined;
|
|
283
283
|
warnings: string[];
|
|
284
284
|
}
|
|
285
285
|
interface SourceData<G extends Geometry = Geometry, P extends GeoJsonProperties = GeoJsonProperties> {
|
|
@@ -486,14 +486,24 @@ interface ConflateResult {
|
|
|
486
486
|
ignored: number;
|
|
487
487
|
};
|
|
488
488
|
matched: Record<MatchType, number>;
|
|
489
|
-
conflated:
|
|
490
|
-
create: number;
|
|
491
|
-
edit: number;
|
|
492
|
-
delete: number;
|
|
493
|
-
perfect: number;
|
|
494
|
-
};
|
|
489
|
+
conflated: ConflationResult;
|
|
495
490
|
};
|
|
496
491
|
}
|
|
492
|
+
interface ConflationResult {
|
|
493
|
+
create: number;
|
|
494
|
+
edit: number;
|
|
495
|
+
delete: number;
|
|
496
|
+
perfect: number;
|
|
497
|
+
}
|
|
498
|
+
interface RecursiveHistoryRow {
|
|
499
|
+
date: string;
|
|
500
|
+
conflated: ConflationResult;
|
|
501
|
+
}
|
|
502
|
+
interface RecursiveHistoryFile {
|
|
503
|
+
/** ISO Date */
|
|
504
|
+
lastUpdated: string;
|
|
505
|
+
rows: RecursiveHistoryRow[];
|
|
506
|
+
}
|
|
497
507
|
interface RunResult {
|
|
498
508
|
conflate?: ConflateResult;
|
|
499
509
|
}
|
|
@@ -521,5 +531,5 @@ declare function readJsonL<T, U extends string>(filePath: string, getKey: (row:
|
|
|
521
531
|
/** writes JSONL to a file by stringifying each line at a time */
|
|
522
532
|
declare function writeJsonL(filePath: string, rows: unknown[]): Promise<void>;
|
|
523
533
|
//#endregion
|
|
524
|
-
export { Action, Callbacks, Config, ConflateResult, ConflationDiff, ConflationResultExtra, DatasetId, Identity, IndexFile, IndexFileProperties, MutliFeatureConflationResult, OsmFeature, OsmFeatureTypeShort, OsmFlags, OsmId, RunOptions, RunResult, SingleFeatureConflationResult, SourceDataFeature, TagDiff, Vec2, getTempFileNames, readJsonL, run, writeJsonL };
|
|
534
|
+
export { Action, Callbacks, Config, ConflateResult, ConflationDiff, ConflationResult, ConflationResultExtra, DatasetId, Identity, IndexFile, IndexFileProperties, MutliFeatureConflationResult, OsmFeature, OsmFeatureTypeShort, OsmFlags, OsmId, RecursiveHistoryFile, RecursiveHistoryRow, RunOptions, RunResult, SingleFeatureConflationResult, SourceDataFeature, TagDiff, Vec2, getTempFileNames, readJsonL, run, writeJsonL };
|
|
525
535
|
//# sourceMappingURL=index.d.cts.map
|
package/dist/index.d.cts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.cts","names":["Geometry","GeoJSON","type","GeoJsonTypes","bbox","BBox","G","P","GeoJsonProperties","Feature","FeatureCollection","Point","MultiPoint","LineString","MultiLineString","Polygon","MultiPolygon","GeometryCollection","GeoJsonObject","coordinates","Position","geometries","name","geometry","id","properties","features","Array"],"sources":["../../../node_modules/.pnpm/@types+geojson@7946.0.16/node_modules/@types/geojson/index.d.ts","../src/types/config.def.ts","../src/constants/defaults.ts","../src/types/internal.def.ts","../src/types/callbacks.def.ts","../src/main.ts","../src/common/jsonl.ts"],"x_google_ignoreList":[0],"mappings":";;;;AAqBgB;AA2ChB;;KAjDY,YAAA,GAAe,OAAO;AAiDd;AAUpB;;;AAVoB,KA3CR,IAAA;;;;;AAqEG;AAMf;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG4B;AAM5B;;;;;;;KAzCY,QAAA;;;;;;;;;UAUK,aAAA;EAAA;EAAA;EAAA;EAsCA;;;EA/BbE,IAAAA,EAAM,YAAA;EA+BqBgB;;;;;AAEN;AAOzB;;EA/BId,IAAAA,GAAO,IAAI;AAAA;;;;KAMH,OAAA,WAAkB,QAAA,UAAkB,QAAA,MAAc,iBAAA,IACxD,CAAA,GACA,OAAA,CAAQ,CAAA,EAAG,CAAA,IACX,iBAAA,CAAkB,CAAA,EAAG,CAAA;;;;;KAMf,QAAA,GAAW,KAAA,GAAQ,UAAA,GAAa,UAAA,GAAa,eAAA,GAAkB,OAAA,GAAU,YAAA,GAAe,kBAAA;AAkCpG;;;;AAAA,UA3BiB,KAAA,SAAc,aAAa;EACxCF,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;AA2BQ;AAOzB;;;AAPyB,UApBR,UAAA,SAAmB,aAAa;EAC7CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;AA2BQ;AAOzB;;UA3BiB,UAAA,SAAmB,aAAa;EAC7CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;AA2BQ;AAOzB;UA3BiB,eAAA,SAAwB,aAAa;EAClDjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;;;UAOA,OAAA,SAAgB,aAAa;EAC1CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;AAkBA;AAGjB;UAdiB,YAAA,SAAqB,aAAa;EAC/CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;;;UAOA,kBAAA,WAA6B,QAAA,GAAW,QAAA,UAAkB,aAAA;EACvEjB,IAAAA;EACAmB,UAAAA,EAAY,CAAA;AAAA;AAAA,KAGJ,iBAAA;EAAA,CAAuBC,IAAY;AAAA;;;;;UAM9B,OAAA,WAAkB,QAAA,UAAkB,QAAA,MAAc,iBAAA,UAA2B,aAAA;EAC1FpB,IAAAA;EAIAqB;;;EAAAA,QAAAA,EAAU,CAAA;EASEhB;;AAAC;AAOjB;EAXIiB,EAAAA;EAW8B;;;EAP9BC,UAAAA,EAAY,CAAA;AAAA;;;;;UAOC,iBAAA,WAA4B,QAAA,UAAkB,QAAA,MAAc,iBAAA,UAA2B,aAAA;EACpGvB,IAAAA;EACAwB,QAAAA,EAAU,KAAA,CAAM,OAAA,CAAQ,CAAA,EAAG,CAAA;AAAA;;;UCrMd,MAAA;EACf,OAAA;EAKA,QAAA;
|
|
1
|
+
{"version":3,"file":"index.d.cts","names":["Geometry","GeoJSON","type","GeoJsonTypes","bbox","BBox","G","P","GeoJsonProperties","Feature","FeatureCollection","Point","MultiPoint","LineString","MultiLineString","Polygon","MultiPolygon","GeometryCollection","GeoJsonObject","coordinates","Position","geometries","name","geometry","id","properties","features","Array"],"sources":["../../../node_modules/.pnpm/@types+geojson@7946.0.16/node_modules/@types/geojson/index.d.ts","../src/types/config.def.ts","../src/constants/defaults.ts","../src/types/internal.def.ts","../src/types/callbacks.def.ts","../src/main.ts","../src/common/jsonl.ts"],"x_google_ignoreList":[0],"mappings":";;;;AAqBgB;AA2ChB;;KAjDY,YAAA,GAAe,OAAO;AAiDd;AAUpB;;;AAVoB,KA3CR,IAAA;;;;;AAqEG;AAMf;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG4B;AAM5B;;;;;;;KAzCY,QAAA;;;;;;;;;UAUK,aAAA;EAAA;EAAA;EAAA;EAsCA;;;EA/BbE,IAAAA,EAAM,YAAA;EA+BqBgB;;;;;AAEN;AAOzB;;EA/BId,IAAAA,GAAO,IAAI;AAAA;;;;KAMH,OAAA,WAAkB,QAAA,UAAkB,QAAA,MAAc,iBAAA,IACxD,CAAA,GACA,OAAA,CAAQ,CAAA,EAAG,CAAA,IACX,iBAAA,CAAkB,CAAA,EAAG,CAAA;;;;;KAMf,QAAA,GAAW,KAAA,GAAQ,UAAA,GAAa,UAAA,GAAa,eAAA,GAAkB,OAAA,GAAU,YAAA,GAAe,kBAAA;AAkCpG;;;;AAAA,UA3BiB,KAAA,SAAc,aAAa;EACxCF,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;AA2BQ;AAOzB;;;AAPyB,UApBR,UAAA,SAAmB,aAAa;EAC7CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;AA2BQ;AAOzB;;UA3BiB,UAAA,SAAmB,aAAa;EAC7CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;AA2BQ;AAOzB;UA3BiB,eAAA,SAAwB,aAAa;EAClDjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;;;UAOA,OAAA,SAAgB,aAAa;EAC1CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;AAkBA;AAGjB;UAdiB,YAAA,SAAqB,aAAa;EAC/CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;;;UAOA,kBAAA,WAA6B,QAAA,GAAW,QAAA,UAAkB,aAAA;EACvEjB,IAAAA;EACAmB,UAAAA,EAAY,CAAA;AAAA;AAAA,KAGJ,iBAAA;EAAA,CAAuBC,IAAY;AAAA;;;;;UAM9B,OAAA,WAAkB,QAAA,UAAkB,QAAA,MAAc,iBAAA,UAA2B,aAAA;EAC1FpB,IAAAA;EAIAqB;;;EAAAA,QAAAA,EAAU,CAAA;EASEhB;;AAAC;AAOjB;EAXIiB,EAAAA;EAW8B;;;EAP9BC,UAAAA,EAAY,CAAA;AAAA;;;;;UAOC,iBAAA,WAA4B,QAAA,UAAkB,QAAA,MAAc,iBAAA,UAA2B,aAAA;EACpGvB,IAAAA;EACAwB,QAAAA,EAAU,KAAA,CAAM,OAAA,CAAQ,CAAA,EAAG,CAAA;AAAA;;;UCrMd,MAAA;EACf,OAAA;EAKA,QAAA;IACE,IAAA;IACA,WAAA;IACA,SAAA;IACA,MAAA;EAAA;EAGF,WAAA;IACE,IAAA;IACA,IAAA;EAAA;EAGF,QAAA;ID2CU;;;;AAAQ;ICrChB,MAAA;MAEM,IAAA;MDoDA;;;;;;;AASK;MCpDL,OAAA,UD0DS;MCxDT,UAAA;IAAA;MAGA,IAAA,cDqDoD;MCnDpD,mBAAA,WDqDI;MCnDJ,mBAAA;IAAA;MAGA,IAAA,cDiDiB;MC/CjB,mBAAA,WD+Ca;MC7Cb,mBAAA;IAAA,GD0CoB1B;ICtC1B,YAAA;IACA,cAAA;EAAA;EDsCEM;ECpCe,MAAA;EAEnB,KAAA;IDmCYA,cCjCV,OAAA,UDkCEI;IChCF,cAAA,UDgCuBH;IC9BvB,iBAAA;IACA,IAAA;MDmCQ,wDChCN,QAAA;MACA,OAAA;IAAA;EAAA;EAIJ,MAAA;IACE,MAAA;IAEA,cAAA;MAAA,CACG,GAAA;IAAA;IDuB+G;;;;;IChBlH,wBAAA;EAAA;EAGF,SAAA;IACE,qBAAA;EAAA;AAAA;;;cCvES,KAAA;AAAA,KACD,IAAA,WAAe,KAAK;AAAA,cAEnB,UAAA;EAAA;;;;;;;;;KAiBD,SAAA,gBAAyB,UAAU;;;QC9BvC,MAAA;EAAA,UACI,MAAA;IAAA,UACE,UAAA;MACR,QAAA;MACA,QAAA;MACA,cAAA;IAAA;EAAA;AAAA;AAAA,UAKW,GAAA,WACL,QAAA,GAAW,QAAA,YACX,iBAAA,GAAoB,iBAAA;EAE9B,SAAA;EACA,EAAA;EACA,UAAA;EACA,aAAA,EAAe,MAAA,CAAO,SAAA,CAAU,SAAA;EAChC,MAAA,EAAQ,MAAA;EACR,SAAA,EAAW,SAAA,CAAU,CAAA,EAAG,CAAA;EACxB,YAAA;EACA,QAAA;AAAA;AAAA,UAGe,UAAA,WACL,QAAA,GAAW,QAAA,YACX,iBAAA,GAAoB,iBAAA;EAAA,CAE7B,SAAA,EAAW,SAAA,GAAY,iBAAA,CAAkB,CAAA,EAAG,CAAA;AAAA;;;;;;aAiCnC,SAAA;EACV,QAAA;EACA,SAAA;EACA,SAAA;EACA,UAAA;EH4BqB;EG1BrB,MAAA;EH0B0C;EGxB1C,KAAA;AAAA;;;KCjEU,OAAA,GAAU,eAAe;;KAGzB,QAAA;EAAoB,CAAA,EAAG,CAAC;EAAA,SAAW,EAAA;AAAA;AAAA,KAEnC,mBAAA;AAAA,KACA,KAAA,MAAW,mBAAmB;AAAA,KAC9B,SAAA,YAAqB,QAAQ;AAAA,KAE7B,IAAA,IAAQ,GAAA,UAAa,GAAG;;KAGxB,MAAA,eACE,eAAe;AAAA,aAEjB,QAAA;EACV,IAAA;EACA,sBAAA;EACA,iBAAA;EACA,SAAA;EACA,iBAAA;AAAA;AAAA,UAGe,UAAA;EACf,EAAA,EAAI,KAAA;EACJ,OAAA;EACA,QAAA,EAAU,IAAA;EACV,IAAA,EAAM,IAAA;EACN,QAAA,GAAW,OAAA,CACT,IAAA,CAAK,YAAA;EAEP,KAAA,WAAgB,QAAA;AAAA;AAAA,UAGD,iBAAA,WACL,QAAA,GAAW,QAAA,YACX,iBAAA,GAAoB,iBAAA,UACtB,OAAA,CAAQ,CAAA,EAAG,CAAA;EJ2CyC;EIzC5D,OAAA;EACA,QAAA,EAAU,IAAA;AAAA;AAAA,KAGP,YAAA,MAAkB,CAAA,GAAI,OAAA,CAAQ,CAAA;AAAA,UAElB,qBAAA;EJsCU;EIpCzB,cAAA,GAAiB,eAAA;EACjB,aAAA,GAAgB,eAAe;EAC/B,QAAA;AAAA;AAAA,UAGe,cAAA;EACf,IAAA,EAAM,OAAA;EACN,QAAA,GAAW,QAAQ;AAAA;AAAA,UAGJ,6BAAA;EACf,QAAA;EACA,KAAA;EACA,IAAA,EAAM,cAAA;EACN,KAAA,GAAQ,qBAAqB;AAAA;AAAA,UAGd,4BAAA;EACf,QAAA;EACA,KAAA;EACA,cAAA;IAAA,CAAmB,EAAA,EAAI,KAAA,GAAQ,cAAA;EAAA;EAC/B,KAAA,GAAQ,qBAAA;AAAA;AAAA,UAIO,SAAA,WAAoB,QAAA,YAAoB,iBAAA;EJiBA;;;;;;EIVvD,iBAAA,CAAkB,UAAA,EAAY,UAAA;EJUDK;;;;EIL7B,oBAAA,CAAqB,aAAA,EAAe,iBAAA,CAAkB,CAAA,EAAG,CAAA;EJKyCK;;AAAkB;AAOtH;;EILE,UAAA,EAAY,GAAA,EAAK,SAAA,GAAY,SAAA;EJKa;;;;;EIE1C,YAAA,EAAc,QAAA;EAGd,aAAA,CAAc,KAAA;IACZ,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,6BAAA;EAEjB,cAAA,EAAgB,KAAA;IACd,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,4BAAA;EAEjB,cAAA,EAAgB,KAAA;IACd,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,6BAAA;EAEjB,eAAA,EAAiB,KAAA;IACf,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,4BAAA;EJJff;;;;AACqB;AAOzB;;EIKE,aAAA,EAAe,KAAA;IACb,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,6BAAA;EJNfA;;;;AACqB;AAOzB;;;EIQE,MAAA,CAAO,KAAA;IACL,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,aAAA,EAAe,UAAA;EAAA,IACb,YAAA,EACC,6BAAA;IJVUkB;;AAAQ;AAOzB;IIQQ,SAAA,EAAW,KAAA;EAAA;EJRgC;;;;;EIkBjD,gBAAA,EAAkB,KAAA;IAChB,QAAA;IACA,KAAA;IACA,QAAA,EAAU,eAAA;IACV,OAAA,EAAS,MAAA,CAAO,KAAA,EAAO,UAAA;IACvB,UAAA,EAAY,UAAA,CAAW,CAAA,EAAG,CAAA;EAAA,IACxB,YAAA,CAAa,eAAA;EJbH;;;;;;EIqBd,gBAAA,EAAkB,KAAA;IAAS,QAAA;IAAkB,KAAA;EAAA,IAAkB,YAAA;IAC7D,YAAA;IACA,aAAA,GAAgB,IAAA;EAAA;EJpBR;;;;AAAmC;EI4B7C,eAAA,KAAoB,YAAA;IAAA,CACjB,QAAA;MAAA,CACE,KAAA,WAAgB,qBAAA;IAAA;EAAA;AAAA;AAAA,UAIN,UAAA;EACf,KAAA,GAAQ,IAAI;EACZ,SAAA;AAAA;AAAA,UAGe,cAAA;EACf,MAAA,EAAQ,MAAA;EACR,QAAA;EACA,aAAA;IACE,IAAA;MACE,aAAA;MACA,GAAA;QAEE,OAAA;QACA,aAAA;QACA,IAAA;QACA,KAAA;QAGA,eAAA;QACA,eAAA;QACA,oBAAA;MAAA;MAEF,OAAA;IAAA;IAEF,OAAA,EAAS,MAAA,CAAO,SAAA;IAChB,SAAA,EAAW,gBAAA;EAAA;AAAA;AAAA,UAIE,gBAAA;EACf,MAAA;EACA,IAAA;EACA,MAAA;EACA,OAAA;AAAA;AAAA,UAGe,mBAAA;EACf,IAAA;EACA,SAAA,EAAW,gBAAgB;AAAA;AAAA,UAGZ,oBAAA;EJjD4DZ;EImD3E,WAAA;EACA,IAAA,EAAM,mBAAmB;AAAA;AAAA,UAGV,SAAA;EACf,QAAA,GAAW,cAAc;AAAA;AAAA,UAGV,mBAAA;EACf,EAAA;EACA,QAAA;EACA,KAAA;EACA,KAAA;EACA,YAAA;EACA,KAAA;EACA,UAAA;AAAA;;KAIU,SAAA,GAAY,iBAAA,CAAkB,OAAA,EAAS,mBAAA;EACjD,SAAA;AAAA;;;iBCvOoB,gBAAA,CACpB,MAAA,EAAQ,MAAA,GACP,OAAA,CAAQ,IAAA,CAAK,GAAA;AAAA,iBAmBM,GAAA,WAAc,QAAA,YAAoB,iBAAA,EACtD,MAAA,EAAQ,MAAA,EACR,SAAA,EAAW,SAAA,CAAU,CAAA,EAAG,CAAA,GACxB,UAAA,GAAY,UAAA,GACX,OAAA,CAAQ,SAAA;;;;iBC3DW,SAAA,sBACpB,QAAA,UACA,MAAA,GAAS,GAAA,EAAK,CAAA,KAAM,CAAA,GACnB,OAAA,CAAQ,MAAA,CAAO,CAAA,EAAG,CAAA;;iBAcC,UAAA,CAAW,QAAA,UAAkB,IAAA,cAAe,OAAA"}
|
package/dist/index.d.mts
CHANGED
|
@@ -180,7 +180,6 @@ interface FeatureCollection<G extends Geometry | null = Geometry, P = GeoJsonPro
|
|
|
180
180
|
interface Config {
|
|
181
181
|
$schema: (string & {}) | 'https://unpkg.com/@osm-conflation-engine/cli/dist/config.schema.json' | 'node_modules/@osm-conflation-engine/cli/config.schema.json';
|
|
182
182
|
metadata: {
|
|
183
|
-
/** format should be `https://github.com/a/b` */git_repository: string;
|
|
184
183
|
name: string;
|
|
185
184
|
description: string;
|
|
186
185
|
wiki_page: string;
|
|
@@ -280,6 +279,7 @@ interface Ctx<G extends Geometry = Geometry, P extends GeoJsonProperties = GeoJs
|
|
|
280
279
|
tempFileNames: Record<Lowercase<FileNames>, string>;
|
|
281
280
|
config: Config;
|
|
282
281
|
callbacks: Callbacks<G, P>;
|
|
282
|
+
formatterUrl: string | undefined;
|
|
283
283
|
warnings: string[];
|
|
284
284
|
}
|
|
285
285
|
interface SourceData<G extends Geometry = Geometry, P extends GeoJsonProperties = GeoJsonProperties> {
|
|
@@ -486,14 +486,24 @@ interface ConflateResult {
|
|
|
486
486
|
ignored: number;
|
|
487
487
|
};
|
|
488
488
|
matched: Record<MatchType, number>;
|
|
489
|
-
conflated:
|
|
490
|
-
create: number;
|
|
491
|
-
edit: number;
|
|
492
|
-
delete: number;
|
|
493
|
-
perfect: number;
|
|
494
|
-
};
|
|
489
|
+
conflated: ConflationResult;
|
|
495
490
|
};
|
|
496
491
|
}
|
|
492
|
+
interface ConflationResult {
|
|
493
|
+
create: number;
|
|
494
|
+
edit: number;
|
|
495
|
+
delete: number;
|
|
496
|
+
perfect: number;
|
|
497
|
+
}
|
|
498
|
+
interface RecursiveHistoryRow {
|
|
499
|
+
date: string;
|
|
500
|
+
conflated: ConflationResult;
|
|
501
|
+
}
|
|
502
|
+
interface RecursiveHistoryFile {
|
|
503
|
+
/** ISO Date */
|
|
504
|
+
lastUpdated: string;
|
|
505
|
+
rows: RecursiveHistoryRow[];
|
|
506
|
+
}
|
|
497
507
|
interface RunResult {
|
|
498
508
|
conflate?: ConflateResult;
|
|
499
509
|
}
|
|
@@ -521,5 +531,5 @@ declare function readJsonL<T, U extends string>(filePath: string, getKey: (row:
|
|
|
521
531
|
/** writes JSONL to a file by stringifying each line at a time */
|
|
522
532
|
declare function writeJsonL(filePath: string, rows: unknown[]): Promise<void>;
|
|
523
533
|
//#endregion
|
|
524
|
-
export { Action, Callbacks, Config, ConflateResult, ConflationDiff, ConflationResultExtra, DatasetId, Identity, IndexFile, IndexFileProperties, MutliFeatureConflationResult, OsmFeature, OsmFeatureTypeShort, OsmFlags, OsmId, RunOptions, RunResult, SingleFeatureConflationResult, SourceDataFeature, TagDiff, Vec2, getTempFileNames, readJsonL, run, writeJsonL };
|
|
534
|
+
export { Action, Callbacks, Config, ConflateResult, ConflationDiff, ConflationResult, ConflationResultExtra, DatasetId, Identity, IndexFile, IndexFileProperties, MutliFeatureConflationResult, OsmFeature, OsmFeatureTypeShort, OsmFlags, OsmId, RecursiveHistoryFile, RecursiveHistoryRow, RunOptions, RunResult, SingleFeatureConflationResult, SourceDataFeature, TagDiff, Vec2, getTempFileNames, readJsonL, run, writeJsonL };
|
|
525
535
|
//# sourceMappingURL=index.d.mts.map
|
package/dist/index.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.mts","names":["Geometry","GeoJSON","type","GeoJsonTypes","bbox","BBox","G","P","GeoJsonProperties","Feature","FeatureCollection","Point","MultiPoint","LineString","MultiLineString","Polygon","MultiPolygon","GeometryCollection","GeoJsonObject","coordinates","Position","geometries","name","geometry","id","properties","features","Array"],"sources":["../../../node_modules/.pnpm/@types+geojson@7946.0.16/node_modules/@types/geojson/index.d.ts","../src/types/config.def.ts","../src/constants/defaults.ts","../src/types/internal.def.ts","../src/types/callbacks.def.ts","../src/main.ts","../src/common/jsonl.ts"],"x_google_ignoreList":[0],"mappings":";;;;AAqBgB;AA2ChB;;KAjDY,YAAA,GAAe,OAAO;AAiDd;AAUpB;;;AAVoB,KA3CR,IAAA;;;;;AAqEG;AAMf;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG4B;AAM5B;;;;;;;KAzCY,QAAA;;;;;;;;;UAUK,aAAA;EAAA;EAAA;EAAA;EAsCA;;;EA/BbE,IAAAA,EAAM,YAAA;EA+BqBgB;;;;;AAEN;AAOzB;;EA/BId,IAAAA,GAAO,IAAI;AAAA;;;;KAMH,OAAA,WAAkB,QAAA,UAAkB,QAAA,MAAc,iBAAA,IACxD,CAAA,GACA,OAAA,CAAQ,CAAA,EAAG,CAAA,IACX,iBAAA,CAAkB,CAAA,EAAG,CAAA;;;;;KAMf,QAAA,GAAW,KAAA,GAAQ,UAAA,GAAa,UAAA,GAAa,eAAA,GAAkB,OAAA,GAAU,YAAA,GAAe,kBAAA;AAkCpG;;;;AAAA,UA3BiB,KAAA,SAAc,aAAa;EACxCF,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;AA2BQ;AAOzB;;;AAPyB,UApBR,UAAA,SAAmB,aAAa;EAC7CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;AA2BQ;AAOzB;;UA3BiB,UAAA,SAAmB,aAAa;EAC7CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;AA2BQ;AAOzB;UA3BiB,eAAA,SAAwB,aAAa;EAClDjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;;;UAOA,OAAA,SAAgB,aAAa;EAC1CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;AAkBA;AAGjB;UAdiB,YAAA,SAAqB,aAAa;EAC/CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;;;UAOA,kBAAA,WAA6B,QAAA,GAAW,QAAA,UAAkB,aAAA;EACvEjB,IAAAA;EACAmB,UAAAA,EAAY,CAAA;AAAA;AAAA,KAGJ,iBAAA;EAAA,CAAuBC,IAAY;AAAA;;;;;UAM9B,OAAA,WAAkB,QAAA,UAAkB,QAAA,MAAc,iBAAA,UAA2B,aAAA;EAC1FpB,IAAAA;EAIAqB;;;EAAAA,QAAAA,EAAU,CAAA;EASEhB;;AAAC;AAOjB;EAXIiB,EAAAA;EAW8B;;;EAP9BC,UAAAA,EAAY,CAAA;AAAA;;;;;UAOC,iBAAA,WAA4B,QAAA,UAAkB,QAAA,MAAc,iBAAA,UAA2B,aAAA;EACpGvB,IAAAA;EACAwB,QAAAA,EAAU,KAAA,CAAM,OAAA,CAAQ,CAAA,EAAG,CAAA;AAAA;;;UCrMd,MAAA;EACf,OAAA;EAKA,QAAA;
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":["Geometry","GeoJSON","type","GeoJsonTypes","bbox","BBox","G","P","GeoJsonProperties","Feature","FeatureCollection","Point","MultiPoint","LineString","MultiLineString","Polygon","MultiPolygon","GeometryCollection","GeoJsonObject","coordinates","Position","geometries","name","geometry","id","properties","features","Array"],"sources":["../../../node_modules/.pnpm/@types+geojson@7946.0.16/node_modules/@types/geojson/index.d.ts","../src/types/config.def.ts","../src/constants/defaults.ts","../src/types/internal.def.ts","../src/types/callbacks.def.ts","../src/main.ts","../src/common/jsonl.ts"],"x_google_ignoreList":[0],"mappings":";;;;AAqBgB;AA2ChB;;KAjDY,YAAA,GAAe,OAAO;AAiDd;AAUpB;;;AAVoB,KA3CR,IAAA;;;;;AAqEG;AAMf;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG4B;AAM5B;;;;;;;KAzCY,QAAA;;;;;;;;;UAUK,aAAA;EAAA;EAAA;EAAA;EAsCA;;;EA/BbE,IAAAA,EAAM,YAAA;EA+BqBgB;;;;;AAEN;AAOzB;;EA/BId,IAAAA,GAAO,IAAI;AAAA;;;;KAMH,OAAA,WAAkB,QAAA,UAAkB,QAAA,MAAc,iBAAA,IACxD,CAAA,GACA,OAAA,CAAQ,CAAA,EAAG,CAAA,IACX,iBAAA,CAAkB,CAAA,EAAG,CAAA;;;;;KAMf,QAAA,GAAW,KAAA,GAAQ,UAAA,GAAa,UAAA,GAAa,eAAA,GAAkB,OAAA,GAAU,YAAA,GAAe,kBAAA;AAkCpG;;;;AAAA,UA3BiB,KAAA,SAAc,aAAa;EACxCF,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;AA2BQ;AAOzB;;;AAPyB,UApBR,UAAA,SAAmB,aAAa;EAC7CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;AA2BQ;AAOzB;;UA3BiB,UAAA,SAAmB,aAAa;EAC7CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;AA2BQ;AAOzB;UA3BiB,eAAA,SAAwB,aAAa;EAClDjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;;;UAOA,OAAA,SAAgB,aAAa;EAC1CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;AAkBA;AAGjB;UAdiB,YAAA,SAAqB,aAAa;EAC/CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;;;UAOA,kBAAA,WAA6B,QAAA,GAAW,QAAA,UAAkB,aAAA;EACvEjB,IAAAA;EACAmB,UAAAA,EAAY,CAAA;AAAA;AAAA,KAGJ,iBAAA;EAAA,CAAuBC,IAAY;AAAA;;;;;UAM9B,OAAA,WAAkB,QAAA,UAAkB,QAAA,MAAc,iBAAA,UAA2B,aAAA;EAC1FpB,IAAAA;EAIAqB;;;EAAAA,QAAAA,EAAU,CAAA;EASEhB;;AAAC;AAOjB;EAXIiB,EAAAA;EAW8B;;;EAP9BC,UAAAA,EAAY,CAAA;AAAA;;;;;UAOC,iBAAA,WAA4B,QAAA,UAAkB,QAAA,MAAc,iBAAA,UAA2B,aAAA;EACpGvB,IAAAA;EACAwB,QAAAA,EAAU,KAAA,CAAM,OAAA,CAAQ,CAAA,EAAG,CAAA;AAAA;;;UCrMd,MAAA;EACf,OAAA;EAKA,QAAA;IACE,IAAA;IACA,WAAA;IACA,SAAA;IACA,MAAA;EAAA;EAGF,WAAA;IACE,IAAA;IACA,IAAA;EAAA;EAGF,QAAA;ID2CU;;;;AAAQ;ICrChB,MAAA;MAEM,IAAA;MDoDA;;;;;;;AASK;MCpDL,OAAA,UD0DS;MCxDT,UAAA;IAAA;MAGA,IAAA,cDqDoD;MCnDpD,mBAAA,WDqDI;MCnDJ,mBAAA;IAAA;MAGA,IAAA,cDiDiB;MC/CjB,mBAAA,WD+Ca;MC7Cb,mBAAA;IAAA,GD0CoB1B;ICtC1B,YAAA;IACA,cAAA;EAAA;EDsCEM;ECpCe,MAAA;EAEnB,KAAA;IDmCYA,cCjCV,OAAA,UDkCEI;IChCF,cAAA,UDgCuBH;IC9BvB,iBAAA;IACA,IAAA;MDmCQ,wDChCN,QAAA;MACA,OAAA;IAAA;EAAA;EAIJ,MAAA;IACE,MAAA;IAEA,cAAA;MAAA,CACG,GAAA;IAAA;IDuB+G;;;;;IChBlH,wBAAA;EAAA;EAGF,SAAA;IACE,qBAAA;EAAA;AAAA;;;cCvES,KAAA;AAAA,KACD,IAAA,WAAe,KAAK;AAAA,cAEnB,UAAA;EAAA;;;;;;;;;KAiBD,SAAA,gBAAyB,UAAU;;;QC9BvC,MAAA;EAAA,UACI,MAAA;IAAA,UACE,UAAA;MACR,QAAA;MACA,QAAA;MACA,cAAA;IAAA;EAAA;AAAA;AAAA,UAKW,GAAA,WACL,QAAA,GAAW,QAAA,YACX,iBAAA,GAAoB,iBAAA;EAE9B,SAAA;EACA,EAAA;EACA,UAAA;EACA,aAAA,EAAe,MAAA,CAAO,SAAA,CAAU,SAAA;EAChC,MAAA,EAAQ,MAAA;EACR,SAAA,EAAW,SAAA,CAAU,CAAA,EAAG,CAAA;EACxB,YAAA;EACA,QAAA;AAAA;AAAA,UAGe,UAAA,WACL,QAAA,GAAW,QAAA,YACX,iBAAA,GAAoB,iBAAA;EAAA,CAE7B,SAAA,EAAW,SAAA,GAAY,iBAAA,CAAkB,CAAA,EAAG,CAAA;AAAA;;;;;;aAiCnC,SAAA;EACV,QAAA;EACA,SAAA;EACA,SAAA;EACA,UAAA;EH4BqB;EG1BrB,MAAA;EH0B0C;EGxB1C,KAAA;AAAA;;;KCjEU,OAAA,GAAU,eAAe;;KAGzB,QAAA;EAAoB,CAAA,EAAG,CAAC;EAAA,SAAW,EAAA;AAAA;AAAA,KAEnC,mBAAA;AAAA,KACA,KAAA,MAAW,mBAAmB;AAAA,KAC9B,SAAA,YAAqB,QAAQ;AAAA,KAE7B,IAAA,IAAQ,GAAA,UAAa,GAAG;;KAGxB,MAAA,eACE,eAAe;AAAA,aAEjB,QAAA;EACV,IAAA;EACA,sBAAA;EACA,iBAAA;EACA,SAAA;EACA,iBAAA;AAAA;AAAA,UAGe,UAAA;EACf,EAAA,EAAI,KAAA;EACJ,OAAA;EACA,QAAA,EAAU,IAAA;EACV,IAAA,EAAM,IAAA;EACN,QAAA,GAAW,OAAA,CACT,IAAA,CAAK,YAAA;EAEP,KAAA,WAAgB,QAAA;AAAA;AAAA,UAGD,iBAAA,WACL,QAAA,GAAW,QAAA,YACX,iBAAA,GAAoB,iBAAA,UACtB,OAAA,CAAQ,CAAA,EAAG,CAAA;EJ2CyC;EIzC5D,OAAA;EACA,QAAA,EAAU,IAAA;AAAA;AAAA,KAGP,YAAA,MAAkB,CAAA,GAAI,OAAA,CAAQ,CAAA;AAAA,UAElB,qBAAA;EJsCU;EIpCzB,cAAA,GAAiB,eAAA;EACjB,aAAA,GAAgB,eAAe;EAC/B,QAAA;AAAA;AAAA,UAGe,cAAA;EACf,IAAA,EAAM,OAAA;EACN,QAAA,GAAW,QAAQ;AAAA;AAAA,UAGJ,6BAAA;EACf,QAAA;EACA,KAAA;EACA,IAAA,EAAM,cAAA;EACN,KAAA,GAAQ,qBAAqB;AAAA;AAAA,UAGd,4BAAA;EACf,QAAA;EACA,KAAA;EACA,cAAA;IAAA,CAAmB,EAAA,EAAI,KAAA,GAAQ,cAAA;EAAA;EAC/B,KAAA,GAAQ,qBAAA;AAAA;AAAA,UAIO,SAAA,WAAoB,QAAA,YAAoB,iBAAA;EJiBA;;;;;;EIVvD,iBAAA,CAAkB,UAAA,EAAY,UAAA;EJUDK;;;;EIL7B,oBAAA,CAAqB,aAAA,EAAe,iBAAA,CAAkB,CAAA,EAAG,CAAA;EJKyCK;;AAAkB;AAOtH;;EILE,UAAA,EAAY,GAAA,EAAK,SAAA,GAAY,SAAA;EJKa;;;;;EIE1C,YAAA,EAAc,QAAA;EAGd,aAAA,CAAc,KAAA;IACZ,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,6BAAA;EAEjB,cAAA,EAAgB,KAAA;IACd,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,4BAAA;EAEjB,cAAA,EAAgB,KAAA;IACd,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,6BAAA;EAEjB,eAAA,EAAiB,KAAA;IACf,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,4BAAA;EJJff;;;;AACqB;AAOzB;;EIKE,aAAA,EAAe,KAAA;IACb,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,6BAAA;EJNfA;;;;AACqB;AAOzB;;;EIQE,MAAA,CAAO,KAAA;IACL,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,aAAA,EAAe,UAAA;EAAA,IACb,YAAA,EACC,6BAAA;IJVUkB;;AAAQ;AAOzB;IIQQ,SAAA,EAAW,KAAA;EAAA;EJRgC;;;;;EIkBjD,gBAAA,EAAkB,KAAA;IAChB,QAAA;IACA,KAAA;IACA,QAAA,EAAU,eAAA;IACV,OAAA,EAAS,MAAA,CAAO,KAAA,EAAO,UAAA;IACvB,UAAA,EAAY,UAAA,CAAW,CAAA,EAAG,CAAA;EAAA,IACxB,YAAA,CAAa,eAAA;EJbH;;;;;;EIqBd,gBAAA,EAAkB,KAAA;IAAS,QAAA;IAAkB,KAAA;EAAA,IAAkB,YAAA;IAC7D,YAAA;IACA,aAAA,GAAgB,IAAA;EAAA;EJpBR;;;;AAAmC;EI4B7C,eAAA,KAAoB,YAAA;IAAA,CACjB,QAAA;MAAA,CACE,KAAA,WAAgB,qBAAA;IAAA;EAAA;AAAA;AAAA,UAIN,UAAA;EACf,KAAA,GAAQ,IAAI;EACZ,SAAA;AAAA;AAAA,UAGe,cAAA;EACf,MAAA,EAAQ,MAAA;EACR,QAAA;EACA,aAAA;IACE,IAAA;MACE,aAAA;MACA,GAAA;QAEE,OAAA;QACA,aAAA;QACA,IAAA;QACA,KAAA;QAGA,eAAA;QACA,eAAA;QACA,oBAAA;MAAA;MAEF,OAAA;IAAA;IAEF,OAAA,EAAS,MAAA,CAAO,SAAA;IAChB,SAAA,EAAW,gBAAA;EAAA;AAAA;AAAA,UAIE,gBAAA;EACf,MAAA;EACA,IAAA;EACA,MAAA;EACA,OAAA;AAAA;AAAA,UAGe,mBAAA;EACf,IAAA;EACA,SAAA,EAAW,gBAAgB;AAAA;AAAA,UAGZ,oBAAA;EJjD4DZ;EImD3E,WAAA;EACA,IAAA,EAAM,mBAAmB;AAAA;AAAA,UAGV,SAAA;EACf,QAAA,GAAW,cAAc;AAAA;AAAA,UAGV,mBAAA;EACf,EAAA;EACA,QAAA;EACA,KAAA;EACA,KAAA;EACA,YAAA;EACA,KAAA;EACA,UAAA;AAAA;;KAIU,SAAA,GAAY,iBAAA,CAAkB,OAAA,EAAS,mBAAA;EACjD,SAAA;AAAA;;;iBCvOoB,gBAAA,CACpB,MAAA,EAAQ,MAAA,GACP,OAAA,CAAQ,IAAA,CAAK,GAAA;AAAA,iBAmBM,GAAA,WAAc,QAAA,YAAoB,iBAAA,EACtD,MAAA,EAAQ,MAAA,EACR,SAAA,EAAW,SAAA,CAAU,CAAA,EAAG,CAAA,GACxB,UAAA,GAAY,UAAA,GACX,OAAA,CAAQ,SAAA;;;;iBC3DW,SAAA,sBACpB,QAAA,UACA,MAAA,GAAS,GAAA,EAAK,CAAA,KAAM,CAAA,GACnB,OAAA,CAAQ,MAAA,CAAO,CAAA,EAAG,CAAA;;iBAcC,UAAA,CAAW,QAAA,UAAkB,IAAA,cAAe,OAAA"}
|
package/dist/index.mjs
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import{basename as e,dirname as t,join as n}from"node:path";import{tmpdir as r}from"node:os";import{createReadStream as i,createWriteStream as a,existsSync as o,promises as s}from"node:fs";import{Ajv as c}from"ajv";import{createInterface as l}from"node:readline/promises";import{POLYGON_TO_CELLS_FLAGS as u,latLngToCell as d,polygonToCellsExperimental as f}from"h3-js";import{setTimeout as p}from"node:timers/promises";import{spawn as m}from"node:child_process";import{geoCentroid as h,geoDistance as g}from"d3-geo";import _ from"pbf2json";import{createHash as v}from"node:crypto";import{getIDToken as ee}from"@actions/core";const y=process.env.NODE_ENV===`test`,b=[`download`,`match`,`conflate`],
|
|
2
|
-
`);n.end(),await new Promise(e=>n.on(`finish`,e))}function E(e,t){let n=t||3;switch(e.type){case`GeometryCollection`:throw Error(`not supported`);case`Point`:return[d(e.coordinates[1],e.coordinates[0],n)];case`MultiPoint`:case`LineString`:case`MultiLineString`:case`Polygon`:return f(e.coordinates,n,u.containmentOverlapping,!0).toSorted();case`MultiPolygon`:{let t=new Set;for(let r of e.coordinates){let e=f(r,n,u.containmentOverlapping,!0);for(let n of e)t.add(n)}return[...t].toSorted()}default:throw TypeError(e)}}const
|
|
1
|
+
import{basename as e,dirname as t,join as n}from"node:path";import{tmpdir as r}from"node:os";import{createReadStream as i,createWriteStream as a,existsSync as o,promises as s}from"node:fs";import{Ajv as c}from"ajv";import{createInterface as l}from"node:readline/promises";import{POLYGON_TO_CELLS_FLAGS as u,latLngToCell as d,polygonToCellsExperimental as f}from"h3-js";import{setTimeout as p}from"node:timers/promises";import{spawn as m}from"node:child_process";import{geoCentroid as h,geoDistance as g}from"d3-geo";import _ from"pbf2json";import{createHash as v}from"node:crypto";import{getIDToken as ee}from"@actions/core";import te from"tag2link"with{type:"json"};const y=process.env.NODE_ENV===`test`,b=[`download`,`match`,`conflate`],ne={PBF:`o.pbf`,OVERPASS:`o.overpass.geo.json`,POSTPASS:`o.postpass.geo.json`,OSM_PROCESSED_WITH_REF:`osm-processed-with-ref.jsonl`,OSM_PROCESSED_NO_REF:`osm-processed-no-ref.jsonl`,OSM_PROCESSED_OTHER:`osm-processed-other.json`,IGNORE_LIST:`ignore-list.json`,MATCHES:`matches.json`},x=`https://osm-conflation-engine.kyle.kiwi`,S=e=>({"User-Agent":`https://github.com/osm-nz/osm-conflation-engine on behalf of ${e.metadata.wiki_page}`});let re=function(e){return e[e.None=0]=`None`,e[e.IsLastEditedByImporter=1]=`IsLastEditedByImporter`,e[e.IsRecentlyChanged=2]=`IsRecentlyChanged`,e[e.IsChecked=4]=`IsChecked`,e[e.IsCheckedRecently=8]=`IsCheckedRecently`,e}({});function C(e){let t=e.config.osm_data?.check_date_key||`check_date`,n=new Set([t,e.config.merge.osm_key]).difference(new Set(e.config.osm_data.tags_to_keep));if(n.size)throw Error(`config.osm_data.tags_to_keep is missing some tags that are referenced in other parts of the config: ${[...n].join(`, `)}`);let r=[];for(let t of e.config.osm_data.tags_to_keep)t.startsWith(`/`)&&t.endsWith(`/`)&&r.push(new RegExp(t.slice(1,-1)));let i={};function a(t){if(t in i)return i[t];let n=e.config.osm_data.tags_to_keep.includes(t);if(!n){for(let e of r)if(e.test(t)){n=!0;break}}return i[t]=n,n}function o(e){let t={};if(e)for(let n in e){let r=e[n];a(n)&&r&&(t[n]=r)}return t}return{pickTags:o}}let ie=function(e){return e[e.OneToOne=1]=`OneToOne`,e[e.OneToMany=2]=`OneToMany`,e[e.ManyToOne=3]=`ManyToOne`,e[e.ManyToMany=4]=`ManyToMany`,e[e.Delete=5]=`Delete`,e[e.Guess=6]=`Guess`,e}({});const ae=e=>e?(Date.now()-+new Date(e))/1e3/60/60/24/365<4?1:2:0;async function w(e,t){let n=l({input:i(e)}),r={};for await(let e of n){let n=JSON.parse(e),i=t(n);r[i]=n}return r}async function T(e,t){let n=a(e,{encoding:`utf8`});for(let e of t)n.write(JSON.stringify(e)),n.write(`
|
|
2
|
+
`);n.end(),await new Promise(e=>n.on(`finish`,e))}function E(e,t){let n=t||3;switch(e.type){case`GeometryCollection`:throw Error(`not supported`);case`Point`:return[d(e.coordinates[1],e.coordinates[0],n)];case`MultiPoint`:case`LineString`:case`MultiLineString`:case`Polygon`:return f(e.coordinates,n,u.containmentOverlapping,!0).toSorted();case`MultiPolygon`:{let t=new Set;for(let r of e.coordinates){let e=f(r,n,u.containmentOverlapping,!0);for(let n of e)t.add(n)}return[...t].toSorted()}default:throw TypeError(e)}}const oe=(e=>(e.setDate(e.getDate()-90),+e))(new Date);function D(e,t,n){n.sectors=E({type:`Point`,coordinates:n.centroid},e.config.merge.sector_resolution);let r=e.config.osm_data?.check_date_key||`check_date`;+new Date(n.metadata?.timestamp??0)>oe&&(n.flags|=2);let i=n.metadata?.user&&(e.callbacks.isImportUser?e.callbacks.isImportUser?.(n.metadata.user):n.metadata?.user?.endsWith(`_import`));(n.metadata?.version===1||i)&&(n.flags|=1),n.tags[r]&&(n.flags|=4),ae(n.tags[r])===1&&(n.flags|=8);let a=n.tags[e.config.merge.osm_key];if(a)if(t.count++,t.duplicateRefs[a])t.duplicateRefs[a].push(n);else if(t.withRef[a])t.duplicateRefs[a]=[t.withRef[a],n],delete t.withRef[a];else if(a.includes(`;`)){let e=a.split(`;`);for(let r of e)t.semi[r]=n}else t.withRef[a]=n;else t.noRef[n.id]=n}async function O(e,t){let{withRef:n,noRef:r,...i}=t;await T(e.tempFileNames.osm_processed_with_ref,Object.values(n)),await T(e.tempFileNames.osm_processed_no_ref,Object.values(r)),await s.writeFile(e.tempFileNames.osm_processed_other,JSON.stringify(i))}async function se(e,t,n){let r=e.tempFileNames.overpass;if(e.use_cache&&o(r)){console.info(`Using cached overpass response`),await O(e,JSON.parse(await s.readFile(r,`utf8`)));return}let i=`
|
|
3
3
|
[out:json][timeout:480];
|
|
4
4
|
nwr["${e.config.merge.osm_key}"];
|
|
5
5
|
out meta center;
|
|
6
|
-
`,a=t?await s.readFile(t,`utf8`):i;console.info(`fetching OSM data via overpass…`);let c=await fetch(`${n||`https://overpass-api.de/api/interpreter`}?data=${encodeURIComponent(a)}`,{headers:S(e.config)});if(!c.ok){let e=await c.text()||c.statusText;throw Error(`HTTP Error ${c.status} ${e}`)}let l=await c.json(),{pickTags:u}=C(e),d={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0};for(let t of l.elements)D(e,d,{id:`${t.type[0]}${t.id}`,centroid:t.type===`node`?[t.lon,t.lat]:t.center?[t.center.lon,t.center.lat]:[0,0],tags:u(t.tags),metadata:{changeset:t.changeset,timestamp:t.timestamp,user:t.user,version:t.version},sectors:[],flags:0});await s.writeFile(r,JSON.stringify(d)),await O(e,d)}const k=`https://osm-internal.download.geofabrik.de`,A=`https://www.openstreetmap.org`,j=e=>e.match(/name="csrf-token" content="([^"]+)"/)[1];async function
|
|
6
|
+
`,a=t?await s.readFile(t,`utf8`):i;console.info(`fetching OSM data via overpass…`);let c=await fetch(`${n||`https://overpass-api.de/api/interpreter`}?data=${encodeURIComponent(a)}`,{headers:S(e.config)});if(!c.ok){let e=await c.text()||c.statusText;throw Error(`HTTP Error ${c.status} ${e}`)}let l=await c.json(),{pickTags:u}=C(e),d={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0};for(let t of l.elements)D(e,d,{id:`${t.type[0]}${t.id}`,centroid:t.type===`node`?[t.lon,t.lat]:t.center?[t.center.lon,t.center.lat]:[0,0],tags:u(t.tags),metadata:{changeset:t.changeset,timestamp:t.timestamp,user:t.user,version:t.version},sectors:[],flags:0});await s.writeFile(r,JSON.stringify(d)),await O(e,d)}const k=`https://osm-internal.download.geofabrik.de`,A=`https://www.openstreetmap.org`,j=e=>e.match(/name="csrf-token" content="([^"]+)"/)[1];async function ce(e,t){if(t.startsWith(k)||e.warnings.push(`(!) Planet PBF url is not from ${k}, so metadata may not be available.`),!process.env.OSM_AUTH)return e.warnings.push(`(!) No OSM credentials, so the planet extract will not include metadata`),{url:t.replace(k,`https://download.geofabrik.de`).replace(`-internal.osm.pbf`,`.osm.pbf`),cookies:``};let n=new URLSearchParams(process.env.OSM_AUTH),r=S(e.config),i=await fetch(`${k}/get_cookie?action=get_authorization_url`,{method:`POST`,headers:r}).then(e=>e.json()),a=await fetch(`${A}/login?cookie_test=true`,{headers:r}),o=a.headers.get(`Set-Cookie`),s=await fetch(`${A}/login`,{method:`POST`,body:new URLSearchParams({username:n.get(`username`),password:n.get(`password`),referer:`/`,commit:`Login`,authenticity_token:j(await a.text())}),headers:{...r,Cookie:o,"Content-Type":`application/x-www-form-urlencoded`}});if(s.status!==200)throw Error(`Login failed`);o+=`; ${s.headers.get(`Set-Cookie`)}`;let c=await fetch(i.authorization_url,{headers:{...r,Cookie:o}}),l=c.url.startsWith(A)?await fetch(i.authorization_url,{method:`POST`,body:new URLSearchParams({client_id:i.client_id,redirect_uri:i.redirect_uri,authenticity_token:j(await c.text()),state:i.state,response_type:`code`,scope:`read_prefs`,nonce:``,code_challenge:``,code_challenge_method:``,commit:`Authorize`}),headers:{...r,Cookie:o,"Content-Type":`application/x-www-form-urlencoded`}}).then(e=>e.text()):await c.text();if(!l.includes(`gf_download_oauth`))throw Error(`No token`);return{url:t,cookies:l}}async function le(n,r){let i=n.tempFileNames.pbf,{cookies:a,url:o}=await ce(n,r);if(await s.mkdir(t(i),{recursive:!0}),await s.rm(i,{recursive:!0,force:!0}),/^https?:\/\//.test(o)){console.info(`downloading from`,o,`to`,i);let n=m(`curl`,[`-L`,o,`-H`,`Cookie: ${a}`,`--output`,e(i)],{cwd:t(i)});n.stdout.pipe(process.stdout),n.stderr.pipe(process.stderr),await new Promise((e,t)=>{n.on(`close`,n=>n?t(n):e())}),await p(5e3)}else console.info(`copying file from`,o,`to`,i),await s.copyFile(o,i);console.info(i,await s.stat(i))}async function ue(e,t,n){let r=e.tempFileNames.postpass;if(e.use_cache&&o(r)){console.info(`Using cached postpass response`),await O(e,JSON.parse(await s.readFile(r,`utf8`)));return}let i=`
|
|
7
7
|
SELECT osm_type, osm_id, tags, ST_PointOnSurface(geom) AS geom
|
|
8
8
|
FROM postpass_pointlinepolygon
|
|
9
9
|
WHERE tags ? '${e.config.merge.osm_key}'
|
|
10
|
-
`,a=t?await s.readFile(t,`utf8`):i;console.info(`fetching OSM data via postpass…`);let c=await fetch(n||`https://postpass.geofabrik.de/api/interpreter`,{method:`POST`,body:new URLSearchParams({data:a}),headers:S(e.config)});if(!c.ok){let e=await c.text()||c.statusText;throw Error(`HTTP Error ${c.status} ${e}`)}let l=await c.json(),{pickTags:u}=C(e),d={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0},f=new Set;for(let t of l.features){let{osm_type:n,osm_id:r,tags:i}=t.properties,a=`${n.toLowerCase()}${r}`;f.has(a)||(f.add(a),D(e,d,{id:a,centroid:t.geometry?.type?h(t.geometry):[0,0],tags:u(i),metadata:void 0,sectors:[],flags:0}))}await s.writeFile(r,JSON.stringify(d)),await O(e,d)}async function ue(e){let t=e.config.merge.osm_key,n=e.config.e2e_tests?.ignore_list_file_path;if(y&&!n)throw Error(`When running in the unit test envrironment, you must supply config.e2e_tests.ignoreListFilePath`);let r=y&&n?JSON.parse(await s.readFile(n,`utf8`)):await fetch(`${x}/api/ignore_list/${encodeURIComponent(t)}?return_ids_only=true`).then(e=>e.json());await s.writeFile(e.tempFileNames.ignore_list,JSON.stringify(r.result,null,2))}async function de(e,t){if(e.use_cache&&o(e.tempFileNames.osm_processed_other)){console.info(`Using cached osm data`);return}let{pickTags:n}=C(e);console.info(`Starting preprocess of OSM data...`),await O(e,await new Promise((r,i)=>{let a={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0},o=!1,s=!1,c=0;_.createReadStream({file:e.tempFileNames.pbf,tags:t,leveldb:`/tmp`,metadata:!0}).on(`data`,t=>{s||=!0;let r=t.type===`node`?t:t.centroid,i=t.meta;i&&(o=!0);let l=n(t.tags);D(e,a,{id:`${t.type[0]}${t.id}`,centroid:[+r.lon,+r.lat],metadata:i?{changeset:i.changeset,timestamp:i.timestamp?new Date(1e3*i.timestamp).toISOString():void 0,user:i.user,version:i.version}:void 0,tags:l,sectors:[],flags:0}),c+=1,c%1e3||process.stdout.write(`.`)}).on(`finish`,()=>{if(o||e.warnings.push(`No metadata extracted from the planet file!`),!s)throw Error(`Nothing extracted from the pbf file, presumably it’s invalid.`);r(a)}).on(`error`,i)}))}async function fe(e){if(await ue(e),e.use_cache&&o(e.tempFileNames.osm_processed_other)){console.info(`Using cached OSM data`);return}switch(e.config.osm_data.source.type){case`pbf`:await ce(e,e.config.osm_data.source.pbf_url),await de(e,e.config.osm_data.source.pbf_filter);break;case`overpass`:await oe(e,e.config.osm_data.source.overpass_query_file,e.config.osm_data.source.overpass_server_url);break;case`postpass`:await le(e,e.config.osm_data.source.postpass_query_file,e.config.osm_data.source.postpass_server_url);break;default:throw TypeError(e.config.osm_data.source)}}function pe(e,t,n){let r=[];for(let i in t.withRef){let a=i,o=t.withRef[a];if(!(a in n)&&!(o.flags&8)){if(e.callbacks.getAltRefs?.(a).some(e=>e in n))continue;r.push(a)}}let i={};for(let t in n){let r=t,a=n[r];for(let t of a.sectors){i[t]||={};let n=e.callbacks.getLocalKeyForSource(a);i[t][n]=r}}let a={},o=[];for(let n of r){let r=t.withRef[n],s=e.callbacks.getLocalKeyForOsm(r),c=new Set(r.sectors.map(e=>i[e]?.[s]).filter(Boolean));if(c.size>1&&console.warn(`${r.id} spans multiple sectors, and there are conflicting localKeys across the sectors. Therefore, it will be straight-up deleted`),c.size===1){let e=[...c][0];t.withRef[e]?o.push(n):a[e]=n}else o.push(n)}return{refsThatChanged:a,toDelete:o}}const me=(e,t,n=5)=>`${e.toFixed(n)},${t.toFixed(n)}`,M=6371008.8;function N(e,t){let n=Math.max(0,t.minLat-e.maxLat,e.minLat-t.maxLat),r=Math.max(0,t.minLng-e.maxLng,e.minLng-t.maxLng);return Math.hypot(n,r)}function P(e){return Math.max(Math.abs(e.maxLat-e.minLat),Math.abs(e.maxLng-e.minLng))}const F=new Map,he=(e,t,n)=>{let r=e.callbacks.getLocalKeyForSource(t);return n.filter(t=>{let n=F.get(t);return n||(n=e.callbacks.getLocalKeyForOsm(t),F.set(t,n)),n===r}).map(e=>{let n=M*g(e.centroid,t.centroid);return Object.assign(e,{offset:n})}).filter(e=>e.offset<200).toSorted((e,t)=>e.offset-t.offset)};async function ge(e,t,n){let r=Object.keys(t).length,i=new Set(JSON.parse(await s.readFile(e.tempFileNames.ignore_list,`utf8`)));console.info(`processing deleted data...`),console.time(`matchDeletions`);let{refsThatChanged:a,toDelete:o}=pe(e,n,t);console.timeEnd(`matchDeletions`),console.info(`Processing data...`);let c=0;console.time(`matchLoop`);let l={1:[],2:[],3:{},4:[],5:o,6:[]},u=Object.values(n.noRef),d={};for(let e in n.semi){let t=e.split(`;`);for(let[n,r]of t.entries())d[r]||=[],d[r].push({value:e,isFirst:!n})}let f={};for(let t in n.semi){let r=n.semi[t],i=r.tags[e.config.merge.osm_key];i&&(f[i]=r)}for(let o in t){let s=o,p=a[s]||s;if(i.has(p)||p in a)continue;let m=n.withRef[p]||e.callbacks.getAltRefs?.(p).map(e=>n.withRef[e]).find(Boolean),h=t[s],g=n.duplicateRefs[p],_=f[p];if(d[p])if(d[p][0].isFirst)p=d[p][0].value;else continue;let v=n.semi[p];if(m&&v){let e=p.split(`;`);l[4].push({osm:[m.id,v.id],source:e})}else if(m)l[1].push({osm:m.id,source:s});else if(g)l[2].push({osm:g.map(e=>e.id),source:p});else if(_)l[1].push({osm:_.id,source:s});else if(v){let e=p.split(`;`);l[3][v.id]||=[],l[3][v.id]?.push(...e)}else{let t=he(e,h,u);t.length===1?l[1].push({source:p,osm:t[0].id}):l[6].push({source:p,osmCandidates:t.map(e=>e.id)})}c+=1,c%1e3||process.stdout.write(`${(c/r*100).toFixed(1)}% `)}return console.timeEnd(`matchLoop`),await s.writeFile(e.tempFileNames.matches,JSON.stringify(l,null,void 0)),{output:l}}const I=2e-4;function L({lat:e,lng:t}){let n=[[t,e+I],[t+I,e],[t,e-I],[t-I,e]];return n.push(n[0]),[n]}function _e({lat:e,lng:t}){let n=[[t+I/2,e+I/2],[t+I/2,e-I/2],[t-I/2,e-I/2],[t-I/2,e+I/2]];return n.push(n[0]),[n]}function R(e){let t={minLat:1/0,minLng:1/0,maxLat:-1/0,maxLng:-1/0};function n([e,n]){let r=+n,i=+e;r<t.minLat&&(t.minLat=r),i<t.minLng&&(t.minLng=i),r>t.maxLat&&(t.maxLat=r),i>t.maxLng&&(t.maxLng=i)}for(let t of e)switch(t.geometry.type){case`Point`:n(t.geometry.coordinates);break;case`MultiPoint`:case`LineString`:t.geometry.coordinates.forEach(n);break;case`MultiLineString`:case`Polygon`:for(let e of t.geometry.coordinates)e.forEach(n);break;case`MultiPolygon`:for(let e of t.geometry.coordinates)for(let t of e)t.forEach(n);break;default:}return t}function z({minLng:e,minLat:t,maxLng:n,maxLat:r}){return{type:`Polygon`,coordinates:[[[e,t],[n,t],[n,r],[e,r],[e,t]]]}}function B(e,t,n,r,i=0){if(!r.length)return{};let a=R(r),o=E(z(a),e.config.merge.sector_resolution),s=i>10;if(s&&console.info(`(!) Couldn't split ${t} (${r.length} left)`),r.length<=110||s)return{[t]:{features:r,bbox:a,sectorIds:o,...n}};if(a.maxLat-a.minLat>a.maxLng-a.minLng){let o=a.minLat+(a.maxLat-a.minLat)/2,s=[[],[]];for(let e of r)s[+(h(e.geometry)[1]<o)].push(e);let[c,l]=s;return{...B(e,`${t}^N`,n,c,i+1),...B(e,`${t}^S`,n,l,i+1)}}let c=a.minLng+(a.maxLng-a.minLng)/2,l=[[],[]];for(let e of r)l[+(h(e.geometry)[0]<c)].push(e);let[u,d]=l;return{...B(e,`${t}^E`,n,u,i+1),...B(e,`${t}^W`,n,d,i+1)}}const{sin:ve,cos:ye,floor:be,sqrt:xe,PI:Se}=Math;function Ce(e){let t=be(xe(e)),n=(e-t**2)*Se/(2*((t+1)**2-t**2-1))||0,[r,i]=[t*ye(n),t*ve(n)];return[+r.toFixed(2),+i.toFixed(2)]}function we(e){for(let t in e){let n=e[t],r={};for(let e of n.features)if(e.geometry.type===`Point`){let[t,n]=e.geometry.coordinates,i=me(n,t,4);r[i]||=[],r[i].push(e)}for(let e of Object.values(r))if(e.length>1)for(let t=0;t<e.length;t+=1){let[n,r]=Ce(t),[i,a]=e[t].geometry.coordinates;e[t].geometry={type:`Point`,coordinates:[i+4e-6*n,a+4e-6*r]}}}return e}function V(e){return v(`sha256`).update(e,`utf8`).digest(`hex`)}function Te(e){let t=0,n=0,r=0,i=0;for(let a of e)switch(a.properties.__action){case`delete`:n+=1;break;case`move`:r+=1;break;case`edit`:i+=1;break;default:t+=1}return{count:[t&&`add ${t}`,n&&`delete ${n}`,r&&`move ${r}`,i&&`edit ${i}`].filter(e=>!!e).join(`, `),totalCount:t+n+r+i}}const Ee={n:`node`,w:`way`,r:`relation`},H=e=>`https://osm.org/${Ee[e[0]]}/${e.slice(1)}`,U=e=>`[${e}](${H(e)})`;function W(e,t){let n=`# Matches – ${re[t]}\n\n`,r=e[t];if(Array.isArray(r))for(let e of r){if(typeof e==`string`){n+=`- ${e}\n`;continue}let t=`source`in e?Array.isArray(e.source)?e.source:[e.source]:[],r=`osm`in e?Array.isArray(e.osm)?e.osm:[e.osm]:`osmCandidates`in e?e.osmCandidates:[],i=`osmCandidates`in e?` or `:` + `;n+=`- ${t.join(` + `)||`?`} ⇄ ${r.map(U).join(i)||`?`}\n`}else for(let e in r){let t=r[e]||[];n+=`- ${t.join(` + `)||`?`} ⇄ ${U(e)}\n`}return n}function G(e,t){let n=`# Conflation – ${t}\n\n`;for(let r in e){let i=`\n## ${r}\n\n`;n+=i;for(let i in e[r]){let a=`\n### ${i}\n\n`;n+=a;for(let a of e[r][i].features){if((a.properties.__action||`create`)!==t)continue;let e=t===`create`?a.id:U(a.id),r=Object.entries(a.properties).filter(([e])=>!e.startsWith(`__`)).map(e=>e.join(`=`)).join(` + `)||`∅`;switch(t){case`create`:case`edit`:case`delete`:n+=`- ${e}\t${r}\n`;break;case`move`:{let[[e,t],[r,i]]=a.geometry.coordinates,o=M*g([e,t],[r,i]),s=`${H(a.id)}?mlat=${i}&mlon=${r}#map=13/${i}/${r}`;n+=`- [${a.id}](${s})\tneeds to move ${o|0}m to ${i},${r}\n`;break}default:throw TypeError(t)}}n.endsWith(a)&&(n=n.slice(0,-a.length))}n.endsWith(i)&&(n=n.slice(0,-i.length))}return n}function De(e,t){let n={"report-match-OneToOne.md":W(e,1),"report-match-OneToMany.md":W(e,2),"report-match-ManyToMany.md":W(e,4),"report-match-ManyToOne.md":W(e,3),"report-match-Delete.md":W(e,5),"report-match-Guess.md":W(e,6),"report-conflation-create.md":G(t,`create`),"report-conflation-edit.md":G(t,`edit`),"report-conflation-move.md":G(t,`move`),"report-conflation-delete.md":G(t,`delete`)};for(let e in n)n[e].length>5e6&&(n[e]=`_file too large_`);return n}function Oe(e){return`${e.replaceAll(/[^\dA-Za-zĀāĒēĪīŌōŪū]/g,``).slice(0,100)}_${V(e).slice(0,6)}`}async function ke(e,t,r,i){let a=e.config.metadata.git_repository.match(`https://github.com/([^/]+)/([^/]+)`);if(!a)throw Error(`git_repository is not a valid github url`);let o=e.config.output?.folder||n(process.cwd(),`output`),c=`datasets`,l=De(r,i),u=Object.entries(i).flatMap(([e,t])=>Object.entries(t).map(([t,n])=>{let r=[e,t].filter(Boolean).join(` - `);return{id:Oe(r),category:e,group:t,title:r,bbox:n.bbox,instructions:n.instructions,...Te(n.features)}})),d={fields:[],results:u.map(e=>({id:e.id,url:`https://${a[1]}.github.io/${a[2]}/${c}/${e.id}.osmPatch.geo.json`,name:e.title,title:e.title,totalCount:e.totalCount,source:``,snippet:e.count,extent:[[e.bbox.minLng,e.bbox.minLat],[e.bbox.maxLng,e.bbox.maxLat]],instructions:e.instructions,groupCategories:[e.category,`/Categories/Addresses`]})).toSorted((e,t)=>e.name.localeCompare(t.name))},f={type:`FeatureCollection`,features:u.map(({bbox:e,...t})=>({type:`Feature`,geometry:z(e),properties:t})).toSorted((e,t)=>e.properties.title.localeCompare(t.properties.title)),__reports:[...Object.keys(l),...e.config.output?.custom_report_file_names||[]]};await s.mkdir(n(o,c),{recursive:!0}),await s.writeFile(n(o,`index.json`),JSON.stringify(d,null,y?2:void 0)),await s.writeFile(n(o,`index.geo.json`),JSON.stringify(f,null,y?2:void 0)),await s.writeFile(n(o,`metrics.json`),JSON.stringify(t,null,2));for(let e in l)await s.writeFile(n(o,e),l[e]);for(let e of u){let t={type:`FeatureCollection`,...i[e.category][e.group]};await s.writeFile(n(o,c,`${e.id}.osmPatch.geo.json`),JSON.stringify(t,null,y?2:void 0))}}function K(e,t){let n={...e};for(let e in t)n[e]?n[e]!==t[e]&&(n[e]+=`;${t[e]}`):n[e]=t[e];return n}function q(e,t){return R([{type:`Feature`,geometry:z(e),properties:{}},{type:`Feature`,geometry:z(t),properties:{}}])}function Ae(e){let t=!0;for(;t;){t=!1;for(let n in e){let r=e[n];if(r.features.length>40)continue;let i=Object.entries(e).filter(([e,t])=>t.sectorIds.some(e=>r.sectorIds.includes(e))&&e!==n&&r.features.length+t.features.length<110).toSorted(([,e],[,t])=>N(r.bbox,e.bbox)-N(r.bbox,t.bbox)).find(([,e])=>{let t=P(q(r.bbox,e.bbox))<=4,n=P(r.bbox)>4&&P(e.bbox)>4;return t||n});if(!i)continue;let[a,o]=i;r.bbox=q(r.bbox,o.bbox),r.sectorIds=[...new Set([...r.sectorIds,...o.sectorIds])],r.changesetTags=K(r.changesetTags||{},o.changesetTags||{}),o.instructions&&(r.instructions||=``,r.instructions+=`\n\n${o.instructions}`,r.instructions.trim()),r.features.push(...o.features),delete e[n],delete e[a];let{name:s}=K({name:n},{name:a});e[s]=r,t=!0}}return e}async function J(e,t,n,r,i){let a=new Set(JSON.parse(await s.readFile(e.tempFileNames.ignore_list,`utf8`))),o=0,c=0,l=0,u=new Set(Object.values(n.semi));function d(e){e.flags&1&&o++,e.flags&2&&c++,e.flags&8&&l++}for(let e of Object.values(n.withRef))d(e);for(let e of Object.values(n.noRef))d(e);for(let e of u)d(e);for(let e of Object.values(n.duplicateRefs))for(let t of e)d(t);let f={config:e.config,warnings:e.warnings,countsByPhase:{init:{sourceDataset:Object.keys(t).length,ignored:a.size,osm:{withRef:Object.keys(n.withRef).length,noRef:Object.keys(n.noRef).length,semi:u.size,duplicateRefs:Object.values(n.duplicateRefs).flat().length,lastEditedByImporter:o,recentlyChanged:c,recentlyChecked:l}},matched:{1:r[1].length,2:r[2].length,3:Object.keys(r[3]).length,4:r[4].length,5:r[5].length,6:r[6].length},conflated:{create:0,edit:0,delete:0,perfect:0}}};for(let e of Object.values(i).flatMap(e=>Object.values(e)))for(let t of e.features)t.properties.__action===`edit`?f.countsByPhase.conflated.edit++:t.properties.__action===`delete`?f.countsByPhase.conflated.delete++:f.countsByPhase.conflated.create++;return f.countsByPhase.conflated.perfect=n.count-f.countsByPhase.conflated.edit-f.countsByPhase.conflated.delete,f}async function je(e){if(y)return;if(!process.env.GITHUB_ACTIONS){console.info(`Skipping run history update, because not running in GitHub Actions`);return}let t=await ee(`osm-conflation-engine`),n=e.config.merge.osm_key,r=await fetch(`${x}/api/run_history/${n}`,{method:`PUT`,headers:{Authorization:`Bearer ${t}`},body:JSON.stringify(e)});if(r.status!==200)throw Error(await r.text())}const Y=`invalid value returned by your callback function`;function Me(e){for(let t in e)if(!(t===`__action`&&e[t]===`edit`))return!1;return!0}function X(e){if(!e)return!1;if(typeof e!=`object`||typeof e.tags!=`object`)throw TypeError(Y);return!!e.geometry||!Me(e.tags)}function Z(e,t,n){if(!t&&!n)throw Error(`requires either osm or source`);t&&(e.tags.__action||=`edit`);let[r,i]=t?.centroid||n?.centroid;return{type:`Feature`,id:t?.id||n?.id,geometry:e.tags.__action===`delete`?{type:`Polygon`,coordinates:_e({lat:i,lng:r})}:e.geometry||(e.tags.__action===`edit`?{type:`Polygon`,coordinates:L({lat:i,lng:r})}:{type:`Point`,coordinates:[r,i]}),properties:e.tags}}async function Ne(e,t,n){let r={...n.noRef};for(let e of[`withRef`,`semi`,`duplicateRefs`])for(let t in n[e]){let i=n[e][t];if(Array.isArray(i))for(let e of i)r[e.id]=e;else r[i.id]=i}let i={};function a(t,n,r){if(!t)return;t.warnings?.length&&e.warnings.push(...t.warnings);let a=t.extraFeatures||t.createFeatures;a&&(i[n]||={},i[n][r]||=[],i[n][r].push(...a))}let o=JSON.parse(await s.readFile(e.tempFileNames.matches,`utf8`));for(let{source:n,osm:s}of o[1]){let o=r[s],c=t[n];if(o.flags&8)continue;let l=await e.callbacks.mergeOneToOne({osm:o,source:c});if(!l)continue;if(typeof l!=`object`)throw TypeError(Y);let u=l.category||``,d=l.group||o.sectors[0];a(l.extra,u,d),X(l.diff)&&(i[u]||={},i[u][d]||=[],i[u][d].push(Z(l.diff,o,c)))}for(let{source:n,osm:s}of o[2]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=t[n],l=await e.callbacks.mergeOneToMany?.({osm:o,source:c});if(!l)continue;if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(Y);let u=l.category||``;a(l.extra,u,l.group||o[0]?.sectors[0]||c.sectors[0]);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(Y);if(!X(r))continue;let a=l.group||n.sectors[0];i[u]||={},i[u][a]||=[],i[u][a].push(Z(r,n,c))}}for(let n in o[3]){let s=n,c=r[s],l=o[3][s].map(e=>t[e]);if(c.flags&8)continue;let u=await e.callbacks.mergeManyToOne?.({osm:c,source:l});if(!u)continue;if(typeof u!=`object`)throw TypeError(Y);let d=u.category||``,f=u.group||c.sectors[0];a(u.extra,d,f),X(u.diff)&&(i[d]||={},i[d][f]||=[],i[d][f].push(Z(u.diff,c)))}for(let{source:n,osm:s}of o[4]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=n.map(e=>t[e]),l=await e.callbacks.mergeManyToMany?.({osm:o,source:c});if(!l)continue;if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(Y);let u=l.category||``;a(l.extra,u,l.group||o[0]?.sectors[0]||c[0].sectors[0]);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(Y);if(!X(r))continue;let a=l.group||n.sectors[0];i[u]||={},i[u][a]||=[],i[u][a].push(Z(r,n))}}for(let t of o[5]){let r=n.withRef[t];if(r.flags&8)continue;let o=await e.callbacks.deleteFeature?.({osm:r});if(!o)continue;if(typeof o!=`object`)throw TypeError(Y);let s=o.category||``,c=o.group||r.sectors[0];a(o.extra,s,c),X(o.diff)&&(i[s]||={},i[s][c]||=[],i[s][c].push(Z(o.diff,r)))}for(let r of o[6]){let o=t[r.source],s=await e.callbacks.create({source:o,osmCandidates:r.osmCandidates.map(e=>n.noRef[e])});if(!s)continue;if(typeof s!=`object`)throw TypeError(Y);let c=s.selection?n.noRef[s.selection]:void 0,l=s.category||``,u=s.group||c?.sectors[0]||`unknown`;a(s.extra,l,u),X(s.diff)&&(i[l]||={},i[l][u]||=[],i[l][u].push(Z(s.diff,c,o)))}let c=await e.callbacks.addCustomLayers?.();for(let e in c)for(let t in c[e])a(c[e][t],e,t);let l={};for(let n in i){l[n]||={};for(let a in i[n]){let{changesetTags:o={},instructions:s}=await e.callbacks.getChangesetTags?.({category:n,group:a})||{};o.created_by||=`osm-conflation-engine`,o.import||=`yes`,o.source||=e.config.metadata.wiki_page;let c=i[n][a];e.callbacks.postprocessLayer&&await e.callbacks.postprocessLayer({category:n,group:a,osmData:r,sourceData:t,features:c});let u=B(e,a,{changesetTags:o,instructions:s},c);Object.assign(l[n],u)}we(l[n]),Ae(l[n])}let u=await J(e,t,n,o,l);return await ke(e,u,o,l),await je(u),u}async function Q(e){console.info(`Reading source data into memory...`);let t=0,n={},r=l({input:i(e.config.source_data.file),crlfDelay:1/0});for await(let i of r){let r=JSON.parse(i),a=r.properties[e.config.merge.dataset_column];n[a]=Object.assign(r,{sectors:E(r.geometry,e.config.merge.sector_resolution),centroid:r.geometry.type===`Point`?r.geometry.coordinates:h(r.geometry)}),t++}console.info(`\tread ${t} rows.`),console.info(`Reading osm data into memory (withRef)...`);let a=await w(e.tempFileNames.osm_processed_with_ref,t=>t.tags[e.config.merge.osm_key]);console.info(`Reading osm data into memory (noRef)...`);let o=await w(e.tempFileNames.osm_processed_no_ref,e=>e.id);console.info(`Reading osm data into memory (other)...`);let c={withRef:a,noRef:o,...JSON.parse(await s.readFile(e.tempFileNames.osm_processed_other,`utf8`))};return console.info(` read data.`),{sourceData:n,osmData:c}}function Pe(e){for(let t of e)if(!b.includes(t))throw Error(`‘${t}’ is not a valid step. The options are: ${b.join(`, `)}`)}async function $(e){let t=V(e.metadata.name).slice(0,12),i=y?`-mock`:``,a=n(r(),`oce-${i}`,t);return await s.mkdir(a,{recursive:!0}),{id:t,tempFolder:a,tempFileNames:Object.fromEntries(Object.entries(te).map(([e,t])=>[e.toLowerCase(),n(a,t)]))}}async function Fe(e,t,n={}){let r=JSON.parse(await s.readFile(new URL(`config.schema.json`,import.meta.url),`utf8`)),i=new c().compile(r);if(!i(e))throw AggregateError(i.errors.map(e=>TypeError(e.message+JSON.stringify(e,null,2),{cause:e})),`Config file does not conform to the schema`);let{id:a,tempFolder:o,tempFileNames:l}=await $(e),u={use_cache:!!n.use_cache,id:a,tempFolder:o,tempFileNames:l,config:e,callbacks:t,warnings:[]};console.info(`temp folder:`,o);let{steps:d=b}=n;Pe(d),d.includes(`download`)&&(console.time(`download`),await fe(u),console.timeEnd(`download`));let f,p;d.includes(`match`)&&({sourceData:f,osmData:p}=await Q(u),console.time(`match`),await ge(u,f,p),console.timeEnd(`match`));let m;return d.includes(`conflate`)&&((!f||!p)&&({sourceData:f,osmData:p}=await Q(u)),console.time(`conflate`),m=await Ne(u,f,p),console.timeEnd(`conflate`)),console.info(`done.`),{conflate:m}}export{ne as OsmFlags,$ as getTempFileNames,w as readJsonL,Fe as run,T as writeJsonL};
|
|
10
|
+
`,a=t?await s.readFile(t,`utf8`):i;console.info(`fetching OSM data via postpass…`);let c=await fetch(n||`https://postpass.geofabrik.de/api/interpreter`,{method:`POST`,body:new URLSearchParams({data:a}),headers:S(e.config)});if(!c.ok){let e=await c.text()||c.statusText;throw Error(`HTTP Error ${c.status} ${e}`)}let l=await c.json(),{pickTags:u}=C(e),d={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0},f=new Set;for(let t of l.features){let{osm_type:n,osm_id:r,tags:i}=t.properties,a=`${n.toLowerCase()}${r}`;f.has(a)||(f.add(a),D(e,d,{id:a,centroid:t.geometry?.type?h(t.geometry):[0,0],tags:u(i),metadata:void 0,sectors:[],flags:0}))}await s.writeFile(r,JSON.stringify(d)),await O(e,d)}async function de(e){let t=e.config.merge.osm_key,n=e.config.e2e_tests?.ignore_list_file_path;if(y&&!n)throw Error(`When running in the unit test envrironment, you must supply config.e2e_tests.ignoreListFilePath`);let r=y&&n?JSON.parse(await s.readFile(n,`utf8`)):await fetch(`${x}/api/ignore_list/${encodeURIComponent(t)}?return_ids_only=true`).then(e=>e.json());await s.writeFile(e.tempFileNames.ignore_list,JSON.stringify(r.result,null,2))}async function fe(e,t){if(e.use_cache&&o(e.tempFileNames.osm_processed_other)){console.info(`Using cached osm data`);return}let{pickTags:n}=C(e);console.info(`Starting preprocess of OSM data...`),await O(e,await new Promise((r,i)=>{let a={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0},o=!1,s=!1,c=0;_.createReadStream({file:e.tempFileNames.pbf,tags:t,leveldb:`/tmp`,metadata:!0}).on(`data`,t=>{s||=!0;let r=t.type===`node`?t:t.centroid,i=t.meta;i&&(o=!0);let l=n(t.tags);D(e,a,{id:`${t.type[0]}${t.id}`,centroid:[+r.lon,+r.lat],metadata:i?{changeset:i.changeset,timestamp:i.timestamp?new Date(1e3*i.timestamp).toISOString():void 0,user:i.user,version:i.version}:void 0,tags:l,sectors:[],flags:0}),c+=1,c%1e3||process.stdout.write(`.`)}).on(`finish`,()=>{if(o||e.warnings.push(`No metadata extracted from the planet file!`),!s)throw Error(`Nothing extracted from the pbf file, presumably it’s invalid.`);r(a)}).on(`error`,i)}))}async function pe(e){if(await de(e),e.use_cache&&o(e.tempFileNames.osm_processed_other)){console.info(`Using cached OSM data`);return}switch(e.config.osm_data.source.type){case`pbf`:await le(e,e.config.osm_data.source.pbf_url),await fe(e,e.config.osm_data.source.pbf_filter);break;case`overpass`:await se(e,e.config.osm_data.source.overpass_query_file,e.config.osm_data.source.overpass_server_url);break;case`postpass`:await ue(e,e.config.osm_data.source.postpass_query_file,e.config.osm_data.source.postpass_server_url);break;default:throw TypeError(e.config.osm_data.source)}}function me(e,t,n){let r=[];for(let i in t.withRef){let a=i,o=t.withRef[a];if(!(a in n)&&!(o.flags&8)){if(e.callbacks.getAltRefs?.(a).some(e=>e in n))continue;r.push(a)}}let i={};for(let t in n){let r=t,a=n[r];for(let t of a.sectors){i[t]||={};let n=e.callbacks.getLocalKeyForSource(a);i[t][n]=r}}let a={},o=[];for(let n of r){let r=t.withRef[n],s=e.callbacks.getLocalKeyForOsm(r),c=new Set(r.sectors.map(e=>i[e]?.[s]).filter(Boolean));if(c.size>1&&console.warn(`${r.id} spans multiple sectors, and there are conflicting localKeys across the sectors. Therefore, it will be straight-up deleted`),c.size===1){let e=[...c][0];t.withRef[e]?o.push(n):a[e]=n}else o.push(n)}return{refsThatChanged:a,toDelete:o}}const he=(e,t,n=5)=>`${e.toFixed(n)},${t.toFixed(n)}`,M=6371008.8;function N(e,t){let n=Math.max(0,t.minLat-e.maxLat,e.minLat-t.maxLat),r=Math.max(0,t.minLng-e.maxLng,e.minLng-t.maxLng);return Math.hypot(n,r)}function P(e){return Math.max(Math.abs(e.maxLat-e.minLat),Math.abs(e.maxLng-e.minLng))}const F=new Map,ge=(e,t,n)=>{let r=e.callbacks.getLocalKeyForSource(t);return n.filter(t=>{let n=F.get(t);return n||(n=e.callbacks.getLocalKeyForOsm(t),F.set(t,n)),n===r}).map(e=>{let n=M*g(e.centroid,t.centroid);return Object.assign(e,{offset:n})}).filter(e=>e.offset<200).toSorted((e,t)=>e.offset-t.offset)};async function _e(e,t,n){let r=Object.keys(t).length,i=new Set(JSON.parse(await s.readFile(e.tempFileNames.ignore_list,`utf8`)));console.info(`processing deleted data...`),console.time(`matchDeletions`);let{refsThatChanged:a,toDelete:o}=me(e,n,t);console.timeEnd(`matchDeletions`),console.info(`Processing data...`);let c=0;console.time(`matchLoop`);let l={1:[],2:[],3:{},4:[],5:o,6:[]},u=Object.values(n.noRef),d={};for(let e in n.semi){let t=e.split(`;`);for(let[n,r]of t.entries())d[r]||=[],d[r].push({value:e,isFirst:!n})}let f={};for(let t in n.semi){let r=n.semi[t],i=r.tags[e.config.merge.osm_key];i&&(f[i]=r)}for(let o in t){let s=o,p=a[s]||s;if(i.has(p)||p in a)continue;let m=n.withRef[p]||e.callbacks.getAltRefs?.(p).map(e=>n.withRef[e]).find(Boolean),h=t[s],g=n.duplicateRefs[p],_=f[p];if(d[p])if(d[p][0].isFirst)p=d[p][0].value;else continue;let v=n.semi[p];if(m&&v){let e=p.split(`;`);l[4].push({osm:[m.id,v.id],source:e})}else if(m)l[1].push({osm:m.id,source:s});else if(g)l[2].push({osm:g.map(e=>e.id),source:p});else if(_)l[1].push({osm:_.id,source:s});else if(v){let e=p.split(`;`);l[3][v.id]||=[],l[3][v.id]?.push(...e)}else{let t=ge(e,h,u);t.length===1?l[1].push({source:p,osm:t[0].id}):l[6].push({source:p,osmCandidates:t.map(e=>e.id)})}c+=1,c%1e3||process.stdout.write(`${(c/r*100).toFixed(1)}% `)}return console.timeEnd(`matchLoop`),await s.writeFile(e.tempFileNames.matches,JSON.stringify(l,null,void 0)),{output:l}}const I=2e-4;function ve({lat:e,lng:t}){let n=[[t,e+I],[t+I,e],[t,e-I],[t-I,e]];return n.push(n[0]),[n]}function ye({lat:e,lng:t}){let n=[[t+I/2,e+I/2],[t+I/2,e-I/2],[t-I/2,e-I/2],[t-I/2,e+I/2]];return n.push(n[0]),[n]}function L(e){let t={minLat:1/0,minLng:1/0,maxLat:-1/0,maxLng:-1/0};function n([e,n]){let r=+n,i=+e;r<t.minLat&&(t.minLat=r),i<t.minLng&&(t.minLng=i),r>t.maxLat&&(t.maxLat=r),i>t.maxLng&&(t.maxLng=i)}for(let t of e)switch(t.geometry.type){case`Point`:n(t.geometry.coordinates);break;case`MultiPoint`:case`LineString`:t.geometry.coordinates.forEach(n);break;case`MultiLineString`:case`Polygon`:for(let e of t.geometry.coordinates)e.forEach(n);break;case`MultiPolygon`:for(let e of t.geometry.coordinates)for(let t of e)t.forEach(n);break;default:}return t}function R({minLng:e,minLat:t,maxLng:n,maxLat:r}){return{type:`Polygon`,coordinates:[[[e,t],[n,t],[n,r],[e,r],[e,t]]]}}function z(e,t,n,r,i=0){if(!r.length)return{};let a=L(r),o=E(R(a),e.config.merge.sector_resolution),s=i>10;if(s&&console.info(`(!) Couldn't split ${t} (${r.length} left)`),r.length<=110||s)return{[t]:{features:r,bbox:a,sectorIds:o,...n}};if(a.maxLat-a.minLat>a.maxLng-a.minLng){let o=a.minLat+(a.maxLat-a.minLat)/2,s=[[],[]];for(let e of r)s[+(h(e.geometry)[1]<o)].push(e);let[c,l]=s;return{...z(e,`${t}^N`,n,c,i+1),...z(e,`${t}^S`,n,l,i+1)}}let c=a.minLng+(a.maxLng-a.minLng)/2,l=[[],[]];for(let e of r)l[+(h(e.geometry)[0]<c)].push(e);let[u,d]=l;return{...z(e,`${t}^E`,n,u,i+1),...z(e,`${t}^W`,n,d,i+1)}}function be(e){let t={},n={};for(let r in e){let i=r.replace(/^([^^]+)(\^\w)+$/,(e,n)=>(t[n]||=0,t[n]+=1,`${n} ${t[n]}`));n[i]=e[r]}return n}const{sin:xe,cos:Se,floor:Ce,sqrt:we,PI:Te}=Math;function Ee(e){let t=Ce(we(e)),n=(e-t**2)*Te/(2*((t+1)**2-t**2-1))||0,[r,i]=[t*Se(n),t*xe(n)];return[+r.toFixed(2),+i.toFixed(2)]}function De(e){for(let t in e){let n=e[t],r={};for(let e of n.features)if(e.geometry.type===`Point`){let[t,n]=e.geometry.coordinates,i=he(n,t,4);r[i]||=[],r[i].push(e)}for(let e of Object.values(r))if(e.length>1)for(let t=0;t<e.length;t+=1){let[n,r]=Ee(t),[i,a]=e[t].geometry.coordinates;e[t].geometry={type:`Point`,coordinates:[i+4e-6*n,a+4e-6*r]}}}return e}function B(e){return v(`sha256`).update(e,`utf8`).digest(`hex`)}function Oe(e){let t=0,n=0,r=0,i=0;for(let a of e)switch(a.properties.__action){case`delete`:n+=1;break;case`move`:r+=1;break;case`edit`:i+=1;break;default:t+=1}return{count:[t&&`add ${t}`,n&&`delete ${n}`,r&&`move ${r}`,i&&`edit ${i}`].filter(e=>!!e).join(`, `),totalCount:t+n+r+i}}const ke={n:`node`,w:`way`,r:`relation`},V=e=>`https://osm.org/${ke[e[0]]}/${e.slice(1)}`,H=e=>`[${e}](${V(e)})`;function U(e,t,n){function r(t){return e.formatterUrl?`[${t}](${e.formatterUrl.replaceAll(`$1`,t)})`:t}let i=`# Matches – ${ie[n]}\n\n`,a=t[n];if(Array.isArray(a))for(let e of a){if(typeof e==`string`){i+=`- ${e}\n`;continue}let t=`source`in e?Array.isArray(e.source)?e.source:[e.source]:[],n=`osm`in e?Array.isArray(e.osm)?e.osm:[e.osm]:`osmCandidates`in e?e.osmCandidates:[],a=`osmCandidates`in e?` or `:` + `;i+=`- ${t.map(r).join(` + `)||`?`} ⇄ ${n.map(H).join(a)||`?`}\n`}else for(let e in a){let t=a[e]||[];i+=`- ${t.map(r).join(` + `)||`?`} ⇄ ${H(e)}\n`}return i}function W(e,t){let n=`# Conflation – ${t}\n\n`;for(let r in e){let i=`\n## ${r}\n\n`;n+=i;for(let i in e[r]){let a=`\n### ${i}\n\n`;n+=a;for(let a of e[r][i].features){if((a.properties.__action||`create`)!==t)continue;let e=t===`create`?a.id:H(a.id),r=Object.entries(a.properties).filter(([e])=>!e.startsWith(`__`)).map(e=>e.join(`=`)).join(` + `)||`∅`;switch(t){case`create`:case`edit`:case`delete`:n+=`- ${e}\t${r}\n`;break;case`move`:{let[[e,t],[r,i]]=a.geometry.coordinates,o=M*g([e,t],[r,i]),s=`${V(a.id)}?mlat=${i}&mlon=${r}#map=13/${i}/${r}`;n+=`- [${a.id}](${s})\tneeds to move ${o|0}m to ${i},${r}\n`;break}default:throw TypeError(t)}}n.endsWith(a)&&(n=n.slice(0,-a.length))}n.endsWith(i)&&(n=n.slice(0,-i.length))}return n}function Ae(e,t,n){let r={"report-match-OneToOne.md":U(e,t,1),"report-match-OneToMany.md":U(e,t,2),"report-match-ManyToMany.md":U(e,t,4),"report-match-ManyToOne.md":U(e,t,3),"report-match-Delete.md":U(e,t,5),"report-match-Guess.md":U(e,t,6),"report-conflation-create.md":W(n,`create`),"report-conflation-edit.md":W(n,`edit`),"report-conflation-move.md":W(n,`move`),"report-conflation-delete.md":W(n,`delete`)};for(let e in r)r[e].length>5e6&&(r[e]=`_file too large_`);return r}const G=`metrics-history.json`,K=()=>({lastUpdated:``,rows:[]});async function je(e,t){let n=e.config.merge.osm_key,r=y?K():await fetch(`${x}/api/static/${n}/${G}`,{headers:S(e.config)}).then(async e=>{if(!e.ok){let t=await e.text()||e.statusText,n=Error(`HTTP Error ${e.status} ${t}`);if(e.status===404)return console.error(ReferenceError(`Recursive History does not exist, presumably this is the first run. History will be reset`,{cause:n})),K();throw n}return await e.json()}),i=y?`MOCK`:new Date().toISOString().split(`T`,1)[0],a={date:i,conflated:t.countsByPhase.conflated},o=r.rows.at(-1);return JSON.stringify(o)===JSON.stringify(a)?console.info(`No change in statistics, not updating table`):r.rows.push(a),r.lastUpdated=i,r}function Me(e){return`${e.replaceAll(/[^\dA-Za-zĀāĒēĪīŌōŪū]/g,``).slice(0,100)}_${B(e).slice(0,6)}`}async function Ne(e,t,r,i){let a=e.config.output?.folder||n(process.cwd(),`output`),o=`datasets`,c=Ae(e,r,i),l=Object.entries(i).flatMap(([e,t])=>Object.entries(t).map(([t,n])=>{let r=[e,t].filter(Boolean).join(` - `);return{id:Me(r),category:e,group:t,title:r,bbox:n.bbox,instructions:n.instructions,...Oe(n.features)}})),u={type:`FeatureCollection`,features:l.map(({bbox:e,...t})=>({type:`Feature`,geometry:R(e),properties:t})).toSorted((e,t)=>e.properties.title.localeCompare(t.properties.title)),__reports:[...Object.keys(c),...e.config.output?.custom_report_file_names||[]]},d=await je(e,t);await s.mkdir(n(a,o),{recursive:!0}),await s.writeFile(n(a,`index.geo.json`),JSON.stringify(u,null,y?2:void 0)),await s.writeFile(n(a,`metrics.json`),JSON.stringify(t,null,2)),await s.writeFile(n(a,G),JSON.stringify(d,null,2));for(let e in c)await s.writeFile(n(a,e),c[e]);for(let e of l){let t={type:`FeatureCollection`,...i[e.category][e.group]};await s.writeFile(n(a,o,`${e.id}.osmPatch.geo.json`),JSON.stringify(t,null,y?2:void 0))}}function q(e,t){let n={...e};for(let e in t)n[e]?n[e]!==t[e]&&(n[e]+=`;${t[e]}`):n[e]=t[e];return n}function J(e,t){return L([{type:`Feature`,geometry:R(e),properties:{}},{type:`Feature`,geometry:R(t),properties:{}}])}function Pe(e){let t=!0;for(;t;){t=!1;for(let n in e){let r=e[n];if(r.features.length>40)continue;let i=Object.entries(e).filter(([e,t])=>t.sectorIds.some(e=>r.sectorIds.includes(e))&&e!==n&&r.features.length+t.features.length<110).toSorted(([,e],[,t])=>N(r.bbox,e.bbox)-N(r.bbox,t.bbox)).find(([,e])=>{let t=P(J(r.bbox,e.bbox))<=4,n=P(r.bbox)>4&&P(e.bbox)>4;return t||n});if(!i)continue;let[a,o]=i;r.bbox=J(r.bbox,o.bbox),r.sectorIds=[...new Set([...r.sectorIds,...o.sectorIds])],r.changesetTags=q(r.changesetTags||{},o.changesetTags||{}),o.instructions&&(r.instructions||=``,r.instructions+=`\n\n${o.instructions}`,r.instructions.trim()),r.features.push(...o.features),delete e[n],delete e[a];let{name:s}=q({name:n},{name:a});e[s]=r,t=!0}}return e}async function Fe(e,t,n,r,i){let a=new Set(JSON.parse(await s.readFile(e.tempFileNames.ignore_list,`utf8`))),o=0,c=0,l=0,u=new Set(Object.values(n.semi));function d(e){e.flags&1&&o++,e.flags&2&&c++,e.flags&8&&l++}for(let e of Object.values(n.withRef))d(e);for(let e of Object.values(n.noRef))d(e);for(let e of u)d(e);for(let e of Object.values(n.duplicateRefs))for(let t of e)d(t);let f={config:e.config,warnings:e.warnings,countsByPhase:{init:{sourceDataset:Object.keys(t).length,ignored:a.size,osm:{withRef:Object.keys(n.withRef).length,noRef:Object.keys(n.noRef).length,semi:u.size,duplicateRefs:Object.values(n.duplicateRefs).flat().length,lastEditedByImporter:o,recentlyChanged:c,recentlyChecked:l}},matched:{1:r[1].length,2:r[2].length,3:Object.keys(r[3]).length,4:r[4].length,5:r[5].length,6:r[6].length},conflated:{create:0,edit:0,delete:0,perfect:0}}};for(let e of Object.values(i).flatMap(e=>Object.values(e)))for(let t of e.features)t.properties.__action===`edit`?f.countsByPhase.conflated.edit++:t.properties.__action===`delete`?f.countsByPhase.conflated.delete++:f.countsByPhase.conflated.create++;return f.countsByPhase.conflated.perfect=n.count-f.countsByPhase.conflated.edit-f.countsByPhase.conflated.delete,f}async function Ie(e){if(y)return;if(!process.env.GITHUB_ACTIONS){console.info(`Skipping run history update, because not running in GitHub Actions`);return}let t=await ee(`osm-conflation-engine`),n=e.config.merge.osm_key,r=await fetch(`${x}/api/run_history/${n}`,{method:`PUT`,headers:{Authorization:`Bearer ${t}`},body:JSON.stringify(e)});if(r.status!==200)throw Error(await r.text())}const Y=`invalid value returned by your callback function`;function Le(e){for(let t in e)if(!(t===`__action`&&e[t]===`edit`))return!1;return!0}function X(e){if(!e)return!1;if(typeof e!=`object`||typeof e.tags!=`object`)throw TypeError(Y);return!!e.geometry||!Le(e.tags)}function Z(e,t,n){if(!t&&!n)throw Error(`requires either osm or source`);t&&(e.tags.__action||=`edit`);let[r,i]=t?.centroid||n?.centroid;return{type:`Feature`,id:t?.id||n?.id,geometry:e.tags.__action===`delete`?{type:`Polygon`,coordinates:ye({lat:i,lng:r})}:e.geometry||(e.tags.__action===`edit`?{type:`Polygon`,coordinates:ve({lat:i,lng:r})}:{type:`Point`,coordinates:[r,i]}),properties:e.tags}}async function Re(e,t,n){let r={...n.noRef};for(let e of[`withRef`,`semi`,`duplicateRefs`])for(let t in n[e]){let i=n[e][t];if(Array.isArray(i))for(let e of i)r[e.id]=e;else r[i.id]=i}let i={};function a(t,n,r){if(!t)return;t.warnings?.length&&e.warnings.push(...t.warnings);let a=t.extraFeatures||t.createFeatures;a&&(i[n]||={},i[n][r]||=[],i[n][r].push(...a))}let o=JSON.parse(await s.readFile(e.tempFileNames.matches,`utf8`));for(let{source:n,osm:s}of o[1]){let o=r[s],c=t[n];if(o.flags&8)continue;let l=await e.callbacks.mergeOneToOne({osm:o,source:c});if(!l)continue;if(typeof l!=`object`)throw TypeError(Y);let u=l.category||``,d=l.group||o.sectors[0];a(l.extra,u,d),X(l.diff)&&(i[u]||={},i[u][d]||=[],i[u][d].push(Z(l.diff,o,c)))}for(let{source:n,osm:s}of o[2]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=t[n],l=await e.callbacks.mergeOneToMany?.({osm:o,source:c});if(!l)continue;if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(Y);let u=l.category||``;a(l.extra,u,l.group||o[0]?.sectors[0]||c.sectors[0]);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(Y);if(!X(r))continue;let a=l.group||n.sectors[0];i[u]||={},i[u][a]||=[],i[u][a].push(Z(r,n,c))}}for(let n in o[3]){let s=n,c=r[s],l=o[3][s].map(e=>t[e]);if(c.flags&8)continue;let u=await e.callbacks.mergeManyToOne?.({osm:c,source:l});if(!u)continue;if(typeof u!=`object`)throw TypeError(Y);let d=u.category||``,f=u.group||c.sectors[0];a(u.extra,d,f),X(u.diff)&&(i[d]||={},i[d][f]||=[],i[d][f].push(Z(u.diff,c)))}for(let{source:n,osm:s}of o[4]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=n.map(e=>t[e]),l=await e.callbacks.mergeManyToMany?.({osm:o,source:c});if(!l)continue;if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(Y);let u=l.category||``;a(l.extra,u,l.group||o[0]?.sectors[0]||c[0].sectors[0]);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(Y);if(!X(r))continue;let a=l.group||n.sectors[0];i[u]||={},i[u][a]||=[],i[u][a].push(Z(r,n))}}for(let t of o[5]){let r=n.withRef[t];if(r.flags&8)continue;let o=await e.callbacks.deleteFeature?.({osm:r});if(!o)continue;if(typeof o!=`object`)throw TypeError(Y);let s=o.category||``,c=o.group||r.sectors[0];a(o.extra,s,c),X(o.diff)&&(i[s]||={},i[s][c]||=[],i[s][c].push(Z(o.diff,r)))}for(let r of o[6]){let o=t[r.source],s=await e.callbacks.create({source:o,osmCandidates:r.osmCandidates.map(e=>n.noRef[e])});if(!s)continue;if(typeof s!=`object`)throw TypeError(Y);let c=s.selection?n.noRef[s.selection]:void 0,l=s.category||``,u=s.group||c?.sectors[0]||`unknown`;a(s.extra,l,u),X(s.diff)&&(i[l]||={},i[l][u]||=[],i[l][u].push(Z(s.diff,c,o)))}let c=await e.callbacks.addCustomLayers?.();for(let e in c)for(let t in c[e])a(c[e][t],e,t);let l={};for(let n in i){l[n]||={};for(let a in i[n]){let{changesetTags:o={},instructions:s}=await e.callbacks.getChangesetTags?.({category:n,group:a})||{};o.created_by||=`osm-conflation-engine`,o.import||=`yes`,o.source||=e.config.metadata.wiki_page;let c=i[n][a];e.callbacks.postprocessLayer&&await e.callbacks.postprocessLayer({category:n,group:a,osmData:r,sourceData:t,features:c});let u=z(e,a,{changesetTags:o,instructions:s},c);u=be(u),Object.assign(l[n],u)}De(l[n]),Pe(l[n])}let u=await Fe(e,t,n,o,l);return await Ne(e,u,o,l),await Ie(u),u}async function Q(e){console.info(`Reading source data into memory...`);let t=0,n={},r=l({input:i(e.config.source_data.file),crlfDelay:1/0});for await(let i of r){let r=JSON.parse(i),a=r.properties[e.config.merge.dataset_column];n[a]=Object.assign(r,{sectors:E(r.geometry,e.config.merge.sector_resolution),centroid:r.geometry.type===`Point`?r.geometry.coordinates:h(r.geometry)}),t++}console.info(`\tread ${t} rows.`),console.info(`Reading osm data into memory (withRef)...`);let a=await w(e.tempFileNames.osm_processed_with_ref,t=>t.tags[e.config.merge.osm_key]);console.info(`Reading osm data into memory (noRef)...`);let o=await w(e.tempFileNames.osm_processed_no_ref,e=>e.id);console.info(`Reading osm data into memory (other)...`);let c={withRef:a,noRef:o,...JSON.parse(await s.readFile(e.tempFileNames.osm_processed_other,`utf8`))};return console.info(` read data.`),{sourceData:n,osmData:c}}const ze=[`deprecated`,`normal`,`preferred`];function Be(e){return te.filter(t=>t.key===`Key:${e}`).toSorted((e,t)=>ze.indexOf(t.rank)-ze.indexOf(e.rank))[0]?.url}function Ve(e){for(let t of e)if(!b.includes(t))throw Error(`‘${t}’ is not a valid step. The options are: ${b.join(`, `)}`)}async function $(e){let t=B(e.metadata.name).slice(0,12),i=y?`-mock`:``,a=n(r(),`oce-${i}`,t);return await s.mkdir(a,{recursive:!0}),{id:t,tempFolder:a,tempFileNames:Object.fromEntries(Object.entries(ne).map(([e,t])=>[e.toLowerCase(),n(a,t)]))}}async function He(e,t,n={}){let r=JSON.parse(await s.readFile(new URL(`config.schema.json`,import.meta.url),`utf8`)),i=new c().compile(r);if(!i(e))throw AggregateError(i.errors.map(e=>TypeError(e.message+JSON.stringify(e,null,2),{cause:e})),`Config file does not conform to the schema`);let{id:a,tempFolder:o,tempFileNames:l}=await $(e),u={use_cache:!!n.use_cache,id:a,tempFolder:o,tempFileNames:l,config:e,callbacks:t,formatterUrl:Be(e.merge.osm_key),warnings:[]};console.info(`temp folder:`,o);let{steps:d=b}=n;Ve(d),d.includes(`download`)&&(console.time(`download`),await pe(u),console.timeEnd(`download`));let f,p;d.includes(`match`)&&({sourceData:f,osmData:p}=await Q(u),console.time(`match`),await _e(u,f,p),console.timeEnd(`match`));let m;return d.includes(`conflate`)&&((!f||!p)&&({sourceData:f,osmData:p}=await Q(u)),console.time(`conflate`),m=await Re(u,f,p),console.timeEnd(`conflate`)),console.info(`done.`),{conflate:m}}export{re as OsmFlags,$ as getTempFileNames,w as readJsonL,He as run,T as writeJsonL};
|
|
11
11
|
//# sourceMappingURL=index.mjs.map
|