@osm-conflation-engine/cli 0.0.0 → 3.0.1

package/dist/config.schema.json

@@ -0,0 +1,236 @@
+{
+  "$ref": "#/definitions/Config",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "definitions": {
+    "Config": {
+      "additionalProperties": false,
+      "properties": {
+        "$schema": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "enum": [
+                "https://unpkg.com/@osm-conflation-engine/cli/dist/config.schema.json",
+                "node_modules/@osm-conflation-engine/cli/config.schema.json"
+              ],
+              "type": "string"
+            }
+          ]
+        },
+        "e2e_tests": {
+          "additionalProperties": false,
+          "properties": {
+            "ignore_list_file_path": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        },
+        "merge": {
+          "additionalProperties": false,
+          "properties": {
+            "dataset_column": {
+              "description": "the column in the dataset",
+              "type": "string"
+            },
+            "hash": {
+              "additionalProperties": false,
+              "properties": {
+                "columns": {
+                  "items": {
+                    "type": "string"
+                  },
+                  "type": "array"
+                },
+                "location": {
+                  "description": "if true, the coordinates will be part of the hash",
+                  "type": "boolean"
+                }
+              },
+              "required": [
+                "location",
+                "columns"
+              ],
+              "type": "object"
+            },
+            "osm_key": {
+              "description": "the key",
+              "type": "string"
+            },
+            "sector_resolution": {
+              "default": 3,
+              "description": "optional, passed to  {@link  https://h3geo.org h3 }",
+              "type": "number"
+            }
+          },
+          "required": [
+            "osm_key",
+            "dataset_column"
+          ],
+          "type": "object"
+        },
+        "metadata": {
+          "additionalProperties": false,
+          "properties": {
+            "description": {
+              "type": "string"
+            },
+            "git_repository": {
+              "description": "format should be `https://github.com/a/b`",
+              "type": "string"
+            },
+            "name": {
+              "type": "string"
+            },
+            "region": {
+              "type": "string"
+            },
+            "wiki_page": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "git_repository",
+            "name",
+            "description",
+            "wiki_page",
+            "region"
+          ],
+          "type": "object"
+        },
+        "o_data": {
+          "additionalProperties": false,
+          "properties": {
+            "check_date_key": {
+              "type": "string"
+            },
+            "source": {
+              "anyOf": [
+                {
+                  "additionalProperties": false,
+                  "properties": {
+                    "pbf_filter": {
+                      "items": {
+                        "type": "string"
+                      },
+                      "type": "array"
+                    },
+                    "pbf_url": {
+                      "type": "string"
+                    },
+                    "type": {
+                      "const": "pbf",
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "type",
+                    "pbf_url",
+                    "pbf_filter"
+                  ],
+                  "type": "object"
+                },
+                {
+                  "additionalProperties": false,
+                  "properties": {
+                    "overpass_query_file": {
+                      "type": "string"
+                    },
+                    "overpass_server_url": {
+                      "type": "string"
+                    },
+                    "type": {
+                      "const": "overpass",
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "type",
+                    "overpass_query_file"
+                  ],
+                  "type": "object"
+                },
+                {
+                  "additionalProperties": false,
+                  "properties": {
+                    "postpass_query_file": {
+                      "type": "string"
+                    },
+                    "postpass_server_url": {
+                      "type": "string"
+                    },
+                    "type": {
+                      "const": "postpass",
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "type",
+                    "postpass_query_file"
+                  ],
+                  "type": "object"
+                }
+              ],
+              "description": "if you choose `pbf`, you should also define an environment variable called `OSM_AUTH`, so that metadata can be considered by the algorithm."
+            },
+            "tags_to_keep": {
+              "description": "if the value starts and ends with `/`, then it's considered a regex",
+              "items": {
+                "type": "string"
+              },
+              "type": "array"
+            }
+          },
+          "required": [
+            "source",
+            "tags_to_keep"
+          ],
+          "type": "object"
+        },
+        "output": {
+          "additionalProperties": false,
+          "properties": {
+            "changeset_tags": {
+              "additionalProperties": {
+                "type": "string"
+              },
+              "type": "object"
+            },
+            "folder": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        },
+        "source_data": {
+          "additionalProperties": false,
+          "properties": {
+            "file": {
+              "type": "string"
+            },
+            "type": {
+              "const": "file",
+              "type": "string"
+            }
+          },
+          "required": [
+            "type",
+            "file"
+          ],
+          "type": "object"
+        }
+      },
+      "required": [
+        "$schema",
+        "metadata",
+        "source_data",
+        "o_data",
+        "merge"
+      ],
+      "type": "object"
+    }
+  }
+}
+

package/dist/index.cjs

@@ -0,0 +1,2 @@
+Object.defineProperty(exports,Symbol.toStringTag,{value:`Module`});var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},s=(n,r,a)=>(a=n==null?{}:e(i(n)),o(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n));let c=require("node:path"),l=require("node:os"),u=require("node:fs"),d=require("ajv"),f=require("node:timers/promises"),p=require("node:child_process"),m=require("pbf2json");m=s(m,1);let h=require("h3-js"),g=require("node:readline/promises"),_=require("d3-geo"),ee=require("node:crypto");const v=process.env.NODE_ENV===`test`,y=[`download`,`match`,`conflate`],te={PBF:`o.pbf`,OVERPASS:`o.geo.json`,POSTPASS:`o.geo.json`,OSM_PROCESSED_WITH_REF:`osm-processed-with-ref.jsonl`,OSM_PROCESSED_NO_REF:`osm-processed-no-ref.jsonl`,OSM_PROCESSED_OTHER:`osm-processed-other.json`,IGNORE_LIST:`ignore-list.json`,MATCHES:`matches.json`},ne=e=>({"User-Agent":`https://github.com/osm-nz/osm-conflation-engine on behalf of ${e.metadata.wiki_page}`}),b=`https://osm-internal.download.geofabrik.de`,x=`https://download.geofabrik.de`,S=`https://www.openstreetmap.org`,C=e=>e.match(/name="csrf-token" content="([^"]+)"/)[1];async function w(e,t){if(!t.startsWith(b)&&!t.startsWith(x)&&e.warnings.push(`(!) Planet PBF url is not from geofabrik.de, so metadata may not be available.`),!process.env.OSM_AUTH)return e.warnings.push(`(!) No OSM credentials, so the planet extract will not include metadata`),{url:t.replace(b,x),cookies:``};let n=new URLSearchParams(process.env.OSM_AUTH),r=ne(e.config),i=await fetch(`${b}/get_cookie?action=get_authorization_url`,{method:`POST`,headers:r}).then(e=>e.json()),a=await fetch(`${S}/login?cookie_test=true`,{headers:r}),o=a.headers.get(`Set-Cookie`),s=await fetch(`${S}/login`,{method:`POST`,body:new URLSearchParams({username:n.get(`username`),password:n.get(`password`),referer:`/`,commit:`Login`,authenticity_token:C(await a.text())}),headers:{...r,Cookie:o,"Content-Type":`application/x-www-form-urlencoded`}});if(s.status!==200)throw Error(`Login failed`);o+=`; ${s.headers.get(`Set-Cookie`)}`;let c=await fetch(i.authorization_url,{headers:{...r,Cookie:o}}),l=c.url.startsWith(S)?await fetch(i.authorization_url,{method:`POST`,body:new URLSearchParams({client_id:i.client_id,redirect_uri:i.redirect_uri,authenticity_token:C(await c.text()),state:i.state,response_type:`code`,scope:`read_prefs`,nonce:``,code_challenge:``,code_challenge_method:``,commit:`Authorize`}),headers:{...r,Cookie:o,"Content-Type":`application/x-www-form-urlencoded`}}).then(e=>e.text()):await c.text();if(!l.includes(`gf_download_oauth`))throw Error(`No token`);return{url:t.replace(x,b),cookies:l}}async function T(e,t){let n=e.tempFileNames.pbf;if(e.use_cache&&(0,u.existsSync)(n)){console.info(`Using cached planet file`);return}let{cookies:r,url:i}=await w(e,t);if(await u.promises.mkdir((0,c.dirname)(n),{recursive:!0}),await u.promises.rm(n,{recursive:!0,force:!0}),/^https?:\/\//.test(i)){console.info(`downloading from`,i,`to`,n);let e=(0,p.spawn)(`curl`,[`-L`,i,`-H`,`Cookie: ${r}`,`--output`,(0,c.basename)(n)],{cwd:(0,c.dirname)(n)});e.stdout.pipe(process.stdout),e.stderr.pipe(process.stderr),await new Promise((t,n)=>{e.on(`close`,e=>e?n(e):t())}),await(0,f.setTimeout)(5e3)}else console.info(`copying file from`,i,`to`,n),await u.promises.copyFile(i,n);console.info(n,await u.promises.stat(n))}async function E(e){let t=e.config.merge.osm_key,n=e.config.e2e_tests?.ignore_list_file_path;if(v&&!n)throw Error(`When running in the unit test envrironment, you must supply config.e2e_tests.ignoreListFilePath`);let r=v&&n?JSON.parse(await u.promises.readFile(n,`utf8`)):await fetch(`https://osm-conflation-engine.kyle.kiwi/api/ignore_list/${encodeURIComponent(t)}?return_ids_only=true`).then(e=>e.json());await u.promises.writeFile(e.tempFileNames.ignore_list,JSON.stringify(r.rows,null,2))}let re=function(e){return e[e.None=0]=`None`,e[e.IsLastEditedByImporter=1]=`IsLastEditedByImporter`,e[e.IsRecentlyChanged=2]=`IsRecentlyChanged`,e[e.IsChecked=4]=`IsChecked`,e[e.IsCheckedRecently=8]=`IsCheckedRecently`,e}({});const ie=e=>e?(Date.now()-+new Date(e))/1e3/60/60/24/365<4?1:2:0;function D(e,t){let n=t||3;switch(e.type){case`GeometryCollection`:throw Error(`not supported`);case`Point`:return[(0,h.latLngToCell)(e.coordinates[1],e.coordinates[0],n)];case`MultiPoint`:case`LineString`:case`MultiLineString`:case`Polygon`:return(0,h.polygonToCellsExperimental)(e.coordinates,n,h.POLYGON_TO_CELLS_FLAGS.containmentOverlapping,!0).toSorted();case`MultiPolygon`:{let t=new Set;for(let r of e.coordinates){let e=(0,h.polygonToCellsExperimental)(r,n,h.POLYGON_TO_CELLS_FLAGS.containmentOverlapping,!0);for(let n of e)t.add(n)}return[...t].toSorted()}default:throw TypeError(e)}}async function O(e,t){let n=(0,g.createInterface)({input:(0,u.createReadStream)(e)}),r={};for await(let e of n){let n=JSON.parse(e),i=t(n);r[i]=n}return r}async function k(e,t){let n=(0,u.createWriteStream)(e,{encoding:`utf8`});for(let e of t)n.write(JSON.stringify(e)),n.write(`
+`);n.end(),await new Promise(e=>n.on(`finish`,e))}const ae=(e=>(e.setDate(e.getDate()-90),e/1e3))(new Date);async function oe(e,t){if(e.use_cache&&(0,u.existsSync)(e.tempFileNames.osm_processed_other)){console.info(`Using cached osm data`);return}console.info(`Starting preprocess of OSM data...`);let{withRef:n,noRef:r,...i}=await new Promise((n,r)=>{let i={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0},a=0,o=!1,s=e.config.o_data.check_date_key||`check_date`,c=new Set([s,e.config.merge.osm_key]).difference(new Set(e.config.o_data.tags_to_keep));if(c.size)throw Error(`config.o_data.tags_to_keep is missing some tags that are referenced in other parts of the config: ${[...c].join(`, `)}`);let l=[];for(let t of e.config.o_data.tags_to_keep)t.startsWith(`/`)&&t.endsWith(`/`)&&l.push(new RegExp(t.slice(1,-1)));let u={};function d(t){if(t in u)return u[t];let n=e.config.o_data.tags_to_keep.includes(t);if(!n){for(let e of l)if(e.test(t)){n=!0;break}}return u[t]=n,n}m.default.createReadStream({file:e.tempFileNames.pbf,tags:t,leveldb:`/tmp`,metadata:!0}).on(`data`,t=>{let n=t.type===`node`?t:t.centroid,r=t.meta;r&&(o=!0);let c={};for(let e in t.tags){let n=t.tags[e];d(e)&&n&&(c[e]=n)}let l={id:`${t.type[0]}${t.id}`,centroid:[n.lon,n.lat],sectors:D({type:`Point`,coordinates:[n.lon,n.lat]},e.config.merge.sector_resolution),metadata:r?{changeset:r.changeset,timestamp:r.timestamp?new Date(1e3*r.timestamp).toISOString():void 0,user:r.user,version:r.version}:void 0,tags:c,flags:0};(r?.timestamp??0)>ae&&(l.flags|=2);let u=r?.user&&(e.callbacks.isImportUser?e.callbacks.isImportUser?.(r?.user):r?.user?.endsWith(`_import`));(r?.version===1||u)&&(l.flags|=1),c[s]&&(l.flags|=4),ie(c[s])===1&&(l.flags|=8);let f=t.tags[e.config.merge.osm_key];if(f)if(i.count++,i.duplicateRefs[f])i.duplicateRefs[f].push(l);else if(i.withRef[f])i.duplicateRefs[f]=[i.withRef[f],l],delete i.withRef[f];else if(f.includes(`;`)){let e=f.split(`;`);for(let t of e)i.semi[t]=l}else i.withRef[f]=l;else i.noRef[l.id]=l;a+=1,a%1e3||process.stdout.write(`.`)}).on(`finish`,()=>{o||e.warnings.push(`No metadata extracted from the planet file!`),n(i)}).on(`error`,r)});await k(e.tempFileNames.osm_processed_with_ref,Object.values(n)),await k(e.tempFileNames.osm_processed_no_ref,Object.values(r)),await u.promises.writeFile(e.tempFileNames.osm_processed_other,JSON.stringify(i))}async function se(e){switch(await E(e),e.config.o_data.source.type){case`pbf`:await T(e,e.config.o_data.source.pbf_url),await oe(e,e.config.o_data.source.pbf_filter);break;case`overpass`:throw Error(`not supported yet`);case`postpass`:throw Error(`not supported yet`);default:e.config.o_data.source}}function A(e,t,n){let r=[];for(let e in t.withRef){let i=e,a=t.withRef[i];!(i in n)&&!(a.flags&8)&&r.push(i)}let i={};for(let t in n){let r=t,a=n[r];for(let t of a.sectors){i[t]||={};let n=e.callbacks.getLocalKeyForSource(a);i[t][n]=r}}let a={},o=[];for(let n of r){let r=t.withRef[n],s=e.callbacks.getLocalKeyForOsm(r),c=new Set(r.sectors.map(e=>i[e]?.[s]).filter(Boolean));if(c.size>1&&console.warn(`${r.id} spans multiple sectors, and there are conflicting localKeys across the sectors. Therefore, it will be straight-up deleted`),c.size===1){let e=[...c][0];t.withRef[e]?o.push(n):a[e]=n}else o.push(n)}return{refsThatChanged:a,toDelete:o}}const j=(e,t,n=5)=>`${e.toFixed(n)},${t.toFixed(n)}`,M=new Map,N=(e,t,n)=>{let r=e.callbacks.getLocalKeyForSource(t);return n.filter(t=>{let n=M.get(t);return n||(n=e.callbacks.getLocalKeyForOsm(t),M.set(t,n)),n===r}).map(e=>{let n=6371008.8*(0,_.geoDistance)(e.centroid,t.centroid);return Object.assign(e,{offset:n})}).filter(e=>e.offset<200).toSorted((e,t)=>e.offset-t.offset)};async function P(e,t,n){let r=Object.keys(t).length,i=new Set(JSON.parse(await u.promises.readFile(e.tempFileNames.ignore_list,`utf8`)));console.info(`processing deleted data...`),console.time(`matchDeletions`);let{refsThatChanged:a,toDelete:o}=A(e,n,t);console.timeEnd(`matchDeletions`),console.info(`Processing data...`);let s=0;console.time(`match`);let c={1:[],2:[],3:{},4:[],5:o,6:[]},l=Object.values(n.noRef),d={};for(let e in n.semi){let t=e.split(`;`);for(let[n,r]of t.entries())d[r]||=[],d[r].push({value:e,isFirst:!n})}for(let o in t){let u=o,f=a[u]||u;if(i.has(f)||f in a)continue;let p=n.withRef[f],m=t[u],h=n.duplicateRefs[f];if(d[f])if(d[f][0].isFirst)f=d[f][0].value;else continue;let g=n.semi[f];if(p&&g){let e=f.split(`;`);c[4].push({osm:[p.id,g.id],source:e})}else if(p)c[1].push({osm:p.id,source:u});else if(h)c[2].push({osm:h.map(e=>e.id),source:f});else if(g){let e=f.split(`;`);c[3][g.id]||=[],c[3][g.id]?.push(...e)}else{let t=N(e,m,l);t.length===1?c[1].push({source:f,osm:t[0].id}):c[6].push({source:f,osmCandidates:t.map(e=>e.id)})}s+=1,s%1e3||process.stdout.write(`${(s/r*100).toFixed(1)}% `)}return console.timeEnd(`match`),await u.promises.writeFile(e.tempFileNames.matches,JSON.stringify(c,null,void 0)),{output:c}}const F=2e-4;function I({lat:e,lng:t}){let n=[[t,e+F],[t+F,e],[t,e-F],[t-F,e]];return n.push(n[0]),[n]}function L({lat:e,lng:t}){let n=[[t+F/2,e+F/2],[t+F/2,e-F/2],[t-F/2,e-F/2],[t-F/2,e+F/2]];return n.push(n[0]),[n]}function R(e){let t={minLat:1/0,minLng:1/0,maxLat:-1/0,maxLng:-1/0};function n([e,n]){let r=+n,i=+e;r<t.minLat&&(t.minLat=r),i<t.minLng&&(t.minLng=i),r>t.maxLat&&(t.maxLat=r),i>t.maxLng&&(t.maxLng=i)}for(let t of e)switch(t.geometry.type){case`Point`:n(t.geometry.coordinates);break;case`LineString`:t.geometry.coordinates.forEach(n);break;case`Polygon`:for(let e of t.geometry.coordinates)e.forEach(n);break;case`MultiPolygon`:for(let e of t.geometry.coordinates)for(let t of e)t.forEach(n);break;default:}return t}function z(e,t,n,r,i=0){if(!r.length)return{};let a=R(r),o=i>10;if(o&&console.info(`(!) Couldn't split ${e} (${r.length} left)`),r.length<=110||o)return{[e]:{features:r,bbox:a,instructions:t,changesetTags:n}};if(a.maxLat-a.minLat>a.maxLng-a.minLng){let o=a.minLat+(a.maxLat-a.minLat)/2,s=[[],[]];for(let e of r)s[+((0,_.geoCentroid)(e.geometry)[1]<o)].push(e);let[c,l]=s;return{...z(`${e}^N`,t,n,c,i+1),...z(`${e}^S`,t,n,l,i+1)}}let s=a.minLng+(a.maxLng-a.minLng)/2,c=[[],[]];for(let e of r)c[+((0,_.geoCentroid)(e.geometry)[0]<s)].push(e);let[l,u]=c;return{...z(`${e}^E`,t,n,l,i+1),...z(`${e}^W`,t,n,u,i+1)}}const{sin:B,cos:V,floor:H,sqrt:U,PI:W}=Math;function G(e){let t=H(U(e)),n=(e-t**2)*W/(2*((t+1)**2-t**2-1))||0,[r,i]=[t*V(n),t*B(n)];return[+r.toFixed(2),+i.toFixed(2)]}function ce(e){for(let t in e){let n=e[t],r={};for(let e of n.features)if(e.geometry.type===`Point`){let[t,n]=e.geometry.coordinates,i=j(n,t,4);r[i]||=[],r[i].push(e)}for(let e of Object.values(r))if(e.length>1)for(let t=0;t<e.length;t+=1){let[n,r]=G(t),[i,a]=e[t].geometry.coordinates;e[t].geometry={type:`Point`,coordinates:[i+4e-6*n,a+4e-6*r]}}}return e}function K(e){return(0,ee.createHash)(`sha256`).update(e,`utf8`).digest(`hex`)}function le(e){let t=0,n=0,r=0,i=0;for(let a of e)switch(a.properties.__action){case`delete`:n+=1;break;case`move`:r+=1;break;case`edit`:i+=1;break;default:t+=1}return{count:[t&&`add ${t}`,n&&`delete ${n}`,r&&`move ${r}`,i&&`edit ${i}`].filter(e=>!!e).join(`, `),totalCount:t+n+r+i}}function q(e){return`${e.replaceAll(/[^\dA-Za-zĀāĒēĪīŌōŪū]/g,``)}_${K(e).slice(0,6)}`}async function ue(e,t){let n=e.config.metadata.git_repository.match(`https://github.com/([^/]+)/([^/]+)`);if(!n)throw Error(`git_repository is not a valid github url`);let r=e.config.output?.folder||(0,c.join)(process.cwd(),`output`),i=`suburbs`,a=Object.entries(t).map(([e,t])=>({suburb:e,bbox:t.bbox,instructions:t.instructions,...le(t.features)})),o={fields:[],results:a.map(e=>{let t=e.suburb.replace(`ZZ `,``).replace(`Z `,``);return{id:q(e.suburb),url:`https://${n[1]}.github.io/${n[2]}/${i}/${q(e.suburb)}.osmPatch.geo.json`,name:t,title:t,totalCount:e.totalCount,source:``,snippet:e.count,extent:[[e.bbox.minLng,e.bbox.minLat],[e.bbox.maxLng,e.bbox.maxLat]],instructions:e.instructions,groupCategories:[e.suburb.startsWith(`ZZ`)?`/Categories/Preview`:`/Categories/Addresses`]}}).toSorted((e,t)=>e.name.localeCompare(t.name))};await u.promises.writeFile((0,c.join)(r,`index.json`),JSON.stringify(o,null,v?2:void 0)),await u.promises.mkdir((0,c.join)(r,i),{recursive:!0});for(let e of a){let{suburb:n}=e,a={type:`FeatureCollection`,...t[n]};a.changesetTags||={attribution:`https://wiki.openstreetmap.org/wiki/Contributors#LINZ`,created_by:`LINZ Data Import 2.0.0`,locale:`en-NZ`,source:`https://wiki.osm.org/LINZ`,comment:n},await u.promises.writeFile((0,c.join)(r,i,`${q(n)}.osmPatch.geo.json`),JSON.stringify(a,null,v?2:void 0))}}const J=`invalid value returned by your callback function`;function Y(e){for(let t in e)if(!(t===`__action`&&e[t]===`edit`))return!1;return!0}function X(e){if(typeof e!=`object`||typeof e.tags!=`object`)throw TypeError(J);return!!e.geometry||!Y(e.tags)}function Z(e,t,n){if(!t&&!n)throw Error(`requires either o or s`);t&&(e.tags.__action||=`edit`);let[r,i]=t?.centroid||n?.centroid;return{type:`Feature`,id:t?.id||n?.id,geometry:e.tags.__action===`delete`?{type:`Polygon`,coordinates:L({lat:i,lng:r})}:e.geometry||(e.tags.__action===`edit`?{type:`Polygon`,coordinates:I({lat:i,lng:r})}:{type:`Point`,coordinates:[r,i]}),properties:e.tags}}async function de(e,t,n){let r={...n.noRef};for(let e of[`withRef`,`semi`,`duplicateRefs`])for(let t in n[e]){let i=n[e][t];if(Array.isArray(i))for(let e of i)r[e.id]=e;else r[i.id]=i}let i={};function a(t,n){t&&(t.warnings?.length&&e.warnings.push(...t.warnings),t.createFeatures&&(i[n]||=[],i[n].push(...t.createFeatures)))}let o=JSON.parse(await u.promises.readFile(e.tempFileNames.matches,`utf8`));for(let{source:n,osm:s}of o[1]){let o=r[s],c=t[n];if(o.flags&8)continue;let l=await e.callbacks.mergeOneToOne({osm:o,source:c});if(typeof l!=`object`)throw TypeError(J);if(!X(l.diff))continue;let u=l.group||o.sectors[0];a(l.extra,u),i[u]||=[],i[u].push(Z(l.diff,o,c))}for(let{source:n,osm:s}of o[2]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=t[n],l=await e.callbacks.mergeOneToMany?.({osm:o,source:c});if(l){if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(J);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(J);if(!X(r))continue;let s=l.group||n.sectors[0];a(l.extra,s),i[s]||=[],i[s].push(Z(r,n,c))}}}for(let n in o[3]){let s=n,c=r[s],l=o[3][s].map(e=>t[e]);if(c.flags&8)continue;let u=await e.callbacks.mergeManyToOne?.({osm:c,source:l});if(!u)continue;if(typeof u!=`object`||typeof u.diff!=`object`)throw TypeError(J);if(!X(u.diff))continue;let d=u.group||c.sectors[0];a(u.extra,d),i[d]||=[],i[d].push(Z(u.diff,c))}for(let{source:n,osm:s}of o[4]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=n.map(e=>t[e]),l=await e.callbacks.mergeManyToMany?.({osm:o,source:c});if(l){if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(J);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(J);if(!X(r))continue;let s=l.group||n.sectors[0];a(l.extra,s),i[s]||=[],i[s].push(Z(r,n))}}}for(let t of o[5]){let r=n.withRef[t];if(r.flags&8)continue;let o=await e.callbacks.deleteFeature?.({osm:r});if(!o)continue;if(typeof o!=`object`||typeof o.diff!=`object`)throw TypeError(J);if(!X(o.diff))continue;let s=o.group||r.sectors[0];a(o.extra,s),i[s]||=[],i[s].push(Z(o.diff,r))}for(let r of o[6]){let o=t[r.source],s=await e.callbacks.create({source:o,osmCandidates:r.osmCandidates.map(e=>n.noRef[e])});if(!s)continue;if(typeof s!=`object`||typeof s.diff!=`object`)throw TypeError(J);if(!X(s.diff))continue;let c=s.selection?n.noRef[s.selection]:void 0,l=s.group||c?.sectors[0]||`unknown`;a(s.extra,l),i[l]||=[],i[l].push(Z(s.diff,c,o))}let s={};for(let n in i){let{changesetTags:a={},instructions:o}=await e.callbacks.getChangesetTags?.(n)||{};a.created_by||=`osm-conflation-engine`,a.import||=`yes`,a.source||=e.config.metadata.wiki_page;let c=i[n];e.callbacks.postprocessLayer&&await e.callbacks.postprocessLayer({group:n,osmData:r,sourceData:t,features:c});let l=z(n,o,a,c);Object.assign(s,l)}ce(s),ue(e,s);let c={create:0,edit:0,delete:0,perfect:0};for(let e in s)for(let t of s[e].features)t.properties.__action===`edit`?c.edit++:t.properties.__action===`delete`?c.delete++:c.create++;return c.perfect=n.count-c.edit-c.delete,{counts:c}}async function Q(e){console.info(`Reading source data into memory...`);let t=0,n={},r=(0,g.createInterface)({input:(0,u.createReadStream)(e.config.source_data.file),crlfDelay:1/0});for await(let i of r){let r=JSON.parse(i),a=r.properties[e.config.merge.dataset_column];n[a]=Object.assign(r,{sectors:D(r.geometry,e.config.merge.sector_resolution),centroid:r.geometry.type===`Point`?r.geometry.coordinates:(0,_.geoCentroid)(r.geometry)}),t++}console.info(`\tread ${t} rows.`),console.info(`Reading osm data into memory (withRef)...`);let i=await O(e.tempFileNames.osm_processed_with_ref,t=>t.tags[e.config.merge.osm_key]);console.info(`Reading osm data into memory (noRef)...`);let a=await O(e.tempFileNames.osm_processed_no_ref,e=>e.id);console.info(`Reading osm data into memory (other)...`);let o={withRef:i,noRef:a,...JSON.parse(await u.promises.readFile(e.tempFileNames.osm_processed_other,`utf8`))};return console.info(`	read data.`),{sourceData:n,osmData:o}}function fe(e){for(let t of e)if(!y.includes(t))throw Error(`‘${t}’ is not a valid step. The options are: ${y.join(`, `)}`)}async function $(e){let t=K(e.metadata.name).slice(0,12),n=v?`-mock`:``,r=(0,c.join)((0,l.tmpdir)(),`oce-${n}`,t);return await u.promises.mkdir(r,{recursive:!0}),{id:t,tempFolder:r,tempFileNames:Object.fromEntries(Object.entries(te).map(([e,t])=>[e.toLowerCase(),(0,c.join)(r,t)]))}}async function pe(e,t,n={}){let r=JSON.parse(await u.promises.readFile(new URL(`config.schema.json`,require("url").pathToFileURL(__filename).href),`utf8`)),i=new d.Ajv().compile(r);if(!i(e))throw AggregateError(i.errors.map(e=>TypeError(e.message+JSON.stringify(e,null,2),{cause:e})),`Config file does not conform to the schema`);let{id:a,tempFolder:o,tempFileNames:s}=await $(e),c={use_cache:!!n.use_cache,id:a,tempFolder:o,tempFileNames:s,config:e,callbacks:t,warnings:[]};console.info(`temp folder:`,o);let{steps:l=y}=n;fe(l),l.includes(`download`)&&(console.time(`download`),await se(c),console.timeEnd(`download`));let f,p;l.includes(`match`)&&({sourceData:f,osmData:p}=await Q(c),console.time(`match`),await P(c,f,p),console.timeEnd(`match`));let m;return l.includes(`conflate`)&&({sourceData:f,osmData:p}=await Q(c),console.time(`conflate`),m=await de(c,f,p),console.timeEnd(`conflate`)),console.info(`done.`),{conflate:m}}exports.OsmFlags=re,exports.getTempFileNames=$,exports.readJsonL=O,exports.run=pe,exports.writeJsonL=k;