@osm-conflation-engine/cli 0.0.0 → 3.0.0

package/dist/index.d.mts

@@ -0,0 +1,436 @@
+import { OsmFeature as OsmFeature$1, OsmPatchFeature, Tags } from "osm-api";
+
+//#region ../../node_modules/.pnpm/@types+geojson@7946.0.16/node_modules/@types/geojson/index.d.ts
+/**
+ * The value values for the "type" property of GeoJSON Objects.
+ * https://tools.ietf.org/html/rfc7946#section-1.4
+ */
+type GeoJsonTypes = GeoJSON["type"];
+/**
+ * Bounding box
+ * https://tools.ietf.org/html/rfc7946#section-5
+ */
+type BBox = [number, number, number, number] | [number, number, number, number, number, number];
+/**
+ * A Position is an array of coordinates.
+ * https://tools.ietf.org/html/rfc7946#section-3.1.1
+ * Array should contain between two and three elements.
+ * The previous GeoJSON specification allowed more elements (e.g., which could be used to represent M values),
+ * but the current specification only allows X, Y, and (optionally) Z to be defined.
+ *
+ * Note: the type will not be narrowed down to `[number, number] | [number, number, number]` due to
+ * marginal benefits and the large impact of breaking change.
+ *
+ * See previous discussions on the type narrowing:
+ * - {@link https://github.com/DefinitelyTyped/DefinitelyTyped/pull/21590|Nov 2017}
+ * - {@link https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/67773|Dec 2023}
+ * - {@link https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/71441| Dec 2024}
+ *
+ * One can use a
+ * {@link https://www.typescriptlang.org/docs/handbook/2/narrowing.html#using-type-predicates|user-defined type guard that returns a type predicate}
+ * to determine if a position is a 2D or 3D position.
+ *
+ * @example
+ * import type { Position } from 'geojson';
+ *
+ * type StrictPosition = [x: number, y: number] | [x: number, y: number, z: number]
+ *
+ * function isStrictPosition(position: Position): position is StrictPosition {
+ *   return position.length === 2 || position.length === 3
+ * };
+ *
+ * let position: Position = [-116.91, 45.54];
+ *
+ * let x: number;
+ * let y: number;
+ * let z: number | undefined;
+ *
+ * if (isStrictPosition(position)) {
+ *   // `tsc` would throw an error if we tried to destructure a fourth parameter
+ * 	 [x, y, z] = position;
+ * } else {
+ * 	 throw new TypeError("Position is not a 2D or 3D point");
+ * }
+ */
+type Position = number[];
+/**
+ * The base GeoJSON object.
+ * https://tools.ietf.org/html/rfc7946#section-3
+ * The GeoJSON specification also allows foreign members
+ * (https://tools.ietf.org/html/rfc7946#section-6.1)
+ * Developers should use "&" type in TypeScript or extend the interface
+ * to add these foreign members.
+ */
+interface GeoJsonObject {
+  // Don't include foreign members directly into this type def.
+  // in order to preserve type safety.
+  // [key: string]: any;
+  /**
+   * Specifies the type of GeoJSON object.
+   */
+  type: GeoJsonTypes;
+  /**
+   * Bounding box of the coordinate range of the object's Geometries, Features, or Feature Collections.
+   * The value of the bbox member is an array of length 2*n where n is the number of dimensions
+   * represented in the contained geometries, with all axes of the most southwesterly point
+   * followed by all axes of the more northeasterly point.
+   * The axes order of a bbox follows the axes order of geometries.
+   * https://tools.ietf.org/html/rfc7946#section-5
+   */
+  bbox?: BBox | undefined;
+}
+/**
+ * Union of GeoJSON objects.
+ */
+type GeoJSON<G extends Geometry | null = Geometry, P = GeoJsonProperties> = G | Feature<G, P> | FeatureCollection<G, P>;
+/**
+ * Geometry object.
+ * https://tools.ietf.org/html/rfc7946#section-3
+ */
+type Geometry = Point | MultiPoint | LineString | MultiLineString | Polygon | MultiPolygon | GeometryCollection;
+/**
+ * Point geometry object.
+ * https://tools.ietf.org/html/rfc7946#section-3.1.2
+ */
+interface Point extends GeoJsonObject {
+  type: "Point";
+  coordinates: Position;
+}
+/**
+ * MultiPoint geometry object.
+ *  https://tools.ietf.org/html/rfc7946#section-3.1.3
+ */
+interface MultiPoint extends GeoJsonObject {
+  type: "MultiPoint";
+  coordinates: Position[];
+}
+/**
+ * LineString geometry object.
+ * https://tools.ietf.org/html/rfc7946#section-3.1.4
+ */
+interface LineString extends GeoJsonObject {
+  type: "LineString";
+  coordinates: Position[];
+}
+/**
+ * MultiLineString geometry object.
+ * https://tools.ietf.org/html/rfc7946#section-3.1.5
+ */
+interface MultiLineString extends GeoJsonObject {
+  type: "MultiLineString";
+  coordinates: Position[][];
+}
+/**
+ * Polygon geometry object.
+ * https://tools.ietf.org/html/rfc7946#section-3.1.6
+ */
+interface Polygon extends GeoJsonObject {
+  type: "Polygon";
+  coordinates: Position[][];
+}
+/**
+ * MultiPolygon geometry object.
+ * https://tools.ietf.org/html/rfc7946#section-3.1.7
+ */
+interface MultiPolygon extends GeoJsonObject {
+  type: "MultiPolygon";
+  coordinates: Position[][][];
+}
+/**
+ * Geometry Collection
+ * https://tools.ietf.org/html/rfc7946#section-3.1.8
+ */
+interface GeometryCollection<G extends Geometry = Geometry> extends GeoJsonObject {
+  type: "GeometryCollection";
+  geometries: G[];
+}
+type GeoJsonProperties = {
+  [name: string]: any;
+} | null;
+/**
+ * A feature object which contains a geometry and associated properties.
+ * https://tools.ietf.org/html/rfc7946#section-3.2
+ */
+interface Feature<G extends Geometry | null = Geometry, P = GeoJsonProperties> extends GeoJsonObject {
+  type: "Feature";
+  /**
+   * The feature's geometry
+   */
+  geometry: G;
+  /**
+   * A value that uniquely identifies this feature in a
+   * https://tools.ietf.org/html/rfc7946#section-3.2.
+   */
+  id?: string | number | undefined;
+  /**
+   * Properties associated with this feature.
+   */
+  properties: P;
+}
+/**
+ * A collection of feature objects.
+ *  https://tools.ietf.org/html/rfc7946#section-3.3
+ */
+interface FeatureCollection<G extends Geometry | null = Geometry, P = GeoJsonProperties> extends GeoJsonObject {
+  type: "FeatureCollection";
+  features: Array<Feature<G, P>>;
+}
+//#endregion
+//#region src/types/config.def.d.ts
+interface Config {
+  $schema: (string & {}) | 'https://unpkg.com/@osm-conflation-engine/cli/dist/config.schema.json' | 'node_modules/@osm-conflation-engine/cli/config.schema.json';
+  metadata: {
+    /** format should be `https://github.com/a/b` */git_repository: string;
+    name: string;
+    description: string;
+    wiki_page: string;
+    region: string;
+  };
+  source_data: {
+    type: 'file';
+    file: string;
+  };
+  o_data: {
+    /**
+     * if you choose `pbf`, you should also define an environment
+     * variable called `OSM_AUTH`, so that metadata can be considered
+     * by the algorithm.
+     */
+    source: {
+      type: 'pbf';
+      pbf_url: string;
+      pbf_filter: string[];
+    } | {
+      type: 'overpass';
+      overpass_query_file: string;
+      overpass_server_url?: string;
+    } | {
+      type: 'postpass';
+      postpass_query_file: string;
+      postpass_server_url?: string;
+    }; /** if the value starts and ends with `/`, then it's considered a regex */
+    tags_to_keep: string[];
+    check_date_key?: string;
+  };
+  merge: {
+    /** the key */osm_key: string; /** the column in the dataset */
+    dataset_column: string; /** optional, passed to {@link https://h3geo.org h3} @default 3 */
+    sector_resolution?: number;
+    hash?: {
+      /** if true, the coordinates will be part of the hash */location: boolean;
+      columns: string[];
+    };
+  };
+  output?: {
+    folder?: string;
+    changeset_tags?: {
+      [key: string]: string;
+    };
+  };
+  e2e_tests?: {
+    ignore_list_file_path?: string;
+  };
+}
+//#endregion
+//#region src/constants/defaults.d.ts
+declare const STEPS: readonly ["download", "match", "conflate"];
+type Step = (typeof STEPS)[number];
+declare const FILE_NAMES: {
+  readonly PBF: "o.pbf";
+  readonly OVERPASS: "o.geo.json";
+  readonly POSTPASS: "o.geo.json";
+  readonly OSM_PROCESSED_WITH_REF: "osm-processed-with-ref.jsonl";
+  readonly OSM_PROCESSED_NO_REF: "osm-processed-no-ref.jsonl";
+  readonly OSM_PROCESSED_OTHER: "osm-processed-other.json";
+  readonly IGNORE_LIST: "ignore-list.json";
+  readonly MATCHES: "matches.json";
+};
+type FileNames = keyof typeof FILE_NAMES;
+//#endregion
+//#region src/types/internal.def.d.ts
+declare global {
+  namespace NodeJS {
+    interface ProcessEnv {
+      NODE_ENV?: 'test';
+      OSM_AUTH?: string;
+    }
+  }
+}
+interface Ctx<G extends Geometry = Geometry, P extends GeoJsonProperties = GeoJsonProperties> {
+  use_cache: boolean;
+  id: string;
+  tempFolder: string;
+  tempFileNames: Record<Lowercase<FileNames>, string>;
+  config: Config;
+  callbacks: Callbacks<G, P>;
+  warnings: string[];
+}
+interface SourceData<G extends Geometry = Geometry, P extends GeoJsonProperties = GeoJsonProperties> {
+  [datasetId: DatasetId]: SourceDataFeature<G, P>;
+}
+//#endregion
+//#region src/types/callbacks.def.d.ts
+type TagDiff = OsmPatchFeature['properties'];
+/** see microsoft/TypeScript#202 */
+type Identity<out T> = {
+  _: T;
+  readonly __: unique symbol;
+};
+type OsmFeatureTypeShort = 'n' | 'w' | 'r';
+type OsmId = `${OsmFeatureTypeShort}${number}`;
+type DatasetId = string & Identity<'DatasetId'>;
+type Vec2 = [lon: number, lat: number];
+declare enum OsmFlags {
+  None = 0,
+  IsLastEditedByImporter = 1,
+  IsRecentlyChanged = 2,
+  IsChecked = 4,
+  IsCheckedRecently = 8
+}
+interface OsmFeature {
+  id: OsmId;
+  sectors: string[];
+  centroid: Vec2;
+  tags: Tags;
+  metadata?: Partial<Pick<OsmFeature$1, 'changeset' | 'version' | 'user' | 'timestamp'>>;
+  flags: number & OsmFlags;
+}
+interface SourceDataFeature<G extends Geometry = Geometry, P extends GeoJsonProperties = GeoJsonProperties> extends Feature<G, P> {
+  /** list of sector IDs (h3 hexes) that this feature passes through */
+  sectors: string[];
+  centroid: Vec2;
+}
+type MaybePromise<T> = T | Promise<T>;
+interface ConflationResultExtra {
+  createFeatures?: OsmPatchFeature[];
+  warnings?: string[];
+}
+interface ConflationDiff {
+  tags: TagDiff;
+  geometry?: Geometry;
+}
+interface SingleFeatureConflationResult {
+  group?: string;
+  diff: ConflationDiff;
+  extra?: ConflationResultExtra;
+}
+interface MutliFeatureConflationResult {
+  group?: string;
+  diffPerFeature: {
+    [id: OsmId]: ConflationDiff;
+  };
+  extra?: ConflationResultExtra;
+}
+interface Callbacks<G extends Geometry, P extends GeoJsonProperties> {
+  /**
+   * called for every OSM feature, you should return a key which is unique
+   * within its sector. For example, this could be the most important tags
+   * concatenated together, if those tags are usually unique within the
+   * local area (the sector).
+   */
+  getLocalKeyForOsm(osmFeature: OsmFeature): string;
+  /**
+   * see docs for {@link getLocalKeyForOsm}. This is the equivilant callback
+   * for the source data, which is called for every feature.
+   */
+  getLocalKeyForSource(sourceFeature: SourceDataFeature<G, P>): string;
+  /**
+   * optional, by default a username which ends in `_import` is considered
+   * to be an importer. Use this callback if you want to customise the
+   * behaviour.
+   */
+  isImportUser?(username: string): boolean;
+  mergeOneToOne(input: {
+    source: SourceDataFeature<G, P>;
+    osm: OsmFeature;
+  }): MaybePromise<SingleFeatureConflationResult | undefined>;
+  mergeOneToMany?(input: {
+    source: SourceDataFeature<G, P>;
+    osm: OsmFeature[];
+  }): MaybePromise<MutliFeatureConflationResult | undefined>;
+  mergeManyToOne?(input: {
+    source: SourceDataFeature<G, P>[];
+    osm: OsmFeature;
+  }): MaybePromise<SingleFeatureConflationResult | undefined>;
+  mergeManyToMany?(input: {
+    source: SourceDataFeature<G, P>[];
+    osm: OsmFeature[];
+  }): MaybePromise<MutliFeatureConflationResult | undefined>;
+  /**
+   * called for OSM features which have a ref tag that does not exist in the
+   * source dataset. You need to decide whether to:
+   * (1) delete this feature; or
+   * (2) only remove the import related tags; or
+   * (3) do nothing.
+   */
+  deleteFeature?(input: {
+    osm: OsmFeature;
+  }): MaybePromise<SingleFeatureConflationResult | undefined>;
+  /**
+   * Calls for each feature in the source dataset where a matching feature
+   * could not be found in OSM. If there are potential matches, these are
+   * supplied using the `osmCandidates` argument. If you choose to accept
+   * one of these candidates, return the `selection` property set to that
+   * feature's {@link OsmId}. If not, return `selection: undefined`, which
+   * will create a new OSM feature instead.
+   */
+  create(input: {
+    source: SourceDataFeature<G, P>;
+    osmCandidates: OsmFeature[];
+  }): MaybePromise<(SingleFeatureConflationResult & {
+    /**
+     * undefined means that no selection was made, so we
+     * will create a new OSM feature instead
+     */
+    selection: OsmId | undefined;
+  }) | undefined>;
+  /**
+   * optional, called after the conflation stage once all diffs have been
+   * allocated to a group. You can apply any addition transformations to
+   * the features in each layer.
+   */
+  postprocessLayer?(input: {
+    group: string;
+    features: OsmPatchFeature[];
+    osmData: Record<OsmId, OsmFeature>;
+    sourceData: SourceData<G, P>;
+  }): MaybePromise<OsmPatchFeature[] | undefined>;
+  /**
+   * optional, called after the conflation stage for every group. You can
+   * return the changeset tags which will be applied to this group, and
+   * optionally `instructions` which are shown in the website UI, before
+   * someone starts working on this group.
+   */
+  getChangesetTags?(group: string): MaybePromise<{
+    instructions?: string;
+    changesetTags?: Tags;
+  }>;
+}
+interface RunOptions {
+  steps?: Step[];
+  use_cache?: boolean;
+}
+interface ConflateResult {
+  counts: {
+    create: number;
+    edit: number;
+    delete: number;
+    perfect: number;
+  };
+}
+interface RunResult {
+  conflate?: ConflateResult;
+}
+//#endregion
+//#region src/main.d.ts
+declare function getTempFileNames(config: Config): Promise<Pick<Ctx, 'id' | 'tempFolder' | 'tempFileNames'>>;
+declare function run<G extends Geometry, P extends GeoJsonProperties>(config: Config, callbacks: Callbacks<G, P>, runOptions?: RunOptions): Promise<RunResult>;
+//#endregion
+//#region src/common/jsonl.d.ts
+/** reads JSONL by parsing one line at a time */
+declare function readJsonL<T, U extends string>(filePath: string, getKey: (row: T) => U): Promise<Record<U, T>>;
+/** writes JSONL to a file by stringifying each line at a time */
+declare function writeJsonL(filePath: string, rows: unknown[]): Promise<void>;
+//#endregion
+export { Callbacks, Config, ConflateResult, ConflationDiff, ConflationResultExtra, DatasetId, Identity, MutliFeatureConflationResult, OsmFeature, OsmFeatureTypeShort, OsmFlags, OsmId, RunOptions, RunResult, SingleFeatureConflationResult, SourceDataFeature, TagDiff, Vec2, getTempFileNames, readJsonL, run, writeJsonL };
+//# sourceMappingURL=index.d.mts.map

package/dist/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","names":["Geometry","GeoJSON","type","GeoJsonTypes","bbox","BBox","G","P","GeoJsonProperties","Feature","FeatureCollection","Point","MultiPoint","LineString","MultiLineString","Polygon","MultiPolygon","GeometryCollection","GeoJsonObject","coordinates","Position","geometries","name","geometry","id","properties","features","Array"],"sources":["../../../node_modules/.pnpm/@types+geojson@7946.0.16/node_modules/@types/geojson/index.d.ts","../src/types/config.def.ts","../src/constants/defaults.ts","../src/types/internal.def.ts","../src/types/callbacks.def.ts","../src/main.ts","../src/common/jsonl.ts"],"x_google_ignoreList":[0],"mappings":";;;;AAqBgB;AA2ChB;;KAjDY,YAAA,GAAe,OAAO;AAiDd;AAUpB;;;AAVoB,KA3CR,IAAA;;;;;AAqEG;AAMf;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG4B;AAM5B;;;;;;;KAzCY,QAAA;;;;;;;;;UAUK,aAAA;EAAA;EAAA;EAAA;EAsCA;;;EA/BbE,IAAAA,EAAM,YAAA;EA+BqBgB;;;;;AAEN;AAOzB;;EA/BId,IAAAA,GAAO,IAAI;AAAA;;;;KAMH,OAAA,WAAkB,QAAA,UAAkB,QAAA,MAAc,iBAAA,IACxD,CAAA,GACA,OAAA,CAAQ,CAAA,EAAG,CAAA,IACX,iBAAA,CAAkB,CAAA,EAAG,CAAA;;;;;KAMf,QAAA,GAAW,KAAA,GAAQ,UAAA,GAAa,UAAA,GAAa,eAAA,GAAkB,OAAA,GAAU,YAAA,GAAe,kBAAA;AAkCpG;;;;AAAA,UA3BiB,KAAA,SAAc,aAAa;EACxCF,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;AA2BQ;AAOzB;;;AAPyB,UApBR,UAAA,SAAmB,aAAa;EAC7CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;AA2BQ;AAOzB;;UA3BiB,UAAA,SAAmB,aAAa;EAC7CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;AA2BQ;AAOzB;UA3BiB,eAAA,SAAwB,aAAa;EAClDjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;;;UAOA,OAAA,SAAgB,aAAa;EAC1CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;AAkBA;AAGjB;UAdiB,YAAA,SAAqB,aAAa;EAC/CjB,IAAAA;EACAiB,WAAAA,EAAa,QAAA;AAAA;;;;;UAOA,kBAAA,WAA6B,QAAA,GAAW,QAAA,UAAkB,aAAA;EACvEjB,IAAAA;EACAmB,UAAAA,EAAY,CAAA;AAAA;AAAA,KAGJ,iBAAA;EAAA,CAAuBC,IAAY;AAAA;;;;;UAM9B,OAAA,WAAkB,QAAA,UAAkB,QAAA,MAAc,iBAAA,UAA2B,aAAA;EAC1FpB,IAAAA;EAIAqB;;;EAAAA,QAAAA,EAAU,CAAA;EASEhB;;AAAC;AAOjB;EAXIiB,EAAAA;EAW8B;;;EAP9BC,UAAAA,EAAY,CAAA;AAAA;;;;;UAOC,iBAAA,WAA4B,QAAA,UAAkB,QAAA,MAAc,iBAAA,UAA2B,aAAA;EACpGvB,IAAAA;EACAwB,QAAAA,EAAU,KAAA,CAAM,OAAA,CAAQ,CAAA,EAAG,CAAA;AAAA;;;UCrMd,MAAA;EACf,OAAA;EAKA,QAAA;IDMU,gDCJR,cAAA;IACA,IAAA;IACA,WAAA;IACA,SAAA;IACA,MAAA;EAAA;EAGF,WAAA;IACE,IAAA;IACA,IAAA;EAAA;EAGF,MAAA;IDyCkB;;AAAA;AAUpB;;IC7CI,MAAA;MAEM,IAAA;MACA,OAAA;MACA,UAAA;IAAA;MAGA,IAAA;MACA,mBAAA;MACA,mBAAA;IAAA;MAGA,IAAA;MACA,mBAAA;MACA,mBAAA;IAAA,GDqDoD;ICjD1D,YAAA;IACA,cAAA;EAAA;EAGF,KAAA;IDgDsB,cC9CpB,OAAA,UD8CE;IC5CF,cAAA,UD4CmB;IC1CnB,iBAAA;IACA,IAAA;MDsC4C1B,wDCnC1C,QAAA;MACA,OAAA;IAAA;EAAA;EAIJ,MAAA;IACE,MAAA;IAEA,cAAA;MAAA,CACG,GAAA;IAAA;EAAA;EAIL,SAAA;IACE,qBAAA;EAAA;AAAA;;;cC3DS,KAAA;AAAA,KACD,IAAA,WAAe,KAAK;AAAA,cAEnB,UAAA;EAAA;;;;;;;;;KAiBD,SAAA,gBAAyB,UAAU;;;QCxBvC,MAAA;EAAA,UACI,MAAA;IAAA,UACE,UAAA;MACR,QAAA;MACA,QAAA;IAAA;EAAA;AAAA;AAAA,UAKW,GAAA,WACL,QAAA,GAAW,QAAA,YACX,iBAAA,GAAoB,iBAAA;EAE9B,SAAA;EACA,EAAA;EACA,UAAA;EACA,aAAA,EAAe,MAAA,CAAO,SAAA,CAAU,SAAA;EAChC,MAAA,EAAQ,MAAA;EACR,SAAA,EAAW,SAAA,CAAU,CAAA,EAAG,CAAA;EACxB,QAAA;AAAA;AAAA,UAGe,UAAA,WACL,QAAA,GAAW,QAAA,YACX,iBAAA,GAAoB,iBAAA;EAAA,CAE7B,SAAA,EAAW,SAAA,GAAY,iBAAA,CAAkB,CAAA,EAAG,CAAA;AAAA;;;KC7BnC,OAAA,GAAU,eAAe;;KAGzB,QAAA;EAAoB,CAAA,EAAG,CAAC;EAAA,SAAW,EAAA;AAAA;AAAA,KAEnC,mBAAA;AAAA,KACA,KAAA,MAAW,mBAAmB;AAAA,KAC9B,SAAA,YAAqB,QAAQ;AAAA,KAE7B,IAAA,IAAQ,GAAA,UAAa,GAAG;AAAA,aAExB,QAAA;EACV,IAAA;EACA,sBAAA;EACA,iBAAA;EACA,SAAA;EACA,iBAAA;AAAA;AAAA,UAGe,UAAA;EACf,EAAA,EAAI,KAAA;EACJ,OAAA;EACA,QAAA,EAAU,IAAA;EACV,IAAA,EAAM,IAAA;EACN,QAAA,GAAW,OAAA,CACT,IAAA,CAAK,YAAA;EAEP,KAAA,WAAgB,QAAA;AAAA;AAAA,UAGD,iBAAA,WACL,QAAA,GAAW,QAAA,YACX,iBAAA,GAAoB,iBAAA,UACtB,OAAA,CAAQ,CAAA,EAAG,CAAA;EJsDF;EIpDjB,OAAA;EACA,QAAA,EAAU,IAAA;AAAA;AAAA,KAGP,YAAA,MAAkB,CAAA,GAAI,OAAA,CAAQ,CAAA;AAAA,UAElB,qBAAA;EACf,cAAA,GAAiB,eAAe;EAChC,QAAA;AAAA;AAAA,UAGe,cAAA;EACf,IAAA,EAAM,OAAA;EACN,QAAA,GAAW,QAAQ;AAAA;AAAA,UAGJ,6BAAA;EACf,KAAA;EACA,IAAA,EAAM,cAAA;EACN,KAAA,GAAQ,qBAAqB;AAAA;AAAA,UAGd,4BAAA;EACf,KAAA;EACA,cAAA;IAAA,CAAmB,EAAA,EAAI,KAAA,GAAQ,cAAA;EAAA;EAC/B,KAAA,GAAQ,qBAAA;AAAA;AAAA,UAIO,SAAA,WAAoB,QAAA,YAAoB,iBAAA;EJ0B7B;AAM5B;;;;;EIzBE,iBAAA,CAAkB,UAAA,EAAY,UAAA;EJyByB;;;;EIpBvD,oBAAA,CAAqB,aAAA,EAAe,iBAAA,CAAkB,CAAA,EAAG,CAAA;EJoB2D;;;;;EIbpH,YAAA,EAAc,QAAA;EAGd,aAAA,CAAc,KAAA;IACZ,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,6BAAA;EAEjB,cAAA,EAAgB,KAAA;IACd,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,4BAAA;EAEjB,cAAA,EAAgB,KAAA;IACd,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,6BAAA;EAEjB,eAAA,EAAiB,KAAA;IACf,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,4BAAA;;;;;;;;EASjB,aAAA,EAAe,KAAA;IACb,GAAA,EAAK,UAAA;EAAA,IACH,YAAA,CAAa,6BAAA;;;;;;;;AJQM;EIEvB,MAAA,CAAO,KAAA;IACL,MAAA,EAAQ,iBAAA,CAAkB,CAAA,EAAG,CAAA;IAC7B,aAAA,EAAe,UAAA;EAAA,IACb,YAAA,EACC,6BAAA;IJCkCkB;;;;IIIjC,SAAA,EAAW,KAAA;EAAA;EJKF;;;;;EIKf,gBAAA,EAAkB,KAAA;IAChB,KAAA;IACA,QAAA,EAAU,eAAA;IACV,OAAA,EAAS,MAAA,CAAO,KAAA,EAAO,UAAA;IACvB,UAAA,EAAY,UAAA,CAAW,CAAA,EAAG,CAAA;EAAA,IACxB,YAAA,CAAa,eAAA;EJDU;;;;;;EIS3B,gBAAA,EAAkB,KAAA,WAAgB,YAAA;IAChC,YAAA;IACA,aAAA,GAAgB,IAAA;EAAA;AAAA;AAAA,UAGH,UAAA;EACf,KAAA,GAAQ,IAAI;EACZ,SAAA;AAAA;AAAA,UAGe,cAAA;EACf,MAAA;IACE,MAAA;IACA,IAAA;IACA,MAAA;IACA,OAAA;EAAA;AAAA;AAAA,UAIa,SAAA;EACf,QAAA,GAAW,cAAc;AAAA;;;iBCrJL,gBAAA,CACpB,MAAA,EAAQ,MAAA,GACP,OAAA,CAAQ,IAAA,CAAK,GAAA;AAAA,iBAmBM,GAAA,WAAc,QAAA,YAAoB,iBAAA,EACtD,MAAA,EAAQ,MAAA,EACR,SAAA,EAAW,SAAA,CAAU,CAAA,EAAG,CAAA,GACxB,UAAA,GAAY,UAAA,GACX,OAAA,CAAQ,SAAA;;;;iBC1DW,SAAA,sBACpB,QAAA,UACA,MAAA,GAAS,GAAA,EAAK,CAAA,KAAM,CAAA,GACnB,OAAA,CAAQ,MAAA,CAAO,CAAA,EAAG,CAAA;;iBAcC,UAAA,CAAW,QAAA,UAAkB,IAAA,cAAe,OAAA"}

package/dist/index.mjs

@@ -0,0 +1,3 @@
+import{basename as e,dirname as t,join as n}from"node:path";import{tmpdir as r}from"node:os";import{createReadStream as i,createWriteStream as a,existsSync as o,promises as s}from"node:fs";import{Ajv as c}from"ajv";import{setTimeout as l}from"node:timers/promises";import{spawn as u}from"node:child_process";import d from"pbf2json";import{POLYGON_TO_CELLS_FLAGS as f,latLngToCell as p,polygonToCellsExperimental as m}from"h3-js";import{createInterface as h}from"node:readline/promises";import{geoCentroid as g,geoDistance as ee}from"d3-geo";import{createHash as te}from"node:crypto";const _=process.env.NODE_ENV===`test`,v=[`download`,`match`,`conflate`],ne={PBF:`o.pbf`,OVERPASS:`o.geo.json`,POSTPASS:`o.geo.json`,OSM_PROCESSED_WITH_REF:`osm-processed-with-ref.jsonl`,OSM_PROCESSED_NO_REF:`osm-processed-no-ref.jsonl`,OSM_PROCESSED_OTHER:`osm-processed-other.json`,IGNORE_LIST:`ignore-list.json`,MATCHES:`matches.json`},y=e=>({"User-Agent":`https://github.com/osm-nz/osm-conflation-engine on behalf of ${e.metadata.wiki_page}`}),b=`https://osm-internal.download.geofabrik.de`,x=`https://download.geofabrik.de`,S=`https://www.openstreetmap.org`,C=e=>e.match(/name="csrf-token" content="([^"]+)"/)[1];async function w(e,t){if(!t.startsWith(b)&&!t.startsWith(x)&&e.warnings.push(`(!) Planet PBF url is not from geofabrik.de, so metadata may not be available.`),!process.env.OSM_AUTH)return e.warnings.push(`(!) No OSM credentials, so the planet extract will not include metadata`),{url:t.replace(b,x),cookies:``};let n=new URLSearchParams(process.env.OSM_AUTH),r=y(e.config),i=await fetch(`${b}/get_cookie?action=get_authorization_url`,{method:`POST`,headers:r}).then(e=>e.json()),a=await fetch(`${S}/login?cookie_test=true`,{headers:r}),o=a.headers.get(`Set-Cookie`),s=await fetch(`${S}/login`,{method:`POST`,body:new URLSearchParams({username:n.get(`username`),password:n.get(`password`),referer:`/`,commit:`Login`,authenticity_token:C(await a.text())}),headers:{...r,Cookie:o,"Content-Type":`application/x-www-form-urlencoded`}});if(s.status!==200)throw Error(`Login failed`);o+=`; ${s.headers.get(`Set-Cookie`)}`;let c=await fetch(i.authorization_url,{headers:{...r,Cookie:o}}),l=c.url.startsWith(S)?await fetch(i.authorization_url,{method:`POST`,body:new URLSearchParams({client_id:i.client_id,redirect_uri:i.redirect_uri,authenticity_token:C(await c.text()),state:i.state,response_type:`code`,scope:`read_prefs`,nonce:``,code_challenge:``,code_challenge_method:``,commit:`Authorize`}),headers:{...r,Cookie:o,"Content-Type":`application/x-www-form-urlencoded`}}).then(e=>e.text()):await c.text();if(!l.includes(`gf_download_oauth`))throw Error(`No token`);return{url:t.replace(x,b),cookies:l}}async function T(n,r){let i=n.tempFileNames.pbf;if(n.use_cache&&o(i)){console.info(`Using cached planet file`);return}let{cookies:a,url:c}=await w(n,r);if(await s.mkdir(t(i),{recursive:!0}),await s.rm(i,{recursive:!0,force:!0}),/^https?:\/\//.test(c)){console.info(`downloading from`,c,`to`,i);let n=u(`curl`,[`-L`,c,`-H`,`Cookie: ${a}`,`--output`,e(i)],{cwd:t(i)});n.stdout.pipe(process.stdout),n.stderr.pipe(process.stderr),await new Promise((e,t)=>{n.on(`close`,n=>n?t(n):e())}),await l(5e3)}else console.info(`copying file from`,c,`to`,i),await s.copyFile(c,i);console.info(i,await s.stat(i))}async function re(e){let t=e.config.merge.osm_key,n=e.config.e2e_tests?.ignore_list_file_path;if(_&&!n)throw Error(`When running in the unit test envrironment, you must supply config.e2e_tests.ignoreListFilePath`);let r=_&&n?JSON.parse(await s.readFile(n,`utf8`)):await fetch(`https://osm-conflation-engine.kyle.kiwi/api/ignore_list/${encodeURIComponent(t)}?return_ids_only=true`).then(e=>e.json());await s.writeFile(e.tempFileNames.ignore_list,JSON.stringify(r.rows,null,2))}let ie=function(e){return e[e.None=0]=`None`,e[e.IsLastEditedByImporter=1]=`IsLastEditedByImporter`,e[e.IsRecentlyChanged=2]=`IsRecentlyChanged`,e[e.IsChecked=4]=`IsChecked`,e[e.IsCheckedRecently=8]=`IsCheckedRecently`,e}({});const ae=e=>e?(Date.now()-+new Date(e))/1e3/60/60/24/365<4?1:2:0;function E(e,t){let n=t||3;switch(e.type){case`GeometryCollection`:throw Error(`not supported`);case`Point`:return[p(e.coordinates[1],e.coordinates[0],n)];case`MultiPoint`:case`LineString`:case`MultiLineString`:case`Polygon`:return m(e.coordinates,n,f.containmentOverlapping,!0).toSorted();case`MultiPolygon`:{let t=new Set;for(let r of e.coordinates){let e=m(r,n,f.containmentOverlapping,!0);for(let n of e)t.add(n)}return[...t].toSorted()}default:throw TypeError(e)}}async function D(e,t){let n=h({input:i(e)}),r={};for await(let e of n){let n=JSON.parse(e),i=t(n);r[i]=n}return r}async function O(e,t){let n=a(e,{encoding:`utf8`});for(let e of t)n.write(JSON.stringify(e)),n.write(`
+`);n.end(),await new Promise(e=>n.on(`finish`,e))}const oe=(e=>(e.setDate(e.getDate()-90),e/1e3))(new Date);async function se(e,t){if(e.use_cache&&o(e.tempFileNames.osm_processed_other)){console.info(`Using cached osm data`);return}console.info(`Starting preprocess of OSM data...`);let{withRef:n,noRef:r,...i}=await new Promise((n,r)=>{let i={withRef:{},noRef:{},duplicateRefs:{},semi:{},count:0},a=0,o=!1,s=e.config.o_data.check_date_key||`check_date`,c=new Set([s,e.config.merge.osm_key]).difference(new Set(e.config.o_data.tags_to_keep));if(c.size)throw Error(`config.o_data.tags_to_keep is missing some tags that are referenced in other parts of the config: ${[...c].join(`, `)}`);let l=[];for(let t of e.config.o_data.tags_to_keep)t.startsWith(`/`)&&t.endsWith(`/`)&&l.push(new RegExp(t.slice(1,-1)));let u={};function f(t){if(t in u)return u[t];let n=e.config.o_data.tags_to_keep.includes(t);if(!n){for(let e of l)if(e.test(t)){n=!0;break}}return u[t]=n,n}d.createReadStream({file:e.tempFileNames.pbf,tags:t,leveldb:`/tmp`,metadata:!0}).on(`data`,t=>{let n=t.type===`node`?t:t.centroid,r=t.meta;r&&(o=!0);let c={};for(let e in t.tags){let n=t.tags[e];f(e)&&n&&(c[e]=n)}let l={id:`${t.type[0]}${t.id}`,centroid:[n.lon,n.lat],sectors:E({type:`Point`,coordinates:[n.lon,n.lat]},e.config.merge.sector_resolution),metadata:r?{changeset:r.changeset,timestamp:r.timestamp?new Date(1e3*r.timestamp).toISOString():void 0,user:r.user,version:r.version}:void 0,tags:c,flags:0};(r?.timestamp??0)>oe&&(l.flags|=2);let u=r?.user&&(e.callbacks.isImportUser?e.callbacks.isImportUser?.(r?.user):r?.user?.endsWith(`_import`));(r?.version===1||u)&&(l.flags|=1),c[s]&&(l.flags|=4),ae(c[s])===1&&(l.flags|=8);let d=t.tags[e.config.merge.osm_key];if(d)if(i.count++,i.duplicateRefs[d])i.duplicateRefs[d].push(l);else if(i.withRef[d])i.duplicateRefs[d]=[i.withRef[d],l],delete i.withRef[d];else if(d.includes(`;`)){let e=d.split(`;`);for(let t of e)i.semi[t]=l}else i.withRef[d]=l;else i.noRef[l.id]=l;a+=1,a%1e3||process.stdout.write(`.`)}).on(`finish`,()=>{o||e.warnings.push(`No metadata extracted from the planet file!`),n(i)}).on(`error`,r)});await O(e.tempFileNames.osm_processed_with_ref,Object.values(n)),await O(e.tempFileNames.osm_processed_no_ref,Object.values(r)),await s.writeFile(e.tempFileNames.osm_processed_other,JSON.stringify(i))}async function k(e){switch(await re(e),e.config.o_data.source.type){case`pbf`:await T(e,e.config.o_data.source.pbf_url),await se(e,e.config.o_data.source.pbf_filter);break;case`overpass`:throw Error(`not supported yet`);case`postpass`:throw Error(`not supported yet`);default:e.config.o_data.source}}function A(e,t,n){let r=[];for(let e in t.withRef){let i=e,a=t.withRef[i];!(i in n)&&!(a.flags&8)&&r.push(i)}let i={};for(let t in n){let r=t,a=n[r];for(let t of a.sectors){i[t]||={};let n=e.callbacks.getLocalKeyForSource(a);i[t][n]=r}}let a={},o=[];for(let n of r){let r=t.withRef[n],s=e.callbacks.getLocalKeyForOsm(r),c=new Set(r.sectors.map(e=>i[e]?.[s]).filter(Boolean));if(c.size>1&&console.warn(`${r.id} spans multiple sectors, and there are conflicting localKeys across the sectors. Therefore, it will be straight-up deleted`),c.size===1){let e=[...c][0];t.withRef[e]?o.push(n):a[e]=n}else o.push(n)}return{refsThatChanged:a,toDelete:o}}const j=(e,t,n=5)=>`${e.toFixed(n)},${t.toFixed(n)}`,M=new Map,N=(e,t,n)=>{let r=e.callbacks.getLocalKeyForSource(t);return n.filter(t=>{let n=M.get(t);return n||(n=e.callbacks.getLocalKeyForOsm(t),M.set(t,n)),n===r}).map(e=>{let n=6371008.8*ee(e.centroid,t.centroid);return Object.assign(e,{offset:n})}).filter(e=>e.offset<200).toSorted((e,t)=>e.offset-t.offset)};async function P(e,t,n){let r=Object.keys(t).length,i=new Set(JSON.parse(await s.readFile(e.tempFileNames.ignore_list,`utf8`)));console.info(`processing deleted data...`),console.time(`matchDeletions`);let{refsThatChanged:a,toDelete:o}=A(e,n,t);console.timeEnd(`matchDeletions`),console.info(`Processing data...`);let c=0;console.time(`match`);let l={1:[],2:[],3:{},4:[],5:o,6:[]},u=Object.values(n.noRef),d={};for(let e in n.semi){let t=e.split(`;`);for(let[n,r]of t.entries())d[r]||=[],d[r].push({value:e,isFirst:!n})}for(let o in t){let s=o,f=a[s]||s;if(i.has(f)||f in a)continue;let p=n.withRef[f],m=t[s],h=n.duplicateRefs[f];if(d[f])if(d[f][0].isFirst)f=d[f][0].value;else continue;let g=n.semi[f];if(p&&g){let e=f.split(`;`);l[4].push({osm:[p.id,g.id],source:e})}else if(p)l[1].push({osm:p.id,source:s});else if(h)l[2].push({osm:h.map(e=>e.id),source:f});else if(g){let e=f.split(`;`);l[3][g.id]||=[],l[3][g.id]?.push(...e)}else{let t=N(e,m,u);t.length===1?l[1].push({source:f,osm:t[0].id}):l[6].push({source:f,osmCandidates:t.map(e=>e.id)})}c+=1,c%1e3||process.stdout.write(`${(c/r*100).toFixed(1)}% `)}return console.timeEnd(`match`),await s.writeFile(e.tempFileNames.matches,JSON.stringify(l,null,void 0)),{output:l}}const F=2e-4;function I({lat:e,lng:t}){let n=[[t,e+F],[t+F,e],[t,e-F],[t-F,e]];return n.push(n[0]),[n]}function L({lat:e,lng:t}){let n=[[t+F/2,e+F/2],[t+F/2,e-F/2],[t-F/2,e-F/2],[t-F/2,e+F/2]];return n.push(n[0]),[n]}function R(e){let t={minLat:1/0,minLng:1/0,maxLat:-1/0,maxLng:-1/0};function n([e,n]){let r=+n,i=+e;r<t.minLat&&(t.minLat=r),i<t.minLng&&(t.minLng=i),r>t.maxLat&&(t.maxLat=r),i>t.maxLng&&(t.maxLng=i)}for(let t of e)switch(t.geometry.type){case`Point`:n(t.geometry.coordinates);break;case`LineString`:t.geometry.coordinates.forEach(n);break;case`Polygon`:for(let e of t.geometry.coordinates)e.forEach(n);break;case`MultiPolygon`:for(let e of t.geometry.coordinates)for(let t of e)t.forEach(n);break;default:}return t}function z(e,t,n,r,i=0){if(!r.length)return{};let a=R(r),o=i>10;if(o&&console.info(`(!) Couldn't split ${e} (${r.length} left)`),r.length<=110||o)return{[e]:{features:r,bbox:a,instructions:t,changesetTags:n}};if(a.maxLat-a.minLat>a.maxLng-a.minLng){let o=a.minLat+(a.maxLat-a.minLat)/2,s=[[],[]];for(let e of r)s[+(g(e.geometry)[1]<o)].push(e);let[c,l]=s;return{...z(`${e}^N`,t,n,c,i+1),...z(`${e}^S`,t,n,l,i+1)}}let s=a.minLng+(a.maxLng-a.minLng)/2,c=[[],[]];for(let e of r)c[+(g(e.geometry)[0]<s)].push(e);let[l,u]=c;return{...z(`${e}^E`,t,n,l,i+1),...z(`${e}^W`,t,n,u,i+1)}}const{sin:B,cos:V,floor:H,sqrt:U,PI:W}=Math;function G(e){let t=H(U(e)),n=(e-t**2)*W/(2*((t+1)**2-t**2-1))||0,[r,i]=[t*V(n),t*B(n)];return[+r.toFixed(2),+i.toFixed(2)]}function ce(e){for(let t in e){let n=e[t],r={};for(let e of n.features)if(e.geometry.type===`Point`){let[t,n]=e.geometry.coordinates,i=j(n,t,4);r[i]||=[],r[i].push(e)}for(let e of Object.values(r))if(e.length>1)for(let t=0;t<e.length;t+=1){let[n,r]=G(t),[i,a]=e[t].geometry.coordinates;e[t].geometry={type:`Point`,coordinates:[i+4e-6*n,a+4e-6*r]}}}return e}function K(e){return te(`sha256`).update(e,`utf8`).digest(`hex`)}function le(e){let t=0,n=0,r=0,i=0;for(let a of e)switch(a.properties.__action){case`delete`:n+=1;break;case`move`:r+=1;break;case`edit`:i+=1;break;default:t+=1}return{count:[t&&`add ${t}`,n&&`delete ${n}`,r&&`move ${r}`,i&&`edit ${i}`].filter(e=>!!e).join(`, `),totalCount:t+n+r+i}}function q(e){return`${e.replaceAll(/[^\dA-Za-zĀāĒēĪīŌōŪū]/g,``)}_${K(e).slice(0,6)}`}async function ue(e,t){let r=e.config.metadata.git_repository.match(`https://github.com/([^/]+)/([^/]+)`);if(!r)throw Error(`git_repository is not a valid github url`);let i=e.config.output?.folder||n(process.cwd(),`output`),a=`suburbs`,o=Object.entries(t).map(([e,t])=>({suburb:e,bbox:t.bbox,instructions:t.instructions,...le(t.features)})),c={fields:[],results:o.map(e=>{let t=e.suburb.replace(`ZZ `,``).replace(`Z `,``);return{id:q(e.suburb),url:`https://${r[1]}.github.io/${r[2]}/${a}/${q(e.suburb)}.osmPatch.geo.json`,name:t,title:t,totalCount:e.totalCount,source:``,snippet:e.count,extent:[[e.bbox.minLng,e.bbox.minLat],[e.bbox.maxLng,e.bbox.maxLat]],instructions:e.instructions,groupCategories:[e.suburb.startsWith(`ZZ`)?`/Categories/Preview`:`/Categories/Addresses`]}}).toSorted((e,t)=>e.name.localeCompare(t.name))};await s.writeFile(n(i,`index.json`),JSON.stringify(c,null,_?2:void 0)),await s.mkdir(n(i,a),{recursive:!0});for(let e of o){let{suburb:r}=e,o={type:`FeatureCollection`,...t[r]};o.changesetTags||={attribution:`https://wiki.openstreetmap.org/wiki/Contributors#LINZ`,created_by:`LINZ Data Import 2.0.0`,locale:`en-NZ`,source:`https://wiki.osm.org/LINZ`,comment:r},await s.writeFile(n(i,a,`${q(r)}.osmPatch.geo.json`),JSON.stringify(o,null,_?2:void 0))}}const J=`invalid value returned by your callback function`;function Y(e){for(let t in e)if(!(t===`__action`&&e[t]===`edit`))return!1;return!0}function X(e){if(typeof e!=`object`||typeof e.tags!=`object`)throw TypeError(J);return!!e.geometry||!Y(e.tags)}function Z(e,t,n){if(!t&&!n)throw Error(`requires either o or s`);t&&(e.tags.__action||=`edit`);let[r,i]=t?.centroid||n?.centroid;return{type:`Feature`,id:t?.id||n?.id,geometry:e.tags.__action===`delete`?{type:`Polygon`,coordinates:L({lat:i,lng:r})}:e.geometry||(e.tags.__action===`edit`?{type:`Polygon`,coordinates:I({lat:i,lng:r})}:{type:`Point`,coordinates:[r,i]}),properties:e.tags}}async function de(e,t,n){let r={...n.noRef};for(let e of[`withRef`,`semi`,`duplicateRefs`])for(let t in n[e]){let i=n[e][t];if(Array.isArray(i))for(let e of i)r[e.id]=e;else r[i.id]=i}let i={};function a(t,n){t&&(t.warnings?.length&&e.warnings.push(...t.warnings),t.createFeatures&&(i[n]||=[],i[n].push(...t.createFeatures)))}let o=JSON.parse(await s.readFile(e.tempFileNames.matches,`utf8`));for(let{source:n,osm:s}of o[1]){let o=r[s],c=t[n];if(o.flags&8)continue;let l=await e.callbacks.mergeOneToOne({osm:o,source:c});if(typeof l!=`object`)throw TypeError(J);if(!X(l.diff))continue;let u=l.group||o.sectors[0];a(l.extra,u),i[u]||=[],i[u].push(Z(l.diff,o,c))}for(let{source:n,osm:s}of o[2]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=t[n],l=await e.callbacks.mergeOneToMany?.({osm:o,source:c});if(l){if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(J);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(J);if(!X(r))continue;let s=l.group||n.sectors[0];a(l.extra,s),i[s]||=[],i[s].push(Z(r,n,c))}}}for(let n in o[3]){let s=n,c=r[s],l=o[3][s].map(e=>t[e]);if(c.flags&8)continue;let u=await e.callbacks.mergeManyToOne?.({osm:c,source:l});if(!u)continue;if(typeof u!=`object`||typeof u.diff!=`object`)throw TypeError(J);if(!X(u.diff))continue;let d=u.group||c.sectors[0];a(u.extra,d),i[d]||=[],i[d].push(Z(u.diff,c))}for(let{source:n,osm:s}of o[4]){let o=s.map(e=>r[e]).filter(e=>!(e.flags&8)),c=n.map(e=>t[e]),l=await e.callbacks.mergeManyToMany?.({osm:o,source:c});if(l){if(typeof l!=`object`||typeof l.diffPerFeature!=`object`)throw TypeError(J);for(let e in l.diffPerFeature){let t=e,n=o.find(e=>e.id===t),r=l.diffPerFeature[t];if(typeof r!=`object`)throw TypeError(J);if(!X(r))continue;let s=l.group||n.sectors[0];a(l.extra,s),i[s]||=[],i[s].push(Z(r,n))}}}for(let t of o[5]){let r=n.withRef[t];if(r.flags&8)continue;let o=await e.callbacks.deleteFeature?.({osm:r});if(!o)continue;if(typeof o!=`object`||typeof o.diff!=`object`)throw TypeError(J);if(!X(o.diff))continue;let s=o.group||r.sectors[0];a(o.extra,s),i[s]||=[],i[s].push(Z(o.diff,r))}for(let r of o[6]){let o=t[r.source],s=await e.callbacks.create({source:o,osmCandidates:r.osmCandidates.map(e=>n.noRef[e])});if(!s)continue;if(typeof s!=`object`||typeof s.diff!=`object`)throw TypeError(J);if(!X(s.diff))continue;let c=s.selection?n.noRef[s.selection]:void 0,l=s.group||c?.sectors[0]||`unknown`;a(s.extra,l),i[l]||=[],i[l].push(Z(s.diff,c,o))}let c={};for(let n in i){let{changesetTags:a={},instructions:o}=await e.callbacks.getChangesetTags?.(n)||{};a.created_by||=`osm-conflation-engine`,a.import||=`yes`,a.source||=e.config.metadata.wiki_page;let s=i[n];e.callbacks.postprocessLayer&&await e.callbacks.postprocessLayer({group:n,osmData:r,sourceData:t,features:s});let l=z(n,o,a,s);Object.assign(c,l)}ce(c),ue(e,c);let l={create:0,edit:0,delete:0,perfect:0};for(let e in c)for(let t of c[e].features)t.properties.__action===`edit`?l.edit++:t.properties.__action===`delete`?l.delete++:l.create++;return l.perfect=n.count-l.edit-l.delete,{counts:l}}async function Q(e){console.info(`Reading source data into memory...`);let t=0,n={},r=h({input:i(e.config.source_data.file),crlfDelay:1/0});for await(let i of r){let r=JSON.parse(i),a=r.properties[e.config.merge.dataset_column];n[a]=Object.assign(r,{sectors:E(r.geometry,e.config.merge.sector_resolution),centroid:r.geometry.type===`Point`?r.geometry.coordinates:g(r.geometry)}),t++}console.info(`\tread ${t} rows.`),console.info(`Reading osm data into memory (withRef)...`);let a=await D(e.tempFileNames.osm_processed_with_ref,t=>t.tags[e.config.merge.osm_key]);console.info(`Reading osm data into memory (noRef)...`);let o=await D(e.tempFileNames.osm_processed_with_ref,e=>e.id);console.info(`Reading osm data into memory (other)...`);let c={withRef:a,noRef:o,...JSON.parse(await s.readFile(e.tempFileNames.osm_processed_other,`utf8`))};return console.info(`	read data.`),{sourceData:n,osmData:c}}function fe(e){for(let t of e)if(!v.includes(t))throw Error(`‘${t}’ is not a valid step. The options are: ${v.join(`, `)}`)}async function $(e){let t=K(e.metadata.name).slice(0,12),i=_?`-mock`:``,a=n(r(),`oce-${i}`,t);return await s.mkdir(a,{recursive:!0}),{id:t,tempFolder:a,tempFileNames:Object.fromEntries(Object.entries(ne).map(([e,t])=>[e.toLowerCase(),n(a,t)]))}}async function pe(e,t,n={}){let r=JSON.parse(await s.readFile(new URL(`config.schema.json`,import.meta.url),`utf8`)),i=new c().compile(r);if(!i(e))throw AggregateError(i.errors.map(e=>TypeError(e.message+JSON.stringify(e,null,2),{cause:e})),`Config file does not conform to the schema`);let{id:a,tempFolder:o,tempFileNames:l}=await $(e),u={use_cache:!!n.use_cache,id:a,tempFolder:o,tempFileNames:l,config:e,callbacks:t,warnings:[]};console.info(`temp folder:`,o);let{steps:d=v}=n;fe(d),d.includes(`download`)&&(console.time(`download`),await k(u),console.timeEnd(`download`));let f,p;d.includes(`match`)&&({sourceData:f,osmData:p}=await Q(u),console.time(`match`),await P(u,f,p),console.timeEnd(`match`));let m;return d.includes(`conflate`)&&({sourceData:f,osmData:p}=await Q(u),console.time(`conflate`),m=await de(u,f,p),console.timeEnd(`conflate`)),console.info(`done.`),{conflate:m}}export{ie as OsmFlags,$ as getTempFileNames,D as readJsonL,pe as run,O as writeJsonL};
+//# sourceMappingURL=index.mjs.map