nswtopo 2.0.0.pre.beta1 → 3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/COPYING +70 -83
  3. data/bin/nswtopo +227 -116
  4. data/docs/README.md +1 -12
  5. data/docs/add.md +1 -1
  6. data/docs/config.md +1 -1
  7. data/docs/contours.md +3 -1
  8. data/docs/init.md +8 -0
  9. data/docs/inspect.md +103 -0
  10. data/docs/move.md +9 -0
  11. data/docs/render.md +16 -7
  12. data/docs/scrape.md +67 -0
  13. data/docs/spot-heights.md +6 -2
  14. data/lib/nswtopo/archive.rb +50 -41
  15. data/lib/nswtopo/chrome.rb +227 -0
  16. data/lib/nswtopo/commands/add.rb +106 -0
  17. data/lib/nswtopo/commands/config.rb +38 -0
  18. data/lib/nswtopo/commands/inspect.rb +74 -0
  19. data/lib/nswtopo/commands/layers.rb +22 -0
  20. data/lib/nswtopo/commands/scrape.rb +79 -0
  21. data/lib/nswtopo/commands.rb +57 -0
  22. data/lib/nswtopo/dither.rb +5 -3
  23. data/lib/nswtopo/font.rb +46 -21
  24. data/lib/nswtopo/formats/gemf.rb +42 -0
  25. data/lib/nswtopo/formats/kmz.rb +26 -24
  26. data/lib/nswtopo/formats/mbtiles.rb +5 -41
  27. data/lib/nswtopo/formats/pdf.rb +82 -17
  28. data/lib/nswtopo/formats/svg.rb +114 -45
  29. data/lib/nswtopo/formats/svgz.rb +2 -2
  30. data/lib/nswtopo/formats/zip.rb +33 -23
  31. data/lib/nswtopo/formats.rb +77 -32
  32. data/lib/nswtopo/geometry/overlap.rb +1 -32
  33. data/lib/nswtopo/geometry/r_tree.rb +16 -10
  34. data/lib/nswtopo/geometry/segment.rb +3 -3
  35. data/lib/nswtopo/geometry/straight_skeleton/nodes.rb +5 -6
  36. data/lib/nswtopo/geometry/vector_sequence.rb +7 -6
  37. data/lib/nswtopo/gis/arcgis/connection.rb +56 -0
  38. data/lib/nswtopo/gis/arcgis/layer/map.rb +163 -0
  39. data/lib/nswtopo/gis/arcgis/layer/query.rb +87 -0
  40. data/lib/nswtopo/gis/arcgis/layer/renderer.rb +66 -0
  41. data/lib/nswtopo/gis/arcgis/layer/statistics.rb +15 -0
  42. data/lib/nswtopo/gis/arcgis/layer.rb +201 -0
  43. data/lib/nswtopo/gis/arcgis/service.rb +57 -0
  44. data/lib/nswtopo/gis/arcgis.rb +3 -0
  45. data/lib/nswtopo/gis/dem.rb +13 -12
  46. data/lib/nswtopo/gis/esri_hdr.rb +8 -2
  47. data/lib/nswtopo/gis/geojson/collection.rb +45 -21
  48. data/lib/nswtopo/gis/geojson/multi_line_string.rb +2 -24
  49. data/lib/nswtopo/gis/geojson/multi_polygon.rb +2 -53
  50. data/lib/nswtopo/gis/geojson/polygon.rb +15 -0
  51. data/lib/nswtopo/gis/geojson.rb +12 -3
  52. data/lib/nswtopo/gis/gps/kml.rb +25 -19
  53. data/lib/nswtopo/gis/gps.rb +2 -0
  54. data/lib/nswtopo/gis/projection.rb +35 -24
  55. data/lib/nswtopo/gis/shapefile.rb +89 -16
  56. data/lib/nswtopo/gis.rb +1 -2
  57. data/lib/nswtopo/helpers/array.rb +0 -11
  58. data/lib/nswtopo/helpers/colour.rb +34 -14
  59. data/lib/nswtopo/layer/arcgis_raster.rb +44 -48
  60. data/lib/nswtopo/layer/colour_mask.rb +5 -0
  61. data/lib/nswtopo/layer/contour.rb +35 -28
  62. data/lib/nswtopo/layer/control.rb +2 -7
  63. data/lib/nswtopo/layer/declination.rb +9 -9
  64. data/lib/nswtopo/layer/feature.rb +36 -22
  65. data/lib/nswtopo/layer/grid.rb +30 -27
  66. data/lib/nswtopo/layer/import.rb +1 -21
  67. data/lib/nswtopo/layer/labels/barrier.rb +39 -0
  68. data/lib/nswtopo/layer/labels.rb +551 -383
  69. data/lib/nswtopo/layer/mask_render.rb +37 -0
  70. data/lib/nswtopo/layer/overlay.rb +2 -2
  71. data/lib/nswtopo/layer/raster.rb +31 -41
  72. data/lib/nswtopo/layer/raster_import.rb +17 -0
  73. data/lib/nswtopo/layer/raster_render.rb +15 -0
  74. data/lib/nswtopo/layer/relief.rb +27 -95
  75. data/lib/nswtopo/layer/spot.rb +63 -62
  76. data/lib/nswtopo/layer/vector/cutout.rb +15 -0
  77. data/lib/nswtopo/layer/vector/knockout.rb +16 -0
  78. data/lib/nswtopo/layer/vector.rb +121 -89
  79. data/lib/nswtopo/layer/vegetation.rb +39 -34
  80. data/lib/nswtopo/layer.rb +30 -16
  81. data/lib/nswtopo/map.rb +202 -109
  82. data/lib/nswtopo/os.rb +5 -27
  83. data/lib/nswtopo/tiled_web_map.rb +54 -0
  84. data/lib/nswtopo/tree_indenter.rb +27 -0
  85. data/lib/nswtopo/version.rb +27 -2
  86. data/lib/nswtopo.rb +6 -199
  87. metadata +41 -22
  88. data/lib/nswtopo/font/chrome.rb +0 -59
  89. data/lib/nswtopo/font/generic.rb +0 -25
  90. data/lib/nswtopo/gis/arcgis_server/connection.rb +0 -52
  91. data/lib/nswtopo/gis/arcgis_server.rb +0 -155
  92. data/lib/nswtopo/gis/geojson/multi_point.rb +0 -12
  93. data/lib/nswtopo/gis/world_file.rb +0 -19
  94. data/lib/nswtopo/layer/labels/fence.rb +0 -20
data/docs/inspect.md ADDED
@@ -0,0 +1,103 @@
1
+ # Description
2
+
3
+ Use the *inspect* command to examine ArcGIS REST endpoints and local GIS data.
4
+
5
+ # ArcGIS REST layers
6
+
7
+ List all the layers in an ArcGIS map or feature service as follows:
8
+
9
+ ```
10
+ $ nswtopo inspect https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer
11
+ layers:
12
+ ├─ 0: HydroPoint
13
+ ├─ 1: AncillaryHydroPoint
14
+ ├─ 2: NamedWatercourse
15
+ ├─ 3: FuzzyExtentWaterLine
16
+ ├─ 4: Coastline
17
+ ├─ 5: HydroLine
18
+ ├─ 6: HydroArea
19
+ └─ 7: FuzzyExtentWaterArea
20
+ ```
21
+
22
+ List the fields for a layer using the layer URL, or with the `--layer` or `--id` option. The layer's fields and their types will be listed:
23
+
24
+ ```
25
+ $ nswtopo inspect https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
26
+ name: HydroArea
27
+ id: 6
28
+ geometry: Polygon
29
+ fields:
30
+ ├─ attributereliabilitydate: Date
31
+ ├─ capturemethodcode: SmallInteger
32
+ │ ...
33
+ ├─ urbanity: String
34
+ └─ verticalaccuracy: Single
35
+ ```
36
+
37
+ Use the `--fields` option to inspect values and counts for one or more fields:
38
+
39
+ ```
40
+ $ nswtopo inspect --fields classsubtype,hydrotype https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
41
+ classsubtype │ 447690 │ 1
42
+ hydrotype │ 9070 │ ├─ 1
43
+ hydrotype │ 438620 │ └─ 2
44
+ classsubtype │ 20084 │ 2
45
+ hydrotype │ 19849 │ ├─ 1
46
+ hydrotype │ 235 │ └─ 2
47
+ ```
48
+
49
+ Field names and all possible values are shown, with counts for each combination of values. In this case, the values are *coded values*. Use `--decode` to decode them:
50
+
51
+ ```
52
+ $ nswtopo inspect --fields classsubtype,hydrotype --decode https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
53
+ classsubtype │ 447690 │ WaterbodyArea
54
+ hydrotype │ 438620 │ ├─ ManMadeWaterBody
55
+ hydrotype │ 9070 │ └─ NaturalWaterBody
56
+ classsubtype │ 20084 │ Watercourse
57
+ hydrotype │ 235 │ ├─ Canal-Drain
58
+ hydrotype │ 19849 │ └─ NaturalWatercourse
59
+ ```
60
+
61
+ List all *coded value* conversions for a layer with the `--codes` option:
62
+
63
+ ```
64
+ $ nswtopo inspect --codes https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
65
+ perenniality:
66
+ ├─ 0 → NotApplicable
67
+ ├─ 1 → Perennial
68
+ ├─ 2 → NonPerennial
69
+ └─ 3 → MainlyDry
70
+ classsubtype:
71
+ ├─ 1 → WaterbodyArea
72
+ │ └─ hydrotype:
73
+ │ ├─ 1 → NaturalWaterBody
74
+ │ └─ 2 → ManMadeWaterBody
75
+ └─ 2 → Watercourse
76
+ └─ hydrotype:
77
+ ├─ 1 → NaturalWatercourse
78
+ └─ 2 → Canal-Drain
79
+ ```
80
+
81
+ Use the `--where` option to restrict output using a SQL expression on the fields.
82
+
83
+ # Local GIS Data
84
+
85
+ Any OGR-readable data can likewise be examined using the `inspect` command. For example, to list layers contained in a spatialite file:
86
+
87
+ ```
88
+ $ nswtopo inspect nsw.sqlite
89
+ layers:
90
+ ├─ cableway (LineString)
91
+ ├─ electricitytransissionline (LineString)
92
+ ├─ railway (LineString)
93
+ └─ trafficcontroldevice (Point)
94
+ ```
95
+
96
+ To list field values for a layer:
97
+
98
+ ```
99
+ $ nswtopo inspect nsw.sqlite --layer cableway --fields classsubtype
100
+ classsubtype │ 4 │ CableCar
101
+ classsubtype │ 19 │ FlyingFox
102
+ classsubtype │ 58 │ SkiLift
103
+ ```
data/docs/move.md ADDED
@@ -0,0 +1,9 @@
1
+ # Description
2
+
3
+ Use the *move* command to rearrange the layer order in a map. Specify the name of the layer you wish to move, and the layer you want to place it before or after:
4
+
5
+ ```
6
+ $ nswtopo move map.tgz relief --after nsw.vegetation-spot5
7
+ ```
8
+
9
+ Use the *info* command to see the names of layers currently in the map file.
data/docs/render.md CHANGED
@@ -1,6 +1,10 @@
1
1
  # Description
2
2
 
3
- Once you've added your layers, use the *render* command to create the map itself. Different output formats are available for various purposes.
3
+ Once you've added your layers, use the *render* command to create the map itself. Different output formats are available for various purposes. In its simplest form, to render a GeoTIFF from a map file:
4
+
5
+ ```
6
+ $ nswtopo render map.tgz map.tif
7
+ ```
4
8
 
5
9
  Specify your output format either as a filename with appropriate extension, or just the format extension itself (in which case the map's file name will be used). You can create multiple outputs at once.
6
10
 
@@ -15,7 +19,8 @@ The following formats are available:
15
19
  * **pdf**: *Portable Document Format*, either in the original vector form, or as a raster by setting a resolution with `--ppi`
16
20
  * **kmz**: for use with *Google Earth* (add as a network link for best results)
17
21
  * **zip**: a tiled format used by the *Avenza Maps* mobile app
18
- * **mbtiles**: a tiled format for use with mobile GPS apps including *Locus Map* and *Guru Maps*
22
+ * **mbtiles**: a tiled format for use with mobile GPS apps including *Locus Map* and *OruxMaps*
23
+ * **gemf**: a fast tiled format compatible with *Locus Map*
19
24
  * **svgz**: a compressed SVG format, viewable directly in some browsers
20
25
  * **png**: the well-known *Portable Network Graphics* format
21
26
  * **jpg**: the well-known *JPEG* format (not well-suited to maps)
@@ -28,9 +33,7 @@ For the *mbtiles* format, resolution values are fixed to zoom levels. The defaul
28
33
 
29
34
  # Setting Up Chrome
30
35
 
31
- To create any output format except SVG, you'll need to have *Google Chrome* installed. Chrome is used by *nswtopo* in headless mode to render the vector SVG format as a raster graphic. *Firefox* can also be used, although it does not render some effects correctly. (On MacOS and Linux, Chrome is also used to measure font metrics during labelling.)
32
-
33
- Add the path for your Chrome executable to your *nswtopo* configuration as follows:
36
+ You'll need to have *Google Chrome* or *Chromium* installed. The Chrome browser is used by *nswtopo* in headless mode to measure font metrics during labelling, and to render the vector SVG format as a raster graphic or PDF. Chrome should be detected automatically, but you can also configure its path manually if necessary:
34
37
 
35
38
  ```
36
39
  $ nswtopo config --chrome "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
@@ -38,6 +41,12 @@ $ nswtopo config --chrome "/Applications/Google Chrome.app/Contents/MacOS/Google
38
41
 
39
42
  # Miscellaneous
40
43
 
41
- For raster formats, use the `--dither` option to create the raster in indexed colour mode. This can reduce file size. For best results, have the `pngquant` program available on your command line for the dithering process.
44
+ After generating your map in SVG format, you can add content outside of *nswtopo* using a vector graphics editor such as Inkscape. You can then generate raster formats from the edited SVG instead of the map file:
42
45
 
43
- After generating your map in SVG format, you can add content outside of *nswtopo* using a vector graphics editor such as Inkscape. Use the `--external` option to render from the edited map, instead of the internal copy maintained by *nswtopo*.
46
+ ```
47
+ $ nswtopo render map.svg map.tif
48
+ ```
49
+
50
+ Maps normally use a white background. To specify a different background colour, use the `--background` option.
51
+
52
+ For raster formats, use the `--dither` option to create the raster in indexed colour mode. This can reduce file size. For best results, have the `pngquant` program available on your command line for the dithering process.
data/docs/scrape.md ADDED
@@ -0,0 +1,67 @@
1
+ # Description
2
+
3
+ Use *scrape* to download data from ArcGIS REST endpoint. Data may be downloaded from both *FeatureServer* and *MapServer* services, including *MapServer* services with no *Query* capability.
4
+
5
+ # Usage
6
+
7
+ Download some layers from an NSW ArcGIS server to a sqlite database `nsw.sqlite`:
8
+
9
+ ```
10
+ $ nswtopo scrape https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/1 nsw.sqlite
11
+ nswtopo: saved 87731 features
12
+ $ nswtopo scrape https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Features_of_Interest_Category/MapServer/6 nsw.sqlite
13
+ nswtopo: saved 6569 features
14
+ $ nswtopo scrape https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/10 nsw.sqlite
15
+ nswtopo: saved 81 features
16
+ ```
17
+
18
+ Provide the full URL for the layer, including id number, or use the `--id` or `--layer` option along with the service URL:
19
+
20
+ ```
21
+ $ nswtopo scrape --layer Railway https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer nsw.sqlite
22
+ nswtopo: saved 12979 features
23
+ ```
24
+
25
+ Shapefile and sqlite output formats are supported. The filename extension determines the format, with `.sqlite` or `.db` selecting sqlite output, `.shp` a single shapefile layer, and any other filename a shapefile directory.
26
+
27
+ Downloaded layers are named according to the ArcGIS layer name, or from the `--layer` option if specified.
28
+
29
+ # Filtering
30
+
31
+ Use the `--coords` option to restrict feature download to a specified bounding box:
32
+
33
+ ```
34
+ $ nswtopo scrape --coords 148.26,-36.52,148.38,-36.47 https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/10 nsw.sqlite
35
+ nswtopo: saved 12 features
36
+ ```
37
+
38
+ Use the `--where` option to restrict the download to certain field values:
39
+
40
+ ```
41
+ $ nswtopo scrape --where "classsubtype = 1" https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/10 nsw.sqlite
42
+ nswtopo: saved 58 features
43
+ ```
44
+
45
+ # Fields
46
+
47
+ If all fields are not needed, apply the `--fields` option to specify a list of fields to be downloaded:
48
+
49
+ ```
50
+ $ nswtopo scrape --fields CONTOUR_TY,ELEVATION https://services.thelist.tas.gov.au/arcgis/rest/services/Public/OpenDataWFS/MapServer/16 tas.sqlite
51
+ nswtopo: saved 528438 features
52
+ ```
53
+
54
+ Some ArcGIS fields contain `coded values`—integers or short strings representing longer, more descriptive strings. These can be decoded during download using the `--decode` option:
55
+
56
+ ```
57
+ $ nswtopo scrape --fields classsubtype,hydrotype --decode https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6 nsw.sqlite
58
+ nswtopo: saved 467774 features
59
+ ```
60
+
61
+ The `--decode` flag is particularly useful with type and subtype fields, where the same subtype code has a different meaning depending on the type code. Use the *inspect* command to view all codings for a given layer.
62
+
63
+ # Scraping from Map Layers
64
+
65
+ Some ArcGIS REST map layers do not expose the *Query* capability, preventing feature queries. It is still possible to download features from such a layer, provided it supports SVG output and dynamic layers.
66
+
67
+ When this situation occurs, you may be prompted to provide a *unique value* field name with the `--unique` option. Choose a layer field which is likely to have a small number of possible values. Using `--fields` to restrict the fields is also recommended. Specify only the fields you need to reduce download time.
data/docs/spot-heights.md CHANGED
@@ -10,14 +10,18 @@ DEM tiles from the ELVIS website are delivered as doubly-zipped files. It's not
10
10
  # Spot Height Configuration
11
11
  Use the `--spacing` option to determine the maximum density of spot heights on the map. The value in millimetres represents the minimum distance between any two spot heights.
12
12
 
13
- Choose the amount of smoothing to apply to the DEM with the `--smooth` option. The value represents a smoothing radius in millimetres. For consistency, the same smoothing radius shoult be used for both contours and spot heights.
13
+ Choose the amount of smoothing to apply to the DEM with the `--smooth` option. The value represents a smoothing radius in millimetres. For consistency in elevation, the same smoothing radius should be used for both contours and spot heights.
14
14
 
15
15
  Use the `--prefer` option to favour `knolls` or `saddles` when selecting spot locations. No preference is taken by default.
16
16
 
17
+ Use the `--extent` option to set a minimum size in millimetres when searching for spot-height features. Smaller, localised knolls and saddles will be rejected.
18
+
19
+ DEM tiles are normally processed at their maximumum native resolution. Change this using the `--resolution` option. A reduced resolution (say 5 metres) can markedly improve processing speed for 1- and 2-metre tiles.
20
+
17
21
  # Layer Position
18
22
 
19
23
  Use an `--after`, `--before` or `--replace` option to insert the spot heights in an appropriate layer position. You will most likely want to replace an existing spot heights layer:
20
24
 
21
25
  ```
22
- $ nswtopo contours --replace nsw.topographic.spot-heights map.tgz DATA_25994.zip
26
+ $ nswtopo spot-heights --replace nsw.topographic.spot-heights map.tgz DATA_25994.zip
23
27
  ```
@@ -1,14 +1,11 @@
1
1
  module NSWTopo
2
2
  class Archive
3
3
  extend Safely
4
+ include Enumerable
5
+ Invalid = Class.new RuntimeError
4
6
 
5
- def initialize(path, tar_in)
6
- @tar_in, @basename, @entries = tar_in, path.basename(".tgz").basename(".tar.gz").to_s, Hash[]
7
- end
8
- attr_reader :basename
9
-
10
- def delete(filename)
11
- @entries[filename] = nil
7
+ def initialize(tar)
8
+ @tar, @entries = tar, Hash[]
12
9
  end
13
10
 
14
11
  def write(filename, content)
@@ -17,15 +14,36 @@ module NSWTopo
17
14
  @entries[filename] = Gem::Package::TarReader::Entry.new header, io
18
15
  end
19
16
 
20
- def mtime(filename)
21
- header = @entries.key?(filename) ? @entries[filename]&.header : @tar_in.seek(filename, &:header)
22
- Time.at header.mtime if header
17
+ def each(&block)
18
+ @tar.rewind
19
+ @tar.each do |entry|
20
+ yield entry unless @entries.key? entry.full_name
21
+ end
22
+ @entries.each do |filename, entry|
23
+ yield entry if entry
24
+ end
25
+ end
26
+
27
+ def delete(filename)
28
+ find do |entry|
29
+ entry.full_name == filename
30
+ end&.tap do
31
+ @entries[filename] = nil
32
+ end
23
33
  end
24
34
 
25
35
  def read(filename)
26
- @entries.key?(filename) ? @entries[filename]&.read : @tar_in.seek(filename, &:read)
27
- ensure
28
- @entries[filename]&.rewind
36
+ find do |entry|
37
+ entry.full_name == filename
38
+ end&.read
39
+ end
40
+
41
+ def mtime(filename)
42
+ find do |entry|
43
+ entry.full_name == filename
44
+ end&.then do |entry|
45
+ Time.at entry.header.mtime
46
+ end
29
47
  end
30
48
 
31
49
  def uptodate?(depender, *dependees)
@@ -36,36 +54,25 @@ module NSWTopo
36
54
  end
37
55
  end
38
56
 
39
- def each(&block)
40
- @tar_in.each do |entry|
41
- yield entry unless @entries.key? entry.full_name
42
- end
43
- @entries.each do |filename, entry|
44
- yield entry if entry
45
- end
46
- end
47
-
48
57
  def changed?
49
- return true if @entries.values.any?
50
- @entries.keys.any? do |filename|
51
- @tar_in.seek(filename, &:itself)
52
- end
58
+ @entries.any?
53
59
  end
54
60
 
55
- def self.open(out_path, in_path = nil, &block)
61
+ def self.open(out_path: nil, in_path: nil, &block)
56
62
  buffer, reader = StringIO.new, in_path ? Zlib::GzipReader : StringIO
57
63
 
58
64
  reader.open(*in_path) do |input|
59
- if in_path
60
- version = input.comment.to_s[/^nswtopo (.+)$/, 1]
61
- raise "unrecognised map file: %s" % in_path unless version
62
- comparison = version.split(?.).map(&:to_i) <=> MIN_VERSION.split(?.).map(&:to_i)
63
- raise "map file too old: version %s, minimum %s required" % [version, MIN_VERSION] unless comparison >= 0
64
- end
65
- Gem::Package::TarReader.new(input) do |tar_in|
66
- archive = new(out_path, tar_in).tap(&block)
67
- Gem::Package::TarWriter.new(buffer) do |tar_out|
68
- archive.each &tar_out.method(:add_entry)
65
+ begin
66
+ version = Version[input.comment]
67
+ raise "map file too old: created with nswtopo %s, minimum %s required" % [version, MIN_VERSION] unless version >= MIN_VERSION
68
+ raise "nswtopo too old: map file created with nswtopo %s, this version %s" % [version, VERSION] unless version <= VERSION
69
+ rescue Version::Error
70
+ raise "unrecognised map file: %s" % in_path
71
+ end if in_path && false != Config["versioning"]
72
+ Gem::Package::TarReader.new(input) do |tar|
73
+ archive = new(tar).tap(&block)
74
+ Gem::Package::TarWriter.new(buffer) do |tar|
75
+ archive.each &tar.method(:add_entry)
69
76
  end if archive.changed?
70
77
  end
71
78
  end
@@ -73,8 +80,8 @@ module NSWTopo
73
80
  Dir.mktmppath do |temp_dir|
74
81
  log_update "nswtopo: saving map..."
75
82
  temp_path = temp_dir / "temp.tgz"
76
- Zlib::GzipWriter.open temp_path, Zlib::BEST_COMPRESSION do |gzip|
77
- gzip.comment = "nswtopo %s" % VERSION
83
+ Zlib::GzipWriter.open temp_path, Config["zlib-level"] || Zlib::BEST_SPEED do |gzip|
84
+ gzip.comment = VERSION.creator_string
78
85
  gzip.write buffer.string
79
86
  rescue Interrupt
80
87
  log_update "nswtopo: interrupted, please wait..."
@@ -82,12 +89,14 @@ module NSWTopo
82
89
  end
83
90
  safely "saving map file, please wait..." do
84
91
  FileUtils.cp temp_path, out_path
92
+ rescue SystemCallError
93
+ raise "couldn't save #{out_path}"
85
94
  end
86
95
  log_success "map saved"
87
- end unless buffer.size.zero?
96
+ end if out_path && buffer.size.nonzero?
88
97
 
89
98
  rescue Zlib::GzipFile::Error
90
- raise "unrecognised map file: %s" % in_path
99
+ raise Invalid
91
100
  end
92
101
  end
93
102
  end
@@ -0,0 +1,227 @@
1
+ module NSWTopo
2
+ class Chrome
3
+ MIN_VERSION = 112
4
+ TIMEOUT_KILL = 5
5
+ TIMEOUT_LOADEVENT = 30
6
+ TIMEOUT_COMMAND = 10
7
+ TIMEOUT_SCREENSHOT = 120
8
+
9
+ ARGS = %W[
10
+ --disable-background-networking
11
+ --disable-component-extensions-with-background-pages
12
+ --disable-component-update
13
+ --disable-default-apps
14
+ --disable-extensions
15
+ --disable-features=site-per-process,Translate
16
+ --disable-lcd-text
17
+ --disable-renderer-backgrounding
18
+ --force-color-profile=srgb
19
+ --force-device-scale-factor=1
20
+ --headless=new
21
+ --hide-scrollbars
22
+ --no-default-browser-check
23
+ --no-first-run
24
+ --no-startup-window
25
+ --remote-debugging-pipe=JSON
26
+ --use-mock-keychain
27
+ ]
28
+
29
+ class Error < RuntimeError
30
+ def initialize(message = "chrome error")
31
+ super
32
+ end
33
+ end
34
+
35
+ def self.mac?
36
+ /darwin/ === RbConfig::CONFIG["host_os"]
37
+ end
38
+
39
+ def self.windows?
40
+ /mingw|mswin|cygwin/ === RbConfig::CONFIG["host_os"]
41
+ end
42
+
43
+ def self.path
44
+ @path ||= case
45
+ when Config["chrome"]
46
+ [Config["chrome"]]
47
+ when mac?
48
+ ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Chromium.app/Contents/MacOS/Chromium"]
49
+ when windows?
50
+ ["C:/Program Files/Google/Chrome/Application/chrome.exe", "C:/Program Files (x86)/Google/Chrome/Application/chrome.exe"]
51
+ else
52
+ ENV["PATH"].split(File::PATH_SEPARATOR).product(%w[chrome google-chrome chromium chromium-browser]).map do |path, binary|
53
+ [path, binary].join(File::SEPARATOR)
54
+ end
55
+ end.find do |path|
56
+ File.executable?(path) && !File.directory?(path)
57
+ end.tap do |path|
58
+ raise Error, "couldn't find chrome" unless path
59
+ stdout, status = Open3.capture2 path, "--version"
60
+ raise Error, "couldn't start chrome" unless status.success?
61
+ version = /(?<major>\d+)(?:\.\d+)*/.match stdout
62
+ raise Error, "couldn't start chrome" unless version
63
+ raise Error, "chrome version #{MIN_VERSION} or higher required" if version[:major].to_i < MIN_VERSION
64
+ end
65
+ end
66
+
67
+ def self.rmdir(tmp)
68
+ Proc.new do
69
+ FileUtils.remove_entry tmp
70
+ rescue SystemCallError
71
+ end
72
+ end
73
+
74
+ def self.kill(pid, *pipes)
75
+ Proc.new do
76
+ if windows?
77
+ *, status = Open3.capture2e *%W[taskkill /f /t /pid #{pid}]
78
+ Process.kill "KILL", pid unless status.success?
79
+ else
80
+ Timeout.timeout(TIMEOUT_KILL, Error) do
81
+ Process.kill "-USR1", Process.getpgid(pid)
82
+ Process.wait pid
83
+ rescue Error
84
+ Process.kill "-KILL", Process.getpgid(pid)
85
+ Process.wait pid
86
+ end
87
+ end
88
+ rescue Errno::ESRCH, Errno::ECHILD
89
+ ensure
90
+ pipes.each(&:close)
91
+ end
92
+ end
93
+
94
+ def close
95
+ Chrome.kill(@pid, @input, @output).call
96
+ Chrome.rmdir(@data_dir).call
97
+ ObjectSpace.undefine_finalizer self
98
+ end
99
+
100
+ def self.with_browser(url, **opts, &block)
101
+ browser = new url, **opts
102
+ block.call browser
103
+ ensure
104
+ browser&.close
105
+ end
106
+
107
+ def initialize(url, width: 800, height: 600, background: { r: 0, g: 0, b: 0, a: 0 }, args: [])
108
+ @id, @data_dir = 0, Dir.mktmpdir("nswtopo_headless_chrome_")
109
+ ObjectSpace.define_finalizer self, Chrome.rmdir(@data_dir)
110
+
111
+ args << "--disable-gpu" if Config["gpu"] == false
112
+ args << "--user-data-dir=#{@data_dir}"
113
+
114
+ input, @input, @output, output = *IO.pipe, *IO.pipe
115
+ input.nonblock, output.nonblock = false, false
116
+ @input.sync = true
117
+
118
+ @pid = Process.spawn Chrome.path, *ARGS, *args, 1 => File::NULL, 2 => File::NULL, 3 => input, 4 => output, :pgroup => Chrome.windows? ? nil : true
119
+ ObjectSpace.define_finalizer self, Chrome.kill(@pid, @input, @output)
120
+ input.close; output.close
121
+
122
+ target_id = command("Target.createTarget", url: url).fetch("targetId")
123
+ @session_id = command("Target.attachToTarget", targetId: target_id, flatten: true).fetch("sessionId")
124
+ command "Page.enable"
125
+ wait "Page.loadEventFired", timeout: TIMEOUT_LOADEVENT
126
+ command "Emulation.setDeviceMetricsOverride", width: width, height: height, deviceScaleFactor: 1, mobile: false
127
+ command "Emulation.setDefaultBackgroundColorOverride", color: background
128
+ @node_id = command("DOM.getDocument").fetch("root").fetch("nodeId")
129
+ rescue SystemCallError
130
+ raise Error, "couldn't start chrome"
131
+ rescue KeyError
132
+ raise Error
133
+ end
134
+
135
+ def send(**message)
136
+ message.merge! sessionId: @session_id if @session_id
137
+ @input.write message.to_json, ?\0
138
+ end
139
+
140
+ def messages
141
+ Enumerator.produce do
142
+ json = @output.readline(?\0).chomp(?\0)
143
+ JSON.parse(json).tap do |message|
144
+ raise Error if message["error"]
145
+ raise Error if message["method"] == "Target.detachedFromTarget"
146
+ end
147
+ rescue JSON::ParserError, EOFError
148
+ raise Error
149
+ end
150
+ end
151
+
152
+ def wait(event, timeout: nil)
153
+ Timeout.timeout(timeout) do
154
+ messages.find do |message|
155
+ message["method"] == event
156
+ end
157
+ end
158
+ rescue Timeout::Error
159
+ raise Error
160
+ end
161
+
162
+ def command(method, timeout: TIMEOUT_COMMAND, **params)
163
+ send id: @id += 1, method: method, params: params
164
+ Timeout.timeout(timeout) do
165
+ messages.find do |message|
166
+ message["id"] == @id
167
+ end
168
+ end.fetch("result")
169
+ rescue Timeout::Error, KeyError
170
+ raise Error
171
+ end
172
+
173
+ def screenshot(png_path)
174
+ data = command("Page.captureScreenshot", timeout: TIMEOUT_SCREENSHOT).fetch("data")
175
+ png_path.binwrite Base64.decode64(data)
176
+ rescue KeyError
177
+ raise Error
178
+ end
179
+
180
+ def print_to_pdf(pdf_path)
181
+ data = command("Page.printToPDF", timeout: nil, preferCSSPageSize: true).fetch("data")
182
+ pdf_path.binwrite Base64.decode64(data)
183
+ rescue KeyError
184
+ raise Error
185
+ end
186
+
187
+ def query_selector_node_id(selector)
188
+ command("DOM.querySelector", selector: selector, nodeId: @node_id).fetch("nodeId")
189
+ rescue KeyError
190
+ raise Error
191
+ end
192
+
193
+ class Node
194
+ def initialize(browser, selector)
195
+ @browser, @node_id = browser, browser.query_selector_node_id(selector)
196
+ end
197
+
198
+ def [](name)
199
+ @browser.command("DOM.getAttributes", nodeId: @node_id).fetch("attributes").each_slice(2).to_h.fetch(name.to_s)
200
+ rescue KeyError
201
+ raise Error
202
+ end
203
+
204
+ def []=(name, value)
205
+ if value.nil?
206
+ @browser.command "DOM.removeAttribute", nodeId: @node_id, name: name
207
+ else
208
+ @browser.command "DOM.setAttributeValue", nodeId: @node_id, name: name, value: value
209
+ end
210
+ end
211
+
212
+ def value=(value)
213
+ @browser.command "DOM.setNodeValue", nodeId: @node_id + 1, value: value
214
+ end
215
+
216
+ def width
217
+ @browser.command("DOM.getBoxModel", nodeId: @node_id).fetch("model").fetch("content").each_slice(2).map(&:first).minmax.reverse.inject(&:-)
218
+ rescue KeyError
219
+ raise Error
220
+ end
221
+ end
222
+
223
+ def query_selector(selector)
224
+ Node.new self, selector
225
+ end
226
+ end
227
+ end