nswtopo 2.0.0.pre.beta1 → 3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/COPYING +70 -83
  3. data/bin/nswtopo +227 -116
  4. data/docs/README.md +1 -12
  5. data/docs/add.md +1 -1
  6. data/docs/config.md +1 -1
  7. data/docs/contours.md +3 -1
  8. data/docs/init.md +8 -0
  9. data/docs/inspect.md +103 -0
  10. data/docs/move.md +9 -0
  11. data/docs/render.md +16 -7
  12. data/docs/scrape.md +67 -0
  13. data/docs/spot-heights.md +6 -2
  14. data/lib/nswtopo/archive.rb +50 -41
  15. data/lib/nswtopo/chrome.rb +227 -0
  16. data/lib/nswtopo/commands/add.rb +106 -0
  17. data/lib/nswtopo/commands/config.rb +38 -0
  18. data/lib/nswtopo/commands/inspect.rb +74 -0
  19. data/lib/nswtopo/commands/layers.rb +22 -0
  20. data/lib/nswtopo/commands/scrape.rb +79 -0
  21. data/lib/nswtopo/commands.rb +57 -0
  22. data/lib/nswtopo/dither.rb +5 -3
  23. data/lib/nswtopo/font.rb +46 -21
  24. data/lib/nswtopo/formats/gemf.rb +42 -0
  25. data/lib/nswtopo/formats/kmz.rb +26 -24
  26. data/lib/nswtopo/formats/mbtiles.rb +5 -41
  27. data/lib/nswtopo/formats/pdf.rb +82 -17
  28. data/lib/nswtopo/formats/svg.rb +114 -45
  29. data/lib/nswtopo/formats/svgz.rb +2 -2
  30. data/lib/nswtopo/formats/zip.rb +33 -23
  31. data/lib/nswtopo/formats.rb +77 -32
  32. data/lib/nswtopo/geometry/overlap.rb +1 -32
  33. data/lib/nswtopo/geometry/r_tree.rb +16 -10
  34. data/lib/nswtopo/geometry/segment.rb +3 -3
  35. data/lib/nswtopo/geometry/straight_skeleton/nodes.rb +5 -6
  36. data/lib/nswtopo/geometry/vector_sequence.rb +7 -6
  37. data/lib/nswtopo/gis/arcgis/connection.rb +56 -0
  38. data/lib/nswtopo/gis/arcgis/layer/map.rb +163 -0
  39. data/lib/nswtopo/gis/arcgis/layer/query.rb +87 -0
  40. data/lib/nswtopo/gis/arcgis/layer/renderer.rb +66 -0
  41. data/lib/nswtopo/gis/arcgis/layer/statistics.rb +15 -0
  42. data/lib/nswtopo/gis/arcgis/layer.rb +201 -0
  43. data/lib/nswtopo/gis/arcgis/service.rb +57 -0
  44. data/lib/nswtopo/gis/arcgis.rb +3 -0
  45. data/lib/nswtopo/gis/dem.rb +13 -12
  46. data/lib/nswtopo/gis/esri_hdr.rb +8 -2
  47. data/lib/nswtopo/gis/geojson/collection.rb +45 -21
  48. data/lib/nswtopo/gis/geojson/multi_line_string.rb +2 -24
  49. data/lib/nswtopo/gis/geojson/multi_polygon.rb +2 -53
  50. data/lib/nswtopo/gis/geojson/polygon.rb +15 -0
  51. data/lib/nswtopo/gis/geojson.rb +12 -3
  52. data/lib/nswtopo/gis/gps/kml.rb +25 -19
  53. data/lib/nswtopo/gis/gps.rb +2 -0
  54. data/lib/nswtopo/gis/projection.rb +35 -24
  55. data/lib/nswtopo/gis/shapefile.rb +89 -16
  56. data/lib/nswtopo/gis.rb +1 -2
  57. data/lib/nswtopo/helpers/array.rb +0 -11
  58. data/lib/nswtopo/helpers/colour.rb +34 -14
  59. data/lib/nswtopo/layer/arcgis_raster.rb +44 -48
  60. data/lib/nswtopo/layer/colour_mask.rb +5 -0
  61. data/lib/nswtopo/layer/contour.rb +35 -28
  62. data/lib/nswtopo/layer/control.rb +2 -7
  63. data/lib/nswtopo/layer/declination.rb +9 -9
  64. data/lib/nswtopo/layer/feature.rb +36 -22
  65. data/lib/nswtopo/layer/grid.rb +30 -27
  66. data/lib/nswtopo/layer/import.rb +1 -21
  67. data/lib/nswtopo/layer/labels/barrier.rb +39 -0
  68. data/lib/nswtopo/layer/labels.rb +551 -383
  69. data/lib/nswtopo/layer/mask_render.rb +37 -0
  70. data/lib/nswtopo/layer/overlay.rb +2 -2
  71. data/lib/nswtopo/layer/raster.rb +31 -41
  72. data/lib/nswtopo/layer/raster_import.rb +17 -0
  73. data/lib/nswtopo/layer/raster_render.rb +15 -0
  74. data/lib/nswtopo/layer/relief.rb +27 -95
  75. data/lib/nswtopo/layer/spot.rb +63 -62
  76. data/lib/nswtopo/layer/vector/cutout.rb +15 -0
  77. data/lib/nswtopo/layer/vector/knockout.rb +16 -0
  78. data/lib/nswtopo/layer/vector.rb +121 -89
  79. data/lib/nswtopo/layer/vegetation.rb +39 -34
  80. data/lib/nswtopo/layer.rb +30 -16
  81. data/lib/nswtopo/map.rb +202 -109
  82. data/lib/nswtopo/os.rb +5 -27
  83. data/lib/nswtopo/tiled_web_map.rb +54 -0
  84. data/lib/nswtopo/tree_indenter.rb +27 -0
  85. data/lib/nswtopo/version.rb +27 -2
  86. data/lib/nswtopo.rb +6 -199
  87. metadata +41 -22
  88. data/lib/nswtopo/font/chrome.rb +0 -59
  89. data/lib/nswtopo/font/generic.rb +0 -25
  90. data/lib/nswtopo/gis/arcgis_server/connection.rb +0 -52
  91. data/lib/nswtopo/gis/arcgis_server.rb +0 -155
  92. data/lib/nswtopo/gis/geojson/multi_point.rb +0 -12
  93. data/lib/nswtopo/gis/world_file.rb +0 -19
  94. data/lib/nswtopo/layer/labels/fence.rb +0 -20
data/docs/inspect.md ADDED
@@ -0,0 +1,103 @@
1
+ # Description
2
+
3
+ Use the *inspect* command to examine ArcGIS REST endpoints and local GIS data.
4
+
5
+ # ArcGIS REST layers
6
+
7
+ List all the layers in an ArcGIS map or feature service as follows:
8
+
9
+ ```
10
+ $ nswtopo inspect https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer
11
+ layers:
12
+ ├─ 0: HydroPoint
13
+ ├─ 1: AncillaryHydroPoint
14
+ ├─ 2: NamedWatercourse
15
+ ├─ 3: FuzzyExtentWaterLine
16
+ ├─ 4: Coastline
17
+ ├─ 5: HydroLine
18
+ ├─ 6: HydroArea
19
+ └─ 7: FuzzyExtentWaterArea
20
+ ```
21
+
22
+ List the fields for a layer using the layer URL, or with the `--layer` or `--id` option. The layer's fields and their types will be listed:
23
+
24
+ ```
25
+ $ nswtopo inspect https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
26
+ name: HydroArea
27
+ id: 6
28
+ geometry: Polygon
29
+ fields:
30
+ ├─ attributereliabilitydate: Date
31
+ ├─ capturemethodcode: SmallInteger
32
+ │ ...
33
+ ├─ urbanity: String
34
+ └─ verticalaccuracy: Single
35
+ ```
36
+
37
+ Use the `--fields` option to inspect values and counts for one or more fields:
38
+
39
+ ```
40
+ $ nswtopo inspect --fields classsubtype,hydrotype https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
41
+ classsubtype │ 447690 │ 1
42
+ hydrotype │ 9070 │ ├─ 1
43
+ hydrotype │ 438620 │ └─ 2
44
+ classsubtype │ 20084 │ 2
45
+ hydrotype │ 19849 │ ├─ 1
46
+ hydrotype │ 235 │ └─ 2
47
+ ```
48
+
49
+ Field names and all possible values are shown, with counts for each combination of values. In this case, the values are *coded values*. Use `--decode` to decode them:
50
+
51
+ ```
52
+ $ nswtopo inspect --fields classsubtype,hydrotype --decode https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
53
+ classsubtype │ 447690 │ WaterbodyArea
54
+ hydrotype │ 438620 │ ├─ ManMadeWaterBody
55
+ hydrotype │ 9070 │ └─ NaturalWaterBody
56
+ classsubtype │ 20084 │ Watercourse
57
+ hydrotype │ 235 │ ├─ Canal-Drain
58
+ hydrotype │ 19849 │ └─ NaturalWatercourse
59
+ ```
60
+
61
+ List all *coded value* conversions for a layer with the `--codes` option:
62
+
63
+ ```
64
+ $ nswtopo inspect --codes https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
65
+ perenniality:
66
+ ├─ 0 → NotApplicable
67
+ ├─ 1 → Perennial
68
+ ├─ 2 → NonPerennial
69
+ └─ 3 → MainlyDry
70
+ classsubtype:
71
+ ├─ 1 → WaterbodyArea
72
+ │ └─ hydrotype:
73
+ │ ├─ 1 → NaturalWaterBody
74
+ │ └─ 2 → ManMadeWaterBody
75
+ └─ 2 → Watercourse
76
+ └─ hydrotype:
77
+ ├─ 1 → NaturalWatercourse
78
+ └─ 2 → Canal-Drain
79
+ ```
80
+
81
+ Use the `--where` option to restrict output using a SQL expression on the fields.
82
+
83
+ # Local GIS Data
84
+
85
+ Any OGR-readable data can likewise be examined using the `inspect` command. For example, to list layers contained in a spatialite file:
86
+
87
+ ```
88
+ $ nswtopo inspect nsw.sqlite
89
+ layers:
90
+ ├─ cableway (LineString)
91
+ ├─ electricitytransissionline (LineString)
92
+ ├─ railway (LineString)
93
+ └─ trafficcontroldevice (Point)
94
+ ```
95
+
96
+ To list field values for a layer:
97
+
98
+ ```
99
+ $ nswtopo inspect nsw.sqlite --layer cableway --fields classsubtype
100
+ classsubtype │ 4 │ CableCar
101
+ classsubtype │ 19 │ FlyingFox
102
+ classsubtype │ 58 │ SkiLift
103
+ ```
data/docs/move.md ADDED
@@ -0,0 +1,9 @@
1
+ # Description
2
+
3
+ Use the *move* command to rearrange the layer order in a map. Specify the name of the layer you wish to move, and the layer you want to place it before or after:
4
+
5
+ ```
6
+ $ nswtopo move map.tgz relief --after nsw.vegetation-spot5
7
+ ```
8
+
9
+ Use the *info* command to see the names of layers currently in the map file.
data/docs/render.md CHANGED
@@ -1,6 +1,10 @@
1
1
  # Description
2
2
 
3
- Once you've added your layers, use the *render* command to create the map itself. Different output formats are available for various purposes.
3
+ Once you've added your layers, use the *render* command to create the map itself. Different output formats are available for various purposes. In its simplest form, to render a GeoTIFF from a map file:
4
+
5
+ ```
6
+ $ nswtopo render map.tgz map.tif
7
+ ```
4
8
 
5
9
  Specify your output format either as a filename with appropriate extension, or just the format extension itself (in which case the map's file name will be used). You can create multiple outputs at once.
6
10
 
@@ -15,7 +19,8 @@ The following formats are available:
15
19
  * **pdf**: *Portable Document Format*, either in the original vector form, or as a raster by setting a resolution with `--ppi`
16
20
  * **kmz**: for use with *Google Earth* (add as a network link for best results)
17
21
  * **zip**: a tiled format used by the *Avenza Maps* mobile app
18
- * **mbtiles**: a tiled format for use with mobile GPS apps including *Locus Map* and *Guru Maps*
22
+ * **mbtiles**: a tiled format for use with mobile GPS apps including *Locus Map* and *OruxMaps*
23
+ * **gemf**: a fast tiled format compatible with *Locus Map*
19
24
  * **svgz**: a compressed SVG format, viewable directly in some browsers
20
25
  * **png**: the well-known *Portable Network Graphics* format
21
26
  * **jpg**: the well-known *JPEG* format (not well-suited to maps)
@@ -28,9 +33,7 @@ For the *mbtiles* format, resolution values are fixed to zoom levels. The defaul
28
33
 
29
34
  # Setting Up Chrome
30
35
 
31
- To create any output format except SVG, you'll need to have *Google Chrome* installed. Chrome is used by *nswtopo* in headless mode to render the vector SVG format as a raster graphic. *Firefox* can also be used, although it does not render some effects correctly. (On MacOS and Linux, Chrome is also used to measure font metrics during labelling.)
32
-
33
- Add the path for your Chrome executable to your *nswtopo* configuration as follows:
36
+ You'll need to have *Google Chrome* or *Chromium* installed. The Chrome browser is used by *nswtopo* in headless mode to measure font metrics during labelling, and to render the vector SVG format as a raster graphic or PDF. Chrome should be detected automatically, but you can also configure its path manually if necessary:
34
37
 
35
38
  ```
36
39
  $ nswtopo config --chrome "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
@@ -38,6 +41,12 @@ $ nswtopo config --chrome "/Applications/Google Chrome.app/Contents/MacOS/Google
38
41
 
39
42
  # Miscellaneous
40
43
 
41
- For raster formats, use the `--dither` option to create the raster in indexed colour mode. This can reduce file size. For best results, have the `pngquant` program available on your command line for the dithering process.
44
+ After generating your map in SVG format, you can add content outside of *nswtopo* using a vector graphics editor such as Inkscape. You can then generate raster formats from the edited SVG instead of the map file:
42
45
 
43
- After generating your map in SVG format, you can add content outside of *nswtopo* using a vector graphics editor such as Inkscape. Use the `--external` option to render from the edited map, instead of the internal copy maintained by *nswtopo*.
46
+ ```
47
+ $ nswtopo render map.svg map.tif
48
+ ```
49
+
50
+ Maps normally use a white background. To specify a different background colour, use the `--background` option.
51
+
52
+ For raster formats, use the `--dither` option to create the raster in indexed colour mode. This can reduce file size. For best results, have the `pngquant` program available on your command line for the dithering process.
data/docs/scrape.md ADDED
@@ -0,0 +1,67 @@
1
+ # Description
2
+
3
+ Use *scrape* to download data from ArcGIS REST endpoint. Data may be downloaded from both *FeatureServer* and *MapServer* services, including *MapServer* services with no *Query* capability.
4
+
5
+ # Usage
6
+
7
+ Download some layers from an NSW ArcGIS server to a sqlite database `nsw.sqlite`:
8
+
9
+ ```
10
+ $ nswtopo scrape https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/1 nsw.sqlite
11
+ nswtopo: saved 87731 features
12
+ $ nswtopo scrape https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Features_of_Interest_Category/MapServer/6 nsw.sqlite
13
+ nswtopo: saved 6569 features
14
+ $ nswtopo scrape https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/10 nsw.sqlite
15
+ nswtopo: saved 81 features
16
+ ```
17
+
18
+ Provide the full URL for the layer, including id number, or use the `--id` or `--layer` option along with the service URL:
19
+
20
+ ```
21
+ $ nswtopo scrape --layer Railway https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer nsw.sqlite
22
+ nswtopo: saved 12979 features
23
+ ```
24
+
25
+ Shapefile and sqlite output formats are supported. The filename extension determines the format, with `.sqlite` or `.db` selecting sqlite output, `.shp` a single shapefile layer, and any other filename a shapefile directory.
26
+
27
+ Downloaded layers are named according to the ArcGIS layer name, or from the `--layer` option if specified.
28
+
29
+ # Filtering
30
+
31
+ Use the `--coords` option to restrict feature download to a specified bounding box:
32
+
33
+ ```
34
+ $ nswtopo scrape --coords 148.26,-36.52,148.38,-36.47 https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/10 nsw.sqlite
35
+ nswtopo: saved 12 features
36
+ ```
37
+
38
+ Use the `--where` option to restrict the download to certain field values:
39
+
40
+ ```
41
+ $ nswtopo scrape --where "classsubtype = 1" https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/10 nsw.sqlite
42
+ nswtopo: saved 58 features
43
+ ```
44
+
45
+ # Fields
46
+
47
+ If all fields are not needed, apply the `--fields` option to specify a list of fields to be downloaded:
48
+
49
+ ```
50
+ $ nswtopo scrape --fields CONTOUR_TY,ELEVATION https://services.thelist.tas.gov.au/arcgis/rest/services/Public/OpenDataWFS/MapServer/16 tas.sqlite
51
+ nswtopo: saved 528438 features
52
+ ```
53
+
54
+ Some ArcGIS fields contain `coded values`—integers or short strings representing longer, more descriptive strings. These can be decoded during download using the `--decode` option:
55
+
56
+ ```
57
+ $ nswtopo scrape --fields classsubtype,hydrotype --decode https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6 nsw.sqlite
58
+ nswtopo: saved 467774 features
59
+ ```
60
+
61
+ The `--decode` flag is particularly useful with type and subtype fields, where the same subtype code has a different meaning depending on the type code. Use the *inspect* command to view all codings for a given layer.
62
+
63
+ # Scraping from Map Layers
64
+
65
+ Some ArcGIS REST map layers do not expose the *Query* capability, preventing feature queries. It is still possible to download features from such a layer, provided it supports SVG output and dynamic layers.
66
+
67
+ When this situation occurs, you may be prompted to provide a *unique value* field name with the `--unique` option. Choose a layer field which is likely to have a small number of possible values. Using `--fields` to restrict the fields is also recommended. Specify only the fields you need to reduce download time.
data/docs/spot-heights.md CHANGED
@@ -10,14 +10,18 @@ DEM tiles from the ELVIS website are delivered as doubly-zipped files. It's not
10
10
  # Spot Height Configuration
11
11
  Use the `--spacing` option to determine the maximum density of spot heights on the map. The value in millimetres represents the minimum distance between any two spot heights.
12
12
 
13
- Choose the amount of smoothing to apply to the DEM with the `--smooth` option. The value represents a smoothing radius in millimetres. For consistency, the same smoothing radius shoult be used for both contours and spot heights.
13
+ Choose the amount of smoothing to apply to the DEM with the `--smooth` option. The value represents a smoothing radius in millimetres. For consistency in elevation, the same smoothing radius should be used for both contours and spot heights.
14
14
 
15
15
  Use the `--prefer` option to favour `knolls` or `saddles` when selecting spot locations. No preference is taken by default.
16
16
 
17
+ Use the `--extent` option to set a minimum size in millimetres when searching for spot-height features. Smaller, localised knolls and saddles will be rejected.
18
+
19
+ DEM tiles are normally processed at their maximumum native resolution. Change this using the `--resolution` option. A reduced resolution (say 5 metres) can markedly improve processing speed for 1- and 2-metre tiles.
20
+
17
21
  # Layer Position
18
22
 
19
23
  Use an `--after`, `--before` or `--replace` option to insert the spot heights in an appropriate layer position. You will most likely want to replace an existing spot heights layer:
20
24
 
21
25
  ```
22
- $ nswtopo contours --replace nsw.topographic.spot-heights map.tgz DATA_25994.zip
26
+ $ nswtopo spot-heights --replace nsw.topographic.spot-heights map.tgz DATA_25994.zip
23
27
  ```
@@ -1,14 +1,11 @@
1
1
  module NSWTopo
2
2
  class Archive
3
3
  extend Safely
4
+ include Enumerable
5
+ Invalid = Class.new RuntimeError
4
6
 
5
- def initialize(path, tar_in)
6
- @tar_in, @basename, @entries = tar_in, path.basename(".tgz").basename(".tar.gz").to_s, Hash[]
7
- end
8
- attr_reader :basename
9
-
10
- def delete(filename)
11
- @entries[filename] = nil
7
+ def initialize(tar)
8
+ @tar, @entries = tar, Hash[]
12
9
  end
13
10
 
14
11
  def write(filename, content)
@@ -17,15 +14,36 @@ module NSWTopo
17
14
  @entries[filename] = Gem::Package::TarReader::Entry.new header, io
18
15
  end
19
16
 
20
- def mtime(filename)
21
- header = @entries.key?(filename) ? @entries[filename]&.header : @tar_in.seek(filename, &:header)
22
- Time.at header.mtime if header
17
+ def each(&block)
18
+ @tar.rewind
19
+ @tar.each do |entry|
20
+ yield entry unless @entries.key? entry.full_name
21
+ end
22
+ @entries.each do |filename, entry|
23
+ yield entry if entry
24
+ end
25
+ end
26
+
27
+ def delete(filename)
28
+ find do |entry|
29
+ entry.full_name == filename
30
+ end&.tap do
31
+ @entries[filename] = nil
32
+ end
23
33
  end
24
34
 
25
35
  def read(filename)
26
- @entries.key?(filename) ? @entries[filename]&.read : @tar_in.seek(filename, &:read)
27
- ensure
28
- @entries[filename]&.rewind
36
+ find do |entry|
37
+ entry.full_name == filename
38
+ end&.read
39
+ end
40
+
41
+ def mtime(filename)
42
+ find do |entry|
43
+ entry.full_name == filename
44
+ end&.then do |entry|
45
+ Time.at entry.header.mtime
46
+ end
29
47
  end
30
48
 
31
49
  def uptodate?(depender, *dependees)
@@ -36,36 +54,25 @@ module NSWTopo
36
54
  end
37
55
  end
38
56
 
39
- def each(&block)
40
- @tar_in.each do |entry|
41
- yield entry unless @entries.key? entry.full_name
42
- end
43
- @entries.each do |filename, entry|
44
- yield entry if entry
45
- end
46
- end
47
-
48
57
  def changed?
49
- return true if @entries.values.any?
50
- @entries.keys.any? do |filename|
51
- @tar_in.seek(filename, &:itself)
52
- end
58
+ @entries.any?
53
59
  end
54
60
 
55
- def self.open(out_path, in_path = nil, &block)
61
+ def self.open(out_path: nil, in_path: nil, &block)
56
62
  buffer, reader = StringIO.new, in_path ? Zlib::GzipReader : StringIO
57
63
 
58
64
  reader.open(*in_path) do |input|
59
- if in_path
60
- version = input.comment.to_s[/^nswtopo (.+)$/, 1]
61
- raise "unrecognised map file: %s" % in_path unless version
62
- comparison = version.split(?.).map(&:to_i) <=> MIN_VERSION.split(?.).map(&:to_i)
63
- raise "map file too old: version %s, minimum %s required" % [version, MIN_VERSION] unless comparison >= 0
64
- end
65
- Gem::Package::TarReader.new(input) do |tar_in|
66
- archive = new(out_path, tar_in).tap(&block)
67
- Gem::Package::TarWriter.new(buffer) do |tar_out|
68
- archive.each &tar_out.method(:add_entry)
65
+ begin
66
+ version = Version[input.comment]
67
+ raise "map file too old: created with nswtopo %s, minimum %s required" % [version, MIN_VERSION] unless version >= MIN_VERSION
68
+ raise "nswtopo too old: map file created with nswtopo %s, this version %s" % [version, VERSION] unless version <= VERSION
69
+ rescue Version::Error
70
+ raise "unrecognised map file: %s" % in_path
71
+ end if in_path && false != Config["versioning"]
72
+ Gem::Package::TarReader.new(input) do |tar|
73
+ archive = new(tar).tap(&block)
74
+ Gem::Package::TarWriter.new(buffer) do |tar|
75
+ archive.each &tar.method(:add_entry)
69
76
  end if archive.changed?
70
77
  end
71
78
  end
@@ -73,8 +80,8 @@ module NSWTopo
73
80
  Dir.mktmppath do |temp_dir|
74
81
  log_update "nswtopo: saving map..."
75
82
  temp_path = temp_dir / "temp.tgz"
76
- Zlib::GzipWriter.open temp_path, Zlib::BEST_COMPRESSION do |gzip|
77
- gzip.comment = "nswtopo %s" % VERSION
83
+ Zlib::GzipWriter.open temp_path, Config["zlib-level"] || Zlib::BEST_SPEED do |gzip|
84
+ gzip.comment = VERSION.creator_string
78
85
  gzip.write buffer.string
79
86
  rescue Interrupt
80
87
  log_update "nswtopo: interrupted, please wait..."
@@ -82,12 +89,14 @@ module NSWTopo
82
89
  end
83
90
  safely "saving map file, please wait..." do
84
91
  FileUtils.cp temp_path, out_path
92
+ rescue SystemCallError
93
+ raise "couldn't save #{out_path}"
85
94
  end
86
95
  log_success "map saved"
87
- end unless buffer.size.zero?
96
+ end if out_path && buffer.size.nonzero?
88
97
 
89
98
  rescue Zlib::GzipFile::Error
90
- raise "unrecognised map file: %s" % in_path
99
+ raise Invalid
91
100
  end
92
101
  end
93
102
  end
@@ -0,0 +1,227 @@
1
+ module NSWTopo
2
+ class Chrome
3
+ MIN_VERSION = 112
4
+ TIMEOUT_KILL = 5
5
+ TIMEOUT_LOADEVENT = 30
6
+ TIMEOUT_COMMAND = 10
7
+ TIMEOUT_SCREENSHOT = 120
8
+
9
+ ARGS = %W[
10
+ --disable-background-networking
11
+ --disable-component-extensions-with-background-pages
12
+ --disable-component-update
13
+ --disable-default-apps
14
+ --disable-extensions
15
+ --disable-features=site-per-process,Translate
16
+ --disable-lcd-text
17
+ --disable-renderer-backgrounding
18
+ --force-color-profile=srgb
19
+ --force-device-scale-factor=1
20
+ --headless=new
21
+ --hide-scrollbars
22
+ --no-default-browser-check
23
+ --no-first-run
24
+ --no-startup-window
25
+ --remote-debugging-pipe=JSON
26
+ --use-mock-keychain
27
+ ]
28
+
29
+ class Error < RuntimeError
30
+ def initialize(message = "chrome error")
31
+ super
32
+ end
33
+ end
34
+
35
+ def self.mac?
36
+ /darwin/ === RbConfig::CONFIG["host_os"]
37
+ end
38
+
39
+ def self.windows?
40
+ /mingw|mswin|cygwin/ === RbConfig::CONFIG["host_os"]
41
+ end
42
+
43
+ def self.path
44
+ @path ||= case
45
+ when Config["chrome"]
46
+ [Config["chrome"]]
47
+ when mac?
48
+ ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Chromium.app/Contents/MacOS/Chromium"]
49
+ when windows?
50
+ ["C:/Program Files/Google/Chrome/Application/chrome.exe", "C:/Program Files (x86)/Google/Chrome/Application/chrome.exe"]
51
+ else
52
+ ENV["PATH"].split(File::PATH_SEPARATOR).product(%w[chrome google-chrome chromium chromium-browser]).map do |path, binary|
53
+ [path, binary].join(File::SEPARATOR)
54
+ end
55
+ end.find do |path|
56
+ File.executable?(path) && !File.directory?(path)
57
+ end.tap do |path|
58
+ raise Error, "couldn't find chrome" unless path
59
+ stdout, status = Open3.capture2 path, "--version"
60
+ raise Error, "couldn't start chrome" unless status.success?
61
+ version = /(?<major>\d+)(?:\.\d+)*/.match stdout
62
+ raise Error, "couldn't start chrome" unless version
63
+ raise Error, "chrome version #{MIN_VERSION} or higher required" if version[:major].to_i < MIN_VERSION
64
+ end
65
+ end
66
+
67
+ def self.rmdir(tmp)
68
+ Proc.new do
69
+ FileUtils.remove_entry tmp
70
+ rescue SystemCallError
71
+ end
72
+ end
73
+
74
+ def self.kill(pid, *pipes)
75
+ Proc.new do
76
+ if windows?
77
+ *, status = Open3.capture2e *%W[taskkill /f /t /pid #{pid}]
78
+ Process.kill "KILL", pid unless status.success?
79
+ else
80
+ Timeout.timeout(TIMEOUT_KILL, Error) do
81
+ Process.kill "-USR1", Process.getpgid(pid)
82
+ Process.wait pid
83
+ rescue Error
84
+ Process.kill "-KILL", Process.getpgid(pid)
85
+ Process.wait pid
86
+ end
87
+ end
88
+ rescue Errno::ESRCH, Errno::ECHILD
89
+ ensure
90
+ pipes.each(&:close)
91
+ end
92
+ end
93
+
94
+ def close
95
+ Chrome.kill(@pid, @input, @output).call
96
+ Chrome.rmdir(@data_dir).call
97
+ ObjectSpace.undefine_finalizer self
98
+ end
99
+
100
+ def self.with_browser(url, **opts, &block)
101
+ browser = new url, **opts
102
+ block.call browser
103
+ ensure
104
+ browser&.close
105
+ end
106
+
107
+ def initialize(url, width: 800, height: 600, background: { r: 0, g: 0, b: 0, a: 0 }, args: [])
108
+ @id, @data_dir = 0, Dir.mktmpdir("nswtopo_headless_chrome_")
109
+ ObjectSpace.define_finalizer self, Chrome.rmdir(@data_dir)
110
+
111
+ args << "--disable-gpu" if Config["gpu"] == false
112
+ args << "--user-data-dir=#{@data_dir}"
113
+
114
+ input, @input, @output, output = *IO.pipe, *IO.pipe
115
+ input.nonblock, output.nonblock = false, false
116
+ @input.sync = true
117
+
118
+ @pid = Process.spawn Chrome.path, *ARGS, *args, 1 => File::NULL, 2 => File::NULL, 3 => input, 4 => output, :pgroup => Chrome.windows? ? nil : true
119
+ ObjectSpace.define_finalizer self, Chrome.kill(@pid, @input, @output)
120
+ input.close; output.close
121
+
122
+ target_id = command("Target.createTarget", url: url).fetch("targetId")
123
+ @session_id = command("Target.attachToTarget", targetId: target_id, flatten: true).fetch("sessionId")
124
+ command "Page.enable"
125
+ wait "Page.loadEventFired", timeout: TIMEOUT_LOADEVENT
126
+ command "Emulation.setDeviceMetricsOverride", width: width, height: height, deviceScaleFactor: 1, mobile: false
127
+ command "Emulation.setDefaultBackgroundColorOverride", color: background
128
+ @node_id = command("DOM.getDocument").fetch("root").fetch("nodeId")
129
+ rescue SystemCallError
130
+ raise Error, "couldn't start chrome"
131
+ rescue KeyError
132
+ raise Error
133
+ end
134
+
135
+ def send(**message)
136
+ message.merge! sessionId: @session_id if @session_id
137
+ @input.write message.to_json, ?\0
138
+ end
139
+
140
+ def messages
141
+ Enumerator.produce do
142
+ json = @output.readline(?\0).chomp(?\0)
143
+ JSON.parse(json).tap do |message|
144
+ raise Error if message["error"]
145
+ raise Error if message["method"] == "Target.detachedFromTarget"
146
+ end
147
+ rescue JSON::ParserError, EOFError
148
+ raise Error
149
+ end
150
+ end
151
+
152
+ def wait(event, timeout: nil)
153
+ Timeout.timeout(timeout) do
154
+ messages.find do |message|
155
+ message["method"] == event
156
+ end
157
+ end
158
+ rescue Timeout::Error
159
+ raise Error
160
+ end
161
+
162
+ def command(method, timeout: TIMEOUT_COMMAND, **params)
163
+ send id: @id += 1, method: method, params: params
164
+ Timeout.timeout(timeout) do
165
+ messages.find do |message|
166
+ message["id"] == @id
167
+ end
168
+ end.fetch("result")
169
+ rescue Timeout::Error, KeyError
170
+ raise Error
171
+ end
172
+
173
+ def screenshot(png_path)
174
+ data = command("Page.captureScreenshot", timeout: TIMEOUT_SCREENSHOT).fetch("data")
175
+ png_path.binwrite Base64.decode64(data)
176
+ rescue KeyError
177
+ raise Error
178
+ end
179
+
180
+ def print_to_pdf(pdf_path)
181
+ data = command("Page.printToPDF", timeout: nil, preferCSSPageSize: true).fetch("data")
182
+ pdf_path.binwrite Base64.decode64(data)
183
+ rescue KeyError
184
+ raise Error
185
+ end
186
+
187
+ def query_selector_node_id(selector)
188
+ command("DOM.querySelector", selector: selector, nodeId: @node_id).fetch("nodeId")
189
+ rescue KeyError
190
+ raise Error
191
+ end
192
+
193
+ class Node
194
+ def initialize(browser, selector)
195
+ @browser, @node_id = browser, browser.query_selector_node_id(selector)
196
+ end
197
+
198
+ def [](name)
199
+ @browser.command("DOM.getAttributes", nodeId: @node_id).fetch("attributes").each_slice(2).to_h.fetch(name.to_s)
200
+ rescue KeyError
201
+ raise Error
202
+ end
203
+
204
+ def []=(name, value)
205
+ if value.nil?
206
+ @browser.command "DOM.removeAttribute", nodeId: @node_id, name: name
207
+ else
208
+ @browser.command "DOM.setAttributeValue", nodeId: @node_id, name: name, value: value
209
+ end
210
+ end
211
+
212
+ def value=(value)
213
+ @browser.command "DOM.setNodeValue", nodeId: @node_id + 1, value: value
214
+ end
215
+
216
+ def width
217
+ @browser.command("DOM.getBoxModel", nodeId: @node_id).fetch("model").fetch("content").each_slice(2).map(&:first).minmax.reverse.inject(&:-)
218
+ rescue KeyError
219
+ raise Error
220
+ end
221
+ end
222
+
223
+ def query_selector(selector)
224
+ Node.new self, selector
225
+ end
226
+ end
227
+ end