nswtopo 2.0.0.pre.beta1 → 3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/COPYING +70 -83
- data/bin/nswtopo +227 -116
- data/docs/README.md +1 -12
- data/docs/add.md +1 -1
- data/docs/config.md +1 -1
- data/docs/contours.md +3 -1
- data/docs/init.md +8 -0
- data/docs/inspect.md +103 -0
- data/docs/move.md +9 -0
- data/docs/render.md +16 -7
- data/docs/scrape.md +67 -0
- data/docs/spot-heights.md +6 -2
- data/lib/nswtopo/archive.rb +50 -41
- data/lib/nswtopo/chrome.rb +227 -0
- data/lib/nswtopo/commands/add.rb +106 -0
- data/lib/nswtopo/commands/config.rb +38 -0
- data/lib/nswtopo/commands/inspect.rb +74 -0
- data/lib/nswtopo/commands/layers.rb +22 -0
- data/lib/nswtopo/commands/scrape.rb +79 -0
- data/lib/nswtopo/commands.rb +57 -0
- data/lib/nswtopo/dither.rb +5 -3
- data/lib/nswtopo/font.rb +46 -21
- data/lib/nswtopo/formats/gemf.rb +42 -0
- data/lib/nswtopo/formats/kmz.rb +26 -24
- data/lib/nswtopo/formats/mbtiles.rb +5 -41
- data/lib/nswtopo/formats/pdf.rb +82 -17
- data/lib/nswtopo/formats/svg.rb +114 -45
- data/lib/nswtopo/formats/svgz.rb +2 -2
- data/lib/nswtopo/formats/zip.rb +33 -23
- data/lib/nswtopo/formats.rb +77 -32
- data/lib/nswtopo/geometry/overlap.rb +1 -32
- data/lib/nswtopo/geometry/r_tree.rb +16 -10
- data/lib/nswtopo/geometry/segment.rb +3 -3
- data/lib/nswtopo/geometry/straight_skeleton/nodes.rb +5 -6
- data/lib/nswtopo/geometry/vector_sequence.rb +7 -6
- data/lib/nswtopo/gis/arcgis/connection.rb +56 -0
- data/lib/nswtopo/gis/arcgis/layer/map.rb +163 -0
- data/lib/nswtopo/gis/arcgis/layer/query.rb +87 -0
- data/lib/nswtopo/gis/arcgis/layer/renderer.rb +66 -0
- data/lib/nswtopo/gis/arcgis/layer/statistics.rb +15 -0
- data/lib/nswtopo/gis/arcgis/layer.rb +201 -0
- data/lib/nswtopo/gis/arcgis/service.rb +57 -0
- data/lib/nswtopo/gis/arcgis.rb +3 -0
- data/lib/nswtopo/gis/dem.rb +13 -12
- data/lib/nswtopo/gis/esri_hdr.rb +8 -2
- data/lib/nswtopo/gis/geojson/collection.rb +45 -21
- data/lib/nswtopo/gis/geojson/multi_line_string.rb +2 -24
- data/lib/nswtopo/gis/geojson/multi_polygon.rb +2 -53
- data/lib/nswtopo/gis/geojson/polygon.rb +15 -0
- data/lib/nswtopo/gis/geojson.rb +12 -3
- data/lib/nswtopo/gis/gps/kml.rb +25 -19
- data/lib/nswtopo/gis/gps.rb +2 -0
- data/lib/nswtopo/gis/projection.rb +35 -24
- data/lib/nswtopo/gis/shapefile.rb +89 -16
- data/lib/nswtopo/gis.rb +1 -2
- data/lib/nswtopo/helpers/array.rb +0 -11
- data/lib/nswtopo/helpers/colour.rb +34 -14
- data/lib/nswtopo/layer/arcgis_raster.rb +44 -48
- data/lib/nswtopo/layer/colour_mask.rb +5 -0
- data/lib/nswtopo/layer/contour.rb +35 -28
- data/lib/nswtopo/layer/control.rb +2 -7
- data/lib/nswtopo/layer/declination.rb +9 -9
- data/lib/nswtopo/layer/feature.rb +36 -22
- data/lib/nswtopo/layer/grid.rb +30 -27
- data/lib/nswtopo/layer/import.rb +1 -21
- data/lib/nswtopo/layer/labels/barrier.rb +39 -0
- data/lib/nswtopo/layer/labels.rb +551 -383
- data/lib/nswtopo/layer/mask_render.rb +37 -0
- data/lib/nswtopo/layer/overlay.rb +2 -2
- data/lib/nswtopo/layer/raster.rb +31 -41
- data/lib/nswtopo/layer/raster_import.rb +17 -0
- data/lib/nswtopo/layer/raster_render.rb +15 -0
- data/lib/nswtopo/layer/relief.rb +27 -95
- data/lib/nswtopo/layer/spot.rb +63 -62
- data/lib/nswtopo/layer/vector/cutout.rb +15 -0
- data/lib/nswtopo/layer/vector/knockout.rb +16 -0
- data/lib/nswtopo/layer/vector.rb +121 -89
- data/lib/nswtopo/layer/vegetation.rb +39 -34
- data/lib/nswtopo/layer.rb +30 -16
- data/lib/nswtopo/map.rb +202 -109
- data/lib/nswtopo/os.rb +5 -27
- data/lib/nswtopo/tiled_web_map.rb +54 -0
- data/lib/nswtopo/tree_indenter.rb +27 -0
- data/lib/nswtopo/version.rb +27 -2
- data/lib/nswtopo.rb +6 -199
- metadata +41 -22
- data/lib/nswtopo/font/chrome.rb +0 -59
- data/lib/nswtopo/font/generic.rb +0 -25
- data/lib/nswtopo/gis/arcgis_server/connection.rb +0 -52
- data/lib/nswtopo/gis/arcgis_server.rb +0 -155
- data/lib/nswtopo/gis/geojson/multi_point.rb +0 -12
- data/lib/nswtopo/gis/world_file.rb +0 -19
- data/lib/nswtopo/layer/labels/fence.rb +0 -20
data/docs/inspect.md
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Description
|
|
2
|
+
|
|
3
|
+
Use the *inspect* command to examine ArcGIS REST endpoints and local GIS data.
|
|
4
|
+
|
|
5
|
+
# ArcGIS REST layers
|
|
6
|
+
|
|
7
|
+
List all the layers in an ArcGIS map or feature service as follows:
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
$ nswtopo inspect https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer
|
|
11
|
+
layers:
|
|
12
|
+
├─ 0: HydroPoint
|
|
13
|
+
├─ 1: AncillaryHydroPoint
|
|
14
|
+
├─ 2: NamedWatercourse
|
|
15
|
+
├─ 3: FuzzyExtentWaterLine
|
|
16
|
+
├─ 4: Coastline
|
|
17
|
+
├─ 5: HydroLine
|
|
18
|
+
├─ 6: HydroArea
|
|
19
|
+
└─ 7: FuzzyExtentWaterArea
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
List the fields for a layer using the layer URL, or with the `--layer` or `--id` option. The layer's fields and their types will be listed:
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
$ nswtopo inspect https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
|
|
26
|
+
name: HydroArea
|
|
27
|
+
id: 6
|
|
28
|
+
geometry: Polygon
|
|
29
|
+
fields:
|
|
30
|
+
├─ attributereliabilitydate: Date
|
|
31
|
+
├─ capturemethodcode: SmallInteger
|
|
32
|
+
│ ...
|
|
33
|
+
├─ urbanity: String
|
|
34
|
+
└─ verticalaccuracy: Single
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Use the `--fields` option to inspect values and counts for one or more fields:
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
$ nswtopo inspect --fields classsubtype,hydrotype https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
|
|
41
|
+
classsubtype │ 447690 │ 1
|
|
42
|
+
hydrotype │ 9070 │ ├─ 1
|
|
43
|
+
hydrotype │ 438620 │ └─ 2
|
|
44
|
+
classsubtype │ 20084 │ 2
|
|
45
|
+
hydrotype │ 19849 │ ├─ 1
|
|
46
|
+
hydrotype │ 235 │ └─ 2
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Field names and all possible values are shown, with counts for each combination of values. In this case, the values are *coded values*. Use `--decode` to decode them:
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
$ nswtopo inspect --fields classsubtype,hydrotype --decode https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
|
|
53
|
+
classsubtype │ 447690 │ WaterbodyArea
|
|
54
|
+
hydrotype │ 438620 │ ├─ ManMadeWaterBody
|
|
55
|
+
hydrotype │ 9070 │ └─ NaturalWaterBody
|
|
56
|
+
classsubtype │ 20084 │ Watercourse
|
|
57
|
+
hydrotype │ 235 │ ├─ Canal-Drain
|
|
58
|
+
hydrotype │ 19849 │ └─ NaturalWatercourse
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
List all *coded value* conversions for a layer with the `--codes` option:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
$ nswtopo inspect --codes https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6
|
|
65
|
+
perenniality:
|
|
66
|
+
├─ 0 → NotApplicable
|
|
67
|
+
├─ 1 → Perennial
|
|
68
|
+
├─ 2 → NonPerennial
|
|
69
|
+
└─ 3 → MainlyDry
|
|
70
|
+
classsubtype:
|
|
71
|
+
├─ 1 → WaterbodyArea
|
|
72
|
+
│ └─ hydrotype:
|
|
73
|
+
│ ├─ 1 → NaturalWaterBody
|
|
74
|
+
│ └─ 2 → ManMadeWaterBody
|
|
75
|
+
└─ 2 → Watercourse
|
|
76
|
+
└─ hydrotype:
|
|
77
|
+
├─ 1 → NaturalWatercourse
|
|
78
|
+
└─ 2 → Canal-Drain
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Use the `--where` option to restrict output using a SQL expression on the fields.
|
|
82
|
+
|
|
83
|
+
# Local GIS Data
|
|
84
|
+
|
|
85
|
+
Any OGR-readable data can likewise be examined using the `inspect` command. For example, to list layers contained in a spatialite file:
|
|
86
|
+
|
|
87
|
+
```
|
|
88
|
+
$ nswtopo inspect nsw.sqlite
|
|
89
|
+
layers:
|
|
90
|
+
├─ cableway (LineString)
|
|
91
|
+
├─ electricitytransissionline (LineString)
|
|
92
|
+
├─ railway (LineString)
|
|
93
|
+
└─ trafficcontroldevice (Point)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
To list field values for a layer:
|
|
97
|
+
|
|
98
|
+
```
|
|
99
|
+
$ nswtopo inspect nsw.sqlite --layer cableway --fields classsubtype
|
|
100
|
+
classsubtype │ 4 │ CableCar
|
|
101
|
+
classsubtype │ 19 │ FlyingFox
|
|
102
|
+
classsubtype │ 58 │ SkiLift
|
|
103
|
+
```
|
data/docs/move.md
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Description
|
|
2
|
+
|
|
3
|
+
Use the *move* command to rearrange the layer order in a map. Specify the name of the layer you wish to move, and the layer you want to place it before or after:
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
$ nswtopo move map.tgz relief --after nsw.vegetation-spot5
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Use the *info* command to see the names of layers currently in the map file.
|
data/docs/render.md
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
# Description
|
|
2
2
|
|
|
3
|
-
Once you've added your layers, use the *render* command to create the map itself. Different output formats are available for various purposes.
|
|
3
|
+
Once you've added your layers, use the *render* command to create the map itself. Different output formats are available for various purposes. In its simplest form, to render a GeoTIFF from a map file:
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
$ nswtopo render map.tgz map.tif
|
|
7
|
+
```
|
|
4
8
|
|
|
5
9
|
Specify your output format either as a filename with appropriate extension, or just the format extension itself (in which case the map's file name will be used). You can create multiple outputs at once.
|
|
6
10
|
|
|
@@ -15,7 +19,8 @@ The following formats are available:
|
|
|
15
19
|
* **pdf**: *Portable Document Format*, either in the original vector form, or as a raster by setting a resolution with `--ppi`
|
|
16
20
|
* **kmz**: for use with *Google Earth* (add as a network link for best results)
|
|
17
21
|
* **zip**: a tiled format used by the *Avenza Maps* mobile app
|
|
18
|
-
* **mbtiles**: a tiled format for use with mobile GPS apps including *Locus Map* and *
|
|
22
|
+
* **mbtiles**: a tiled format for use with mobile GPS apps including *Locus Map* and *OruxMaps*
|
|
23
|
+
* **gemf**: a fast tiled format compatible with *Locus Map*
|
|
19
24
|
* **svgz**: a compressed SVG format, viewable directly in some browsers
|
|
20
25
|
* **png**: the well-known *Portable Network Graphics* format
|
|
21
26
|
* **jpg**: the well-known *JPEG* format (not well-suited to maps)
|
|
@@ -28,9 +33,7 @@ For the *mbtiles* format, resolution values are fixed to zoom levels. The defaul
|
|
|
28
33
|
|
|
29
34
|
# Setting Up Chrome
|
|
30
35
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
Add the path for your Chrome executable to your *nswtopo* configuration as follows:
|
|
36
|
+
You'll need to have *Google Chrome* or *Chromium* installed. The Chrome browser is used by *nswtopo* in headless mode to measure font metrics during labelling, and to render the vector SVG format as a raster graphic or PDF. Chrome should be detected automatically, but you can also configure its path manually if necessary:
|
|
34
37
|
|
|
35
38
|
```
|
|
36
39
|
$ nswtopo config --chrome "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
|
|
@@ -38,6 +41,12 @@ $ nswtopo config --chrome "/Applications/Google Chrome.app/Contents/MacOS/Google
|
|
|
38
41
|
|
|
39
42
|
# Miscellaneous
|
|
40
43
|
|
|
41
|
-
|
|
44
|
+
After generating your map in SVG format, you can add content outside of *nswtopo* using a vector graphics editor such as Inkscape. You can then generate raster formats from the edited SVG instead of the map file:
|
|
42
45
|
|
|
43
|
-
|
|
46
|
+
```
|
|
47
|
+
$ nswtopo render map.svg map.tif
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Maps normally use a white background. To specify a different background colour, use the `--background` option.
|
|
51
|
+
|
|
52
|
+
For raster formats, use the `--dither` option to create the raster in indexed colour mode. This can reduce file size. For best results, have the `pngquant` program available on your command line for the dithering process.
|
data/docs/scrape.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Description
|
|
2
|
+
|
|
3
|
+
Use *scrape* to download data from ArcGIS REST endpoint. Data may be downloaded from both *FeatureServer* and *MapServer* services, including *MapServer* services with no *Query* capability.
|
|
4
|
+
|
|
5
|
+
# Usage
|
|
6
|
+
|
|
7
|
+
Download some layers from an NSW ArcGIS server to a sqlite database `nsw.sqlite`:
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
$ nswtopo scrape https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/1 nsw.sqlite
|
|
11
|
+
nswtopo: saved 87731 features
|
|
12
|
+
$ nswtopo scrape https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Features_of_Interest_Category/MapServer/6 nsw.sqlite
|
|
13
|
+
nswtopo: saved 6569 features
|
|
14
|
+
$ nswtopo scrape https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/10 nsw.sqlite
|
|
15
|
+
nswtopo: saved 81 features
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Provide the full URL for the layer, including id number, or use the `--id` or `--layer` option along with the service URL:
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
$ nswtopo scrape --layer Railway https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer nsw.sqlite
|
|
22
|
+
nswtopo: saved 12979 features
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Shapefile and sqlite output formats are supported. The filename extension determines the format, with `.sqlite` or `.db` selecting sqlite output, `.shp` a single shapefile layer, and any other filename a shapefile directory.
|
|
26
|
+
|
|
27
|
+
Downloaded layers are named according to the ArcGIS layer name, or from the `--layer` option if specified.
|
|
28
|
+
|
|
29
|
+
# Filtering
|
|
30
|
+
|
|
31
|
+
Use the `--coords` option to restrict feature download to a specified bounding box:
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
$ nswtopo scrape --coords 148.26,-36.52,148.38,-36.47 https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/10 nsw.sqlite
|
|
35
|
+
nswtopo: saved 12 features
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Use the `--where` option to restrict the download to certain field values:
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
$ nswtopo scrape --where "classsubtype = 1" https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Transport_Theme/MapServer/10 nsw.sqlite
|
|
42
|
+
nswtopo: saved 58 features
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
# Fields
|
|
46
|
+
|
|
47
|
+
If all fields are not needed, apply the `--fields` option to specify a list of fields to be downloaded:
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
$ nswtopo scrape --fields CONTOUR_TY,ELEVATION https://services.thelist.tas.gov.au/arcgis/rest/services/Public/OpenDataWFS/MapServer/16 tas.sqlite
|
|
51
|
+
nswtopo: saved 528438 features
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Some ArcGIS fields contain `coded values`—integers or short strings representing longer, more descriptive strings. These can be decoded during download using the `--decode` option:
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
$ nswtopo scrape --fields classsubtype,hydrotype --decode https://portal.spatial.nsw.gov.au/server/rest/services/NSW_Water_Theme/MapServer/6 nsw.sqlite
|
|
58
|
+
nswtopo: saved 467774 features
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The `--decode` flag is particularly useful with type and subtype fields, where the same subtype code has a different meaning depending on the type code. Use the *inspect* command to view all codings for a given layer.
|
|
62
|
+
|
|
63
|
+
# Scraping from Map Layers
|
|
64
|
+
|
|
65
|
+
Some ArcGIS REST map layers do not expose the *Query* capability, preventing feature queries. It is still possible to download features from such a layer, provided it supports SVG output and dynamic layers.
|
|
66
|
+
|
|
67
|
+
When this situation occurs, you may be prompted to provide a *unique value* field name with the `--unique` option. Choose a layer field which is likely to have a small number of possible values. Using `--fields` to restrict the fields is also recommended. Specify only the fields you need to reduce download time.
|
data/docs/spot-heights.md
CHANGED
|
@@ -10,14 +10,18 @@ DEM tiles from the ELVIS website are delivered as doubly-zipped files. It's not
|
|
|
10
10
|
# Spot Height Configuration
|
|
11
11
|
Use the `--spacing` option to determine the maximum density of spot heights on the map. The value in millimetres represents the minimum distance between any two spot heights.
|
|
12
12
|
|
|
13
|
-
Choose the amount of smoothing to apply to the DEM with the `--smooth` option. The value represents a smoothing radius in millimetres. For consistency, the same smoothing radius
|
|
13
|
+
Choose the amount of smoothing to apply to the DEM with the `--smooth` option. The value represents a smoothing radius in millimetres. For consistency in elevation, the same smoothing radius should be used for both contours and spot heights.
|
|
14
14
|
|
|
15
15
|
Use the `--prefer` option to favour `knolls` or `saddles` when selecting spot locations. No preference is taken by default.
|
|
16
16
|
|
|
17
|
+
Use the `--extent` option to set a minimum size in millimetres when searching for spot-height features. Smaller, localised knolls and saddles will be rejected.
|
|
18
|
+
|
|
19
|
+
DEM tiles are normally processed at their maximumum native resolution. Change this using the `--resolution` option. A reduced resolution (say 5 metres) can markedly improve processing speed for 1- and 2-metre tiles.
|
|
20
|
+
|
|
17
21
|
# Layer Position
|
|
18
22
|
|
|
19
23
|
Use an `--after`, `--before` or `--replace` option to insert the spot heights in an appropriate layer position. You will most likely want to replace an existing spot heights layer:
|
|
20
24
|
|
|
21
25
|
```
|
|
22
|
-
$ nswtopo
|
|
26
|
+
$ nswtopo spot-heights --replace nsw.topographic.spot-heights map.tgz DATA_25994.zip
|
|
23
27
|
```
|
data/lib/nswtopo/archive.rb
CHANGED
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
module NSWTopo
|
|
2
2
|
class Archive
|
|
3
3
|
extend Safely
|
|
4
|
+
include Enumerable
|
|
5
|
+
Invalid = Class.new RuntimeError
|
|
4
6
|
|
|
5
|
-
def initialize(
|
|
6
|
-
@
|
|
7
|
-
end
|
|
8
|
-
attr_reader :basename
|
|
9
|
-
|
|
10
|
-
def delete(filename)
|
|
11
|
-
@entries[filename] = nil
|
|
7
|
+
def initialize(tar)
|
|
8
|
+
@tar, @entries = tar, Hash[]
|
|
12
9
|
end
|
|
13
10
|
|
|
14
11
|
def write(filename, content)
|
|
@@ -17,15 +14,36 @@ module NSWTopo
|
|
|
17
14
|
@entries[filename] = Gem::Package::TarReader::Entry.new header, io
|
|
18
15
|
end
|
|
19
16
|
|
|
20
|
-
def
|
|
21
|
-
|
|
22
|
-
|
|
17
|
+
def each(&block)
|
|
18
|
+
@tar.rewind
|
|
19
|
+
@tar.each do |entry|
|
|
20
|
+
yield entry unless @entries.key? entry.full_name
|
|
21
|
+
end
|
|
22
|
+
@entries.each do |filename, entry|
|
|
23
|
+
yield entry if entry
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def delete(filename)
|
|
28
|
+
find do |entry|
|
|
29
|
+
entry.full_name == filename
|
|
30
|
+
end&.tap do
|
|
31
|
+
@entries[filename] = nil
|
|
32
|
+
end
|
|
23
33
|
end
|
|
24
34
|
|
|
25
35
|
def read(filename)
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
36
|
+
find do |entry|
|
|
37
|
+
entry.full_name == filename
|
|
38
|
+
end&.read
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def mtime(filename)
|
|
42
|
+
find do |entry|
|
|
43
|
+
entry.full_name == filename
|
|
44
|
+
end&.then do |entry|
|
|
45
|
+
Time.at entry.header.mtime
|
|
46
|
+
end
|
|
29
47
|
end
|
|
30
48
|
|
|
31
49
|
def uptodate?(depender, *dependees)
|
|
@@ -36,36 +54,25 @@ module NSWTopo
|
|
|
36
54
|
end
|
|
37
55
|
end
|
|
38
56
|
|
|
39
|
-
def each(&block)
|
|
40
|
-
@tar_in.each do |entry|
|
|
41
|
-
yield entry unless @entries.key? entry.full_name
|
|
42
|
-
end
|
|
43
|
-
@entries.each do |filename, entry|
|
|
44
|
-
yield entry if entry
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
|
|
48
57
|
def changed?
|
|
49
|
-
|
|
50
|
-
@entries.keys.any? do |filename|
|
|
51
|
-
@tar_in.seek(filename, &:itself)
|
|
52
|
-
end
|
|
58
|
+
@entries.any?
|
|
53
59
|
end
|
|
54
60
|
|
|
55
|
-
def self.open(out_path, in_path
|
|
61
|
+
def self.open(out_path: nil, in_path: nil, &block)
|
|
56
62
|
buffer, reader = StringIO.new, in_path ? Zlib::GzipReader : StringIO
|
|
57
63
|
|
|
58
64
|
reader.open(*in_path) do |input|
|
|
59
|
-
|
|
60
|
-
version = input.comment
|
|
61
|
-
raise "
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
65
|
+
begin
|
|
66
|
+
version = Version[input.comment]
|
|
67
|
+
raise "map file too old: created with nswtopo %s, minimum %s required" % [version, MIN_VERSION] unless version >= MIN_VERSION
|
|
68
|
+
raise "nswtopo too old: map file created with nswtopo %s, this version %s" % [version, VERSION] unless version <= VERSION
|
|
69
|
+
rescue Version::Error
|
|
70
|
+
raise "unrecognised map file: %s" % in_path
|
|
71
|
+
end if in_path && false != Config["versioning"]
|
|
72
|
+
Gem::Package::TarReader.new(input) do |tar|
|
|
73
|
+
archive = new(tar).tap(&block)
|
|
74
|
+
Gem::Package::TarWriter.new(buffer) do |tar|
|
|
75
|
+
archive.each &tar.method(:add_entry)
|
|
69
76
|
end if archive.changed?
|
|
70
77
|
end
|
|
71
78
|
end
|
|
@@ -73,8 +80,8 @@ module NSWTopo
|
|
|
73
80
|
Dir.mktmppath do |temp_dir|
|
|
74
81
|
log_update "nswtopo: saving map..."
|
|
75
82
|
temp_path = temp_dir / "temp.tgz"
|
|
76
|
-
Zlib::GzipWriter.open temp_path, Zlib::
|
|
77
|
-
gzip.comment =
|
|
83
|
+
Zlib::GzipWriter.open temp_path, Config["zlib-level"] || Zlib::BEST_SPEED do |gzip|
|
|
84
|
+
gzip.comment = VERSION.creator_string
|
|
78
85
|
gzip.write buffer.string
|
|
79
86
|
rescue Interrupt
|
|
80
87
|
log_update "nswtopo: interrupted, please wait..."
|
|
@@ -82,12 +89,14 @@ module NSWTopo
|
|
|
82
89
|
end
|
|
83
90
|
safely "saving map file, please wait..." do
|
|
84
91
|
FileUtils.cp temp_path, out_path
|
|
92
|
+
rescue SystemCallError
|
|
93
|
+
raise "couldn't save #{out_path}"
|
|
85
94
|
end
|
|
86
95
|
log_success "map saved"
|
|
87
|
-
end
|
|
96
|
+
end if out_path && buffer.size.nonzero?
|
|
88
97
|
|
|
89
98
|
rescue Zlib::GzipFile::Error
|
|
90
|
-
raise
|
|
99
|
+
raise Invalid
|
|
91
100
|
end
|
|
92
101
|
end
|
|
93
102
|
end
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
module NSWTopo
|
|
2
|
+
class Chrome
|
|
3
|
+
MIN_VERSION = 112
|
|
4
|
+
TIMEOUT_KILL = 5
|
|
5
|
+
TIMEOUT_LOADEVENT = 30
|
|
6
|
+
TIMEOUT_COMMAND = 10
|
|
7
|
+
TIMEOUT_SCREENSHOT = 120
|
|
8
|
+
|
|
9
|
+
ARGS = %W[
|
|
10
|
+
--disable-background-networking
|
|
11
|
+
--disable-component-extensions-with-background-pages
|
|
12
|
+
--disable-component-update
|
|
13
|
+
--disable-default-apps
|
|
14
|
+
--disable-extensions
|
|
15
|
+
--disable-features=site-per-process,Translate
|
|
16
|
+
--disable-lcd-text
|
|
17
|
+
--disable-renderer-backgrounding
|
|
18
|
+
--force-color-profile=srgb
|
|
19
|
+
--force-device-scale-factor=1
|
|
20
|
+
--headless=new
|
|
21
|
+
--hide-scrollbars
|
|
22
|
+
--no-default-browser-check
|
|
23
|
+
--no-first-run
|
|
24
|
+
--no-startup-window
|
|
25
|
+
--remote-debugging-pipe=JSON
|
|
26
|
+
--use-mock-keychain
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
class Error < RuntimeError
|
|
30
|
+
def initialize(message = "chrome error")
|
|
31
|
+
super
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def self.mac?
|
|
36
|
+
/darwin/ === RbConfig::CONFIG["host_os"]
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def self.windows?
|
|
40
|
+
/mingw|mswin|cygwin/ === RbConfig::CONFIG["host_os"]
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.path
|
|
44
|
+
@path ||= case
|
|
45
|
+
when Config["chrome"]
|
|
46
|
+
[Config["chrome"]]
|
|
47
|
+
when mac?
|
|
48
|
+
["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Chromium.app/Contents/MacOS/Chromium"]
|
|
49
|
+
when windows?
|
|
50
|
+
["C:/Program Files/Google/Chrome/Application/chrome.exe", "C:/Program Files (x86)/Google/Chrome/Application/chrome.exe"]
|
|
51
|
+
else
|
|
52
|
+
ENV["PATH"].split(File::PATH_SEPARATOR).product(%w[chrome google-chrome chromium chromium-browser]).map do |path, binary|
|
|
53
|
+
[path, binary].join(File::SEPARATOR)
|
|
54
|
+
end
|
|
55
|
+
end.find do |path|
|
|
56
|
+
File.executable?(path) && !File.directory?(path)
|
|
57
|
+
end.tap do |path|
|
|
58
|
+
raise Error, "couldn't find chrome" unless path
|
|
59
|
+
stdout, status = Open3.capture2 path, "--version"
|
|
60
|
+
raise Error, "couldn't start chrome" unless status.success?
|
|
61
|
+
version = /(?<major>\d+)(?:\.\d+)*/.match stdout
|
|
62
|
+
raise Error, "couldn't start chrome" unless version
|
|
63
|
+
raise Error, "chrome version #{MIN_VERSION} or higher required" if version[:major].to_i < MIN_VERSION
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def self.rmdir(tmp)
|
|
68
|
+
Proc.new do
|
|
69
|
+
FileUtils.remove_entry tmp
|
|
70
|
+
rescue SystemCallError
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def self.kill(pid, *pipes)
|
|
75
|
+
Proc.new do
|
|
76
|
+
if windows?
|
|
77
|
+
*, status = Open3.capture2e *%W[taskkill /f /t /pid #{pid}]
|
|
78
|
+
Process.kill "KILL", pid unless status.success?
|
|
79
|
+
else
|
|
80
|
+
Timeout.timeout(TIMEOUT_KILL, Error) do
|
|
81
|
+
Process.kill "-USR1", Process.getpgid(pid)
|
|
82
|
+
Process.wait pid
|
|
83
|
+
rescue Error
|
|
84
|
+
Process.kill "-KILL", Process.getpgid(pid)
|
|
85
|
+
Process.wait pid
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
rescue Errno::ESRCH, Errno::ECHILD
|
|
89
|
+
ensure
|
|
90
|
+
pipes.each(&:close)
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def close
|
|
95
|
+
Chrome.kill(@pid, @input, @output).call
|
|
96
|
+
Chrome.rmdir(@data_dir).call
|
|
97
|
+
ObjectSpace.undefine_finalizer self
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def self.with_browser(url, **opts, &block)
|
|
101
|
+
browser = new url, **opts
|
|
102
|
+
block.call browser
|
|
103
|
+
ensure
|
|
104
|
+
browser&.close
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def initialize(url, width: 800, height: 600, background: { r: 0, g: 0, b: 0, a: 0 }, args: [])
|
|
108
|
+
@id, @data_dir = 0, Dir.mktmpdir("nswtopo_headless_chrome_")
|
|
109
|
+
ObjectSpace.define_finalizer self, Chrome.rmdir(@data_dir)
|
|
110
|
+
|
|
111
|
+
args << "--disable-gpu" if Config["gpu"] == false
|
|
112
|
+
args << "--user-data-dir=#{@data_dir}"
|
|
113
|
+
|
|
114
|
+
input, @input, @output, output = *IO.pipe, *IO.pipe
|
|
115
|
+
input.nonblock, output.nonblock = false, false
|
|
116
|
+
@input.sync = true
|
|
117
|
+
|
|
118
|
+
@pid = Process.spawn Chrome.path, *ARGS, *args, 1 => File::NULL, 2 => File::NULL, 3 => input, 4 => output, :pgroup => Chrome.windows? ? nil : true
|
|
119
|
+
ObjectSpace.define_finalizer self, Chrome.kill(@pid, @input, @output)
|
|
120
|
+
input.close; output.close
|
|
121
|
+
|
|
122
|
+
target_id = command("Target.createTarget", url: url).fetch("targetId")
|
|
123
|
+
@session_id = command("Target.attachToTarget", targetId: target_id, flatten: true).fetch("sessionId")
|
|
124
|
+
command "Page.enable"
|
|
125
|
+
wait "Page.loadEventFired", timeout: TIMEOUT_LOADEVENT
|
|
126
|
+
command "Emulation.setDeviceMetricsOverride", width: width, height: height, deviceScaleFactor: 1, mobile: false
|
|
127
|
+
command "Emulation.setDefaultBackgroundColorOverride", color: background
|
|
128
|
+
@node_id = command("DOM.getDocument").fetch("root").fetch("nodeId")
|
|
129
|
+
rescue SystemCallError
|
|
130
|
+
raise Error, "couldn't start chrome"
|
|
131
|
+
rescue KeyError
|
|
132
|
+
raise Error
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def send(**message)
|
|
136
|
+
message.merge! sessionId: @session_id if @session_id
|
|
137
|
+
@input.write message.to_json, ?\0
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def messages
|
|
141
|
+
Enumerator.produce do
|
|
142
|
+
json = @output.readline(?\0).chomp(?\0)
|
|
143
|
+
JSON.parse(json).tap do |message|
|
|
144
|
+
raise Error if message["error"]
|
|
145
|
+
raise Error if message["method"] == "Target.detachedFromTarget"
|
|
146
|
+
end
|
|
147
|
+
rescue JSON::ParserError, EOFError
|
|
148
|
+
raise Error
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def wait(event, timeout: nil)
|
|
153
|
+
Timeout.timeout(timeout) do
|
|
154
|
+
messages.find do |message|
|
|
155
|
+
message["method"] == event
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
rescue Timeout::Error
|
|
159
|
+
raise Error
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def command(method, timeout: TIMEOUT_COMMAND, **params)
|
|
163
|
+
send id: @id += 1, method: method, params: params
|
|
164
|
+
Timeout.timeout(timeout) do
|
|
165
|
+
messages.find do |message|
|
|
166
|
+
message["id"] == @id
|
|
167
|
+
end
|
|
168
|
+
end.fetch("result")
|
|
169
|
+
rescue Timeout::Error, KeyError
|
|
170
|
+
raise Error
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def screenshot(png_path)
|
|
174
|
+
data = command("Page.captureScreenshot", timeout: TIMEOUT_SCREENSHOT).fetch("data")
|
|
175
|
+
png_path.binwrite Base64.decode64(data)
|
|
176
|
+
rescue KeyError
|
|
177
|
+
raise Error
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def print_to_pdf(pdf_path)
|
|
181
|
+
data = command("Page.printToPDF", timeout: nil, preferCSSPageSize: true).fetch("data")
|
|
182
|
+
pdf_path.binwrite Base64.decode64(data)
|
|
183
|
+
rescue KeyError
|
|
184
|
+
raise Error
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def query_selector_node_id(selector)
|
|
188
|
+
command("DOM.querySelector", selector: selector, nodeId: @node_id).fetch("nodeId")
|
|
189
|
+
rescue KeyError
|
|
190
|
+
raise Error
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
class Node
|
|
194
|
+
def initialize(browser, selector)
|
|
195
|
+
@browser, @node_id = browser, browser.query_selector_node_id(selector)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def [](name)
|
|
199
|
+
@browser.command("DOM.getAttributes", nodeId: @node_id).fetch("attributes").each_slice(2).to_h.fetch(name.to_s)
|
|
200
|
+
rescue KeyError
|
|
201
|
+
raise Error
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def []=(name, value)
|
|
205
|
+
if value.nil?
|
|
206
|
+
@browser.command "DOM.removeAttribute", nodeId: @node_id, name: name
|
|
207
|
+
else
|
|
208
|
+
@browser.command "DOM.setAttributeValue", nodeId: @node_id, name: name, value: value
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def value=(value)
|
|
213
|
+
@browser.command "DOM.setNodeValue", nodeId: @node_id + 1, value: value
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def width
|
|
217
|
+
@browser.command("DOM.getBoxModel", nodeId: @node_id).fetch("model").fetch("content").each_slice(2).map(&:first).minmax.reverse.inject(&:-)
|
|
218
|
+
rescue KeyError
|
|
219
|
+
raise Error
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def query_selector(selector)
|
|
224
|
+
Node.new self, selector
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
end
|