@unvt/charites 0.3.0 → 0.4.0

package/docs/source/usage/example2.rst

@@ -0,0 +1,166 @@
+Example 2: Import existing style.json and edit it with charites
+===================================================================
+This is an example of using charites for importing an existing style.json and editing it as yaml files, and exporting them back to a json style. 
+
+Condition
+------------------------
+- OS: Windows 10
+- Charites version: 0.3.0
+- Running on WindowsPowershell
+
+
+Steps
+-----
+
+
+Step 1. Import style json into a series of YAML files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If your style json is in the Internet, it may need to downloat it from the Internet before you convert because charites cannot directly convert json a file in the Internet. Use curl.exe (in WindowsPowershell) or curl (in bash environment).
+
+.. code-block:: 
+
+    curl.exe -O https://ubukawa.github.io/cmv-test/style1.json
+    charites convert style1.json editing/style.yml
+
+Then, you will see a series of yaml files are converted from the source json. You will obtain an overall YAML that contains a list of style layers with the style yamls in the layer directory.  
+In this case, we will have "style.yml" in "editing" directory together with a lot of yaml files for layers in "layers" directory.
+
+.. image:: ./img/example02-001.png
+   :scale: 75%
+   :align: center
+
+.. image:: ./img/example02-002.png
+   :scale: 75%
+   :align: center
+
+(Tips) It is good to know that if your source json has some folder structure in its layer ids, that structure would be maintained when you import. 
+
+.. image:: ./img/example02-005.png
+   :scale: 75%
+   :align: center
+
+.. image:: ./img/example02-003.png
+   :scale: 75%
+   :align: center
+
+.. image:: ./img/example02-004.png
+   :scale: 75%
+   :align: center
+
+Step 2. Checking an overall yaml (style.yml in this case)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Check the style.yml by opening it with Notepad or any text editor. You will see it like this:
+
+.. image:: ./img/example02-006.png
+   :scale: 75%
+   :align: center
+
+Although it is not often seen, we need to check one thing. Sometimes, if id for each layer is too long, YAML automatically uses ">-" and break the line to improve human's readability. ">-" would pretend an actual line break in YAML, however, it will cause some problem for Charites, so please fix line brakes with ">-" in the layers if you find them.
+This is often seen in style files from Esri vector tile because it has detailed folder structure sometimes.
+
+.. image:: ./img/example02-007.png
+   :scale: 75%
+   :align: center
+
+.. image:: ./img/example02-008.png
+   :scale: 75%
+   :align: center
+
+You can extract layers and use "replace" tool to remove ">-" and unnecessary line changes.
+
+.. image:: ./img/example02-009.png
+   :scale: 75%
+   :align: center
+
+.. image:: ./img/example02-010.png
+   :scale: 75%
+   :align: center
+
+If you obtain the list of layers without ">-", you are ready to go to the next step.
+
+.. image:: ./img/example02-011.png
+   :scale: 75%
+   :align: center
+
+.. note::
+
+    You may also need to check paths in "sprite", "glyphs" and "sources." They would be written as relative paths, so you may need to adjust these paths so that charites in your local PC can access to them. In addition, if you are importing styles from ArcGIS server, reference to the source should be modifyed (use "tiles" instead of "url"). The "tiles" URL will be provided by adding "/tile/{z}/{y}/{x}.pbf" in the end of Esri vector tile server URL.
+
+
+Step 3. Serve YAML files for live preview
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Run the "charites serve" command to serve your YAML files in the localhost. By default, its port is 8080.
+
+.. code-block:: 
+
+    charites serve editing/style.yml
+
+A window would automatically appears in the windows environment, but if not please open "http://localhost:8080" to access your live view. You can stop the live preview by pressing ctrl + c.
+
+.. image:: ./img/example02-012.png
+   :scale: 75%
+   :align: center
+
+Step 4. Editing YAML files to change style
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+You can change/edit the map style by editing YAML files. If you edit and save your update, charites automatically finds your updates to reflect them into the live view.
+
+.. image:: ./img/example02-016.png
+   :scale: 75%
+   :align: center
+
+Here are some tips for efficient style editing.
+
+(Tips 1) Legend control
+########################
+There is a legend control in the lower left of the screen. You can turn on/off any layers so that your edit would be easy. It will also help you to identify layers that should be edited.
+You can enlarge the window by clicking the button, and reduce its size by clicking "x" in the window. You can also choose to list only rendered layers or not in the window. 
+
+.. image:: ./img/example02-013.png
+   :scale: 75%
+   :align: center
+
+(Tips 2) Showing tile boundary
+###############################
+It would be alos helpful to show the tile boundary. There are checkbox in the upper left, and you can turn on/off. You will see tile border and its xyx in z/x/y order together with tile size. This would be useful when you also think about optimization of your vector tile.
+
+.. image:: ./img/example02-014.png
+   :scale: 75%
+   :align: center
+
+(Tips 3) Showing collision boxes
+#################################
+It would be alos helpful to show the collision boxes. If a box is red, it indicates a collision with others. When you adjust label location/size, this is good.
+
+.. image:: ./img/example02-015.png
+   :scale: 75%
+   :align: center
+
+(Tips 4) Use of "#" (comment tag) to specify error
+###################################################
+If there is a layer contains some error, the web map may not appears. In such a case, use a YAML comment tag "#" to comment out a layer (or layers) to specify the layer contains an error. The comment tag is also useful to tentatively change colors, etc.
+
+(Tips 5) Use of inclusion of other YAML file for efficinet edit
+################################################################
+If you repeatedly use the same expression (color, font, etc) for several layers, it is conveninet to use a reference to a common expression. You can do it by using inclusion of  an independent YAML file with "!!inc/file hogehoge.yml". This is particular important if you manage your style with YAML files.
+
+.. image:: ./img/example02-017.png
+   :scale: 75%
+   :align: center
+
+.. image:: ./img/example02-018.png
+   :scale: 75%
+   :align: center
+
+Step 5. Exporting style json from YAML files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Once you have edited your style with YAML files, you can now export it back to json by running the following command.
+
+.. code-block:: 
+
+    charites build editing/style.yml out.json
+
+Congratulations. You will obtain the updated style json, and all done.
+

package/docs/source/usage/index.rst

@@ -8,3 +8,4 @@ Usage
   commandline_interface
   global_options
   examples
+  example2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@unvt/charites",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "",
   "bin": {
     "charites": "dist/cli.js"
@@ -9,7 +9,7 @@
     "build": "tsc -p .",
     "watch": "tsc -w",
     "lint": "eslint --fix .",
-    "test": "npm run build && mocha -r ts-node/register test/*.ts",
+    "test": "npm run build && mocha --timeout 5000 -r ts-node/register test/*.ts",
     "test:watch": "npm test -- --watch --watch-files src/**/*.ts --watch-files test/**/*.ts",
     "test:e2e": "npx playwright test",
     "command": "./node_modules/.bin/ts-node ./src/cli.ts"
@@ -17,41 +17,43 @@
   "author": "",
   "license": "MIT",
   "dependencies": {
-    "@mapbox/mapbox-gl-style-spec": "^13.22.0",
-    "@maplibre/maplibre-gl-style-spec": "^14.0.2",
-    "@unvt/sprite-one": "^0.0.8",
+    "@mapbox/mapbox-gl-style-spec": "^13.27.0",
+    "@maplibre/maplibre-gl-style-spec": "^17.0.2",
     "@types/jsonminify": "^0.4.1",
-    "axios": "^0.24.0",
-    "commander": "^8.2.0",
-    "glob": "^7.2.0",
+    "@unvt/sprite-one": "^0.0.9",
+    "axios": "^1.2.2",
+    "commander": "^9.4.1",
+    "glob": "^8.0.3",
     "js-yaml": "^4.0.0",
-    "jsonminify": "^0.4.1",
-    "node-watch": "^0.7.2",
+    "jsonminify": "^0.4.2",
+    "node-watch": "^0.7.3",
     "open": "^8.2.1",
-    "ws": "^8.2.2",
+    "ws": "^8.11.0",
     "yaml-include": "^1.2.1"
   },
   "devDependencies": {
-    "@playwright/test": "^1.28.1",
-    "@types/chai": "^4.2.11",
+    "@playwright/test": "^1.29.1",
+    "@types/chai": "^4.3.4",
     "@types/chai-as-promised": "^7.1.5",
     "@types/fs-extra": "^9.0.13",
-    "@types/js-yaml": "^4.0.3",
-    "@types/mocha": "^7.0.2",
-    "@types/node": "^14.0.14",
-    "@types/ws": "^8.2.0",
-    "@typescript-eslint/eslint-plugin": "^4.16.1",
-    "@typescript-eslint/parser": "^4.16.1",
-    "chai": "^4.2.0",
+    "@types/js-yaml": "^4.0.5",
+    "@types/mapbox__point-geometry": "^0.1.2",
+    "@types/mocha": "^10.0.1",
+    "@types/node": "^18.11.18",
+    "@types/ws": "^8.5.4",
+    "@typescript-eslint/eslint-plugin": "^5.47.1",
+    "@typescript-eslint/parser": "^5.47.1",
+    "chai": "^4.3.7",
     "chai-as-promised": "^7.1.1",
-    "eslint": "^7.21.0",
-    "eslint-config-prettier": "^6.15.0",
-    "eslint-plugin-prettier": "^3.3.1",
-    "fs-extra": "^10.0.0",
-    "kill-port-process": "^3.0.1",
-    "mocha": "^8.0.1",
-    "prettier": "^2.5.0",
-    "ts-node": "^8.10.2",
-    "typescript": "^3.9.5"
+    "eslint": "^8.31.0",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-plugin-prettier": "^4.2.1",
+    "fs-extra": "^11.1.0",
+    "kill-port-process": "^3.1.0",
+    "mocha": "^10.2.0",
+    "node-abort-controller": "^3.0.1",
+    "prettier": "^2.8.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.4"
   }
 }

package/provider/default/app.js

@@ -9,9 +9,7 @@
   if (zoom) options.zoom = zoom
   const map = new maplibregl.Map(options)
 
-  window._charites.initializeWebSocket((message) => {
-    map.setStyle(JSON.parse(message.data))
-  })
+  window._charites.initializeWebSocket(map)
 
   map.addControl(new maplibregl.NavigationControl(), 'top-right')
 

package/provider/default/index.html

@@ -2,10 +2,10 @@
 <html id="charites-default">
   <head>
     <link rel="stylesheet" href="/app.css" />
-    <link href='https://unpkg.com/maplibre-gl@1.15.2/dist/maplibre-gl.css' rel='stylesheet' />
+    <link href='https://unpkg.com/maplibre-gl@2.4.0/dist/maplibre-gl.css' rel='stylesheet' />
     <link href='https://watergis.github.io/maplibre-gl-legend/maplibre-gl-legend.css' rel='stylesheet' />
     <title>Charites Live Preview</title>
-    <script src='https://unpkg.com/maplibre-gl@1.15.2/dist/maplibre-gl.js'></script>
+    <script src='https://unpkg.com/maplibre-gl@2.4.0/dist/maplibre-gl.js'></script>
     <script src="https://watergis.github.io/maplibre-gl-legend/maplibre-gl-legend.js"></script>
   </head>
   <body>

package/provider/default/shared.js

@@ -1,9 +1,18 @@
 ;(async () => {
   window._charites = {
-    initializeWebSocket: function (onmessage) {
+    initializeWebSocket: function (map) {
       const socket = new WebSocket(`ws://${window.location.host}`)
 
-      socket.addEventListener('message', onmessage)
+      socket.addEventListener('message', (message) => {
+        const data = JSON.parse(message.data)
+        if (data.event === 'styleUpdate') {
+          map.setStyle(`http://${window.location.host}/style.json`)
+        } else if (data.event === 'spriteUpdate') {
+          map.setStyle(`http://${window.location.host}/style.json`, {
+            diff: false,
+          })
+        }
+      })
     },
     parseMapStyle: async function () {
       const mapStyleUrl = `http://${window.location.host}/style.json`

package/provider/geolonia/app.js

@@ -9,9 +9,7 @@
   if (zoom) options.zoom = zoom
   const map = new geolonia.Map(options)
 
-  window._charites.initializeWebSocket((message) => {
-    map.setStyle(JSON.parse(message.data))
-  })
+  window._charites.initializeWebSocket(map)
 
   map.addControl(
     new MaplibreLegendControl(

package/provider/mapbox/app.js

@@ -11,9 +11,7 @@
   if (zoom) options.zoom = zoom
   const map = new mapboxgl.Map(options)
 
-  window._charites.initializeWebSocket((message) => {
-    map.setStyle(JSON.parse(message.data))
-  })
+  window._charites.initializeWebSocket(map)
 
   map.addControl(new mapboxgl.NavigationControl(), 'top-right')
 

package/src/cli/init.ts

@@ -6,7 +6,7 @@ const program = new Command()
 program
   .name('init')
   .arguments('<file>')
-  .description('initialize a style JSON')
+  .description('initialize a style YAML')
   .option(
     '-t, --tilejson-urls <tilejson_urls>',
     'an URL for TileJSON. It will create empty layers from vector_layers property of TileJSON. Please use comma (,) in case multiple TileJSONs require.',

package/src/cli/serve.ts

@@ -17,12 +17,19 @@ program
     '--mapbox-access-token [mapboxAccessToken]',
     'Your Mapbox Access Token (required if using the `mapbox` provider)',
   )
+  .option(
+    '-i, --sprite-input [<icon input directory>]',
+    'directory path of icon source to build icons. The default <icon source> is `icons/`',
+  )
   .option('--port [port]', 'Specify custom port')
-  .action((source: string, serveOptions: serveOptions) => {
+  .option('--no-open', "Don't open the preview in the default browser")
+  .action(async (source: string, serveOptions: serveOptions) => {
     const options: serveOptions = program.opts()
     options.provider = serveOptions.provider
     options.mapboxAccessToken = serveOptions.mapboxAccessToken
     options.port = serveOptions.port
+    options.spriteInput = serveOptions.spriteInput
+    options.open = serveOptions.open
     if (!fs.existsSync(defaultSettings.configFile)) {
       fs.writeFileSync(
         defaultSettings.configFile,
@@ -30,7 +37,7 @@ program
       )
     }
     try {
-      serve(source, program.opts())
+      await serve(source, program.opts())
     } catch (e) {
       error(e)
     }

package/src/commands/build.ts

@@ -6,7 +6,7 @@ import { buildSprite } from '../lib/build-sprite'
 import { getSpriteSlug } from '../lib/get-sprite-slug'
 import { defaultValues } from '../lib/defaultValues'
 import jsonminify from 'jsonminify'
-import { StyleSpecification } from '@maplibre/maplibre-gl-style-spec/types'
+import { StyleSpecification } from '@maplibre/maplibre-gl-style-spec'
 import watch from 'node-watch'
 
 export interface buildOptions {
@@ -94,11 +94,7 @@ export async function build(
     }
   }
 
-  try {
-    fs.writeFileSync(destinationPath, style)
-  } catch (err) {
-    throw `${destinationPath}: Permission denied`
-  }
+  fs.writeFileSync(destinationPath, style)
 }
 
 export function buildWatch(

package/src/commands/convert.ts

@@ -45,11 +45,7 @@ export function convert(source: string, destination: string) {
       const style = JSON.parse(lines.join(''))
       const destinationPath = getDestinationPath(destination)
 
-      try {
-        writeYaml(destinationPath, style, false)
-      } catch (err) {
-        throw `${destinationPath}: Permission denied`
-      }
+      writeYaml(destinationPath, style, false)
     })
   } else {
     sourcePath = path.resolve(process.cwd(), source)
@@ -67,10 +63,6 @@ export function convert(source: string, destination: string) {
 
     const destinationPath = getDestinationPath(destination, sourcePath)
 
-    try {
-      writeYaml(destinationPath, style, false)
-    } catch (err) {
-      throw `${destinationPath}: Permission denied`
-    }
+    writeYaml(destinationPath, style, false)
   }
 }

package/src/commands/init.ts

@@ -1,4 +1,4 @@
-import { StyleSpecification } from '@maplibre/maplibre-gl-style-spec/types'
+import { StyleSpecification } from '@maplibre/maplibre-gl-style-spec'
 import { writeYaml } from '../lib/yaml-writer'
 import {
   TileJSONImporter,

package/src/commands/serve.ts

@@ -1,5 +1,6 @@
 import path from 'path'
 import fs from 'fs'
+import os from 'os'
 import http from 'http'
 import open from 'open'
 import { WebSocketServer } from 'ws'
@@ -8,14 +9,17 @@ import watch from 'node-watch'
 import { parser } from '../lib/yaml-parser'
 import { validateStyle } from '../lib/validate-style'
 import { defaultValues } from '../lib/defaultValues'
+import { buildSprite } from '../lib/build-sprite'
 
 export interface serveOptions {
   provider?: string
   mapboxAccessToken?: string
   port?: string
+  spriteInput?: string
+  open?: boolean
 }
 
-export function serve(source: string, options: serveOptions) {
+export async function serve(source: string, options: serveOptions) {
   let port = process.env.PORT || 8080
   if (options.port) {
     port = Number(options.port)
@@ -42,11 +46,44 @@ export function serve(source: string, options: serveOptions) {
     throw `Provider is mapbox, but the Mapbox Access Token is not set. Please provide it using --mapbox-access-token, or set it in \`~/.charites/config.yml\` (see the Global configuration section of the documentation for more information)`
   }
 
-  const server = http.createServer((req, res) => {
+  let spriteOut: string | undefined = undefined
+  let spriteRefresher: (() => Promise<void>) | undefined = undefined
+  if (options.spriteInput) {
+    spriteOut = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'charites-'))
+    spriteRefresher = async () => {
+      if (
+        typeof options.spriteInput === 'undefined' ||
+        typeof spriteOut === 'undefined'
+      ) {
+        return
+      }
+      await buildSprite(options.spriteInput, spriteOut, 'sprite')
+    }
+    await spriteRefresher()
+  }
+
+  const server = http.createServer(async (req, res) => {
     const url = (req.url || '').replace(/\?.*/, '')
     const defaultProviderDir = path.join(defaultValues.providerDir, 'default')
     const providerDir = path.join(defaultValues.providerDir, provider)
 
+    if (
+      typeof spriteOut !== 'undefined' &&
+      url.match(/^\/sprite(@2x)?\.(json|png)/)
+    ) {
+      res.statusCode = 200
+      if (url.endsWith('.json')) {
+        res.setHeader('Content-Type', 'application/json; charset=UTF-8')
+      } else {
+        res.setHeader('Content-Type', 'image/png')
+      }
+      res.setHeader('Cache-Control', 'no-store')
+      const filename = path.basename(url)
+      const fsStream = fs.createReadStream(path.join(spriteOut, filename))
+      fsStream.pipe(res)
+      return
+    }
+
     switch (url) {
       case '/':
         res.statusCode = 200
@@ -61,12 +98,18 @@ export function serve(source: string, options: serveOptions) {
         let style
         try {
           style = parser(sourcePath)
+          if (typeof spriteOut !== 'undefined') {
+            style.sprite = `http://${
+              req.headers.host || `localhost:${port}`
+            }/sprite`
+          }
           validateStyle(style, provider)
         } catch (error) {
           console.log(error)
         }
         res.statusCode = 200
         res.setHeader('Content-Type', 'application/json; charset=UTF-8')
+        res.setHeader('Cache-Control', 'no-store')
         res.end(JSON.stringify(style))
         break
       case '/app.css':
@@ -116,30 +159,56 @@ export function serve(source: string, options: serveOptions) {
     console.log(`Provider: ${provider}`)
     console.log(`Loading your style: ${sourcePath}`)
     console.log(`Your map is running on http://localhost:${port}/\n`)
-    open(`http://localhost:${port}`)
+    if (options.open) {
+      open(`http://localhost:${port}`)
+    }
   })
 
   const wss = new WebSocketServer({ server })
 
   wss.on('connection', (ws) => {
-    watch(
+    const watcher = watch(
       path.dirname(sourcePath),
-      { recursive: true, filter: /\.yml$/ },
+      { recursive: true, filter: /\.yml$|\.svg$/i },
       (event, file) => {
         console.log(`${(event || '').toUpperCase()}: ${file}`)
         try {
-          const style = parser(sourcePath)
-          try {
-            validateStyle(style, provider)
-          } catch (error) {
-            console.log(error)
+          if (file?.toLowerCase().endsWith('.yml')) {
+            ws.send(
+              JSON.stringify({
+                event: 'styleUpdate',
+              }),
+            )
+          } else if (
+            file?.toLowerCase().endsWith('.svg') &&
+            typeof spriteRefresher !== 'undefined'
+          ) {
+            spriteRefresher().then(() => {
+              ws.send(
+                JSON.stringify({
+                  event: 'spriteUpdate',
+                }),
+              )
+            })
           }
-          ws.send(JSON.stringify(style))
         } catch (e) {
           // Nothing to do
         }
       },
     )
+    ws.on('close', () => {
+      watcher.close()
+    })
+  })
+
+  process.on('SIGINT', () => {
+    console.log('Cleaning up...')
+    server.close()
+    if (typeof spriteOut !== 'undefined') {
+      fs.rmSync(spriteOut, { recursive: true })
+      spriteOut = undefined
+    }
+    process.exit(0)
   })
 
   return server

package/src/lib/build-sprite.ts

@@ -1,5 +1,5 @@
 import { generateSprite } from '@unvt/sprite-one'
-const path = require('path')
+import path from 'path'
 
 export async function buildSprite(
   svgPath: string,
@@ -8,10 +8,6 @@ export async function buildSprite(
 ): Promise<void> {
   const pxRatios = [1, 2]
   const outPath = path.join(publicPath, iconSlug)
-  try {
-    await generateSprite(outPath, [svgPath], pxRatios)
-  } catch (error) {
-    throw error
-  }
+  await generateSprite(outPath, [svgPath], pxRatios)
   return
 }

package/src/lib/get-sprite-slug.ts

@@ -1,4 +1,4 @@
-import { StyleSpecification } from '@maplibre/maplibre-gl-style-spec/types'
+import { StyleSpecification } from '@maplibre/maplibre-gl-style-spec'
 
 export function getSpriteSlug(style: StyleSpecification): string | false {
   if (!style.hasOwnProperty('sprite')) {