@lvce-editor/shared-process 0.5.0 → 0.6.0

package/extensions/builtin.language-basics-yaml/src/tokenizeYaml.js

@@ -66,7 +66,7 @@ const RE_SEMICOLON = /^;/
 const RE_COMMA = /^,/
 const RE_ANYTHING = /^.*/
 const RE_ANYTHING_BUT_COMMENT = /^[^#]+/
-const RE_NUMERIC = /^(([0-9]+\.?[0-9]*)|(\.[0-9]+))\b/
+const RE_NUMERIC = /^(([0-9]+\.?[0-9]*)|(\.[0-9]+))(?=\s|$)/
 const RE_ANYTHING_UNTIL_CLOSE_BRACE = /^[^\}]+/
 const RE_BLOCK_COMMENT_START = /^\/\*/
 const RE_BLOCK_COMMENT_END = /^\*\//
@@ -86,7 +86,7 @@ const RE_LANGUAGE_CONSTANT = /^(?:true|false|null)(?!#)/
 const RE_COLON = /^:/
 const RE_DASH = /^\-/
 const RE_WORDS = /^[\w\s]*\w/
-const RE_MULTI_LINE_STRING_START = /^(?:\||>)/
+const RE_MULTI_LINE_STRING_START = /^(?:\||>\-|>)/
 const RE_KEY_PRE = /^\s*(\-\s*)?/
 
 export const initialLineState = {
@@ -188,7 +188,6 @@ export const tokenizeLine = (line, lineState) => {
           token = TokenType.Comment
           state = State.InsideLineComment
         } else if ((next = part.match(RE_MULTI_LINE_STRING_START))) {
-          part
           token = TokenType.Punctuation
           state = State.AfterPipe
         } else if ((next = part.match(RE_PROPERTY_VALUE_1))) {
@@ -234,6 +233,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_NUMERIC))) {
           token = TokenType.Numeric
           state = State.TopLevelContent
+        } else if ((next = part.match(RE_MULTI_LINE_STRING_START))) {
+          token = TokenType.Punctuation
+          state = State.AfterPipe
         } else if ((next = part.match(RE_ANYTHING))) {
           token = TokenType.PropertyValueString
           state = State.TopLevelContent
@@ -279,6 +281,9 @@ export const tokenizeLine = (line, lineState) => {
         } else if ((next = part.match(RE_PROPERTY_NAME))) {
           token = TokenType.PropertyName
           state = State.AfterPropertyName
+        } else if ((next = part.match(RE_DASH))) {
+          token = TokenType.Punctuation
+          state = State.AfterDash
         } else if ((next = part.match(RE_ANYTHING))) {
           token = TokenType.Text
           state = State.TopLevelContent
@@ -308,6 +313,8 @@ export const tokenizeLine = (line, lineState) => {
       break
     case State.AfterPipe:
       keyOffset = getKeyOffset(line)
+      keyOffset
+      line
       state = State.InsideMultiLineString
       break
     default:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lvce-editor/shared-process",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "",
   "main": "index.js",
   "type": "module",
@@ -20,6 +20,8 @@
     "node": ">=16"
   },
   "dependencies": {
+    "@lvce-editor/extension-host": "0.6.0",
+    "@lvce-editor/pty-host": "0.6.0",
     "chokidar": "^3.5.3",
     "debug": "^4.3.4",
     "execa": "^6.1.0",
@@ -34,12 +36,11 @@
     "parse-json": "^6.0.2",
     "tail": "^2.2.4",
     "tar-fs": "^2.1.1",
+    "tmp-promise": "^3.0.3",
     "trash": "^8.1.0",
     "verror": "^1.10.1",
-    "ws": "^8.8.1",
-    "xdg-basedir": "^5.1.0",
-    "@lvce-editor/pty-host": "0.5.0",
-    "@lvce-editor/extension-host": "0.5.0"
+    "ws": "^8.9.0",
+    "xdg-basedir": "^5.1.0"
   },
   "optionalDependencies": {
     "electron-clipboard-ex": "^1.3.3",

package/src/parts/Cli/Cli.js

@@ -0,0 +1,24 @@
+const getModule = (argv0) => {
+  switch (argv0) {
+    case 'install':
+      return import('../CliInstall/CliInstall.js')
+    case 'list':
+      return import('../CliList/CliList.js')
+    case 'link':
+      return import('../CliLink/CliLink.js')
+    default:
+      throw new Error('command not found')
+  }
+}
+
+export const handleCliArgs = async (argv, console, process) => {
+  const argv0 = argv[0]
+  const module = await getModule(argv0)
+  // console.log('handle cli args')
+  try {
+    await module.handleCliArgs(argv, console, process)
+  } catch (error) {
+    console.error(error)
+    process.exitCode = 1
+  }
+}

package/src/parts/CliInstall/CliInstall.js

@@ -0,0 +1,10 @@
+import * as ExtensionInstall from '../ExtensionInstall/ExtensionInstall.js'
+
+export const handleCliArgs = async (argv, console) => {
+  const extension = argv[1]
+  if (!extension) {
+    console.error('extension argument is required')
+    return
+  }
+  await ExtensionInstall.install(extension)
+}

package/src/parts/CliLink/CliLink.js

@@ -0,0 +1,6 @@
+import * as ExtensionLink from '../ExtensionLink/ExtensionLink.js'
+
+export const handleCliArgs = async (argv, console) => {
+  const path = argv.length > 1 ? argv[1] : process.cwd()
+  await ExtensionLink.link(path)
+}

package/src/parts/CliList/CliList.js

@@ -0,0 +1,15 @@
+import * as ExtensionList from '../ExtensionList/ExtensionList.js'
+
+const getOutputLine = (extension) => {
+  return `${extension.id}: ${extension.version}`
+}
+
+const getOutput = (extensions) => {
+  return extensions.map(getOutputLine).join('\n')
+}
+
+export const handleCliArgs = async (argv, console) => {
+  const extensions = await ExtensionList.list()
+  const output = getOutput(extensions)
+  console.info(output)
+}

package/src/parts/Download/Download.js

@@ -1,9 +1,7 @@
-import got from 'got'
-import { createReadStream, createWriteStream } from 'node:fs'
+import got, { RequestError } from 'got'
+import { createWriteStream } from 'node:fs'
 import { mkdir, rm } from 'node:fs/promises'
 import { pipeline } from 'node:stream/promises'
-import { createBrotliDecompress } from 'node:zlib'
-import tar from 'tar-fs'
 import VError from 'verror'
 import * as Path from '../Path/Path.js'
 
@@ -17,15 +15,9 @@ export const download = async (url, outFile) => {
     } catch {
       // ignore
     }
+    if (error instanceof RequestError) {
+      throw new VError(`Failed to download "${url}": ${error.message}`)
+    }
     throw new VError(error, `Failed to download "${url}"`)
   }
 }
-
-export const extract = async (inFile, outDir) => {
-  await mkdir(outDir, { recursive: true })
-  await pipeline(
-    createReadStream(inFile),
-    createBrotliDecompress(),
-    tar.extract(outDir)
-  )
-}

package/src/parts/DownloadAndExtract/DownloadAndExtract.js

@@ -0,0 +1,14 @@
+import got from 'got'
+import { pipeline } from 'stream/promises'
+import tar from 'tar-fs'
+import { createGunzip } from 'zlib'
+
+export const downloadAndExtractTarGz = async ({ url, outDir, strip }) => {
+  await pipeline(
+    got.stream(url),
+    createGunzip(),
+    tar.extract(outDir, {
+      strip,
+    })
+  )
+}

package/src/parts/Error/FileSystemError.js

@@ -0,0 +1,10 @@
+import VError from 'verror'
+
+export class FileSystemError extends VError {
+  constructor(error, message) {
+    super(error, message)
+    if (error && error.code) {
+      this.code = error.code
+    }
+  }
+}

package/src/parts/ExtensionInstall/ExtensionInstall.ipc.js

@@ -0,0 +1,6 @@
+import * as ExensionInstall from './ExtensionInstall.js'
+
+// prettier-ignore
+export const Commands = {
+  'ExtensionInstall.install': ExensionInstall.install,
+}

package/src/parts/ExtensionInstall/ExtensionInstall.js

@@ -0,0 +1,25 @@
+import * as ExtensionInstallParseInput from '../ExtensionInstallParseInput/ExtensionInstallParseInput.js'
+
+const getModule = (type) => {
+  switch (type) {
+    case ExtensionInstallParseInput.InstallType.GithubRepository:
+      return import(
+        '../ExtensionInstallFromGitHub/ExtensionInstallFromGitHub.js'
+      )
+    case ExtensionInstallParseInput.InstallType.Url:
+      return import('../ExtensionInstallFromUrl/ExtensionInstallFromUrl.js')
+    default:
+      throw new Error('module not found')
+  }
+}
+
+export const install = async (input) => {
+  const parsed = ExtensionInstallParseInput.parse(input)
+  if (parsed.type === ExtensionInstallParseInput.InstallType.ParsingError) {
+    throw new Error(`Failed to parse input: ${parsed.options.message}`)
+  }
+  console.log({ parsed })
+  const module = await getModule(parsed.type)
+  // @ts-ignore
+  await module.install(parsed.options)
+}

package/src/parts/ExtensionInstallFromGitHub/ExtensionInstallFromGitHub.js

@@ -0,0 +1,34 @@
+import VError from 'verror'
+import * as DownloadAndExtract from '../DownloadAndExtract/DownloadAndExtract.js'
+import * as FileSystem from '../FileSystem/FileSystem.js'
+import * as Path from '../Path/Path.js'
+import * as Platform from '../Platform/Platform.js'
+
+export const install = async ({ user, repo, branch }) => {
+  try {
+    const cachedExtensionsPath = Platform.getCachedExtensionsPath()
+    const url = `https://codeload.github.com/${user}/${repo}/tar.gz/${branch}`
+    const cachedExtensionPath = Path.join(
+      cachedExtensionsPath,
+      `github-${user}-${repo}-${branch}`
+    )
+    await DownloadAndExtract.downloadAndExtractTarGz({
+      url,
+      outDir: cachedExtensionPath,
+      strip: 1,
+    })
+    const extensionsPath = Platform.getExtensionsPath()
+    const manifestPath = Path.join(cachedExtensionPath, 'extension.json')
+    const manifestContent = await FileSystem.readFile(manifestPath)
+    const manifestJson = JSON.parse(manifestContent)
+    const id = manifestJson.id
+    if (!id) {
+      throw new Error('missing id in extension manifest')
+    }
+    const outDir = Path.join(extensionsPath, id)
+    await FileSystem.remove(outDir)
+    await FileSystem.rename(cachedExtensionPath, outDir)
+  } catch (error) {
+    throw new VError(error, `Failed to install ${user}/${repo}`)
+  }
+}

package/src/parts/ExtensionInstallFromUrl/ExtensionInstallFromUrl.js

@@ -0,0 +1,32 @@
+import { readFile, rename, rm } from 'fs/promises'
+import { join } from 'path'
+import VError from 'verror'
+import * as Download from '../Download/Download.js'
+import * as Extract from '../Extract/Extract.js'
+import * as Platform from '../Platform/Platform.js'
+import * as TmpFile from '../TmpFile/TmpFile.js'
+
+export const install = async ({ url }) => {
+  try {
+    // TODO cache extension tar br file, probably by url <cachedExtensions>/user/repo/releases/<tag>/<fileName>
+    // const cachedExtensionsPath = Platform.getCachedExtensionsPath()
+    // const outFile = join(cachedExtensionsPath, 'installed-extension.tar.br')
+    const extensionsPath = Platform.getExtensionsPath()
+    const tmpFile = await TmpFile.getTmpFile()
+    await Download.download(url, tmpFile)
+    const tmpDir = await TmpFile.getTmpDir()
+    await Extract.extractTarBr(tmpFile, tmpDir)
+    const manifestPath = join(tmpDir, 'extension.json')
+    const manifestContent = await readFile(manifestPath, 'utf8')
+    const manifestJson = JSON.parse(manifestContent)
+    const id = manifestJson.id
+    if (!id) {
+      throw new Error('missing id in extension manifest')
+    }
+    const outDir = join(extensionsPath, id)
+    await rm(outDir, { recursive: true, force: true })
+    await rename(tmpDir, outDir)
+  } catch (error) {
+    throw new VError(error, `Failed to install "${url}"`)
+  }
+}

package/src/parts/ExtensionInstallParseInput/ExtensionInstallParseInput.js

@@ -0,0 +1,84 @@
+export const InstallType = {
+  Url: 1,
+  GithubRepository: 2,
+  ParsingError: 3,
+}
+
+const parseUrlGithub = (input) => {
+  const parts = input.split('/')
+  const slashCount = parts.length
+  if (slashCount === 5) {
+    const user = parts[3]
+    const repo = parts[4]
+    return {
+      type: InstallType.GithubRepository,
+      options: {
+        user,
+        repo,
+        branch: 'HEAD',
+      },
+    }
+  }
+  if (slashCount === 7) {
+    const user = parts[3]
+    const repo = parts[4]
+    const type = parts[5]
+    const commit = parts[6]
+    if (type === 'tree') {
+      return {
+        type: InstallType.GithubRepository,
+        options: {
+          user,
+          repo,
+          branch: commit,
+        },
+      }
+    }
+    if (type === 'pull') {
+      return {
+        type: InstallType.ParsingError,
+        options: {
+          message: 'Cannot download from Pull Request',
+        },
+      }
+    }
+  }
+  return {
+    type: InstallType.ParsingError,
+    options: {
+      message: 'Failed to parse github url',
+    },
+  }
+}
+
+const parseUrl = (input) => {
+  if (input.startsWith('https://github.com')) {
+    return parseUrlGithub(input)
+  }
+  if (input.endsWith('.tar.br')) {
+    return {
+      type: InstallType.Url,
+      options: {
+        url: input,
+      },
+    }
+  }
+  return {
+    type: InstallType.ParsingError,
+    options: {
+      message: 'Failed to parse url',
+    },
+  }
+}
+
+export const parse = (input) => {
+  if (input.startsWith('https://')) {
+    return parseUrl(input)
+  }
+  return {
+    type: InstallType.ParsingError,
+    options: {
+      message: 'Failed to parse input',
+    },
+  }
+}

package/src/parts/ExtensionLink/ExtensionLink.js

@@ -0,0 +1,32 @@
+import VError from 'verror'
+import * as FileSystem from '../FileSystem/FileSystem.js'
+import * as FileSystemErrorCodes from '../FileSystemErrorCodes/FileSystemErrorCodes.js'
+import * as Path from '../Path/Path.js'
+import * as Platform from '../Platform/Platform.js'
+import * as SymLink from '../SymLink/SymLink.js'
+
+const linkFallBack = async (path) => {
+  try {
+    const extensionsPath = Platform.getExtensionsPath()
+    const baseName = Path.basename(path)
+    const to = Path.join(extensionsPath, baseName)
+    await FileSystem.remove(to)
+    await SymLink.createSymLink(path, to)
+  } catch (error) {
+    throw new VError(error, `Failed to link extension`)
+  }
+}
+
+export const link = async (path) => {
+  try {
+    const extensionsPath = Platform.getExtensionsPath()
+    const baseName = Path.basename(path)
+    const to = Path.join(extensionsPath, baseName)
+    await SymLink.createSymLink(path, to)
+  } catch (error) {
+    if (error && error.code === FileSystemErrorCodes.EEXIST) {
+      return linkFallBack(path)
+    }
+    throw new VError(error, `Failed to link extension`)
+  }
+}

package/src/parts/ExtensionList/ExtensionList.js

@@ -0,0 +1,66 @@
+import { readdir, readFile } from 'node:fs/promises'
+import { basename, dirname, join } from 'node:path'
+import VError from 'verror'
+import * as Platform from '../Platform/Platform.js'
+import * as JsonFile from '../JsonFile/JsonFile.js'
+
+const extensionPath = Platform.getExtensionsPath()
+
+const getAbsolutePath = (dirent) => {
+  return join(extensionPath, dirent)
+}
+
+const getManifestVersion = (json) => {
+  if (json && json.version && typeof json.version === 'string') {
+    return json.version
+  }
+  return 'n/a'
+}
+
+const getManifestId = (json, jsonPath) => {
+  if (json && json.id && typeof json.id === 'string') {
+    return json.id
+  }
+  return basename(jsonPath)
+}
+
+const getManifestInfo = async (extensionPath) => {
+  const manifestPath = join(extensionPath, 'extension.json')
+  const json = await JsonFile.readJson(manifestPath)
+  const id = getManifestId(json, extensionPath)
+  const version = getManifestVersion(json)
+  return {
+    id,
+    version,
+  }
+}
+
+const isFulfilled = (result) => {
+  return result.status === 'fulfilled'
+}
+
+const getValue = (result) => {
+  return result.value
+}
+
+const getManifestInfos = async (manifestPaths) => {
+  const manifestInfos = await Promise.allSettled(
+    manifestPaths.map(getManifestInfo)
+  )
+  const results = manifestInfos.filter(isFulfilled).map(getValue)
+  return results
+}
+
+export const list = async () => {
+  try {
+    const dirents = await readdir(extensionPath)
+    const extensionPaths = dirents.map(getAbsolutePath)
+    const infos = await getManifestInfos(extensionPaths)
+    return infos
+  } catch (error) {
+    if (error && error.code === 'ENOENT') {
+      return []
+    }
+    throw new VError(error, `Failed to list extensions`)
+  }
+}

package/src/parts/ExtensionManagement/ExtensionManagement.js

@@ -36,7 +36,8 @@ export const install = async (id) => {
   // TODO this should be a stateless function, renderer-worker should have info on marketplace url
   // TODO use command.execute
   try {
-    const { download, extract } = await import('../Download/Download.js')
+    const { download } = await import('../Download/Download.js')
+    const { extractTarBr } = await import('../Extract/Extract.js')
     const marketplaceUrl = Platform.getMarketplaceUrl()
     Debug.debug(`ExtensionManagement#install ${id}`)
     const cachedExtensionsPath = Platform.getCachedExtensionsPath()
@@ -50,7 +51,7 @@ export const install = async (id) => {
         Path.join(cachedExtensionsPath, `${id}.tar.br`)
       )
       Debug.debug(`ExtensionManagement#extract ${id}`)
-      await extract(
+      await extractTarBr(
         Path.join(cachedExtensionsPath, `${id}.tar.br`),
         Path.join(extensionsPath, id)
       )
@@ -258,7 +259,7 @@ export const getExtensions = async () => {
   return [...builtinExtensions, ...installedExtensions]
 }
 
-const RE_THEME = /[a-z]+\.[a-z\-]*theme-[a-z\d\-]+$/
+const RE_THEME = /theme-[a-z\d\-]+$/
 const isTheme = (extensionId) => {
   return RE_THEME.test(extensionId)
 }

package/src/parts/Extract/Extract.js

@@ -0,0 +1,34 @@
+import { createReadStream } from 'node:fs'
+import { mkdir } from 'node:fs/promises'
+import { pipeline } from 'node:stream/promises'
+import { createBrotliDecompress, createGunzip } from 'node:zlib'
+import tar from 'tar-fs'
+import VError from 'verror'
+
+export const extractTarBr = async (inFile, outDir) => {
+  try {
+    await mkdir(outDir, { recursive: true })
+    await pipeline(
+      createReadStream(inFile),
+      createBrotliDecompress(),
+      tar.extract(outDir)
+    )
+  } catch (error) {
+    throw new VError(error, `Failed to extract ${inFile}`)
+  }
+}
+
+export const extractTarGz = async ({ inFile, outDir, strip }) => {
+  try {
+    await mkdir(outDir, { recursive: true })
+    await pipeline(
+      createReadStream(inFile),
+      createGunzip(),
+      tar.extract(outDir, {
+        strip,
+      })
+    )
+  } catch (error) {
+    throw new VError(error, `Failed to extract ${inFile}`)
+  }
+}

package/src/parts/FileSystem/FileSystem.js

@@ -185,10 +185,22 @@ export const mkdir = async (path) => {
   }
 }
 
+const fallbackRename = async (oldPath, newPath) => {
+  try {
+    await fs.cp(oldPath, newPath, { recursive: true })
+    await fs.rm(oldPath, { recursive: true })
+  } catch (error) {
+    throw new VError(error, `Failed to rename "${oldPath}" to "${newPath}"`)
+  }
+}
+
 export const rename = async (oldPath, newPath) => {
   try {
     await fs.rename(oldPath, newPath)
   } catch (error) {
+    if (error && error.code === FileSystemErrorCodes.EXDEV) {
+      return fallbackRename(oldPath, newPath)
+    }
     throw new VError(error, `Failed to rename "${oldPath}" to "${newPath}"`)
   }
 }

package/src/parts/FileSystemErrorCodes/FileSystemErrorCodes.js

@@ -1 +1,5 @@
 export const ENOENT = 'ENOENT'
+
+export const EXDEV = 'EXDEV'
+
+export const EEXIST = 'EEXIST'

package/src/parts/Path/Path.js

@@ -15,3 +15,7 @@ export const join = (...parts) => {
 export const dirname = (path) => {
   return NodePath.dirname(path)
 }
+
+export const basename = (path) => {
+  return NodePath.basename(path)
+}

package/src/parts/Preferences/Preferences.js

@@ -37,6 +37,17 @@ export const getDefaultPreferences = async () => {
 // TODO efficiently load preferences -> first load cached preferences
 //                                   -> on idle check preferences
 
+const getOverrides = () => {
+  const argvSliced = process.argv.slice(2)
+  const overrides = {}
+  for (const argv of argvSliced) {
+    console.log({ argv })
+    if (argv.startsWith('--theme=')) {
+      overrides['workbench.colorTheme'] = argv.slice('--theme='.length)
+    }
+  }
+  return overrides
+}
 // TODO compare with timestamp/hash that preferences are fresh
 export const getAll = async () => {
   try {
@@ -50,8 +61,13 @@ export const getAll = async () => {
     // }
     const defaultPreferences = await getDefaultPreferences()
     const userPreferences = await getUserPreferences()
+    const overrides = getOverrides()
     // TODO separate backend and frontend preferences, ui only needs frontend preferences
-    const preferences = { ...defaultPreferences, ...userPreferences }
+    const preferences = {
+      ...defaultPreferences,
+      ...userPreferences,
+      ...overrides,
+    }
     // try {
     //   await mkdir(dirname(CACHED_SETTINGS_PATH), { recursive: true })
     //   await writeFile(

package/src/parts/SymLink/SymLink.js

@@ -0,0 +1,15 @@
+import { symlink } from 'node:fs/promises'
+import { FileSystemError } from '../Error/FileSystemError.js'
+
+// TODO maybe move this to FileSystem module
+
+export const createSymLink = async (target, path) => {
+  try {
+    await symlink(target, path)
+  } catch (error) {
+    throw new FileSystemError(
+      error,
+      `Failed to create symbolic link from ${target} to ${path}`
+    )
+  }
+}

package/src/parts/TmpFile/TmpFile.js

@@ -0,0 +1,15 @@
+import { file, dir } from 'tmp-promise'
+
+export const getTmpFile = async () => {
+  const { path } = await file({
+    prefix: 'lvce-extension-',
+  })
+  return path
+}
+
+export const getTmpDir = async () => {
+  const { path } = await dir({
+    prefix: 'lvce-extension-',
+  })
+  return path
+}