@lvce-editor/server 0.0.28 → 0.0.31

package/static/0e4d46a/packages/renderer-worker/src/parts/Editor/EditorText.js

@@ -0,0 +1,304 @@
+import * as TextDocument from '../TextDocument/TextDocument.js'
+
+/**
+ *
+ * @param {(line:string, lineState)=>any} tokenizeLine
+ * @param {string} line
+ * @param {any} lineState
+ * @returns
+ */
+const safeTokenizeLine = (tokenizeLine, line, lineState) => {
+  try {
+    lineState = tokenizeLine(line, lineState)
+    if (!lineState || !lineState.tokens || !lineState.state) {
+      throw new Error('invalid tokenization result')
+    }
+  } catch (error) {
+    console.error(error)
+    return {
+      tokens: [{ length: line.length, type: 0 }],
+      lineState,
+    }
+  }
+  return lineState
+}
+
+// const getTokens = (editor) => {
+//   const tokens = []
+//   const lines = editor.lines
+//   let lineState = editor.tokenizer.initialLineState
+//   // TODO lineCache should probably only store endOfLineState
+//   // because storing tokens could maybe result in high
+//   // memory usage (e.g. 10000 lines * 10 tokens per line = 100 000 objects stored)
+//   // on the other hand, scrolling should be really fast, and needlessly
+//   // recomputing tokens would be a waste of cpu time and cause lots of garbage collection
+//   const lineCache = editor.lineCache
+//   // TODO only compute tokens in viewport
+//   // console.log('minmax', editor.minLineY, editor.maxLineY)
+//   // const cachedLineStates = Object.create(null)
+//   // if(cachedLineStates[i])
+//   const tokenizeLine = editor.tokenizer.tokenizeLine
+//   for (let i = 0; i < lines.length; i++) {
+//     if (lineCache[i]) {
+//       // console.log('cached line')
+//       tokens.push(lineCache[i].tokens)
+//       continue
+//     }
+//     // TODO use TextDocument.getLine so that text document buffer implementation
+//     // can be changed (e.g. vscode has piece tree, codemirror has something like chunked arrays / string[][])
+//     const line = lines[i]
+//     lineState = safeTokenizeLine(tokenizeLine, line, lineState)
+//     const newTokens = lineState.tokens
+//     lineCache[i] = lineState
+//     tokens.push(newTokens)
+//   }
+//   return tokens
+// }
+
+const getTokensFromCache = (result) => {
+  return result.tokens
+}
+
+// TODO vscode has an interesting approach for tokenizing:
+// first, the viewport is tokenized from startLine to endLine
+// the first iteration might not be accurate because for example
+// there can be a open multiline comment at the start of the file
+// which is not taken into account for this first tokenization
+// but after some time (onIdle), the background tokenizer is invoked
+// that gives accurate results (but can take much longer since it
+// might need to parse from the start of the file)
+//
+// another approach would be to show plain text (non-highlighted)
+// when scrolling fast and the tokenizer is too slow to parse from
+// the start of the file
+//
+// the approach implemented below for tokenizing the viewport
+// is just to parse from the start of the file if necessary
+// that doesn't scale well for large files but it is simpler
+// to implement for now
+
+// TODO only send changed lines to renderer process instead of all lines in viewport
+const getTokensViewport = (editor, startLineIndex, endLineIndex) => {
+  const invalidStartIndex = editor.invalidStartIndex
+  const lineCache = editor.lineCache
+  if (endLineIndex <= invalidStartIndex) {
+    return lineCache
+      .slice(startLineIndex + 1, endLineIndex + 1)
+      .map(getTokensFromCache)
+  }
+  const tokenizeLine = editor.tokenizer.tokenizeLine
+  if (startLineIndex <= invalidStartIndex) {
+    for (let i = invalidStartIndex; i < endLineIndex; i++) {
+      const lineState =
+        i === 0 ? editor.tokenizer.initialLineState : editor.lineCache[i]
+      const line = editor.lines[i]
+      const result = safeTokenizeLine(tokenizeLine, line, lineState)
+      // TODO if lineCacheEnd matches the one before, skip tokenizing lines after
+      lineCache[i + 1] = result
+    }
+    editor.invalidStartIndex = endLineIndex
+    return lineCache
+      .slice(startLineIndex + 1, endLineIndex + 1)
+      .map(getTokensFromCache)
+  }
+  // TODO combine for loops
+  for (let i = invalidStartIndex; i < startLineIndex; i++) {
+    const lineState = editor.lineCache[i]
+    const line = editor.lines[i]
+    const result = safeTokenizeLine(tokenizeLine, line, lineState)
+    lineCache[i + 1] = result
+  }
+  for (let i = startLineIndex; i < endLineIndex; i++) {
+    const lineState = editor.lineCache[i]
+    const line = editor.lines[i]
+    const result = safeTokenizeLine(tokenizeLine, line, lineState)
+    lineCache[i + 1] = result
+    editor.invalidStartIndex = endLineIndex
+  }
+  return lineCache
+    .slice(startLineIndex + 1, endLineIndex + 1)
+    .map(getTokensFromCache)
+}
+
+const invalidateLine = (editor, index) => {
+  editor.validLines[index] = false
+  if (index < editor.invalidStartIndex) {
+    editor.invalidStartIndex = index
+  }
+}
+
+const applyChangesToSyntaxHighlighting = (editor, changes) => {
+  // TODO invalidate lines that are affected
+}
+
+// const getTokensIncremental = (editor, min, max) => {
+//   const currentLength = editor.lineStateCache.length
+//   const tokens = []
+//   const lines = editor.lines
+//   let lineState = editor.tokenizer.initialLineState
+//   for (let i = currentLength; i < max; i++) {
+//     const line = lines[i]
+//     try {
+//       lineState = editor.tokenizer.tokenizeLine(line, lineState)
+//       if (!lineState || !lineState.tokens || !lineState.state) {
+//         throw new Error('invalid tokenization result')
+//       }
+//     } catch (error) {
+//       tokens.push([{ length: line.length, type: 0 }])
+//       console.error(error)
+//       // renderWithoutSyntaxHighlighting(state, firstRow, lastRow)
+//       continue
+//     }
+//     const newTokens = lineState.tokens
+//     tokens.push(newTokens)
+//   }
+//   return tokens
+// }
+
+// const getLineInfosIncremental = (editor, tokens, minLineY, maxLineY) => {
+//   const result = []
+//   const lines = editor.lines
+//   const TokenMap = editor.tokenizer.TokenMap
+//   for (let i = minLineY; i < maxLineY; i++) {
+//     result.push(getLineInfo(lines[i], tokens[i], TokenMap))
+//   }
+//   return result
+// }
+
+const getDecorationClassName = (type) => {
+  switch (type) {
+    case 2816:
+    case 2817:
+    case 2824:
+    case 2825:
+    case 2856:
+    case 2857:
+    case 3072:
+    case 3073:
+    case 3077:
+    case 3088:
+      return 'Function'
+    case 1792:
+    case 1793:
+      return 'Parameter'
+    case 512:
+    case 513:
+    case 769:
+    case 1024:
+    case 1536:
+    case 1537:
+    case 1544:
+    case 1545:
+      return 'Type'
+    case 2048:
+    case 2049:
+    case 2056:
+    case 2057:
+    case 2064:
+    case 2080:
+    case 2081:
+    case 2088:
+    case 2089:
+    case 2313:
+    case 2560:
+    case 2561:
+    case 2569:
+    case 2584:
+      return 'VariableName'
+    case 256:
+    case 257:
+    case 272:
+      return 'Class'
+    default:
+      return `Unknown-${type}`
+  }
+}
+
+const getLineInfo = (line, tokens, decorations, TokenMap, lineOffset) => {
+  let start = 0
+  let end = 0
+  const lineInfo = []
+  let decorationIndex = 0
+  for (; decorationIndex < decorations.length; decorationIndex += 3) {
+    const decorationOffset = decorations[decorationIndex]
+    if (decorationOffset >= lineOffset) {
+      break
+    }
+  }
+
+  // console.log({ tokens, decorations })
+  for (const token of tokens) {
+    const decorationOffset = decorations[decorationIndex]
+    let extraClassName = ''
+    if (
+      decorationOffset !== undefined &&
+      decorationOffset - lineOffset === start
+    ) {
+      const decorationLength = decorations[++decorationIndex]
+      const decorationType = decorations[++decorationIndex]
+      const decorationModifiers = decorations[++decorationIndex]
+      // console.log('MATCHING DECORATION', {
+      //   decorationIndex,
+      //   decorationLength,
+      //   decorationType,
+      //   decorationModifiers,
+      // })
+      extraClassName = getDecorationClassName(decorationType)
+    }
+
+    end += token.length
+    const text = line.slice(start, end)
+    const className = `Token ${
+      extraClassName || TokenMap[token.type] || 'Unknown'
+    }`
+    lineInfo.push(text, className)
+    start = end
+  }
+  return lineInfo
+}
+
+// TODO need lots of tests for this
+const getLineInfosViewport = (
+  editor,
+  tokens,
+  minLineY,
+  maxLineY,
+  minLineOffset
+) => {
+  const result = []
+  const lines = editor.lines
+  const TokenMap = editor.tokenizer.TokenMap
+  const decorations = editor.decorations
+  let offset = minLineOffset
+  for (let i = minLineY; i < maxLineY; i++) {
+    const line = lines[i]
+    result.push(
+      getLineInfo(line, tokens[i - minLineY], decorations, TokenMap, offset)
+    )
+    offset += line.length + 1
+  }
+  return result
+}
+
+export const getVisible = (editor) => {
+  // console.log({ editor })
+  // TODO should separate rendering from business logic somehow
+  // currently hard to test because need to mock editor height, top, left,
+  // invalidStartIndex, lineCache, etc. just for testing editorType
+  // editor.invalidStartIndex = changes[0].start.rowIndex
+  const minLineY = editor.minLineY
+  const maxLineY = Math.min(
+    editor.minLineY + editor.numberOfVisibleLines,
+    editor.lines.length
+  )
+  const tokens = getTokensViewport(editor, minLineY, maxLineY)
+  const minLineOffset = TextDocument.offsetAt(editor, minLineY, 0)
+  const textInfos = getLineInfosViewport(
+    editor,
+    tokens,
+    minLineY,
+    maxLineY,
+    minLineOffset
+  )
+  return textInfos
+}

package/static/0e4d46a/packages/renderer-worker/src/parts/EditorCommand/EditorCommandCancelSelection.js

@@ -0,0 +1,21 @@
+import * as Editor from '../Editor/Editor.js'
+import * as EditorSelection from '../EditorSelection/EditorSelection.js'
+
+export const editorCancelSelection = (editor) => {
+  const selections = editor.selections
+  if (
+    selections.length === 4 &&
+    selections[0] === selections[2] &&
+    selections[1] === selections[3]
+  ) {
+    return editor
+  }
+  const newSelections = EditorSelection.alloc(4)
+  EditorSelection.moveRangeToPosition(
+    newSelections,
+    0,
+    selections[0],
+    selections[1]
+  )
+  return Editor.scheduleSelections(editor, newSelections)
+}

package/static/0e4d46a/packages/renderer-worker/src/parts/EditorCommand/EditorCommandCompletion.js

@@ -0,0 +1,230 @@
+import * as Assert from '../Assert/Assert.js'
+import * as FuzzySearch from '../FuzzySearch/FuzzySearch.js'
+import * as GlobalEventBus from '../GlobalEventBus/GlobalEventBus.js'
+import * as RendererProcess from '../RendererProcess/RendererProcess.js'
+import * as ViewletManager from '../ViewletManager/ViewletManager.js'
+import * as Viewlet from '../Viewlet/Viewlet.js'
+import * as EditorBlur from './EditorCommandBlur.js'
+import * as EditorPosition from './EditorCommandPosition.js'
+import * as ViewletState from '../ViewletStates/ViewletStates.js'
+
+const handleBlur = () => {
+  close()
+}
+
+/**
+ *
+ * @param {string} line
+ * @param {number} columnIndex
+ * @returns
+ */
+const getLeadingWord = (line, columnIndex) => {
+  console.log({ line })
+  Assert.string(line)
+  Assert.number(columnIndex)
+  for (let i = columnIndex - 1; i >= 0; i--) {
+    if (!RE_CHARACTER.test(line[i])) {
+      console.log({ i, columnIndex })
+      return line.slice(i + 1, columnIndex)
+    }
+  }
+  return line.slice(0, columnIndex)
+}
+
+const filterCompletionItems = (completionItems, leadingWord) => {
+  const filterCompletionItem = (completionItem) => {
+    const labelMatch = FuzzySearch.fuzzySearch(
+      leadingWord,
+      completionItem.label
+    )
+    return labelMatch
+  }
+  console.log({ completionItems, leadingWord })
+  const filtered = completionItems.filter(filterCompletionItem)
+  return filtered
+}
+
+const handleSelectionChange = (editor, selectionChanges) => {
+  const selection = selectionChanges[0]
+  const end = selection.end
+  if (end.rowIndex !== state.rowIndex) {
+    close()
+    return
+  }
+  const line = editor.lines[end.rowIndex]
+  const leadingWord = getLeadingWord(line, end.columnIndex)
+  const oldStart = state.columnIndex - state.leadingWord.length
+  const newStart = end.columnIndex - leadingWord.length
+  if (oldStart !== newStart) {
+    close()
+    return
+  }
+  // TODO move completions widget and filter
+
+  // TODO this doesn't work for variable width characters (unicode/emoji)
+
+  const filteredCompletionItems = filterCompletionItems(
+    state.completionItems,
+    leadingWord
+  )
+  state.filteredCompletionItems = filteredCompletionItems
+  state.leadingWord = leadingWord
+  console.log({ leadingWord })
+  const x = EditorPosition.x(editor, editor.cursor)
+  const y = EditorPosition.y(editor, editor.cursor)
+  RendererProcess.invoke(
+    /* Viewlet.send */ 'Viewlet.send',
+    /* id */ 'EditorCompletion',
+    /* method */ 'show',
+    /* x */ x,
+    /* y */ y,
+    /* completionItems */ filteredCompletionItems,
+    /* reason */ 1
+  )
+}
+
+const handleCursorChange = (anyEditor, cursorChange) => {
+  console.log('cursor changed')
+}
+
+// TODO there are still race conditions in this function:
+// - when open is called twice, previous dom nodes can either be reused or the previous dom nodes must be disposed
+
+export const open = async (editor, openingReason = 1) => {
+  console.log('open editor completion')
+  const viewlet = ViewletManager.create(
+    ViewletManager.getModule,
+    'EditorCompletion',
+    'Widget',
+    'builtin://',
+    0,
+    0,
+    0,
+    0
+  )
+
+  await ViewletManager.load(viewlet)
+  return editor
+
+  // if (state.isOpened) {
+  //   // TODO filter
+  //   console.log('is already open')
+  //   return
+  // }
+  // const loadingDelay = Preferences.get('completions.loadingDelay')
+
+  // console.log({ loadingDelay })
+  // state.abortController.abort()
+  // const abortController = new AbortController()
+  // state.abortController = abortController
+  // EditorBlur.registerListener(handleBlur) // TODO don't like side effect here
+
+  // // TODO when editor is disposed, close completions
+
+  // GlobalEventBus.addListener('editor.selectionChange', handleSelectionChange)
+  // GlobalEventBus.addListener('editor.cursorChange', handleCursorChange)
+  // state.isOpened = true
+  // state.openingReason = openingReason
+  // state.focusedIndex = 0
+  // state.rowIndex = editor.cursor.rowIndex
+  // state.columnIndex = editor.cursor.columnIndex
+  // state.editor = editor
+  // const line = editor.lines[editor.cursor.rowIndex]
+  // state.leadingWord = getLeadingWord(line, editor.cursor.columnIndex)
+  // // TODO race condition: when blur occurs before items arrive, should not show items
+  // await RendererProcess.invoke(
+  //   /* Viewlet.load */ 3030,
+  //   /* id */ 'EditorCompletion'
+  // )
+  // if (abortController.signal.aborted) {
+  //   console.info('canceled')
+  //   return
+  // }
+  // await RendererProcess.invoke(
+  //   /* Viewlet.openWidget */ 3030,
+  //   /* id */ 'EditorCompletion'
+  // )
+  // if (abortController.signal.aborted) {
+  //   console.info('canceled')
+  //   return
+  // }
+  // state.loadingTimeout = setTimeout(showLoading, loadingDelay)
+  // if (state.loadingTimeout !== -1) {
+  //   clearTimeout(state.loadingTimeout)
+  // }
+  // if (abortController.signal.aborted) {
+  //   console.info('canceled')
+  //   return
+  // }
+  // state.completionItems = newCompletionItems
+
+  // const filteredCompletionItems = filterCompletionItems(
+  //   state.completionItems,
+  //   state.leadingWord
+  // )
+  // state.filteredCompletionItems = filteredCompletionItems
+
+  // const x = EditorPosition.x(editor, editor.cursor)
+  // const y = EditorPosition.y(editor, editor.cursor)
+  // // TODO race condition: might already be dispose after each of these calls
+
+  // if (newCompletionItems.length === 0 && state.openingReason === 0) {
+  //   return
+  // }
+
+  // await RendererProcess.invoke(
+  //   /* Viewlet.send */ 'Viewlet.send',
+  //   /* id */ 'EditorCompletion',
+  //   /* method */ 'show',
+  //   /* x */ x,
+  //   /* y */ y,
+  //   /* items */ filteredCompletionItems,
+  //   /* reason */ state.openingReason,
+  //   /* focusedIndex */ state.focusedIndex
+  // )
+}
+
+export const advance = (word) => {
+  const includesWord = (completionItem) => {
+    return completionItem.includes(word)
+  }
+  const filteredCompletions = state.completionItems.filter(includesWord)
+  RendererProcess.invoke(
+    /* EditorCompletion.open */ 836,
+    /* items */ filteredCompletions,
+    /* reason */ state.openingReason
+  )
+}
+
+// export const onDidType = () => {}
+
+// TODO possible race condition
+// also should cancel when new open request comes in
+
+const RE_CHARACTER = new RegExp(/^\p{L}/, 'u')
+
+const isCharacter = (text) => {
+  return RE_CHARACTER.test(text)
+}
+
+// TODO trigger characters should be specific per language id
+const isTriggerCharacter = (text) => {
+  return text === '<' || text === '.'
+}
+
+export const openFromType = async (editor, text) => {
+  // if (isCharacter(text) || isTriggerCharacter(text)) {
+  //   await open(editor, 0)
+  // } else if (state.isOpened) {
+  //   await close()
+  // }
+  return ViewletState.getState('EditorText')
+}
+
+export const close = async (editor) => {
+  EditorBlur.removeListener(handleBlur)
+  GlobalEventBus.removeListener('editor.selectionChange', handleSelectionChange)
+  GlobalEventBus.removeListener('editor.cursorChange', handleCursorChange)
+  await Viewlet.dispose('EditorCompletion')
+  return editor
+}

package/static/0e4d46a/packages/renderer-worker/src/parts/EditorCommand/EditorCommandComposition.js

@@ -0,0 +1,53 @@
+import * as Editor from '../Editor/Editor.js'
+
+export const state = {
+  isComposing: false,
+  compositionText: '',
+}
+
+export const editorCompositionStart = (editor, event) => {
+  state.isComposing = true
+  return editor
+}
+
+const getCompositionChanges = (selections, data) => {
+  const changes = []
+  for (let i = 0; i < selections.length; i += 4) {
+    const selectionStartRow = selections[i]
+    const selectionStartColumn = selections[i + 1]
+    const selectionEndRow = selections[i + 2]
+    const selectionEndColumn = selections[i + 3]
+    const startColumnIndex = selectionStartColumn - state.compositionText.length
+    changes.push({
+      start: {
+        rowIndex: selectionStartRow,
+        columnIndex: startColumnIndex,
+      },
+      end: {
+        rowIndex: selectionEndRow,
+        columnIndex: selectionEndColumn,
+      },
+      inserted: [data],
+      deleted: [state.compositionText],
+      origin: 'compositionUpdate',
+    })
+  }
+  return changes
+}
+
+export const editorCompositionUpdate = (editor, data) => {
+  const selections = editor.selections
+  const changes = getCompositionChanges(selections, data)
+  console.log('composition update', changes)
+  state.compositionText = data
+  return Editor.scheduleDocumentAndCursorsSelections(editor, changes)
+}
+
+export const editorCompositionEnd = (editor, data) => {
+  const selections = editor.selections
+  const changes = getCompositionChanges(selections, data)
+  console.log('composition end', { changes })
+  state.isComposing = false
+  state.compositionText = ''
+  return Editor.scheduleDocumentAndCursorsSelections(editor, changes)
+}

package/static/0e4d46a/packages/renderer-worker/src/parts/EditorCommand/EditorCommandCopyLineDown.js

@@ -0,0 +1,25 @@
+import * as TextDocument from '../TextDocument/TextDocument.js'
+import * as Editor from '../Editor/Editor.js'
+import * as Assert from '../Assert/Assert.js'
+// TODO handle multiline selection
+// TODO handle multiple cursors
+
+export const editorCopyLineDown = (editor) => {
+  const selections = editor.selections
+  const primarySelectionIndex = editor.primarySelectionIndex
+  const rowIndex = selections[primarySelectionIndex]
+  Assert.number(rowIndex)
+  const position = {
+    rowIndex,
+    columnIndex: 0,
+  }
+  const changes = [
+    {
+      inserted: [TextDocument.getLine(editor, rowIndex), ''],
+      deleted: [''],
+      start: position,
+      end: position,
+    },
+  ]
+  return Editor.scheduleDocumentAndCursorsSelections(editor, changes)
+}

package/static/0e4d46a/packages/renderer-worker/src/parts/EditorCommand/EditorCommandCursorCharacterRight.js

@@ -0,0 +1,9 @@
+import * as EditorCursorHorizontalRight from './EditorCommandCursorHorizontalRight.js'
+import * as EditorDelta from './EditorCommandDelta.js'
+
+export const editorCursorCharacterRight = (editor) => {
+  return EditorCursorHorizontalRight.editorCursorHorizontalRight(
+    editor,
+    EditorDelta.characterRight
+  )
+}

package/static/0e4d46a/packages/renderer-worker/src/parts/EditorCommand/EditorCommandCursorDown.js

@@ -0,0 +1,28 @@
+import * as Editor from '../Editor/Editor.js'
+import * as EditorSelection from '../EditorSelection/EditorSelection.js'
+
+const moveSelectionDown = (
+  selections,
+  i,
+  selectionStartRow,
+  selectionStartColumn,
+  selectionEndRow,
+  selectionEndColumn
+) => {
+  EditorSelection.moveRangeToPosition(
+    selections,
+    i,
+    selectionEndRow + 1,
+    selectionEndColumn
+  )
+}
+
+const getNewSelections = (selections) => {
+  return EditorSelection.map(selections, moveSelectionDown)
+}
+
+export const editorCursorDown = (editor) => {
+  const selections = editor.selections
+  const newSelections = getNewSelections(selections)
+  return Editor.scheduleSelections(editor, newSelections)
+}