@opensumi/ide-ai-native 3.0.3 → 3.0.4-next-1716367246.0

package/src/browser/inline-completions/types.ts

@@ -0,0 +1,169 @@
+import { URI } from '@opensumi/ide-core-common';
+
+export enum TokenizerName {
+  cl100k_base = 'cl100k_base',
+  gpt2 = 'gpt2',
+  r50k_base = 'r50k_base',
+  p50k_base = 'p50k_base',
+  p50k_edit = 'p50k_edit',
+  o200k_base = 'o200k_base',
+}
+
+export enum StrategyType {
+  InterceptBasedOnChar = 'InterceptBasedOnChar',
+  InterceptBasedOnLine = 'InterceptBasedOnLine',
+  InterceptBasedOnFunction = 'InterceptBasedOnFunction',
+}
+
+export enum SnippetSelectionOption {
+  BestMatch = 'bestMatch',
+  TopK = 'topK',
+}
+
+export enum NeighboringTabsOption {
+  openFileHistory = 'openFileHistory',
+  editFileRecent = 'editFileRecent',
+  editFileHistory = 'editFileHistory',
+}
+
+export enum WishListAttributeName {
+  beforeCursor = 'beforeCursor',
+  afterCursor = 'afterCursor',
+  importedFile = 'importedFile',
+  languageMarker = 'languageMarker',
+  pathMarker = 'pathMarker',
+  similarFile = 'similarFile',
+}
+
+// Sort Option
+export enum SortOption {
+  Ascending = 'ascending',
+  Descending = 'descending',
+  None = 'none',
+}
+
+export interface BaseWishListType {
+  attributeName: WishListAttributeName;
+  enable?: boolean; // 默认为 true
+  ignoreDisabledStateHigherThanVersion?: string;
+  extOption?: { [key: string]: any };
+  priority: number;
+  strategy: StrategyType;
+}
+
+export interface ImportedFileOptions {
+  maxTime: number;
+  importedFilesMaxNum: number;
+  language: string[];
+  maxPercent: number;
+  patternPrefix: string;
+  patternSuffix: string;
+}
+
+export enum SnippetSemantics {
+  Function = 'function',
+  Snippet = 'snippet',
+  Variable = 'variable',
+  Parameter = 'parameter',
+  Method = 'method',
+  Class = 'class',
+  Module = 'module',
+  Alias = 'alias',
+  Enum = 'enum member',
+  Interface = 'interface',
+}
+
+export interface SimilarFileOptions {
+  maxPercent: number;
+  snippetSelectionMode: SnippetSelectionOption;
+  neighboringTabsMaxNum: number;
+  neighboringTabsOption: NeighboringTabsOption[];
+  patternPrefix: string;
+  patternSuffix: string;
+  similarityThreshold: number;
+  snippetMaxNum: number;
+  windowSize: number;
+  maxTime?: number;
+}
+
+/**
+ * 补全配置项
+ * promptTokenSize = beforeCursorTokenSize + afterCursorTokenSize + importedFileTokenSize + pathMarkerTokenSize + languageMarkerTokenSize + similarFileTokenSize
+ * maxPromptTokenSize >= promptTokenSize
+ */
+export interface ICompletionModel {
+  maxPromptTokenSize: number;
+  tokenizerName: TokenizerName;
+  maxExecuteTimeMillSecond: number;
+  wishList: {
+    afterCursor: BaseWishListType & {
+      extOption: {
+        suffixPercent: number; // 0.25
+        minBlockSize: number; // 20
+      };
+    };
+    beforeCursor: BaseWishListType & {
+      extOption: {
+        prefixPercent: number; // 0.75,
+        minBlockSize: number; // 20,
+      };
+    };
+    importedFile: BaseWishListType & {
+      extOption: ImportedFileOptions;
+    };
+    languageMarker: BaseWishListType & {
+      extOption: {
+        language: string[];
+        maxPercent: number;
+      };
+    };
+    pathMarker: BaseWishListType;
+    similarFile: BaseWishListType & {
+      extOption: SimilarFileOptions;
+    };
+  };
+}
+
+export interface ICompletionContext {
+  prompt: string;
+  prefix: string;
+  suffix: string;
+  filename: string;
+  language: string;
+  uri: URI;
+}
+
+export interface CursorContext {
+  context: string;
+  lineCount: number;
+  tokenLength: number;
+  tokenizerName: TokenizerName;
+}
+
+export interface SimilarSnippet {
+  score: number;
+  startLine: number;
+  endLine: number;
+}
+
+export interface MatchSimilarSnippet extends SimilarSnippet {
+  snippet: string;
+  semantics: SnippetSemantics;
+}
+
+export interface ResourceDocument {
+  source: string;
+  uri: URI;
+  languageId: string;
+  offset: number;
+}
+
+export interface MarkerItem {
+  attributeName: WishListAttributeName;
+  priority: number;
+  maxPercent: number;
+  enable: boolean;
+  content?: string;
+  similarSnippets?: [string, MatchSimilarSnippet[]][];
+  importedFiles?: (string | string[])[][];
+}

package/src/browser/languages/parser.ts

@@ -9,6 +9,8 @@ import { SupportedTreeSitterLanguages, TreeSitterLanguageFacts } from './tree-si
 import { ICodeBlockInfo, IOtherBlockInfo } from './tree-sitter/language-facts/base';
 import { WasmModuleManager } from './tree-sitter/wasm-manager';
 
+export const DEFAULT_MIN_BLOCK_COUNT = 20;
+
 @Injectable({ multiple: true })
 export class LanguageParser implements IDisposable {
   private parser: Parser;
@@ -220,6 +222,151 @@ export class LanguageParser implements IDisposable {
     return null;
   }
 
+  async trimSuffixSyntaxErrors(code: string, minBlockCount = DEFAULT_MIN_BLOCK_COUNT): Promise<string> {
+    await this.parserLoaded.promise;
+    const tree = this.parser?.parse(code);
+    const rootNode = tree?.rootNode;
+    if (!rootNode) {
+      return code;
+    }
+    const { namedChildren } = rootNode;
+
+    if (!rootNode.hasError || namedChildren.length <= minBlockCount) {
+      return this.trimToLastCompleteBlock(rootNode, true);
+    }
+
+    let lastErrorNode: Parser.SyntaxNode | null = null;
+    let targetNode: Parser.SyntaxNode | null = null;
+    for (let i = namedChildren.length - 1; i >= minBlockCount; i--) {
+      const child = namedChildren[i];
+      if (child.hasError && i !== 0) {
+        lastErrorNode = child;
+      } else if (lastErrorNode) {
+        targetNode = child;
+        break;
+      }
+    }
+
+    if (!targetNode) {
+      targetNode = namedChildren[minBlockCount - 1];
+    }
+
+    const lines = code.split('\n');
+    const remainingLines = lines.slice(0, targetNode.endPosition.row + 1);
+    const lastLine = remainingLines[remainingLines.length - 1].slice(0, targetNode.endPosition.column);
+    const correctCode = `${remainingLines.length > 1 ? `${remainingLines.slice(0, -1).join('\n')}\n` : ''}${lastLine}`;
+    return correctCode;
+  }
+
+  async trimPrefixSyntaxErrors(code: string, minBlockCount = DEFAULT_MIN_BLOCK_COUNT): Promise<string> {
+    await this.parserLoaded.promise;
+    const tree = this.parser?.parse(code);
+    const rootNode = tree?.rootNode;
+    if (!rootNode) {
+      return code;
+    }
+    const { namedChildren } = rootNode;
+
+    if (!rootNode.hasError || namedChildren.length <= minBlockCount) {
+      return this.trimToLastCompleteBlock(rootNode);
+    }
+
+    let firstErrorNode: Parser.SyntaxNode | null = null;
+    let targetNode: Parser.SyntaxNode | null = null;
+    for (let i = 0; i < namedChildren.length - minBlockCount; i++) {
+      const child = namedChildren[i];
+      if (child.hasError && i !== namedChildren.length - 1) {
+        firstErrorNode = child;
+      } else if (firstErrorNode) {
+        targetNode = child;
+        break;
+      }
+    }
+
+    if (!targetNode) {
+      targetNode = namedChildren[namedChildren.length - minBlockCount];
+    }
+
+    const lines = code.split('\n');
+    const remainingLines = lines.slice(targetNode.startPosition.row);
+    const firstLine = remainingLines[0].slice(targetNode.startPosition.column);
+    const correctCode = `${firstLine}${remainingLines.length > 1 ? `\n${remainingLines.slice(1).join('\n')}` : ''}`;
+    return correctCode;
+  }
+
+  trimToLastCompleteBlock(rootNode: Parser.SyntaxNode, reverse = false) {
+    const { namedChildren } = rootNode;
+    if (reverse) {
+      const lastNamedChild = namedChildren[namedChildren.length - 2];
+      if (lastNamedChild) {
+        const lines = rootNode.text.split('\n');
+        const remainingLines = lines.slice(0, lastNamedChild.endPosition.row + 1);
+        const lastLine = remainingLines[remainingLines.length - 1].slice(0, lastNamedChild.endPosition.column);
+        const correctCode = `${
+          remainingLines.length > 1 ? `${remainingLines.slice(0, -1).join('\n')}\n` : ''
+        }${lastLine}`;
+        return correctCode;
+      }
+    } else {
+      const firstNamedChild = namedChildren[1];
+      if (firstNamedChild) {
+        const lines = rootNode.text.split('\n');
+        const remainingLines = lines.slice(firstNamedChild.startPosition.row);
+        const firstLine = remainingLines[0].slice(firstNamedChild.startPosition.column);
+        const correctCode = `${firstLine}${remainingLines.length > 1 ? `\n${remainingLines.slice(1).join('\n')}` : ''}`;
+        return correctCode;
+      }
+    }
+    return rootNode.text;
+  }
+
+  async extractImportPaths(code: string) {
+    const paths: string[] = [];
+    if (this.language === 'typescript') {
+      await this.parserLoaded.promise;
+      const tree = this.parser?.parse(code);
+      const rootNode = tree?.rootNode;
+      if (rootNode) {
+        for (let i = 0; i < rootNode?.childCount; i++) {
+          const sourceChild = rootNode.child(i);
+          if (sourceChild?.type === 'import_statement') {
+            let importPath = sourceChild.child(3)?.text;
+            if (importPath?.includes("'") || importPath?.includes('"')) {
+              importPath = importPath.slice(1, -1);
+            }
+            if (importPath) {
+              paths.push(importPath);
+            }
+          }
+        }
+      }
+    }
+    return paths;
+  }
+
+  async extractInterfaceOrTypeCode(code: string) {
+    const snippets: string[] = [];
+    if (this.language === 'typescript') {
+      await this.parserLoaded.promise;
+      const tree = this.parser?.parse(code);
+      const rootNode = tree?.rootNode;
+      if (rootNode) {
+        for (let i = 0; i < rootNode?.childCount; i++) {
+          const sourceChild = rootNode.child(i);
+          if (sourceChild?.type === 'export_statement') {
+            const exportChild = sourceChild.child(1);
+            if (exportChild?.type === 'interface_declaration' || exportChild?.type === 'type_alias_declaration') {
+              snippets.push(exportChild.text);
+            }
+          } else if (sourceChild?.type === 'interface_declaration' || sourceChild?.type === 'type_alias_declaration') {
+            snippets.push(sourceChild.text);
+          }
+        }
+      }
+    }
+    return snippets;
+  }
+
   dispose() {
     this.parser.delete();
     this.lruCache.clear();

package/src/browser/languages/tree-sitter/wasm-manager.ts

@@ -1,7 +1,7 @@
 import Parser from 'web-tree-sitter';
 
 import { Autowired, Injectable } from '@opensumi/di';
-import { EKnownResources, IRendererRuntime } from '@opensumi/ide-core-browser/lib/application/runtime/types';
+import { EKnownResources, RendererRuntime } from '@opensumi/ide-core-browser/lib/application/runtime/types';
 import { Deferred } from '@opensumi/ide-utils';
 
 /**
@@ -13,8 +13,8 @@ export class WasmModuleManager {
 
   private cachedRuntime: Map<string, Deferred<ArrayBuffer>> = new Map();
 
-  @Autowired(IRendererRuntime)
-  rendererRuntime: IRendererRuntime;
+  @Autowired(RendererRuntime)
+  rendererRuntime: RendererRuntime;
 
   private async resolveResourceUri() {
     const uri = await this.rendererRuntime.provideResourceUri(EKnownResources.TreeSitterWasmDirectory);

package/src/common/utils.ts

@@ -0,0 +1,23 @@
+import { IEditorDocumentModel } from '@opensumi/ide-editor';
+
+export const isDocumentTooLarge = (document: IEditorDocumentModel) => {
+  try {
+    document.getText();
+  } catch (e) {
+    if (e instanceof RangeError) {
+      return true;
+    }
+  }
+  return false;
+};
+
+const MIN_PROMPT_CHARS = 10;
+
+export const isDocumentTooShort = (document: IEditorDocumentModel) => document.getText().length < MIN_PROMPT_CHARS;
+
+export const isDocumentValid = (document: IEditorDocumentModel) => {
+  if (isDocumentTooLarge(document) || isDocumentTooShort(document)) {
+    return false;
+  }
+  return true;
+};