@opensumi/ide-ai-native 3.0.3 → 3.0.4-next-1716367246.0

package/src/browser/inline-completions/completeProvider.ts

@@ -1,6 +1,6 @@
 import { Autowired, INJECTOR_TOKEN, Injectable, Injector } from '@opensumi/di';
 import { AI_INLINE_COMPLETION_REPORTER } from '@opensumi/ide-core-browser/lib/ai-native/command';
-import { WithEventBus, uuid } from '@opensumi/ide-core-common';
+import { URI, WithEventBus, uuid } from '@opensumi/ide-core-common';
 import { IAICompletionResultModel } from '@opensumi/ide-core-common/lib/types/ai-native';
 import { AISerivceType, IAIReporter } from '@opensumi/ide-core-common/lib/types/ai-native/reporter';
 import { IEditor } from '@opensumi/ide-editor';
@@ -8,12 +8,13 @@ import * as monaco from '@opensumi/ide-monaco';
 
 import { AINativeContextKey } from '../contextkey/ai-native.contextkey.service';
 
+import { DEFAULT_COMPLECTION_MODEL } from './constants';
 import { CompletionRequestBean, InlayList, InlineCompletionItem } from './model/competionModel';
 import { PromptCache } from './promptCache';
-import { prePromptHandler, preSuffixHandler } from './provider';
+import { getPrefixPrompt, getSuffixPrompt } from './provider';
 import { AICompletionsService } from './service/ai-completions.service';
+import { ICompletionContext } from './types';
 
-let lastRequestId: any;
 let timer: any;
 let accept = 0;
 
@@ -29,7 +30,7 @@ export const getAcceptText = (cuont: number) => {
   accept = accept + cuont;
 };
 
-class RequestImp {
+class CompletionRequest {
   isCancelFlag: boolean;
   // todo
   constructor(
@@ -37,6 +38,7 @@ class RequestImp {
     public _isManual: boolean,
     public aiCompletionsService: AICompletionsService,
     public promptCache: PromptCache,
+    public injector: Injector,
   ) {
     this.isCancelFlag = false;
   }
@@ -48,18 +50,12 @@ class RequestImp {
     token: monaco.CancellationToken,
   ) {
     const { model: editor } = this;
-    const beginRequestTime = Date.now();
     if (!editor) {
       return [];
     }
 
     const startRange = new monaco.Range(0, 0, position.lineNumber, position.column);
-    let prompt = model.getValueInRange(startRange!);
-
-    // 如果是空白页面,默认加个回车符?
-    if (this.model.getValue() === '') {
-      prompt += '\n';
-    }
+    const prefix = model.getValueInRange(startRange);
 
     const endRange = new monaco.Range(
       position.lineNumber,
@@ -69,19 +65,18 @@ class RequestImp {
     );
     let suffix = model.getValueInRange(endRange);
 
-    // 改用analysisCodeFuseLanguage  现在目前后端需要语言小写
     const languageId = model.getLanguageId();
-    // if (completionRtModel !== undefined && completionRtModel.sessionId === null) {
-    //     //此处未采纳上一个补全结果,那么设置endtime,并提交到服务端
-    //     completionRtModel.endRenderingTime = Date.now();
-    //     submitCompleteRtData(completionRtModel);
-    // }
-    // TODO 缺少数据前处理
-    // prompt = prePromptHandler(prompt);
-    // suffix = preSuffixHandler(suffix);
-
-    prompt = prePromptHandler(prompt);
-    suffix = preSuffixHandler(suffix);
+    const context: ICompletionContext = {
+      filename: model.uri.toString().split('/').pop()!,
+      language: languageId,
+      prefix,
+      suffix,
+      prompt: '',
+      uri: URI.from(model.uri),
+    };
+
+    const prompt = await getPrefixPrompt(context, DEFAULT_COMPLECTION_MODEL, this.injector, token);
+    suffix = await getSuffixPrompt(context, DEFAULT_COMPLECTION_MODEL, this.injector, token);
 
     // 组装请求参数,向远程发起请求
     const completionRequestBean: CompletionRequestBean = {
@@ -94,7 +89,6 @@ class RequestImp {
 
     this.aiCompletionsService.updateStatusBarItem('running', true);
     const beginAlgTime = Date.now();
-    let status = 0; // 0: 远程请求的结果 1: 网络缓存中的结果
     if (this.isCancelFlag) {
       return [];
     }
@@ -107,7 +101,6 @@ class RequestImp {
     // 如果存在缓存
     if (cacheData) {
       rs = cacheData;
-      status = 1;
     } else {
       try {
         rs = await this.aiCompletionsService.complete(completionRequestBean, model, position, token);
@@ -120,7 +113,6 @@ class RequestImp {
         this.aiCompletionsService.hideStatusBarItem();
         return [];
       }
-      status = 0;
     }
 
     if (!(rs && rs.sessionId)) {
@@ -146,11 +138,6 @@ class RequestImp {
     if (rs && rs.codeModelList && rs.codeModelList.length > 0) {
       this.promptCache.setCache(prompt, rs);
     }
-    let codeModelSize = 0;
-    if (rs.codeModelList !== null) {
-      codeModelSize = rs.codeModelList.length;
-    }
-
     // 返回补全结果为空直接返回
     if (rs.codeModelList.length === 0) {
       this.aiCompletionsService.reporterEnd(relationId, {
@@ -225,7 +212,6 @@ class RequestImp {
         },
       });
     }
-    lastRequestId = rs.sessionId;
     return result;
   }
   cancelRequest() {
@@ -238,7 +224,7 @@ class ReqStack {
   constructor(private readonly aiReporter) {
     this.queue = [];
   }
-  addReq(reqRequest: RequestImp) {
+  addReq(reqRequest: CompletionRequest) {
     this.queue.push(reqRequest);
   }
   runReq(position: monaco.Position, model: monaco.editor.ITextModel, token: monaco.CancellationToken) {
@@ -287,7 +273,6 @@ export class AIInlineCompletionsProvider extends WithEventBus {
     this.reqStack = new ReqStack(this.aiReporter);
   }
 
-  // 取消请求
   cancelRequest() {
     this.aiCompletionsService.cancelRequest();
     if (this.reqStack) {
@@ -349,7 +334,13 @@ export class AIInlineCompletionsProvider extends WithEventBus {
       };
     }
     // 放入队列
-    const requestImp = new RequestImp(model, _isManual, this.aiCompletionsService, this.promptCache);
+    const requestImp = new CompletionRequest(
+      model,
+      _isManual,
+      this.aiCompletionsService,
+      this.promptCache,
+      this.injector,
+    );
     this.reqStack.addReq(requestImp);
     // 如果是自动补全等待300ms
     if (!_isManual) {

package/src/browser/inline-completions/constants.ts

@@ -1,9 +1,89 @@
-interface ICompletionModel {
-  completionPromptMaxLineSize: number;
-  completionSuffixMaxLineSize: number;
-}
+import {
+  ICompletionModel,
+  NeighboringTabsOption,
+  SnippetSelectionOption,
+  StrategyType,
+  TokenizerName,
+  WishListAttributeName,
+} from './types';
 
-export const completionModel: ICompletionModel = {
-  completionPromptMaxLineSize: 1024,
-  completionSuffixMaxLineSize: 500,
+export const DEFAULT_COMPLECTION_MODEL: ICompletionModel = {
+  maxPromptTokenSize: 3000,
+  maxExecuteTimeMillSecond: 1000,
+  tokenizerName: TokenizerName.cl100k_base,
+  wishList: {
+    afterCursor: {
+      enable: true,
+      attributeName: WishListAttributeName.afterCursor,
+      extOption: {
+        suffixPercent: 0.25,
+        minBlockSize: 25,
+      },
+      priority: 1,
+      strategy: StrategyType.InterceptBasedOnLine,
+    },
+    beforeCursor: {
+      enable: true,
+      attributeName: WishListAttributeName.beforeCursor,
+      extOption: {
+        prefixPercent: 0.75,
+        minBlockSize: 75,
+      },
+      priority: 0,
+      strategy: StrategyType.InterceptBasedOnLine,
+    },
+    importedFile: {
+      attributeName: WishListAttributeName.importedFile,
+      enable: true,
+      extOption: {
+        maxTime: 200,
+        importedFilesMaxNum: 10,
+        language: ['typescript'],
+        maxPercent: 0.8,
+        patternPrefix: 'Import Files from ',
+        patternSuffix: '\n',
+      },
+      priority: 3,
+      strategy: StrategyType.InterceptBasedOnLine,
+    },
+    languageMarker: {
+      attributeName: WishListAttributeName.languageMarker,
+      enable: true,
+      extOption: {
+        language: ['python', 'html'],
+        maxPercent: 0,
+      },
+      priority: 4,
+      strategy: StrategyType.InterceptBasedOnLine,
+    },
+    pathMarker: {
+      attributeName: WishListAttributeName.pathMarker,
+      enable: true,
+      priority: 1,
+      strategy: StrategyType.InterceptBasedOnLine,
+    },
+    similarFile: {
+      attributeName: WishListAttributeName.similarFile,
+      enable: true,
+      extOption: {
+        maxTime: 200,
+        fileMaxLength: 5000,
+        maxPercent: 0.3,
+        neighboringTabsMaxNum: 20,
+        neighboringTabsOption: [
+          NeighboringTabsOption.openFileHistory,
+          NeighboringTabsOption.editFileRecent,
+          NeighboringTabsOption.editFileHistory,
+        ],
+        patternPrefix: 'Compare this snippet from ',
+        patternSuffix: '\n',
+        similarityThreshold: 0.6,
+        snippetMaxNum: 4,
+        windowSize: 60,
+        snippetSelectionMode: SnippetSelectionOption.BestMatch,
+      },
+      priority: 2,
+      strategy: StrategyType.InterceptBasedOnLine,
+    },
+  },
 };

package/src/browser/inline-completions/prompt/const.ts

@@ -0,0 +1,286 @@
+export const LANGUAGE_COMMENT_MARKERS: {
+  [key: string]: {
+    start: string;
+    end: string;
+  };
+} = {
+  abap: {
+    start: '"',
+    end: '',
+  },
+  bat: {
+    start: 'REM',
+    end: '',
+  },
+  bibtex: {
+    start: '%',
+    end: '',
+  },
+  blade: {
+    start: '#',
+    end: '',
+  },
+  c: {
+    start: '//',
+    end: '',
+  },
+  clojure: {
+    start: ';',
+    end: '',
+  },
+  coffeescript: {
+    start: '//',
+    end: '',
+  },
+  cpp: {
+    start: '//',
+    end: '',
+  },
+  csharp: {
+    start: '//',
+    end: '',
+  },
+  css: {
+    start: '/*',
+    end: '*/',
+  },
+  dart: {
+    start: '//',
+    end: '',
+  },
+  dockerfile: {
+    start: '#',
+    end: '',
+  },
+  elixir: {
+    start: '#',
+    end: '',
+  },
+  erb: {
+    start: '<%#',
+    end: '%>',
+  },
+  erlang: {
+    start: '%',
+    end: '',
+  },
+  fsharp: {
+    start: '//',
+    end: '',
+  },
+  go: {
+    start: '//',
+    end: '',
+  },
+  groovy: {
+    start: '//',
+    end: '',
+  },
+  haml: {
+    start: '-#',
+    end: '',
+  },
+  handlebars: {
+    start: '{{!',
+    end: '}}',
+  },
+  haskell: {
+    start: '--',
+    end: '',
+  },
+  html: {
+    start: '\x3c!--',
+    end: '--\x3e',
+  },
+  ini: {
+    start: ';',
+    end: '',
+  },
+  java: {
+    start: '//',
+    end: '',
+  },
+  javascript: {
+    start: '//',
+    end: '',
+  },
+  javascriptreact: {
+    start: '//',
+    end: '',
+  },
+  jsonc: {
+    start: '//',
+    end: '',
+  },
+  jsx: {
+    start: '//',
+    end: '',
+  },
+  julia: {
+    start: '#',
+    end: '',
+  },
+  kotlin: {
+    start: '//',
+    end: '',
+  },
+  latex: {
+    start: '%',
+    end: '',
+  },
+  less: {
+    start: '//',
+    end: '',
+  },
+  lua: {
+    start: '--',
+    end: '',
+  },
+  makefile: {
+    start: '#',
+    end: '',
+  },
+  markdown: {
+    start: '[]: #',
+    end: '',
+  },
+  'objective-c': {
+    start: '//',
+    end: '',
+  },
+  'objective-cpp': {
+    start: '//',
+    end: '',
+  },
+  perl: {
+    start: '#',
+    end: '',
+  },
+  php: {
+    start: '//',
+    end: '',
+  },
+  powershell: {
+    start: '#',
+    end: '',
+  },
+  pug: {
+    start: '//',
+    end: '',
+  },
+  python: {
+    start: '#',
+    end: '',
+  },
+  ql: {
+    start: '//',
+    end: '',
+  },
+  r: {
+    start: '#',
+    end: '',
+  },
+  razor: {
+    start: '\x3c!--',
+    end: '--\x3e',
+  },
+  ruby: {
+    start: '#',
+    end: '',
+  },
+  rust: {
+    start: '//',
+    end: '',
+  },
+  sass: {
+    start: '//',
+    end: '',
+  },
+  scala: {
+    start: '//',
+    end: '',
+  },
+  scss: {
+    start: '//',
+    end: '',
+  },
+  shellscript: {
+    start: '#',
+    end: '',
+  },
+  slim: {
+    start: '/',
+    end: '',
+  },
+  solidity: {
+    start: '//',
+    end: '',
+  },
+  sql: {
+    start: '--',
+    end: '',
+  },
+  stylus: {
+    start: '//',
+    end: '',
+  },
+  svelte: {
+    start: '\x3c!--',
+    end: '--\x3e',
+  },
+  swift: {
+    start: '//',
+    end: '',
+  },
+  terraform: {
+    start: '#',
+    end: '',
+  },
+  tex: {
+    start: '%',
+    end: '',
+  },
+  typescript: {
+    start: '//',
+    end: '',
+  },
+  typescriptreact: {
+    start: '//',
+    end: '',
+  },
+  vb: {
+    start: "'",
+    end: '',
+  },
+  verilog: {
+    start: '//',
+    end: '',
+  },
+  'vue-html': {
+    start: '\x3c!--',
+    end: '--\x3e',
+  },
+  vue: {
+    start: '//',
+    end: '',
+  },
+  xml: {
+    start: '\x3c!--',
+    end: '--\x3e',
+  },
+  xsl: {
+    start: '\x3c!--',
+    end: '--\x3e',
+  },
+  yaml: {
+    start: '#',
+    end: '',
+  },
+  diff: {
+    start: '#',
+    end: '',
+  },
+};
+
+export const EXCLUDED_NEIGHBORS = ['node_modules', 'dist', 'site-packages'];
+export const MAX_NEIGHBOR_AGGREGATE_LENGTH = 2e5;

package/src/browser/inline-completions/prompt/importedFiles.ts

@@ -0,0 +1,92 @@
+import { Injector } from '@opensumi/di';
+import { URI } from '@opensumi/ide-core-common';
+import { IFileServiceClient } from '@opensumi/ide-file-service';
+import { Path } from '@opensumi/ide-utils/lib/path';
+import { IWorkspaceService } from '@opensumi/ide-workspace';
+
+import { LanguageParserService } from '../../languages/service';
+import { ICompletionContext, ImportedFileOptions, ResourceDocument } from '../types';
+
+import { LANGUAGE_TO_SUFFIX } from './languages';
+
+export const getImportedFilesInterface = async (
+  code: string,
+  resource: ResourceDocument,
+  options: ImportedFileOptions,
+  injector: Injector,
+): Promise<(string | string[])[][]> => {
+  const languageParserService = injector.get(LanguageParserService) as LanguageParserService;
+  let matches: string[] = [];
+  const snippets: (string | string[])[][] = [];
+  const languageParser = languageParserService.createParser(resource.languageId);
+  if (languageParser) {
+    matches = await languageParser.extractImportPaths(code);
+  }
+  const maxPromptTime = options.maxTime ?? 200;
+  const startTime = Date.now();
+  for (const file of matches) {
+    let filePath = '';
+    const basePath = resource.uri.parent.codeUri.fsPath;
+    if (file.startsWith('@/') || file.startsWith('~') || !file.startsWith('.')) {
+      // skip alias
+      continue;
+    }
+    if (file === '.') {
+      filePath = new Path(basePath).join('index.ts').toString();
+    } else {
+      filePath = new Path(basePath).join(`${file}${LANGUAGE_TO_SUFFIX[resource.languageId]}`).toString();
+    }
+    try {
+      const fileService = injector.get(IFileServiceClient) as IFileServiceClient;
+      const workspaceService = injector.get(IWorkspaceService) as IWorkspaceService;
+      const fileUri = URI.file(filePath);
+      const content = await fileService.readFile(fileUri.codeUri.fsPath);
+      const interfaceCode = await languageParser?.extractInterfaceOrTypeCode(content.content.toString());
+      if (interfaceCode && interfaceCode.length > 0) {
+        const relative = await workspaceService.asRelativePath(fileUri);
+        if (relative?.path) {
+          snippets.push([relative.path, interfaceCode]);
+        }
+      }
+    } catch {}
+    if (Date.now() - startTime > maxPromptTime) {
+      break;
+    }
+  }
+  return snippets;
+};
+
+export const extractLocalImportContext = async (
+  resource: ResourceDocument,
+  options: ImportedFileOptions,
+  injector: Injector,
+): Promise<(string | string[])[][]> => {
+  // 从代码中提取出所有的 import 语句，并进一步提取 interface 相关代码声明
+  const { source } = resource;
+
+  if (resource.languageId === 'typescript') {
+    return getImportedFilesInterface(source, resource, options, injector);
+  }
+  return [];
+};
+
+export const getImportedFile = async (
+  context: ICompletionContext,
+  options: ImportedFileOptions,
+  injector: Injector,
+): Promise<(string | string[])[][]> => {
+  if (context.language && !options.language.includes(context.language)) {
+    return [];
+  }
+  const content = context.prefix;
+  return extractLocalImportContext(
+    {
+      source: content,
+      uri: context.uri,
+      languageId: context.language,
+      offset: content.length,
+    },
+    options,
+    injector,
+  );
+};