npm - @opensumi/ide-ai-native - Versions diffs - 3.0.3 → 3.0.4-next-1716367246.0 - Mend

@opensumi/ide-ai-native 3.0.3 → 3.0.4-next-1716367246.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (77) hide show

package/src/browser/inline-completions/prompt/prompt.ts ADDED Viewed

@@ -0,0 +1,297 @@
+// @ts-ignore
+import { Tiktoken } from 'js-tiktoken';
+import { Injector } from '@opensumi/di';
+import { LanguageParser } from '../../languages/parser';
+import { LanguageParserService } from '../../languages/service';
+import { StrategyType, WishListAttributeName } from '../types';
+import { LANGUAGE_COMMENT_MARKERS } from './const';
+import { getTokenizer } from './tokenizer';
+import type { ICompletionContext, ICompletionModel, MarkerItem } from '../types';
+import type * as monaco from '@opensumi/ide-monaco';
+export const addComment = (commentText?: string) => {
+  if (commentText) {
+    return `${commentText}\n`;
+  }
+  return '';
+};
+export const getMarkerByLanguage = (text: string, language: string) => {
+  const marker = LANGUAGE_COMMENT_MARKERS[language];
+  if (marker) {
+    const end = marker.end === '' ? '' : ` ${marker.end}`;
+    return `${marker.start} ${text}${end}`;
+  }
+  return '';
+};
+export const getLanguageMarker = (language: string) => {
+  if (!language) {
+    return '';
+  }
+  const supportLanguage: {
+    [key: string]: string;
+  } = {
+    html: '<!DOCTYPE html>',
+    python: '#!/usr/bin/env python3',
+    ruby: '#!/usr/bin/env ruby',
+    shellscript: '#!/bin/sh',
+    yaml: '# YAML data',
+  };
+  let marker = '';
+  if (supportLanguage[language]) {
+    return supportLanguage[language];
+  }
+  marker = `Language: ${language}`;
+  return getMarkerByLanguage(marker, language);
+};
+// 根据语言类型获取不同的注释内容
+export const getPathMarker = (path: string, language: string) => path && getMarkerByLanguage(`Path: ${path}`, language);
+export const getMarkerForSnippets = (text: string, language: string) => {
+  const lines = text.split('\n');
+  return lines.map((line) => getMarkerByLanguage(line, language)).join('\n');
+};
+export const getCroppedTextByLine = (text: string, maxTokenSize: number, textTokens: number[][], reverse = false) => {
+  const currentTokenSize = textTokens.reduce((prev, cur) => prev + cur.length, 0);
+  if (currentTokenSize < maxTokenSize) {
+    return text;
+  }
+  const lines = text.split('\n');
+  const endLine = lines.length;
+  let currenSize = 0;
+  if (reverse) {
+    let index = 0;
+    for (; index < endLine; index++) {
+      const tokens = textTokens[index];
+      currenSize += tokens.length;
+      if (currenSize < maxTokenSize) {
+        continue;
+      } else {
+        break;
+      }
+    }
+    if (index === endLine) {
+      return text;
+    }
+    return lines.slice(0, index).join('\n');
+  }
+  let index = endLine - 1;
+  for (; index >= 0; index--) {
+    const tokens = textTokens[index];
+    currenSize += tokens.length;
+    if (currenSize < maxTokenSize) {
+      continue;
+    } else {
+      break;
+    }
+  }
+  if (index === -1) {
+    return text;
+  }
+  return lines.slice(index).join('\n');
+};
+/**
+ * 裁剪字符函数
+ * @param text 文本内容
+ * @param maxTokenSize 最大 token 数量
+ * @param strategy 分割策略
+ * @param language 语言类型，按函数分割仅在 typescript/javascript/typescriptreact 中支持
+ * @param reverse 是否反向裁剪，即从后往前裁剪
+ */
+export const getCroppedText = async (
+  text: string,
+  maxTokenSize: number,
+  textTokens: number[][],
+  strategy = StrategyType.InterceptBasedOnLine,
+  tokenizer: Tiktoken,
+  parser?: LanguageParser,
+  minBlockSize = 20,
+  reverse = false,
+  token?: monaco.CancellationToken,
+): Promise<string> => {
+  let tokens: number[];
+  if (strategy === StrategyType.InterceptBasedOnLine) {
+    // 按行进行裁剪
+    text = getCroppedTextByLine(text, maxTokenSize, textTokens, reverse);
+  } else if (strategy === StrategyType.InterceptBasedOnFunction && !reverse) {
+    tokens = tokenizer.encode(text);
+    // 按函数进行裁剪
+    while (tokens.length > maxTokenSize) {
+      if (reverse) {
+        try {
+          text = text.slice(0, Math.ceil((maxTokenSize / tokens.length) * tokens.length));
+          text = (await parser?.trimSuffixSyntaxErrors(text, minBlockSize)) || text;
+        } catch {
+          text = getCroppedTextByLine(text, maxTokenSize, textTokens, reverse);
+        }
+      } else {
+        try {
+          text = text.slice(-Math.ceil((maxTokenSize / tokens.length) * tokens.length));
+          text = (await parser?.trimPrefixSyntaxErrors(text, minBlockSize)) || text;
+          if (token?.isCancellationRequested) {
+            return '';
+          }
+        } catch {
+          text = getCroppedTextByLine(text, maxTokenSize, textTokens, reverse);
+        }
+      }
+      tokens = tokenizer.encode(text);
+    }
+  } else {
+    tokens = tokenizer.encode(text);
+    // 按字符进行裁剪
+    while (tokens.length > maxTokenSize) {
+      const splitWords = text.split('');
+      if (reverse) {
+        text = splitWords.slice(0, -(tokens.length - maxTokenSize)).join('');
+      } else {
+        text = splitWords.slice(tokens.length - maxTokenSize).join('');
+      }
+      tokens = tokenizer.encode(text);
+    }
+  }
+  return text;
+};
+export const getBeforePrompt = async (
+  promptPriority: MarkerItem[],
+  context: ICompletionContext,
+  promptConfig: ICompletionModel,
+  leftTokenSize: number,
+  injector: Injector,
+  token: monaco.CancellationToken,
+) => {
+  const { tokenizerName, wishList } = promptConfig;
+  const tokenizer = getTokenizer(tokenizerName);
+  // 根据优先级降序排序
+  const sortedPromptPriority = promptPriority.sort((a, b) => b.priority - a.priority);
+  const promptList: {
+    [key in WishListAttributeName]?: string;
+  } = {};
+  for (const current of sortedPromptPriority) {
+    if (!leftTokenSize) {
+      return;
+    }
+    if (current.content) {
+      const tokens = current.content.split('\n').map((line) => tokenizer.encode(line));
+      const currentTokenSize = tokens.reduce((prev, cur) => prev + cur.length, 0);
+      const maxTokenSize = Math.ceil(current.maxPercent * leftTokenSize);
+      if (currentTokenSize > leftTokenSize) {
+        const languageParserService = injector.get(LanguageParserService) as LanguageParserService;
+        const languageParser = languageParserService.createParser(context.language);
+        promptList[current.attributeName] = await getCroppedText(
+          current.content,
+          maxTokenSize,
+          tokens,
+          promptConfig.wishList.beforeCursor.strategy,
+          tokenizer,
+          languageParser,
+          wishList.beforeCursor.extOption.minBlockSize ?? 75,
+          false,
+          token,
+        );
+        if (token.isCancellationRequested) {
+          return;
+        }
+        leftTokenSize -= maxTokenSize;
+      } else {
+        leftTokenSize -= currentTokenSize;
+        promptList[current.attributeName] = `${current.content}`;
+      }
+    } else if (current.importedFiles && current.importedFiles.length > 0) {
+      let currentImportedFile = '';
+      const maxImportedFileTokenSize = Math.ceil(current.maxPercent * leftTokenSize);
+      let currentImportedFileTokenSize = 0;
+      for (const [filename, importedFile] of current.importedFiles) {
+        const importedFileMarker = getMarkerByLanguage(
+          `${promptConfig.wishList.importedFile.extOption.patternPrefix}${filename}:${
+            promptConfig.wishList.importedFile.extOption.patternSuffix || '\n'
+          }${getMarkerForSnippets((importedFile as string[]).join('\n'), context.language)}\n`,
+          context.language,
+        );
+        const tokens = tokenizer.encode(importedFileMarker);
+        if (tokens.length > leftTokenSize || currentImportedFileTokenSize + tokens.length > maxImportedFileTokenSize) {
+          break;
+        }
+        leftTokenSize -= tokens.length;
+        currentImportedFileTokenSize += tokens.length;
+        currentImportedFile += importedFileMarker;
+      }
+      if (currentImportedFile) {
+        // 移除末尾多余的 \n
+        currentImportedFile = currentImportedFile.slice(0, -'\n'.length);
+        promptList[current.attributeName] = `${currentImportedFile}`;
+      }
+    } else if (current.similarSnippets && current.similarSnippets.length > 0) {
+      let currentSnippet = '';
+      const maxSnippetSize = Math.ceil(current.maxPercent * leftTokenSize);
+      let currentSnippetSize = 0;
+      for (const [filename, snippets] of current.similarSnippets) {
+        const similarSnippetMarker = snippets
+          .map((snippet) =>
+            getMarkerByLanguage(
+              `${promptConfig.wishList.similarFile.extOption.patternPrefix}${filename}:${
+                promptConfig.wishList.similarFile.extOption.patternSuffix || '\n'
+              }${getMarkerForSnippets(snippet.snippet, context.language)}`,
+              context.language,
+            ),
+          )
+          .join('\n');
+        const tokens = tokenizer.encode(similarSnippetMarker);
+        if (tokens.length > leftTokenSize || currentSnippetSize + tokens.length > maxSnippetSize) {
+          break;
+        }
+        leftTokenSize -= tokens.length;
+        currentSnippetSize += tokens.length;
+        currentSnippet += similarSnippetMarker;
+      }
+      if (currentSnippet) {
+        promptList[current.attributeName] = `${currentSnippet}`;
+      }
+    }
+  }
+  return `${addComment(promptList.languageMarker)}${addComment(promptList.pathMarker)}${addComment(
+    promptList.importedFile,
+  )}${addComment(promptList.similarFile)}${addComment(promptList.beforeCursor)}`.slice(0, -'\n'.length);
+};
+export const getAfterPrompt = async (
+  context: ICompletionContext,
+  promptConfig: ICompletionModel,
+  injector: Injector,
+  token: monaco.CancellationToken,
+) => {
+  const { maxPromptTokenSize, tokenizerName, wishList } = promptConfig;
+  const tokenizer = getTokenizer(tokenizerName);
+  let afterCursor = context.suffix;
+  const afterCursorOptions = promptConfig.wishList.afterCursor.extOption;
+  const afterCursorTokens = afterCursor.split('\n').map((line) => tokenizer.encode(line));
+  const currentAfterCursorTokenSize = afterCursorTokens.reduce((prev, cur) => prev + cur.length, 0);
+  const afterCursorMaxTokenSize = Math.ceil(afterCursorOptions.suffixPercent * maxPromptTokenSize);
+  if (currentAfterCursorTokenSize > afterCursorMaxTokenSize) {
+    const languageParserService = injector.get(LanguageParserService) as LanguageParserService;
+    const languageParser = languageParserService.createParser(context.language);
+    afterCursor = await getCroppedText(
+      afterCursor,
+      afterCursorMaxTokenSize,
+      afterCursorTokens,
+      promptConfig.wishList.afterCursor.strategy,
+      tokenizer,
+      languageParser,
+      wishList.afterCursor.extOption.minBlockSize ?? 25,
+      true,
+      token,
+    );
+  }
+  return afterCursor;
+};

package/src/browser/inline-completions/prompt/similarSnippets.ts ADDED Viewed

@@ -0,0 +1,130 @@
+/* eslint-disable no-case-declarations */
+import { Injector } from '@opensumi/di';
+import { IEditorDocumentModel, WorkbenchEditorService } from '@opensumi/ide-editor';
+import { IWorkspaceService } from '@opensumi/ide-workspace';
+import { isDocumentValid } from '../../../common/utils';
+import {
+  ICompletionContext,
+  MatchSimilarSnippet,
+  NeighboringTabsOption,
+  ResourceDocument,
+  SimilarFileOptions,
+} from '../types';
+import { MAX_NEIGHBOR_AGGREGATE_LENGTH } from './const';
+import { FixedWindowSizeJaccardMatcher } from './jaccardMatcher';
+export const getOpenedTabFileList = (docuemnts: IEditorDocumentModel[]) => {
+  // 过滤超大文档
+  const recentFiles = docuemnts.filter((document) => isDocumentValid(document));
+  return recentFiles;
+};
+export const getRecentEditFileList = () => [];
+export const getNeighboringDocument = async (type: NeighboringTabsOption, injector: Injector) => {
+  switch (type) {
+    case NeighboringTabsOption.openFileHistory:
+      const editorService = injector.get(WorkbenchEditorService) as WorkbenchEditorService;
+      const documents = await editorService.getAllOpenedDocuments();
+      return getOpenedTabFileList(documents);
+    case NeighboringTabsOption.editFileHistory:
+      // 暂不支持
+      return [];
+    case NeighboringTabsOption.editFileRecent:
+      return [];
+    default:
+      return [];
+  }
+};
+export const getNeighboringResource = async (
+  targetFileName: string,
+  languageId: string,
+  option: SimilarFileOptions,
+  injector: Injector,
+) => {
+  let documentList: IEditorDocumentModel[] = [];
+  for (const type of option.neighboringTabsOption) {
+    documentList = documentList.concat(await getNeighboringDocument(type, injector));
+  }
+  documentList = documentList.filter(
+    (document) => document.languageId === languageId && !document.uri.displayName.endsWith(targetFileName),
+  );
+  // 移除重复项
+  documentList = documentList.filter(
+    (document, index) => documentList.findIndex((item) => item.uri === document.uri) === index,
+  );
+  let neighboringTabsMaxNum = 20;
+  if (typeof option.neighboringTabsMaxNum === 'number') {
+    neighboringTabsMaxNum = option.neighboringTabsMaxNum;
+  }
+  documentList = documentList.slice(0, neighboringTabsMaxNum);
+  const resources: ResourceDocument[] = documentList.map((document) => {
+    const content = document.getText();
+    return {
+      source: content,
+      uri: document.uri,
+      languageId: document.languageId,
+      offset: content.length,
+    };
+  });
+  return resources;
+};
+export const getSimilarSnippets = async (
+  context: ICompletionContext,
+  options: SimilarFileOptions,
+  injector: Injector,
+) => {
+  const editorService = injector.get(WorkbenchEditorService) as WorkbenchEditorService;
+  const workspaceService = injector.get(IWorkspaceService) as IWorkspaceService;
+  // 过滤文件，仅保留相同后缀文件
+  // 仅保留 20 项，顺序取决于配置顺序
+  const neighboringFiles = await getNeighboringResource(context.filename, context.language, options, injector);
+  const snippetLength = (context.prefix + context.suffix).split('\n').length;
+  // 窗口大小最大不大于服务端配置的 windowSize 大小
+  const windowSize = snippetLength > options.windowSize ? options.windowSize : snippetLength;
+  const doMatcher = FixedWindowSizeJaccardMatcher.factory(windowSize);
+  const bestMatchSnippets: [string, MatchSimilarSnippet[]][] = [];
+  const currentDocument = editorService.currentEditor?.currentDocumentModel;
+  if (!currentDocument) {
+    return [];
+  }
+  const content = currentDocument.getText();
+  const matcher = doMatcher.to({
+    source: content,
+    uri: currentDocument.uri,
+    languageId: currentDocument.languageId,
+    offset: content.length,
+  });
+  let currentLength = 0;
+  const maxPromptTime = options.maxTime ?? 1000;
+  const startTime = Date.now();
+  for (const file of neighboringFiles) {
+    currentLength += file.source.length;
+    const snippet = matcher.findMatches(file, options.snippetSelectionMode);
+    if (snippet && snippet.length) {
+      const relative = await workspaceService.asRelativePath(file.uri);
+      if (relative?.path) {
+        bestMatchSnippets.push([relative.path, snippet]);
+      }
+    }
+    if (Date.now() - startTime > maxPromptTime) {
+      break;
+    }
+    // 计算相似性片段总长度不能超过 MAX_NEIGHBOR_AGGREGATE_LENGTH
+    if (currentLength > MAX_NEIGHBOR_AGGREGATE_LENGTH) {
+      break;
+    }
+  }
+  // 对相似片段以 score 降序排序，仅取前 snippetMaxNum 项
+  const sortedSnippets = bestMatchSnippets
+    .filter((item) => item[1][0] && item[1][0].score > (options.similarityThreshold ?? 0.6))
+    .sort((a, b) => b[1][0].score - a[1][0].score)
+    .slice(0, options.snippetMaxNum || 4);
+  return sortedSnippets;
+};

package/src/browser/inline-completions/prompt/tokenizer.ts ADDED Viewed

@@ -0,0 +1,16 @@
+// @ts-ignore
+import { Tiktoken, getEncoding } from 'js-tiktoken';
+import { TokenizerName } from '../types';
+const TOKENIZER_CACHE = new Map<TokenizerName, Tiktoken>();
+export const getTokenizer = (tokenizerName = TokenizerName.cl100k_base) => {
+  let tokenizer = TOKENIZER_CACHE.get(tokenizerName);
+  if (tokenizer) {
+    return tokenizer;
+  }
+  tokenizer = getEncoding('cl100k_base');
+  TOKENIZER_CACHE.set(tokenizerName, tokenizer);
+  return tokenizer;
+};

package/src/browser/inline-completions/provider.ts CHANGED Viewed

@@ -1,47 +1,112 @@
-import * as constants from './constants';
-export function prePromptHandler(prompt: string): string {
-  // remove all empty lines
-  prompt = prompt.replace(/^s*[\n]/gm, '');
-  const arr = prompt.split('\n');
-  // if the number of lines is greater than n, take the last n lines
-  if (arr.length > constants.completionModel.completionPromptMaxLineSize) {
-    prompt = arr.slice(-constants.completionModel.completionPromptMaxLineSize).join('\n');
-  }
-  return prompt;
-}
+import { Injector } from '@opensumi/di';
-export function preSuffixHandler(suffix: string): string {
-  suffix = suffix.replace(/^s*[\n]/gm, '');
-  const arr = suffix.split('\n');
-  if (arr.length > constants.completionModel.completionSuffixMaxLineSize) {
-    suffix = arr.slice(-constants.completionModel.completionSuffixMaxLineSize).join('\n');
-  }
-  return suffix;
-}
+import { getImportedFile } from './prompt/importedFiles';
+import { getAfterPrompt, getBeforePrompt, getLanguageMarker, getPathMarker } from './prompt/prompt';
+import { getSimilarSnippets } from './prompt/similarSnippets';
+import { ICompletionContext, ICompletionModel, MarkerItem, MatchSimilarSnippet } from './types';
-export class ReqStack {
-  queue: any[];
-  constructor() {
-    this.queue = [];
-  }
-  addReq(reqRequest: { sendRequest: any; cancelRequest: any }) {
-    this.queue.push(reqRequest);
-  }
-  runReq() {
-    if (this.queue.length === 0) {
-      return;
-    }
-    const fn = this.queue.pop();
-    return fn.sendRequest();
+import type * as monaco from '@opensumi/ide-monaco';
+export async function getPrefixPrompt(
+  context: ICompletionContext,
+  promptConfig: ICompletionModel,
+  injector: Injector,
+  token: monaco.CancellationToken,
+): Promise<string> {
+  const beforeCursor = context.prefix;
+  const smiilarSnippetPriority = promptConfig.wishList.similarFile.priority;
+  const beforeCursorPriority = promptConfig.wishList.beforeCursor.priority;
+  const importedFilePriority = promptConfig.wishList.importedFile.priority;
+  const pathMarkderPriority = promptConfig.wishList.pathMarker.priority;
+  const languageMarkerPriority = promptConfig.wishList.languageMarker.priority;
+  const { maxPromptTokenSize } = promptConfig;
+  const prefixPercent = 1 - promptConfig.wishList.afterCursor.extOption.suffixPercent ?? 0.25;
+  const beforeCursorMaxTokenSize = Math.ceil(prefixPercent * maxPromptTokenSize);
+  const leftTokenSize = beforeCursorMaxTokenSize;
+  // Language Marker
+  const languageMarker = promptConfig.wishList.languageMarker.enable ? getLanguageMarker(context.language) : '';
+  // Path Marker
+  const pathMarker = promptConfig.wishList.pathMarker.enable ? getPathMarker(context.filename, context.language) : '';
+  // Similar Snippet
+  const similarFileOptions = {
+    ...promptConfig.wishList.similarFile.extOption,
+    maxTime: Math.min(
+      promptConfig.wishList.similarFile.extOption.maxTime || 200,
+      promptConfig.maxExecuteTimeMillSecond,
+    ),
+  };
+  const now = Date.now();
+  const similarFileSnippets = promptConfig.wishList.similarFile.enable
+    ? await getSimilarSnippets(context, similarFileOptions, injector)
+    : [];
+  if (token?.isCancellationRequested) {
+    return beforeCursor;
   }
-  cancleRqe() {
-    if (this.queue.length === 0) {
-      return;
-    }
-    this.queue.forEach((item) => {
-      item.cancelRequest();
-    });
-    this.queue = [];
+  const costTime = Date.now() - now;
+  // Imported File
+  const importedFileOptions = {
+    ...promptConfig.wishList.importedFile.extOption,
+    maxTime: Math.min(
+      promptConfig.wishList.importedFile.extOption.maxTime,
+      promptConfig.maxExecuteTimeMillSecond - costTime,
+    ),
+  };
+  const importedFiles: (string | string[])[][] = promptConfig.wishList.importedFile.enable
+    ? await getImportedFile(context, importedFileOptions, injector)
+    : [];
+  if (token?.isCancellationRequested) {
+    return beforeCursor;
   }
+  // 根据内容优先级拼接 prompt
+  const promptPriority: MarkerItem[] = [
+    {
+      attributeName: promptConfig.wishList.languageMarker.attributeName,
+      priority: languageMarkerPriority,
+      maxPercent: promptConfig.wishList.languageMarker.extOption.maxPercent ?? 1,
+      enable: promptConfig.wishList.languageMarker.enable ?? false,
+      content: languageMarker,
+    },
+    {
+      attributeName: promptConfig.wishList.pathMarker.attributeName,
+      priority: pathMarkderPriority,
+      maxPercent: promptConfig.wishList.pathMarker.extOption?.maxPercent ?? 1,
+      enable: promptConfig.wishList.pathMarker.enable ?? false,
+      content: pathMarker,
+    },
+    {
+      attributeName: promptConfig.wishList.importedFile.attributeName,
+      priority: importedFilePriority,
+      enable: promptConfig.wishList.importedFile.enable ?? false,
+      maxPercent: promptConfig.wishList.importedFile.extOption.maxPercent ?? 1,
+      importedFiles,
+    },
+    {
+      attributeName: promptConfig.wishList.similarFile.attributeName,
+      priority: smiilarSnippetPriority,
+      enable: promptConfig.wishList.similarFile.enable ?? false,
+      similarSnippets: similarFileSnippets as [string, MatchSimilarSnippet[]][],
+      maxPercent: promptConfig.wishList.similarFile.extOption.maxPercent ?? 1,
+    },
+    {
+      attributeName: promptConfig.wishList.beforeCursor.attributeName,
+      priority: beforeCursorPriority,
+      maxPercent: promptConfig.wishList.beforeCursor.extOption.maxPercent ?? 1,
+      enable: promptConfig.wishList.beforeCursor.enable ?? true,
+      content: beforeCursor,
+    },
+  ];
+  return (await getBeforePrompt(promptPriority, context, promptConfig, leftTokenSize, injector, token)) || beforeCursor;
+}
+export async function getSuffixPrompt(
+  context: ICompletionContext,
+  promptConfig: ICompletionModel,
+  injector: Injector,
+  token: monaco.CancellationToken,
+): Promise<string> {
+  return (await getAfterPrompt(context, promptConfig, injector, token)) || '';
 }