@zxyycom/vibe-check 0.0.1 → 0.0.2

package/src/package-checks/function-metrics/analyzer/readers/python.ts

@@ -0,0 +1,431 @@
+/**
+ * Derived from terryyin/lizard 1.24.0.
+ * Source: lizard_languages/python.py.
+ * Upstream revision: 308b1c3efd8c1c69bcc3eb82deeaec64fd3662ec.
+ * SPDX-License-Identifier: MIT
+ * Modified: translated to TypeScript for the product-owned analyzer.
+ */
+
+import type { FileInfoBuilder } from "../analysis-context.ts";
+import type { TokenStream } from "../contracts.ts";
+import {
+  CodeReader,
+  CodeStateMachine,
+  isPythonWhitespace,
+  type TokenFactory
+} from "../shared/code-reader.ts";
+import { ScriptLanguageMixIn } from "../shared/script-language.ts";
+
+const SOURCE_SOFT_KEYWORD_VARIABLE_NEXT = new Set([
+  "=",
+  ".",
+  ":",
+  ",",
+  "+=",
+  "-=",
+  "*=",
+  "/=",
+  "%=",
+  "//=",
+  "**=",
+  "&=",
+  "|=",
+  "^=",
+  "<<=",
+  ">>=",
+  ":="
+]);
+const TRIPLE_QUOTE_EXPRESSION_PREDECESSORS = new Set([
+  "=",
+  "+=",
+  "-=",
+  "*=",
+  "/=",
+  "%=",
+  "//=",
+  "**=",
+  "&=",
+  "|=",
+  "^=",
+  "<<=",
+  ">>=",
+  "(",
+  "return",
+  ",",
+  "[",
+  "+",
+  "-",
+  "*",
+  "/",
+  "%"
+]);
+
+function count_spaces(token: string): number {
+  return token.replaceAll("\t", "        ").length;
+}
+
+class PythonIndents {
+  private readonly indents = [0];
+  private readonly context: FileInfoBuilder;
+
+  public constructor(context: FileInfoBuilder) {
+    this.context = context;
+  }
+
+  public set_nesting(spaces: number, token = ""): void {
+    while (this.indents.at(-1)! > spaces && !token.startsWith(")")) {
+      this.indents.pop();
+      this.context.popNesting();
+    }
+    if (this.indents.at(-1)! < spaces) {
+      this.indents.push(spaces);
+      this.context.addBareNesting();
+    }
+  }
+
+  public reset(): void {
+    this.set_nesting(0);
+  }
+}
+
+export class PythonReader extends CodeReader {
+  public static override ext: readonly string[] = ["py"];
+  public static override languageNames: readonly string[] = ["python"];
+  public static override controlFlowKeywords = new Set([
+    "if",
+    "elif",
+    "for",
+    "while",
+    "except",
+    "finally"
+  ]);
+  public static override logicalOperators = new Set(["and", "or"]);
+  public static override caseKeywords = new Set<string>();
+  public static override ternaryOperators = new Set<string>();
+
+  public static readonly _SOFT_KW_VARIABLE_NEXT = SOURCE_SOFT_KEYWORD_VARIABLE_NEXT;
+
+  private _last_meaningful_token: string | undefined;
+  private _keyword_case = false;
+  private _keyword_match = false;
+
+  public constructor(context: FileInfoBuilder, definition: typeof CodeReader = PythonReader) {
+    super(context, definition);
+    this.parallelStates = [new PythonStates(context, this)];
+  }
+
+  /** str/bytes f-string prefixes, excluding bare bytes/raw strings. */
+  public static readonly _FSTRING_PREFIXES = new Set(["f", "rf", "fr", "bf", "fb"]);
+
+  public static override generateTokens(
+    sourceCode: string,
+    addition = "",
+    tokenFactory?: TokenFactory
+  ): Generator<string> {
+    const tokens = CodeReader.generateTokens(
+      sourceCode,
+      String.raw`|#[^\n]*|(?:"""(?:\.|[^"]|"(?!"")|""(?!"))*""")|(?:'''(?:\.|[^']|'(?!'')|''(?!'))*''')` +
+        addition,
+      tokenFactory
+    );
+    return PythonReader._expand_fstring_interpolations(tokens, tokenFactory);
+  }
+
+  /** Re-tokenize f-string expressions so their control flow reaches the reader. */
+  private static *_expand_fstring_interpolations(
+    tokens: TokenStream,
+    tokenFactory?: TokenFactory
+  ): Generator<string> {
+    let prefix: string | undefined;
+    for (const token of tokens) {
+      if (prefix !== undefined) {
+        const held = prefix;
+        prefix = undefined;
+        if (token.startsWith('"') || token.startsWith("'")) {
+          yield held;
+          yield* PythonReader._tokenize_fstring_body(token, tokenFactory);
+          continue;
+        }
+        yield held;
+      }
+      if (PythonReader._FSTRING_PREFIXES.has(token.toLowerCase())) {
+        prefix = token;
+        continue;
+      }
+      yield token;
+    }
+    if (prefix !== undefined) yield prefix;
+  }
+
+  /** Advance beyond one quoted literal while scanning an f-string expression. */
+  private static _skip_quoted_literal(body: string, position: number): number {
+    const length = body.length;
+    let currentPosition = position;
+    const tripleQuote = body.slice(currentPosition, currentPosition + 3);
+    if (currentPosition + 2 < length && (tripleQuote === '"""' || tripleQuote === "'''")) {
+      currentPosition += 3;
+      while (currentPosition < length) {
+        if (body.slice(currentPosition, currentPosition + 3) === tripleQuote) {
+          return currentPosition + 3;
+        }
+        if (body[currentPosition] === "\\") {
+          currentPosition += 2;
+          continue;
+        }
+        currentPosition += 1;
+      }
+      return length;
+    }
+    if (body[currentPosition] === '"' || body[currentPosition] === "'") {
+      const quote = body[currentPosition];
+      currentPosition += 1;
+      while (currentPosition < length) {
+        if (body[currentPosition] === "\\") {
+          currentPosition += 2;
+          continue;
+        }
+        if (body[currentPosition] === quote) return currentPosition + 1;
+        currentPosition += 1;
+      }
+      return length;
+    }
+    return currentPosition;
+  }
+
+  /** Find the matching brace of an interpolation while honoring quoted literals. */
+  private static _find_interpolation_end(body: string, start: number): number {
+    let depth = 1;
+    let position = start + 1;
+    while (position < body.length && depth > 0) {
+      if (body[position] === '"' || body[position] === "'") {
+        position = PythonReader._skip_quoted_literal(body, position);
+        continue;
+      }
+      if (body[position] === "{") depth += 1;
+      else if (body[position] === "}") depth -= 1;
+      position += 1;
+    }
+    return position;
+  }
+
+  /** Preserve literal chunks and recursively tokenize each f-string expression. */
+  private static *_tokenize_fstring_body(
+    token: string,
+    tokenFactory?: TokenFactory
+  ): Generator<string> {
+    const quote = token.startsWith('"""') || token.startsWith("'''") ? token.slice(0, 3) : token[0];
+    const body = token.slice(quote.length, -quote.length);
+    let literal = "";
+    let produced = false;
+    let position = 0;
+    while (position < body.length) {
+      const doubledBrace = body.slice(position, position + 2);
+      if (doubledBrace === "{{" || doubledBrace === "}}") {
+        literal += body[position];
+        position += 2;
+        continue;
+      }
+      if (body[position] === "{") {
+        const end = PythonReader._find_interpolation_end(body, position);
+        if (literal) {
+          yield `${quote}${literal}${quote}`;
+          literal = "";
+        }
+        yield* PythonReader.generateTokens(body.slice(position + 1, end - 1), "", tokenFactory);
+        produced = true;
+        position = end;
+        continue;
+      }
+      literal += body[position];
+      position += 1;
+    }
+    if (!produced) {
+      yield token;
+      return;
+    }
+    if (literal) yield `${quote}${literal}${quote}`;
+  }
+
+  public *preprocess(tokens: TokenStream): Generator<string> {
+    const indents = new PythonIndents(this.context);
+    let currentLeadingSpaces = 0;
+    let readingLeadingSpace = true;
+
+    for (const token of this._soft_keyword_lookahead(tokens)) {
+      if (token !== "\n") {
+        if (readingLeadingSpace) {
+          if (isPythonWhitespace(token)) {
+            currentLeadingSpaces += count_spaces(token);
+          } else {
+            if (!token.startsWith("#")) {
+              const currentFunction = this.context.currentFunction;
+              if (currentFunction.name === "*global*" || currentFunction.longName.endsWith(")")) {
+                indents.set_nesting(currentLeadingSpaces, token);
+              }
+            }
+            readingLeadingSpace = false;
+          }
+        }
+      } else {
+        readingLeadingSpace = true;
+        currentLeadingSpaces = 0;
+      }
+      if (!isPythonWhitespace(token) || token === "\n") yield token;
+    }
+    indents.reset();
+  }
+
+  public override get_comment_from_token(token: string): string | undefined {
+    return ScriptLanguageMixIn.get_comment_from_token(token);
+  }
+
+  public override process_token(token: string): boolean {
+    if (token === "case" && this._keyword_case) this.context.addCondition();
+
+    if (isTripleQuotedString(token) && token.length >= 6) {
+      const state = this.parallelStates[0];
+      if (state instanceof PythonStates && !state.isFirstLine) {
+        if (!TRIPLE_QUOTE_EXPRESSION_PREDECESSORS.has(this._last_meaningful_token ?? "")) {
+          this.context.addNloc(-token.split("\n").length);
+        }
+      }
+    }
+
+    if (token !== "\n" && !isPythonWhitespace(token)) this._last_meaningful_token = token;
+    return false;
+  }
+
+  private *_soft_keyword_lookahead(tokens: TokenStream): Generator<string> {
+    let atLineStart = true;
+    const iterator = tokens[Symbol.iterator]();
+    let next = iterator.next();
+
+    while (!next.done) {
+      const token = next.value;
+      if (token === "\n") {
+        atLineStart = true;
+        yield token;
+        next = iterator.next();
+      } else if (isPythonWhitespace(token)) {
+        yield token;
+        next = iterator.next();
+      } else if (atLineStart && (token === "case" || token === "match")) {
+        const lookahead: string[] = [];
+        let nextReal: string | undefined;
+        next = iterator.next();
+        while (!next.done) {
+          lookahead.push(next.value);
+          if (next.value !== "\n" && !isPythonWhitespace(next.value)) {
+            nextReal = next.value;
+            next = iterator.next();
+            break;
+          }
+          next = iterator.next();
+        }
+
+        let isKeyword: boolean;
+        if (nextReal !== undefined && PythonReader._SOFT_KW_VARIABLE_NEXT.has(nextReal)) {
+          isKeyword = false;
+        } else if (nextReal === "(" || nextReal === "[") {
+          isKeyword = false;
+          let depth = 1;
+          while (!next.done) {
+            const lookaheadToken = next.value;
+            lookahead.push(lookaheadToken);
+            if (["(", "[", "{"].includes(lookaheadToken)) depth += 1;
+            else if ([")", "]", "}"].includes(lookaheadToken)) depth -= 1;
+            else if (lookaheadToken === ":" && depth === 0) {
+              isKeyword = true;
+              next = iterator.next();
+              break;
+            } else if (lookaheadToken === "\n" && depth === 0) {
+              next = iterator.next();
+              break;
+            }
+            next = iterator.next();
+          }
+        } else {
+          isKeyword = true;
+        }
+
+        if (token === "case") this._keyword_case = isKeyword;
+        else this._keyword_match = isKeyword;
+        atLineStart = false;
+        yield token;
+        yield* lookahead;
+      } else {
+        if (token === "case") this._keyword_case = false;
+        else if (token === "match") this._keyword_match = false;
+        atLineStart = false;
+        yield token;
+        next = iterator.next();
+      }
+    }
+  }
+}
+
+export class PythonStates extends CodeStateMachine {
+  private _awaiting_first_line = false;
+
+  public get isFirstLine(): boolean {
+    return this._awaiting_first_line;
+  }
+
+  public readonly reader: PythonReader | undefined;
+
+  public constructor(context: FileInfoBuilder, reader?: PythonReader) {
+    super(context);
+    this.reader = reader;
+    this.next(this._state_global);
+  }
+
+  public override readonly _state_global = (token: string): void => {
+    if (token === "def") this.next(this._function);
+  };
+
+  protected readonly _function = (token: string): void => {
+    if (token !== "(") {
+      this.context.restartNewFunction(token);
+      this.context.addToLongFunctionName("(");
+    } else {
+      this.next(this._dec);
+    }
+  };
+
+  private readonly _dec = (token: string): void => {
+    if (token === ")") {
+      this.next(this._state_colon);
+    } else if (token === "[") {
+      this.next(this._state_parameterized_type_annotation);
+    } else {
+      this.context.parameter(token);
+      return;
+    }
+    this.context.addToLongFunctionName(` ${token}`);
+  };
+
+  private readonly _state_colon = (token: string): void => {
+    if (token === ":") {
+      this._awaiting_first_line = true;
+      this.next(this._state_first_line);
+    } else {
+      this.next(this._state_global);
+    }
+  };
+
+  private readonly _state_first_line = (token: string): void => {
+    this._awaiting_first_line = false;
+    this.next(this._state_global);
+    if (isTripleQuotedString(token)) this.context.addNloc(-token.split("\n").length);
+    this._state_global(token);
+  };
+
+  private readonly _state_parameterized_type_annotation = (token: string): void => {
+    this.context.addToLongFunctionName(` ${token}`);
+    if (token === "]") this.next(this._dec);
+  };
+}
+
+function isTripleQuotedString(token: string): boolean {
+  return token.startsWith('"""') || token.startsWith("'''");
+}

package/src/package-checks/function-metrics/analyzer/readers/r.ts

@@ -0,0 +1,234 @@
+/**
+ * Derived from terryyin/lizard 1.24.0.
+ * Source: lizard_languages/r.py.
+ * Upstream revision: 308b1c3efd8c1c69bcc3eb82deeaec64fd3662ec.
+ * SPDX-License-Identifier: MIT
+ * Modified: translated to TypeScript for the product-owned analyzer.
+ */
+
+import type { FileInfoBuilder } from "../analysis-context.ts";
+import type { TokenStream } from "../contracts.ts";
+import { CodeReader, CodeStateMachine, isPythonWhitespace } from "../shared/code-reader.ts";
+import { ScriptLanguageMixIn } from "../shared/script-language.ts";
+
+export class RReader extends CodeReader {
+  public static override ext: readonly string[] = ["r", "R"];
+  public static override languageNames: readonly string[] = ["r", "R"];
+  public static override controlFlowKeywords = new Set([
+    "if",
+    "else if",
+    "for",
+    "while",
+    "repeat",
+    "switch",
+    "tryCatch",
+    "try",
+    "ifelse"
+  ]);
+  public static override logicalOperators = new Set(["&&", "||", "&", "|"]);
+  public static override caseKeywords = new Set<string>();
+  public static override ternaryOperators = new Set<string>();
+
+  public constructor(context: FileInfoBuilder) {
+    super(context, RReader);
+    this.parallelStates = [new RStates(context)];
+  }
+
+  public *preprocess(tokens: TokenStream): Generator<string> {
+    yield* tokens;
+  }
+
+  public static override generateTokens(sourceCode: string): Generator<string> {
+    return ScriptLanguageMixIn.generate_common_tokens(
+      sourceCode,
+      String.raw`|<-|->|%[a-zA-Z_*/>]+%|\.\.\.|:::|::`
+    );
+  }
+
+  public override get_comment_from_token(token: string): string | undefined {
+    return ScriptLanguageMixIn.get_comment_from_token(token);
+  }
+}
+
+class RStates extends CodeStateMachine {
+  private recent_tokens: string[] = [];
+  private brace_count = 0;
+  private in_braced_function = false;
+  private additional_function_names: string[] = [];
+
+  public constructor(context: FileInfoBuilder) {
+    super(context);
+  }
+
+  public override _state_global(token: string): void {
+    if (!isPythonWhitespace(token) && token !== "\n") {
+      this.recent_tokens.push(token);
+      if (this.recent_tokens.length > 10) this.recent_tokens.shift();
+    }
+
+    if (token !== "function") return;
+    if (this.recent_tokens.length >= 2) {
+      const assignmentOperator = this.recent_tokens.at(-2);
+      if (assignmentOperator === "<-" || assignmentOperator === "=") {
+        const functionNames = this._extract_function_names();
+        this._start_function(functionNames[0] ?? "(anonymous)");
+        this.next(this._function_params);
+        this.additional_function_names = functionNames.length > 1 ? functionNames.slice(1) : [];
+        return;
+      }
+    }
+
+    this._start_function("(anonymous)");
+    this.next(this._function_params);
+  }
+
+  public override statemachine_before_return(): void {
+    if (
+      this.state === this._function_body ||
+      this.state === this._check_right_assignment ||
+      this.state === this._read_right_assignment_name
+    ) {
+      this._finalize_function_with_multiple_assignments();
+    }
+  }
+
+  private readonly _function_params = (token: string): void => {
+    if (token === "(") {
+      this.context.addToLongFunctionName("(");
+      this.next(this._read_params);
+    } else {
+      this.next(this._function_body, token);
+    }
+  };
+
+  private readonly _read_params = (token: string): void => {
+    if (token === ")") {
+      this.context.addToLongFunctionName(")");
+      this.next(this._function_body);
+    } else if (token !== "\n" && !isPythonWhitespace(token)) {
+      this.context.parameter(token);
+      if (token !== "(") this.context.addToLongFunctionName(` ${token}`);
+    }
+  };
+
+  private readonly _function_body = (token: string): void => {
+    if (!isPythonWhitespace(token) && token !== "\n") {
+      this.recent_tokens.push(token);
+      if (this.recent_tokens.length > 10) this.recent_tokens.shift();
+    }
+
+    if (token === "{") {
+      if (this.brace_count === 0) this.in_braced_function = true;
+      this.brace_count += 1;
+    } else if (token === "}") {
+      this.brace_count -= 1;
+      if (this.brace_count === 0 && this.in_braced_function) {
+        this._end_current_function();
+        return;
+      }
+    }
+
+    if (token === "function" && this.recent_tokens.length >= 2) {
+      const assignmentOperator = this.recent_tokens.at(-2);
+      if (assignmentOperator === "<-" || assignmentOperator === "=") {
+        this.context.endOfFunction();
+        const functionNames = this._extract_function_names();
+        this._start_function(functionNames[0] ?? "(anonymous)");
+        this.next(this._function_params);
+        this.brace_count = 0;
+        this.in_braced_function = false;
+        this.additional_function_names = functionNames.length > 1 ? functionNames.slice(1) : [];
+      }
+    } else if (token === "\n" && !this.in_braced_function) {
+      this._end_current_function();
+    }
+  };
+
+  private readonly _check_right_assignment = (token: string): void => {
+    if (isPythonWhitespace(token) || token === "\n" || token.startsWith("#")) return;
+    if (token === "->") {
+      this.next(this._read_right_assignment_name);
+      return;
+    }
+    this._finalize_function_with_multiple_assignments();
+    this.next(this.globalState, token);
+  };
+
+  private readonly _read_right_assignment_name = (token: string): void => {
+    if (isPythonWhitespace(token) || token === "\n") return;
+    if (isRNameFragment(token)) {
+      this.context.currentFunction.name = token;
+      this._finalize_function_with_multiple_assignments();
+      this.next(this.globalState);
+      return;
+    }
+    this._finalize_function_with_multiple_assignments();
+    this.next(this.globalState, token);
+  };
+
+  private _extract_function_names(): string[] {
+    if (this.recent_tokens.length < 3) return ["(anonymous)"];
+
+    const functionNames: string[] = [];
+    let index = this.recent_tokens.length - 3;
+    let currentNameTokens: string[] = [];
+    while (index >= 0) {
+      const token = this.recent_tokens[index];
+      if (token === "<-" || token === "=") {
+        if (currentNameTokens.length > 0) {
+          functionNames.push(currentNameTokens.reverse().join(""));
+          currentNameTokens = [];
+        }
+        index -= 1;
+        continue;
+      }
+      if (["function", "(", ")", "{", "}", "\n"].includes(token)) break;
+      if (isRNameFragment(token)) {
+        currentNameTokens.push(token);
+        index -= 1;
+      } else {
+        break;
+      }
+    }
+    if (currentNameTokens.length > 0) functionNames.push(currentNameTokens.reverse().join(""));
+    const ordered = functionNames.reverse();
+    return ordered.length > 0 ? ordered : ["(anonymous)"];
+  }
+
+  private _extract_function_name(): string {
+    return this._extract_function_names()[0] ?? "(anonymous)";
+  }
+
+  private _start_function(name: string): void {
+    this.context.restartNewFunction(name);
+  }
+
+  private _end_current_function(): void {
+    this.next(this._check_right_assignment);
+    this.brace_count = 0;
+    this.in_braced_function = false;
+  }
+
+  private _finalize_function_with_multiple_assignments(): void {
+    const currentFunction = this.context.currentFunction;
+    this.context.endOfFunction();
+
+    for (const functionName of this.additional_function_names) {
+      this.context.restartNewFunction(functionName);
+      this.context.currentFunction.cyclomaticComplexity = currentFunction.cyclomaticComplexity;
+      this.context.currentFunction.startLine = currentFunction.startLine;
+      this.context.currentFunction.endLine = currentFunction.endLine;
+      this.context.endOfFunction();
+    }
+    this.additional_function_names = [];
+  }
+}
+
+function isRNameFragment(token: string): boolean {
+  return (
+    token
+      .replaceAll("_", "a")
+      .replaceAll(".", "a")
+      .match(/^[\p{L}\p{N}]+$/u) !== null || token === "."
+  );
+}