@marimo-team/frontend 0.14.18-dev9 → 0.15.0

package/src/core/codemirror/language/languages/markdown.ts

@@ -0,0 +1,261 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { autocompletion } from "@codemirror/autocomplete";
+import { markdown, markdownLanguage } from "@codemirror/lang-markdown";
+import { python, pythonLanguage } from "@codemirror/lang-python";
+import { StreamLanguage } from "@codemirror/language";
+import { languages } from "@codemirror/language-data";
+import { stexMath } from "@codemirror/legacy-modes/mode/stex";
+import type { Extension } from "@codemirror/state";
+import { type EditorView, ViewPlugin } from "@codemirror/view";
+import dedent from "string-dedent";
+import type { CellId } from "@/core/cells/ids";
+import type { CompletionConfig } from "@/core/config/config-schema";
+import type { HotkeyProvider } from "@/core/hotkeys/hotkeys";
+import { markdownAutoRunExtension } from "../../cells/extensions";
+import { pythonCompletionSource } from "../../completion/completer";
+import { conditionalCompletion } from "../../completion/utils";
+import type { PlaceholderType } from "../../config/types";
+import { markdownCompletionSources } from "../../markdown/completions";
+import { enhancedMarkdownExtension } from "../../markdown/extension";
+import { parsePython } from "../embedded/embedded-python";
+import { parseLatex } from "../embedded/latex";
+import { languageMetadataField } from "../metadata";
+import type { LanguageAdapter } from "../types";
+import {
+  QUOTE_PREFIX_KINDS,
+  type QuotePrefixKind,
+  splitQuotePrefix,
+} from "../utils/quotes";
+
+export interface MarkdownLanguageAdapterMetadata {
+  quotePrefix: QuotePrefixKind;
+}
+
+const quoteKinds = [
+  ['"""', '"""'],
+  ["'''", "'''"],
+  ['"', '"'],
+  ["'", "'"],
+];
+
+// explode into all combinations
+const pairs = QUOTE_PREFIX_KINDS.flatMap((prefix) =>
+  quoteKinds.map(([start, end]) => [prefix + start, end]),
+);
+
+const regexes = pairs.map(
+  ([start, end]) =>
+    // mo.md( + any number of spaces + start + capture + any number of spaces + end)
+    [
+      start,
+      new RegExp(`^mo\\.md\\(\\s*${start}(.*)${end}\\s*\\)$`, "s"),
+    ] as const,
+);
+
+/**
+ * Language adapter for Markdown.
+ */
+export class MarkdownLanguageAdapter
+  implements LanguageAdapter<MarkdownLanguageAdapterMetadata>
+{
+  readonly type = "markdown";
+  readonly defaultCode = 'mo.md(r"""\n""")';
+  readonly defaultMetadata: MarkdownLanguageAdapterMetadata = {
+    quotePrefix: "r",
+  };
+
+  static fromMarkdown(markdown: string) {
+    return `mo.md(r"""\n${markdown}\n""")`;
+  }
+
+  transformIn(
+    pythonCode: string,
+  ): [string, number, MarkdownLanguageAdapterMetadata] {
+    pythonCode = pythonCode.trim();
+
+    const metadata = { ...this.defaultMetadata };
+
+    // empty string
+    if (pythonCode === "") {
+      return ["", 0, metadata];
+    }
+
+    for (const [start, regex] of regexes) {
+      const match = pythonCode.match(regex);
+      if (match) {
+        const innerCode = match[1];
+        const [quotePrefix, quoteType] = splitQuotePrefix(start);
+        metadata.quotePrefix = quotePrefix;
+        const unescapedCode = innerCode.replaceAll(`\\${quoteType}`, quoteType);
+
+        const offset = pythonCode.indexOf(innerCode);
+        // string-dedent expects the first and last line to be empty / contain only whitespace, so we pad with \n
+        return [dedent(`\n${unescapedCode}\n`).trim(), offset, metadata];
+      }
+    }
+
+    // no match
+    return [pythonCode, 0, metadata];
+  }
+
+  transformOut(
+    code: string,
+    metadata: MarkdownLanguageAdapterMetadata,
+  ): [string, number] {
+    // Empty string
+    if (code === "") {
+      // Need at least a space, otherwise the output will be 6 quotes
+      code = " ";
+    }
+
+    const { quotePrefix } = metadata;
+
+    // We always transform back with triple quotes, as to avoid needing to
+    // escape single quotes.
+    const escapedCode = code.replaceAll('"""', String.raw`\"""`);
+
+    // If its one line and not bounded by quotes, write it as single line
+    const isOneLine = !code.includes("\n");
+    const boundedByQuote = code.startsWith('"') || code.endsWith('"');
+    if (isOneLine && !boundedByQuote) {
+      const start = `mo.md(${quotePrefix}"""`;
+      const end = `""")`;
+      return [start + escapedCode + end, start.length];
+    }
+
+    // Multiline code
+    const start = `mo.md(\n    ${quotePrefix}"""\n`;
+    const end = `\n"""\n)`;
+    return [start + escapedCode + end, start.length + 1];
+  }
+
+  isSupported(pythonCode: string): boolean {
+    pythonCode = pythonCode.trim();
+
+    // Empty strings are supported
+    if (pythonCode === "") {
+      return true;
+    }
+
+    // Must start with mo.md(
+    if (!pythonCode.startsWith("mo.md(")) {
+      return false;
+    }
+
+    // Empty function calls are supported
+    if (pythonCode === "mo.md()") {
+      return true;
+    }
+
+    // Parse the code using Lezer and check for the exact match of mo.md() signature
+    const tree = pythonLanguage.parser.parse(pythonCode);
+
+    // This is the exact match of mo.md() signature
+    const enterOrder: Array<{ match: string | RegExp; stop?: boolean }> = [
+      { match: "Script" },
+      { match: "ExpressionStatement" },
+      { match: "CallExpression" },
+      { match: "MemberExpression" },
+      { match: "VariableName" },
+      { match: "." },
+      { match: "PropertyName" },
+      { match: "ArgList" },
+      { match: "(" },
+      { match: /String|FormatString/, stop: true },
+      { match: ")" },
+    ];
+
+    let isValid = true;
+
+    // Parse the code using Lezer to check for multiple function calls and string content
+    tree.iterate({
+      enter: (node) => {
+        const current = enterOrder.shift();
+        if (current === undefined) {
+          // If our list is empty, but we are still going
+          // then this is not a valid call
+          isValid = false;
+          return false;
+        }
+
+        const match = current.match;
+
+        if (typeof match === "string") {
+          isValid = isValid && match === node.name;
+          return isValid && !current.stop;
+        }
+
+        if (!match.test(node.name)) {
+          isValid = false;
+          return isValid && !current.stop;
+        }
+
+        return isValid && !current.stop;
+      },
+    });
+
+    return isValid;
+  }
+
+  getExtension(
+    _cellId: CellId,
+    _completionConfig: CompletionConfig,
+    hotkeys: HotkeyProvider,
+    _: PlaceholderType,
+  ): Extension[] {
+    const markdownLanguageData = markdown().language.data;
+    let view: EditorView | undefined;
+
+    // Only activate completions for f-strings
+    const isFStringActive = () => {
+      if (!view) {
+        return true;
+      }
+
+      const metadata = view?.state.field(languageMetadataField);
+      if (metadata === undefined) {
+        return false;
+      }
+      return metadata.quotePrefix?.includes("f") ?? false;
+    };
+
+    return [
+      ViewPlugin.define((_view) => {
+        view = _view;
+        return {};
+      }),
+      markdown({
+        base: markdownLanguage,
+        codeLanguages: languages,
+        extensions: [
+          // Embedded LateX in Markdown
+          parseLatex(StreamLanguage.define(stexMath).parser),
+          // Embedded Python in Markdown
+          parsePython(pythonLanguage.parser, isFStringActive),
+        ],
+      }),
+      enhancedMarkdownExtension(hotkeys),
+      // Completions for markdown
+      markdownCompletionSources.map((source) =>
+        markdownLanguageData.of({ autocomplete: source }),
+      ),
+      // Completions for embedded Python
+      python().language.data.of({
+        autocomplete: conditionalCompletion({
+          completion: pythonCompletionSource,
+          predicate: isFStringActive,
+        }),
+      }),
+
+      autocompletion({
+        // We remove the default keymap because we use our own which
+        // handles the Escape key correctly in Vim
+        defaultKeymap: false,
+        activateOnTyping: true,
+      }),
+      // Markdown autorun
+      markdownAutoRunExtension({ predicate: () => !isFStringActive() }),
+    ];
+  }
+}

package/src/core/codemirror/language/languages/python.ts

@@ -0,0 +1,329 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { autocompletion } from "@codemirror/autocomplete";
+import {
+  globalCompletion,
+  localCompletionSource,
+  pythonLanguage,
+} from "@codemirror/lang-python";
+import {
+  foldInside,
+  foldNodeProp,
+  LanguageSupport,
+} from "@codemirror/language";
+import { type Extension, Prec } from "@codemirror/state";
+import {
+  documentUri,
+  LanguageServerClient,
+  languageServerWithClient,
+} from "@marimo-team/codemirror-languageserver";
+import type { CellId } from "@/core/cells/ids";
+import { hasCapability } from "@/core/config/capabilities";
+import type {
+  CompletionConfig,
+  DiagnosticsConfig,
+  LSPConfig,
+} from "@/core/config/config-schema";
+import type { HotkeyProvider } from "@/core/hotkeys/hotkeys";
+import { getRequestClient } from "@/core/network/requests";
+import { Logger } from "@/utils/Logger";
+import { once } from "@/utils/once";
+import { cellActionsState } from "../../cells/state";
+import { pythonCompletionSource } from "../../completion/completer";
+import type { PlaceholderType } from "../../config/types";
+import { FederatedLanguageServerClient } from "../../lsp/federated-lsp";
+import { NotebookLanguageServerClient } from "../../lsp/notebook-lsp";
+import { createTransport } from "../../lsp/transports";
+import { CellDocumentUri, type ILanguageServerClient } from "../../lsp/types";
+import { getLSPDocumentRootUri } from "../../lsp/utils";
+import {
+  clickablePlaceholderExtension,
+  smartPlaceholderExtension,
+} from "../../placeholder/extensions";
+import type { LanguageAdapter } from "../types";
+
+const pylspClient = once((lspConfig: LSPConfig) => {
+  const lspClientOpts = {
+    transport: createTransport("pylsp"),
+    rootUri: getLSPDocumentRootUri(),
+    workspaceFolders: [],
+  };
+  const config = lspConfig?.pylsp;
+
+  const ignoredStyleRules = [
+    // Notebooks are not really public modules and are better documented
+    // by having a markdown cell with explanations instead
+    "D100", // Missing docstring in public module
+    "D103", // Missing docstring in public function
+  ];
+  const ignoredFlakeRules = [
+    // The final cell in the notebook is not required to have a new line
+    "W292", // No newline at end of file
+    // Modules can be imported in any cell
+    "E402", // Module level import not at top of file
+  ];
+  const ignoredRuffRules = [
+    // Even ruff documentation of this rule explains it is not useful in notebooks
+    "B018", // Useless expression
+    // isort
+    "I001", // Import block is un-sorted or un-formatted
+  ];
+  const settings = {
+    pylsp: {
+      plugins: {
+        marimo_plugin: {
+          enabled: true,
+        },
+        jedi: {
+          // Modules which should be imported and use compile-time, rather
+          // than static analysis; this is a trade-off between being able
+          // to access more information set on runtime (e.g. via setattr)
+          // vs being able to read the information from the source code
+          // (e.g. comments with documentation for attributes).
+          auto_import_modules: ["numpy"],
+        },
+        jedi_completion: {
+          // Ensure that parameters are included for completion snippets.
+          include_params: true,
+          // Include snippets and signatures it at most 50 suggestions.
+          resolve_at_most: 50,
+        },
+        flake8: {
+          enabled: config?.enable_flake8,
+          extendIgnore: ignoredFlakeRules,
+        },
+        pydocstyle: {
+          enabled: config?.enable_pydocstyle,
+          // not `addIgnore`, see https://github.com/python-lsp/python-lsp-server/issues/626
+          ignore: ignoredStyleRules,
+        },
+        pylint: {
+          enabled: config?.enable_pylint,
+        },
+        pyflakes: {
+          enabled: config?.enable_pyflakes,
+        },
+        pylsp_mypy: {
+          enabled: config?.enable_mypy,
+          live_mode: true,
+        },
+        ruff: {
+          enabled: config?.enable_ruff,
+          extendIgnore: [
+            ...ignoredFlakeRules,
+            ...ignoredStyleRules,
+            ...ignoredRuffRules,
+          ],
+        },
+        signature: {
+          formatter: config?.enable_ruff ? "ruff" : "black",
+          line_length: 88,
+        },
+      },
+    },
+  };
+
+  // We wrap the client in a NotebookLanguageServerClient to add some
+  // additional functionality to handle multiple cells
+  return new NotebookLanguageServerClient(
+    new LanguageServerClient({
+      ...lspClientOpts,
+      autoClose: false,
+    }),
+    settings,
+  );
+});
+
+const tyLspClient = once((_: LSPConfig) => {
+  const lspClientOpts = {
+    transport: createTransport("ty"),
+    rootUri: getLSPDocumentRootUri(),
+    workspaceFolders: [],
+  };
+
+  // We wrap the client in a NotebookLanguageServerClient to add some
+  // additional functionality to handle multiple cells
+  return new NotebookLanguageServerClient(
+    new LanguageServerClient({
+      ...lspClientOpts,
+      autoClose: false,
+      getWorkspaceConfiguration: (_) => [{ disableLanguageServices: true }],
+    }),
+    {},
+  );
+});
+
+const pyrightClient = once((_: LSPConfig) => {
+  const lspClientOpts = {
+    transport: createTransport("basedpyright"),
+    rootUri: getLSPDocumentRootUri(),
+    workspaceFolders: [],
+  };
+
+  // We wrap the client in a NotebookLanguageServerClient to add some
+  // additional functionality to handle multiple cells
+  return new NotebookLanguageServerClient(
+    new LanguageServerClient({
+      ...lspClientOpts,
+      autoClose: false,
+    }),
+    {},
+  );
+});
+
+/**
+ * Language adapter for Python.
+ */
+export class PythonLanguageAdapter implements LanguageAdapter<{}> {
+  readonly type = "python";
+  readonly defaultCode = "";
+  readonly defaultMetadata = {};
+
+  transformIn(code: string): [string, number, {}] {
+    return [code, 0, {}];
+  }
+
+  transformOut(code: string, _metadata: {}): [string, number] {
+    return [code, 0];
+  }
+
+  isSupported(_code: string): boolean {
+    return true;
+  }
+
+  getExtension(
+    cellId: CellId,
+    completionConfig: CompletionConfig,
+    hotkeys: HotkeyProvider,
+    placeholderType: PlaceholderType,
+    lspConfig: LSPConfig & { diagnostics: DiagnosticsConfig },
+  ): Extension[] {
+    const getCompletionsExtension = () => {
+      const autocompleteOptions = {
+        // We remove the default keymap because we use our own which
+        // handles the Escape key correctly in Vim
+        defaultKeymap: false,
+        // Whether or not to require keypress to activate autocompletion (default
+        // keymap is Ctrl+Space)
+        activateOnTyping: completionConfig.activate_on_typing,
+        // The Cell component handles the blur event. `closeOnBlur` is too
+        // aggressive and doesn't let the user click into the completion info
+        // element (which contains the docstring/type --- users might want to
+        // copy paste from the docstring). The main issue is that the completion
+        // tooltip is not part of the editable DOM tree:
+        // https://discuss.codemirror.net/t/adding-click-event-listener-to-autocomplete-tooltip-info-panel-is-not-working/4741
+        closeOnBlur: false,
+      };
+
+      const hoverOptions = {
+        hideOnChange: true,
+      };
+
+      const clients: ILanguageServerClient[] = [];
+
+      if (lspConfig?.pylsp?.enabled && hasCapability("pylsp")) {
+        clients.push(pylspClient(lspConfig));
+      }
+      if (lspConfig?.ty?.enabled && hasCapability("ty")) {
+        clients.push(tyLspClient(lspConfig));
+      }
+      if (lspConfig?.basedpyright?.enabled && hasCapability("basedpyright")) {
+        clients.push(pyrightClient(lspConfig));
+      }
+
+      if (clients.length > 0) {
+        const client =
+          clients.length === 1
+            ? (clients[0] as NotebookLanguageServerClient)
+            : new FederatedLanguageServerClient(clients);
+
+        return [
+          languageServerWithClient({
+            client: client as unknown as LanguageServerClient,
+            languageId: "python",
+            allowHTMLContent: true,
+            hoverConfig: hoverOptions,
+            completionConfig: autocompleteOptions,
+            // Default to false
+            diagnosticsEnabled: lspConfig.diagnostics?.enabled ?? false,
+            sendIncrementalChanges: false,
+            signatureHelpEnabled: true,
+            signatureActivateOnTyping: false,
+            keyboardShortcuts: {
+              signatureHelp: hotkeys.getHotkey("cell.signatureHelp").key,
+              goToDefinition: hotkeys.getHotkey("cell.goToDefinition").key,
+              rename: hotkeys.getHotkey("cell.renameSymbol").key,
+            },
+            // Match completions before the cursor is at the end of a word,
+            // after a dot, after a slash, after a comma.
+            completionMatchBefore: /(\w+|\w+\.|\(|\/|,)$/,
+            onGoToDefinition: (result) => {
+              Logger.debug("onGoToDefinition", result);
+              if (client.documentUri === result.uri) {
+                // Local definition
+                return;
+              }
+              getRequestClient().openFile({
+                path: result.uri.replace("file://", ""),
+              });
+            },
+          }),
+          documentUri.of(CellDocumentUri.of(cellId)),
+        ];
+      }
+
+      return autocompletion({
+        ...autocompleteOptions,
+        override: [pythonCompletionSource],
+      });
+    };
+
+    return [
+      getCompletionsExtension(),
+      customPythonLanguageSupport(),
+      getPlaceholderExtension(placeholderType),
+    ];
+  }
+}
+
+function getPlaceholderExtension(placeholderType: PlaceholderType): Extension {
+  if (placeholderType === "marimo-import") {
+    return Prec.highest(smartPlaceholderExtension("import marimo as mo"));
+  }
+
+  if (placeholderType === "ai") {
+    return clickablePlaceholderExtension({
+      beforeText: "Start coding or ",
+      linkText: "generate",
+      afterText: " with AI.",
+      onClick: (ev) => {
+        const cellActions = ev.state.facet(cellActionsState);
+        cellActions.aiCellCompletion();
+      },
+    });
+  }
+
+  return [];
+}
+
+// Customize python to support folding some additional syntax nodes
+const customizedPython = pythonLanguage.configure({
+  props: [
+    foldNodeProp.add({
+      ParenthesizedExpression: foldInside,
+      // Fold function calls whose arguments are split over multiple lines
+      ArgList: foldInside,
+    }),
+  ],
+});
+
+/**
+ * This provide LanguageSupport for Python, but with a custom LRLanguage
+ * that supports folding additional syntax nodes at the top-level.
+ */
+export function customPythonLanguageSupport(): LanguageSupport {
+  return new LanguageSupport(customizedPython, [
+    customizedPython.data.of({ autocomplete: localCompletionSource }),
+    customizedPython.data.of({ autocomplete: globalCompletion }),
+  ]);
+}

package/src/core/codemirror/language/languages/sql/completion-sources.tsx

@@ -0,0 +1,99 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import type { Completion, CompletionSource } from "@codemirror/autocomplete";
+import {
+  keywordCompletionSource,
+  schemaCompletionSource,
+} from "@codemirror/lang-sql";
+import type { EditorState } from "@codemirror/state";
+import { DefaultSqlTooltipRenders } from "@marimo-team/codemirror-sql";
+import { languageMetadataField } from "../../metadata";
+import { SCHEMA_CACHE } from "./completion-store";
+import type { SQLLanguageAdapterMetadata } from "./sql";
+
+function getSQLMetadata(state: EditorState): SQLLanguageAdapterMetadata {
+  return state.field(languageMetadataField) as SQLLanguageAdapterMetadata;
+}
+
+/**
+ * Custom schema completion source that dynamically gets the Dialect and SQL tables.
+ */
+export function tablesCompletionSource(): CompletionSource {
+  return (ctx) => {
+    const metadata = getSQLMetadata(ctx.state);
+    const connectionName = metadata.engine;
+    const config = SCHEMA_CACHE.getCompletionSource(connectionName);
+
+    if (!config) {
+      return null;
+    }
+
+    return schemaCompletionSource(config)(ctx);
+  };
+}
+
+/**
+ * Custom keyword completion source that dynamically gets the Dialect.
+ * This also ignores keyword completions on table columns.
+ */
+export function customKeywordCompletionSource(): CompletionSource {
+  return (ctx) => {
+    const metadata = getSQLMetadata(ctx.state);
+    const connectionName = metadata.engine;
+    const dialect = SCHEMA_CACHE.getDialect(connectionName);
+
+    // We want to ignore keyword completions on something like
+    // `WHERE my_table.col`
+    //                    ^cursor
+    const textBefore = ctx.matchBefore(/\.\w*/);
+    if (textBefore) {
+      // If there is a match, we are typing after a dot,
+      // so we don't want to trigger SQL keyword completion
+      return null;
+    }
+
+    const keywordRenderer = (label: string, type: string): Completion => {
+      return {
+        label,
+        type,
+        info: async () => {
+          const keywordDocs = await getKeywordDocs();
+          const keywordInfo = keywordDocs[label.toLocaleLowerCase()];
+          if (!keywordInfo) {
+            return null;
+          }
+
+          const dom = document.createElement("div");
+          dom.innerHTML = DefaultSqlTooltipRenders.keyword({
+            keyword: label,
+            info: keywordInfo,
+          });
+          return dom;
+        },
+      };
+    };
+
+    const uppercaseKeywords = true;
+    const result = keywordCompletionSource(
+      dialect,
+      uppercaseKeywords,
+      keywordRenderer,
+    )(ctx);
+    return result;
+  };
+}
+
+// e.g. lazily load keyword docs
+const getKeywordDocs = async (): Promise<Record<string, unknown>> => {
+  const keywords = await import(
+    "@marimo-team/codemirror-sql/data/common-keywords.json"
+  );
+  // Include DuckDB for now, but we can remove this once we have a better way to handle dialect-specific keywords
+  const duckdbKeywords = await import(
+    "@marimo-team/codemirror-sql/data/duckdb-keywords.json"
+  );
+  return {
+    ...keywords.default.keywords,
+    ...duckdbKeywords.default.keywords,
+  };
+};