@nshiab/simple-data-analysis-core 2.1.11 → 2.1.13

package/src/helpers/normalizeCacheCode.ts

@@ -0,0 +1,348 @@
+type Token = {
+  kind: "word" | "punctuation" | "literal" | "template";
+  text: string;
+  lineBreak: boolean;
+};
+
+const lineTerminator = /[\r\n\u2028\u2029]/u;
+const word = /^[$_\p{ID_Start}][$\u200c\u200d\p{ID_Continue}]*/u;
+const number =
+  /^(?:0[xX][\da-fA-F_]+n?|0[bB][01_]+n?|0[oO][0-7_]+n?|(?:\d[\d_]*(?:\.[\d_]*)?|\.[\d_]+)(?:[eE][+-]?[\d_]+)?n?)/;
+const punctuation =
+  /^(?:>>>=|===|!==|\*\*=|&&=|\|\|=|\?\?=|>>>|<<=|>>=|=>|==|!=|<=|>=|\+\+|--|&&|\|\||\?\?|\?\.(?!\d)|\*\*|<<|>>|\+=|-=|\*=|%=|&=|\|=|\^=|\.\.\.|[{}()[\].,;:?~!+*%<>=&|^\-#])/;
+const expressionPrefixes = new Set([
+  "(",
+  "[",
+  "{",
+  ",",
+  ";",
+  ":",
+  "?",
+  "=",
+  "=>",
+  "!",
+  "~",
+  "+",
+  "-",
+  "*",
+  "**",
+  "%",
+  "&",
+  "|",
+  "^",
+  "&&",
+  "||",
+  "??",
+  "==",
+  "===",
+  "!=",
+  "!==",
+  "<",
+  ">",
+  "<=",
+  ">=",
+  "<<",
+  ">>",
+  ">>>",
+  "+=",
+  "-=",
+  "*=",
+  "**=",
+  "%=",
+  "&=",
+  "|=",
+  "^=",
+  "&&=",
+  "||=",
+  "??=",
+  "<<=",
+  ">>=",
+  ">>>=",
+  "/",
+  "/=",
+]);
+const expressionKeywords = new Set([
+  "return",
+  "throw",
+  "case",
+  "delete",
+  "void",
+  "typeof",
+  "new",
+  "in",
+  "instanceof",
+  "yield",
+  "await",
+  "else",
+  "do",
+]);
+const restrictedLineBreaks = new Set([
+  "return",
+  "throw",
+  "break",
+  "continue",
+  "yield",
+  "async",
+  "++",
+  "--",
+]);
+
+/**
+ * Normalizes comments, whitespace, and known optional punctuation without
+ * dependencies or executing user code. Literal contents and token boundaries
+ * stay intact. Potentially significant line breaks are retained; ambiguous
+ * slash syntax falls back to exact source instead of risking a stale cache hit.
+ */
+export default function normalizeCacheCode(source: string): string {
+  let position = 0;
+  let uncertain = false;
+
+  function quoted(quote: string): string | undefined {
+    const start = position++;
+    while (position < source.length) {
+      const character = source[position++];
+      if (character === "\\") {
+        position++;
+      } else if (character === quote) {
+        return source.slice(start, position);
+      } else if (lineTerminator.test(character)) {
+        return undefined;
+      }
+    }
+  }
+
+  function template(): string | undefined {
+    const parts: string[] = [];
+    let start = ++position;
+    while (position < source.length) {
+      if (source[position] === "\\") {
+        position += 2;
+      } else if (source[position] === "`") {
+        parts.push(source.slice(start, position++));
+        return JSON.stringify(parts);
+      } else if (source.startsWith("${", position)) {
+        parts.push(source.slice(start, position));
+        position += 2;
+        const tokens = scan(true);
+        if (uncertain) return undefined;
+        parts.push(serialize(tokens));
+        start = position;
+      } else {
+        position++;
+      }
+    }
+  }
+
+  function regex(): string | undefined {
+    const start = position++;
+    let inClass = false;
+    while (position < source.length) {
+      const character = source[position++];
+      if (lineTerminator.test(character)) return undefined;
+      if (character === "\\") {
+        position++;
+      } else if (character === "[") {
+        // Nested character classes depend on the v flag. Leave these opaque.
+        if (inClass) return undefined;
+        inClass = true;
+      } else if (character === "]") {
+        inClass = false;
+      } else if (character === "/" && !inClass) {
+        const flags = source.slice(position).match(word)?.[0] ?? "";
+        position += flags.length;
+        return source.slice(start, position);
+      }
+    }
+  }
+
+  function scan(interpolation = false): Token[] {
+    const tokens: Token[] = [];
+    let braces = 0;
+    let lineBreak = false;
+    while (position < source.length) {
+      const rest = source.slice(position);
+      const character = source[position];
+      if (/\s/u.test(character)) {
+        lineBreak ||= lineTerminator.test(character);
+        position++;
+        continue;
+      }
+      if (rest.startsWith("//")) {
+        while (position < source.length && !lineTerminator.test(source[position])) {
+          position++;
+        }
+        continue;
+      }
+      if (rest.startsWith("/*")) {
+        const end = source.indexOf("*/", position + 2);
+        if (end === -1) break;
+        lineBreak ||= lineTerminator.test(source.slice(position, end + 2));
+        position = end + 2;
+        continue;
+      }
+      if (interpolation && character === "}" && braces === 0) {
+        position++;
+        return tokens;
+      }
+      // Legacy HTML comments and native function text are not ordinary tokens.
+      if (rest.startsWith("<!--") || rest.startsWith("-->") || rest.startsWith("[native code]"))
+        break;
+
+      const previous = tokens.at(-1);
+      let kind: Token["kind"] = "punctuation";
+      let text: string | undefined;
+      if (character === "'" || character === '"') {
+        kind = "literal";
+        text = quoted(character);
+      } else if (character === "`") {
+        kind = "template";
+        text = template();
+      } else if (character === "/") {
+        const property = [".", "?."].includes(tokens.at(-2)?.text ?? "");
+        if (
+          !previous ||
+          (previous.kind === "punctuation" && expressionPrefixes.has(previous.text)) ||
+          (previous.kind === "word" && !property && expressionKeywords.has(previous.text))
+        ) {
+          kind = "literal";
+          text = regex();
+        } else if (previous.kind === "literal" || previous.kind === "template") {
+          text = rest.startsWith("/=") ? "/=" : "/";
+          position += text.length;
+        } else {
+          // After a closing block/parenthesis or contextual keyword, deciding
+          // between division and a regex can require a full JavaScript parser.
+          break;
+        }
+      } else {
+        text = rest.match(word)?.[0];
+        if (text !== undefined) {
+          kind = "word";
+        } else {
+          text = rest.match(number)?.[0];
+          if (text !== undefined) kind = "literal";
+          else text = rest.match(punctuation)?.[0];
+        }
+        if (text !== undefined) position += text.length;
+      }
+      if (text === undefined) {
+        uncertain = true;
+        break;
+      }
+      if (kind === "punctuation") {
+        if (text === "{") braces++;
+        if (text === "}") braces--;
+      }
+      tokens.push({ kind, text, lineBreak });
+      lineBreak = false;
+    }
+    if (position < source.length || interpolation) uncertain = true;
+    return tokens;
+  }
+
+  const tokens = scan();
+  return uncertain
+    ? JSON.stringify(["source", source])
+    : JSON.stringify(["tokens", serialize(tokens)]);
+}
+
+function serialize(tokens: Token[]): string {
+  tokens = normalizePunctuation(tokens);
+  return JSON.stringify(
+    tokens.map((token, index) => {
+      const previous = tokens[index - 1];
+      // Only discard newlines where continuation is certain. Keeping other
+      // newlines permits conservative misses rather than unsafe cache hits.
+      const significant =
+        token.lineBreak &&
+        previous !== undefined &&
+        (restrictedLineBreaks.has(previous.text) ||
+          ["++", "--"].includes(token.text) ||
+          !(
+            (previous.kind === "punctuation" &&
+              (expressionPrefixes.has(previous.text) ||
+                [".", "?.", "}"].includes(previous.text))) ||
+            (token.kind === "punctuation" &&
+              ([")", "]", "}", ",", ";", ".", "?."].includes(token.text) ||
+                (expressionPrefixes.has(token.text) &&
+                  !["(", "[", "{", "!", "~"].includes(token.text))))
+          ));
+      return [token.kind, token.text, significant];
+    }),
+  );
+}
+
+function normalizePunctuation(input: Token[]): Token[] {
+  const tokens = input.map((token) => ({ ...token }));
+  const result: Token[] = [];
+  const brackets: { text: string; control: boolean }[] = [];
+  const controls = new Set(["if", "while", "for", "with", "switch", "catch"]);
+  let previousControl = false;
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i];
+    const next = tokens[i + 1];
+    const previous = result.at(-1);
+    if (token.kind === "literal" && /^["']/.test(token.text) && !token.text.includes("\\")) {
+      token.text = JSON.stringify(token.text.slice(1, -1));
+    }
+    // A single arrow parameter has the same binding with or without parentheses.
+    if (
+      token.text === "(" &&
+      tokens[i + 1]?.kind === "word" &&
+      tokens[i + 2]?.text === ")" &&
+      tokens[i + 3]?.text === "=>"
+    ) {
+      result.push({ ...tokens[i + 1], lineBreak: token.lineBreak });
+      i += 2;
+      previousControl = false;
+      continue;
+    }
+    // Retain commas representing array holes, including the final hole.
+    if (
+      token.text === "," &&
+      [")", "]", "}"].includes(next?.text ?? "") &&
+      previous &&
+      ![",", "[", "("].includes(previous.text)
+    )
+      continue;
+
+    const expressionEnd =
+      previous &&
+      (previous.kind !== "punctuation" ||
+        previous.text === "]" ||
+        (previous.text === ")" && !previousControl));
+    if (
+      token.text === ";" &&
+      brackets.at(-1)?.text === "{" &&
+      expressionEnd &&
+      !restrictedLineBreaks.has(previous.text)
+    ) {
+      if (next?.text === "}") continue;
+      // A call followed by an identifier starts a new statement. Preserve that
+      // boundary as a newline so explicit and automatically inserted semicolons
+      // agree. Never do this in a for header or after a control header.
+      if (
+        previous.text === ")" &&
+        next?.kind === "word" &&
+        !["in", "instanceof", "of"].includes(next.text)
+      ) {
+        next.lineBreak = true;
+        continue;
+      }
+    }
+    previousControl = false;
+    if (["(", "[", "{"].includes(token.text)) {
+      brackets.push({
+        text: token.text,
+        control:
+          token.text === "(" &&
+          (controls.has(previous?.text ?? "") ||
+            (previous?.text === "await" && result.at(-2)?.text === "for")),
+      });
+    } else if ([")", "]", "}"].includes(token.text)) {
+      previousControl = brackets.pop()?.control ?? false;
+    }
+    result.push(token);
+  }
+  return result;
+}

package/src/helpers/observeQuery.ts

@@ -0,0 +1,64 @@
+import stringifyUnknown from "./stringifyUnknown.ts";
+import type { DuckDBConnection, DuckDBValue } from "@duckdb/node-api";
+
+const explainableStatements = new Set([
+  "CREATE",
+  "DELETE",
+  "FROM",
+  "INSERT",
+  "MERGE",
+  "SELECT",
+  "UPDATE",
+  "WITH",
+]);
+
+/**
+ * Logs optional SQL diagnostics without changing the statement that will run.
+ */
+export default async function observeQuery(
+  connection: DuckDBConnection,
+  query: string,
+  values: DuckDBValue[],
+  options: { logSQL: boolean; explainSQL: boolean },
+): Promise<void> {
+  if (options.explainSQL && (await canExplain(connection, query))) {
+    try {
+      const explain = `EXPLAIN ${query}`;
+      const result =
+        values.length === 0 ? await connection.run(explain) : await connection.run(explain, values);
+      const rows = await result.getRowsJS();
+      const plan = rows.map((row) => stringifyUnknown(row[1] ?? row[0] ?? "")).join("\n");
+      console.log(`EXPLAIN ${query}\n${plan}`);
+    } catch (error) {
+      console.warn(
+        `SDA: Could not explain the following SQL; executing it normally.\n${query}\n${String(
+          error,
+        )}`,
+      );
+    }
+  }
+
+  // Keep this immediately before connection.run()/stream() so the last SQL
+  // diagnostic identifies the statement responsible for an execution error.
+  if (options.logSQL) {
+    console.log(query);
+    if (values.length > 0) {
+      console.log("Bound values:", values);
+    }
+  }
+}
+
+async function canExplain(connection: DuckDBConnection, query: string): Promise<boolean> {
+  const keyword = query
+    .trimStart()
+    .match(/^([A-Za-z]+)/)?.[1]
+    .toUpperCase();
+  if (keyword === undefined || !explainableStatements.has(keyword)) {
+    return false;
+  }
+  try {
+    return (await connection.extractStatements(query)).count === 1;
+  } catch {
+    return false;
+  }
+}

package/src/helpers/optimizeUmapLayout.ts

@@ -0,0 +1,103 @@
+import type prepareUmapLayoutOptions from "./prepareUmapLayoutOptions.ts";
+import umapRandom from "./umapRandom.ts";
+
+// Internal interface. Graph entries are directed, in a stable order; the fuzzy
+// union graph supplies both orientations. Coordinates are interleaved x,y.
+// Inputs are borrowed read-only. State is O(edges + vertices), never O(n²).
+// Implements reference Euclidean UMAP SGD: scheduled attractive edges, sampled
+// repulsion, clipped gradients, symmetric attractive updates, linear decay.
+// Algorithm: https://umap-learn.readthedocs.io/en/latest/how_umap_works.html
+export default function optimizeUmapLayout(
+  graph: { source: Uint32Array; target: Uint32Array; weight: Float64Array },
+  initial: Float64Array,
+  options: ReturnType<typeof prepareUmapLayoutOptions>,
+): Float64Array {
+  const {
+    epochs,
+    seed,
+    learningRate,
+    negativeSamples,
+    curve: [a, b],
+  } = options;
+  const count = initial.length / 2;
+  if (!Number.isInteger(count) || count < 3 || count > 0xffffffff) {
+    throw new Error("Expected interleaved coordinates for at least three vertices.");
+  }
+  for (const value of initial) {
+    if (!Number.isFinite(value)) {
+      throw new Error("Initial coordinates must be finite.");
+    }
+  }
+  const { source, target, weight } = graph;
+  if (!weight.length || source.length !== weight.length || target.length !== weight.length)
+    throw new Error("Invalid graph column lengths.");
+  let maximum = 0;
+  for (let edge = 0; edge < weight.length; edge++) {
+    if (source[edge] >= count || target[edge] >= count || source[edge] === target[edge])
+      throw new Error("Graph contains an invalid vertex or self edge.");
+    if (!Number.isFinite(weight[edge]) || weight[edge] < 0 || weight[edge] > 1)
+      throw new Error("Graph weights must be between zero and one.");
+    maximum = Math.max(maximum, weight[edge]);
+  }
+  if (!maximum) throw new Error("Graph must contain a positive edge.");
+  const period = new Float64Array(weight.length);
+  const nextPositive = new Float64Array(weight.length);
+  const nextNegative = new Float64Array(weight.length);
+  // Reference pruning uses 500 when explicitly requesting at most ten epochs.
+  const threshold = maximum / (epochs > 10 ? epochs : 500);
+  for (let edge = 0; edge < weight.length; edge++) {
+    period[edge] =
+      weight[edge] > 0 && weight[edge] >= threshold
+        ? epochs / ((weight[edge] / maximum) * epochs)
+        : Infinity;
+    nextPositive[edge] = period[edge];
+    nextNegative[edge] = period[edge] / negativeSamples;
+  }
+  const coordinates = initial.slice();
+  const random = umapRandom(seed ^ 0x7f4a7c15);
+  const clip = (gradient: number) => Math.max(-4, Math.min(4, gradient));
+  let alpha = learningRate;
+  for (let epoch = 0; epoch < epochs; epoch++) {
+    for (let edge = 0; edge < weight.length; edge++) {
+      if (nextPositive[edge] > epoch) continue;
+      const u = source[edge] * 2,
+        v = target[edge] * 2;
+      let dx = coordinates[u] - coordinates[v],
+        dy = coordinates[u + 1] - coordinates[v + 1];
+      let squared = dx * dx + dy * dy;
+      if (squared > 0) {
+        const attraction = (-2 * a * b * Math.pow(squared, b - 1)) / (1 + a * Math.pow(squared, b));
+        const gx = clip(attraction * dx) * alpha,
+          gy = clip(attraction * dy) * alpha;
+        coordinates[u] += gx;
+        coordinates[u + 1] += gy;
+        coordinates[v] -= gx;
+        coordinates[v + 1] -= gy;
+      }
+      nextPositive[edge] += period[edge];
+      const negativePeriod = period[edge] / negativeSamples;
+      const draws = Math.floor((epoch - nextNegative[edge]) / negativePeriod);
+      for (let draw = 0; draw < draws; draw++) {
+        const other = Math.floor(random() * count) * 2;
+        dx = coordinates[u] - coordinates[other];
+        dy = coordinates[u + 1] - coordinates[other + 1];
+        squared = dx * dx + dy * dy;
+        // Coincident points get zero repulsive gradient, as in current Python
+        // UMAP; this intentionally differs from UMAP-JS's constant +4 fallback.
+        if (squared > 0) {
+          const repulsion = (2 * b) / ((0.001 + squared) * (1 + a * Math.pow(squared, b)));
+          coordinates[u] += clip(repulsion * dx) * alpha;
+          coordinates[u + 1] += clip(repulsion * dy) * alpha;
+        }
+      }
+      nextNegative[edge] += draws * negativePeriod;
+    }
+    alpha = learningRate * (1 - epoch / epochs);
+  }
+  for (const value of coordinates) {
+    if (!Number.isFinite(value)) {
+      throw new Error("UMAP optimization produced a non-finite coordinate.");
+    }
+  }
+  return coordinates;
+}