@svgrid/grid 2.6.21 → 2.7.0

package/src/filtering/excel-filters.ts

@@ -1,295 +1,325 @@
-export type ExcelFilterOperator =
-  | 'contains'
-  | 'notContains'
-  | 'equals'
-  | 'notEquals'
-  | 'startsWith'
-  | 'endsWith'
-  | 'regex'
-  | 'in'
-  | 'notIn'
-  | 'greaterThan'
-  | 'lessThan'
-  | 'between'
-  | 'isBlank'
-  | 'isNotBlank'
-
-export type ExcelFilter = {
-  id: string
-  operator: ExcelFilterOperator
-  value?: unknown
-  valueTo?: unknown
-}
-
-export type ExcelFilterOptions = {
-  /**
-   * BCP-47 locale tag (or fallback list) for accent- and case-
-   * insensitive text comparison. When set, "café", "Café" and "CAFÉ"
-   * all match "cafe" without strain.
-   *
-   * Implementation: NFD-decompose both sides, strip combining marks
-   * (diacritics), then locale-aware lowercase. This is the standard
-   * "lowercase + asciifolding" pipeline used by Elasticsearch, Postgres
-   * (with `unaccent`), and Lucene. Cheap, deterministic, no Collator
-   * round-trip.
-   */
-  locale?: string | ReadonlyArray<string>
-}
-
-// U+0300..U+036F is the Combining Diacritical Marks block. After NFD
-// decomposition accented characters become "base char + combining mark";
-// stripping the marks leaves the bare ASCII base.
-const DIACRITIC_RE = /[̀-ͯ]/g
-
-/** NFD-decompose, strip combining marks (diacritics), then locale-aware
- *  lowercase. The locale-aware lowercasing handles Turkish dotted-I /
- *  dotless-i correctly when the consumer threads `"tr"` through. */
-export function normalizeForFilter(
-  s: string,
-  locale?: string | ReadonlyArray<string>,
-): string {
-  if (!s) return ''
-  const stripped = s.normalize('NFD').replace(DIACRITIC_RE, '')
-  return locale
-    ? stripped.toLocaleLowerCase(locale as string | string[])
-    : stripped.toLowerCase()
-}
-
-/**
- * Delimiter used to serialise the `in` / `notIn` value list into the single
- * `value` string that the filter model carries.
- *
- * This has to be single-line safe: the tool panel and the filter menu show the
- * serialised list in a plain `<input type="text">`, and the HTML input value
- * sanitiser STRIPS newlines - so a newline-joined list came back out of the DOM
- * as one run-together token. `splitInTokens` accepts newlines too, so values
- * serialised by older builds still parse.
- */
-export const IN_TOKEN_SEP = ', '
-
-/**
- * Split a serialised `in` / `notIn` value into its individual tokens.
- *
- * The grammar is CSV-shaped: comma OR newline separates, and a token may be
- * double-quoted to carry a separator literally (`""` escapes a quote inside
- * one). Quoting is what makes a comma separator safe - facet values commonly
- * contain commas of their own, because numeric bucket labels are built with
- * `toLocaleString` ("1,234 - 5,678") and date labels with a short month
- * ("Aug 17, 2026"). A quote only opens a token when it is the token's first
- * non-space character, so an unquoted `5" pipe` stays literal.
- */
-export function splitInTokens(value: unknown): string[] {
-  const { tokens, trailing } = scanInTokens(value)
-  if (trailing) tokens.push(trailing)
-  return tokens
-}
-
-/**
- * Serialise a list of tokens back into the single `in` / `notIn` value,
- * quoting any token that carries a separator or a quote of its own.
- */
-export function joinInTokens(tokens: ReadonlyArray<string>): string {
-  return tokens
-    .map((t) => t.trim())
-    .filter((t) => t.length > 0)
-    .map((t) => (/["\n,]/.test(t) ? `"${t.replace(/"/g, '""')}"` : t))
-    .join(IN_TOKEN_SEP)
-}
-
-/**
- * The token still being typed at the end of an `in` / `notIn` value - the text
- * after the last separator. `'AAPL, MS'` -> `'MS'`; a value ending in a
- * separator (or empty) has no trailing token.
- *
- * Inputs that hold the whole token list use this to drive the value-suggestion
- * dropdown: the trailing fragment is the search query, not a committed token.
- */
-export function trailingInToken(value: unknown): string {
-  return scanInTokens(value).trailing
-}
-
-/**
- * One pass over a serialised `in` / `notIn` value, splitting it into the
- * tokens that are terminated by a separator and the unterminated fragment at
- * the end. Every token helper above is a view onto this, so they can never
- * disagree about where a quoted token starts or ends.
- */
-function scanInTokens(value: unknown): { tokens: string[]; trailing: string } {
-  const raw = String(value ?? '')
-  const tokens: string[] = []
-  let buf = ''
-  let quoted = false
-  let wasQuoted = false
-  let closed = false
-
-  // A quoted token keeps its inner spacing; an unquoted one is trimmed.
-  const take = () => {
-    const t = wasQuoted ? buf : buf.trim()
-    buf = ''
-    wasQuoted = false
-    closed = false
-    return t
-  }
-
-  for (let i = 0; i < raw.length; i += 1) {
-    const ch = raw[i]
-    if (quoted) {
-      if (ch !== '"') { buf += ch; continue }
-      // A doubled quote is an escaped literal quote, not the closing one.
-      if (raw[i + 1] === '"') { buf += '"'; i += 1; continue }
-      quoted = false
-      closed = true
-      continue
-    }
-    if (ch === ',' || ch === '\n') {
-      const t = take()
-      if (t.length) tokens.push(t)
-      continue
-    }
-    // Only opens a quoted token at the token's start - `5" pipe` is literal.
-    if (ch === '"' && !closed && !buf.trim()) {
-      quoted = true
-      wasQuoted = true
-      buf = ''
-      continue
-    }
-    buf += ch
-  }
-
-  return { tokens, trailing: take() }
-}
-
-// Re-exported here so consumers can reach the catalogue through the
-// `@svgrid/grid/filtering` subpath. Importing it from the package root would
-// pull in SvGrid.svelte, which is a runtime cost (and a bundler hazard) for
-// anything that only wants operator semantics.
-export {
-  ALL_FILTER_OPERATORS,
-  SET_OPERATOR_IDS,
-  VALUELESS_OPERATOR_IDS,
-  RANGE_OPERATOR_IDS,
-  isSetOperator,
-  isValuelessOperator,
-  isRangeOperator,
-  type FilterValueType,
-} from './filter-operator-catalogue.js'
-
-/** A filter with its needle-side work already done. Call per row. */
-export type CompiledExcelFilter = (cellValue: unknown) => boolean
-
-/**
- * Compile a filter once, then test many rows against it.
- *
- * Everything that depends only on the FILTER - folding the needle, splitting
- * `in` tokens, building the regex, coercing range endpoints - happens here,
- * so the per-row closure does the minimum. Filtering 100k rows used to redo
- * all of it 100k times.
- *
- * `applyExcelFilter` is defined in terms of this, so there is exactly one
- * copy of the operator semantics.
- */
-export function compileExcelFilter(
-  filter: ExcelFilter,
-  options?: ExcelFilterOptions,
-): CompiledExcelFilter {
-  const locale = options?.locale
-  const fold = (v: unknown) => normalizeForFilter(String(v ?? ''), locale)
-  const needle = fold(filter.value)
-
-  switch (filter.operator) {
-    case 'contains':
-      return (cellValue) => fold(cellValue).includes(needle)
-    case 'notContains':
-      // An empty needle is no constraint (mirrors `contains` returning true),
-      // so nothing is excluded until the user types something.
-      if (needle === '') return () => true
-      return (cellValue) => !fold(cellValue).includes(needle)
-    case 'equals':
-      return (cellValue) => fold(cellValue) === needle
-    case 'notEquals':
-      if (needle === '') return () => true
-      return (cellValue) => fold(cellValue) !== needle
-    case 'startsWith':
-      return (cellValue) => fold(cellValue).startsWith(needle)
-    case 'endsWith':
-      return (cellValue) => fold(cellValue).endsWith(needle)
-    case 'regex': {
-      // Case-insensitive by default (matches the accent/case-folded feel of
-      // the other text operators). An invalid pattern matches nothing rather
-      // than throwing, so a half-typed regex never crashes the row model.
-      // Note this tests the RAW text, not the folded text.
-      const pattern = String(filter.value ?? '')
-      if (!pattern) return () => true
-      let re: RegExp
-      try {
-        re = new RegExp(pattern, 'i')
-      } catch {
-        return () => false
-      }
-      return (cellValue) => re.test(String(cellValue ?? ''))
-    }
-    case 'in':
-    case 'notIn': {
-      const tokens = splitInTokens(filter.value)
-      if (tokens.length === 0) return () => true
-      // A Set of pre-folded tokens turns the old per-row `tokens.some(...)`
-      // (which re-folded every token for every row) into one lookup.
-      const wanted = new Set(tokens.map((t) => normalizeForFilter(t, locale)))
-      if (filter.operator === 'in') return (cellValue) => wanted.has(fold(cellValue))
-      return (cellValue) => !wanted.has(fold(cellValue))
-    }
-    case 'greaterThan': {
-      const b = Number(filter.value)
-      const bFinite = Number.isFinite(b)
-      const bText = String(filter.value ?? '')
-      return (cellValue) => {
-        const a = Number(cellValue)
-        if (bFinite && Number.isFinite(a)) return a > b
-        return String(cellValue ?? '') > bText
-      }
-    }
-    case 'lessThan': {
-      const b = Number(filter.value)
-      const bFinite = Number.isFinite(b)
-      const bText = String(filter.value ?? '')
-      return (cellValue) => {
-        const a = Number(cellValue)
-        if (bFinite && Number.isFinite(a)) return a < b
-        return String(cellValue ?? '') < bText
-      }
-    }
-    case 'between': {
-      // Two paths: numeric (coerce both endpoints with the historical
-      // `?? 0` fallback) and string (used when EITHER endpoint is a
-      // non-numeric string like an ISO date). Numeric wins when its
-      // operands are finite so callers passing numeric ages, prices, etc.
-      // see the historical inclusive-range behaviour.
-      const lo = filter.value   == null ? 0 : Number(filter.value)
-      const hi = filter.valueTo == null ? 0 : Number(filter.valueTo)
-      const rangeFinite = Number.isFinite(lo) && Number.isFinite(hi)
-      // Either endpoint is non-numeric (ISO date string, etc). Compare
-      // lexicographically - YYYY-MM-DD orders chronologically so the
-      // result matches user intent.
-      const sl = String(filter.value ?? '')
-      const sh = String(filter.valueTo ?? '')
-      return (cellValue) => {
-        const a = Number(cellValue ?? 0)
-        if (rangeFinite && Number.isFinite(a)) return a >= lo && a <= hi
-        const s = String(cellValue ?? '')
-        return s >= sl && s <= sh
-      }
-    }
-    case 'isBlank':
-      return (cellValue) => String(cellValue ?? '').trim().length === 0
-    case 'isNotBlank':
-      return (cellValue) => String(cellValue ?? '').trim().length > 0
-  }
-}
-
-export function applyExcelFilter(
-  cellValue: unknown,
-  filter: ExcelFilter,
-  options?: ExcelFilterOptions,
-) {
-  return compileExcelFilter(filter, options)(cellValue)
-}
+export type ExcelFilterOperator =
+  | 'contains'
+  | 'notContains'
+  | 'equals'
+  | 'notEquals'
+  | 'startsWith'
+  | 'endsWith'
+  | 'regex'
+  | 'in'
+  | 'notIn'
+  | 'greaterThan'
+  | 'lessThan'
+  | 'between'
+  | 'isBlank'
+  | 'isNotBlank'
+
+export type ExcelFilter = {
+  id: string
+  operator: ExcelFilterOperator
+  value?: unknown
+  valueTo?: unknown
+}
+
+export type ExcelFilterOptions = {
+  /**
+   * BCP-47 locale tag (or fallback list) for accent- and case-
+   * insensitive text comparison. When set, "café", "Café" and "CAFÉ"
+   * all match "cafe" without strain.
+   *
+   * Implementation: NFD-decompose both sides, strip combining marks
+   * (diacritics), then locale-aware lowercase. This is the standard
+   * "lowercase + asciifolding" pipeline used by Elasticsearch, Postgres
+   * (with `unaccent`), and Lucene. Cheap, deterministic, no Collator
+   * round-trip.
+   */
+  locale?: string | ReadonlyArray<string>
+}
+
+// U+0300..U+036F is the Combining Diacritical Marks block. After NFD
+// decomposition accented characters become "base char + combining mark";
+// stripping the marks leaves the bare ASCII base.
+const DIACRITIC_RE = /[̀-ͯ]/g
+
+/** NFD-decompose, strip combining marks (diacritics), then locale-aware
+ *  lowercase. The locale-aware lowercasing handles Turkish dotted-I /
+ *  dotless-i correctly when the consumer threads `"tr"` through. */
+export function normalizeForFilter(
+  s: string,
+  locale?: string | ReadonlyArray<string>,
+): string {
+  if (!s) return ''
+
+  // Fast path for pure-ASCII input, which is the overwhelming majority of what
+  // a grid filters. Both of the expensive steps are provably no-ops there:
+  // ASCII is already NFD-normalised, and the combining-marks block this strips
+  // (U+0300..U+036F) contains no ASCII. So lowercasing alone is the same
+  // answer, and `normalize-fast-path.test.ts` checks that exhaustively over
+  // every ASCII code point rather than taking the argument on trust.
+  //
+  // Worth the branch: this runs once per cell per filter - 100,000 times for a
+  // one-column filter over a 100k grid - and a CPU profile of a browser filter
+  // put more time in this function than in any other.
+  //
+  // Note the locale still routes through toLocaleLowerCase even on the fast
+  // path: 'I'.toLocaleLowerCase('tr') is 'ı', so ASCII does NOT imply the
+  // locale can be ignored.
+  if (isAsciiOnly(s)) {
+    return locale ? s.toLocaleLowerCase(locale as string | string[]) : s.toLowerCase()
+  }
+
+  const stripped = s.normalize('NFD').replace(DIACRITIC_RE, '')
+  return locale
+    ? stripped.toLocaleLowerCase(locale as string | string[])
+    : stripped.toLowerCase()
+}
+
+/**
+ * True when every code unit is ASCII. A plain char-code scan, which is far
+ * cheaper than the `normalize()` + regex pass it lets us skip.
+ */
+function isAsciiOnly(s: string): boolean {
+  for (let i = 0; i < s.length; i++) {
+    if (s.charCodeAt(i) > 127) return false
+  }
+  return true
+}
+
+/**
+ * Delimiter used to serialise the `in` / `notIn` value list into the single
+ * `value` string that the filter model carries.
+ *
+ * This has to be single-line safe: the tool panel and the filter menu show the
+ * serialised list in a plain `<input type="text">`, and the HTML input value
+ * sanitiser STRIPS newlines - so a newline-joined list came back out of the DOM
+ * as one run-together token. `splitInTokens` accepts newlines too, so values
+ * serialised by older builds still parse.
+ */
+export const IN_TOKEN_SEP = ', '
+
+/**
+ * Split a serialised `in` / `notIn` value into its individual tokens.
+ *
+ * The grammar is CSV-shaped: comma OR newline separates, and a token may be
+ * double-quoted to carry a separator literally (`""` escapes a quote inside
+ * one). Quoting is what makes a comma separator safe - facet values commonly
+ * contain commas of their own, because numeric bucket labels are built with
+ * `toLocaleString` ("1,234 - 5,678") and date labels with a short month
+ * ("Aug 17, 2026"). A quote only opens a token when it is the token's first
+ * non-space character, so an unquoted `5" pipe` stays literal.
+ */
+export function splitInTokens(value: unknown): string[] {
+  const { tokens, trailing } = scanInTokens(value)
+  if (trailing) tokens.push(trailing)
+  return tokens
+}
+
+/**
+ * Serialise a list of tokens back into the single `in` / `notIn` value,
+ * quoting any token that carries a separator or a quote of its own.
+ */
+export function joinInTokens(tokens: ReadonlyArray<string>): string {
+  return tokens
+    .map((t) => t.trim())
+    .filter((t) => t.length > 0)
+    .map((t) => (/["\n,]/.test(t) ? `"${t.replace(/"/g, '""')}"` : t))
+    .join(IN_TOKEN_SEP)
+}
+
+/**
+ * The token still being typed at the end of an `in` / `notIn` value - the text
+ * after the last separator. `'AAPL, MS'` -> `'MS'`; a value ending in a
+ * separator (or empty) has no trailing token.
+ *
+ * Inputs that hold the whole token list use this to drive the value-suggestion
+ * dropdown: the trailing fragment is the search query, not a committed token.
+ */
+export function trailingInToken(value: unknown): string {
+  return scanInTokens(value).trailing
+}
+
+/**
+ * One pass over a serialised `in` / `notIn` value, splitting it into the
+ * tokens that are terminated by a separator and the unterminated fragment at
+ * the end. Every token helper above is a view onto this, so they can never
+ * disagree about where a quoted token starts or ends.
+ */
+function scanInTokens(value: unknown): { tokens: string[]; trailing: string } {
+  const raw = String(value ?? '')
+  const tokens: string[] = []
+  let buf = ''
+  let quoted = false
+  let wasQuoted = false
+  let closed = false
+
+  // A quoted token keeps its inner spacing; an unquoted one is trimmed.
+  const take = () => {
+    const t = wasQuoted ? buf : buf.trim()
+    buf = ''
+    wasQuoted = false
+    closed = false
+    return t
+  }
+
+  for (let i = 0; i < raw.length; i += 1) {
+    const ch = raw[i]
+    if (quoted) {
+      if (ch !== '"') { buf += ch; continue }
+      // A doubled quote is an escaped literal quote, not the closing one.
+      if (raw[i + 1] === '"') { buf += '"'; i += 1; continue }
+      quoted = false
+      closed = true
+      continue
+    }
+    if (ch === ',' || ch === '\n') {
+      const t = take()
+      if (t.length) tokens.push(t)
+      continue
+    }
+    // Only opens a quoted token at the token's start - `5" pipe` is literal.
+    if (ch === '"' && !closed && !buf.trim()) {
+      quoted = true
+      wasQuoted = true
+      buf = ''
+      continue
+    }
+    buf += ch
+  }
+
+  return { tokens, trailing: take() }
+}
+
+// Re-exported here so consumers can reach the catalogue through the
+// `@svgrid/grid/filtering` subpath. Importing it from the package root would
+// pull in SvGrid.svelte, which is a runtime cost (and a bundler hazard) for
+// anything that only wants operator semantics.
+export {
+  ALL_FILTER_OPERATORS,
+  SET_OPERATOR_IDS,
+  VALUELESS_OPERATOR_IDS,
+  RANGE_OPERATOR_IDS,
+  isSetOperator,
+  isValuelessOperator,
+  isRangeOperator,
+  type FilterValueType,
+} from './filter-operator-catalogue.js'
+
+/** A filter with its needle-side work already done. Call per row. */
+export type CompiledExcelFilter = (cellValue: unknown) => boolean
+
+/**
+ * Compile a filter once, then test many rows against it.
+ *
+ * Everything that depends only on the FILTER - folding the needle, splitting
+ * `in` tokens, building the regex, coercing range endpoints - happens here,
+ * so the per-row closure does the minimum. Filtering 100k rows used to redo
+ * all of it 100k times.
+ *
+ * `applyExcelFilter` is defined in terms of this, so there is exactly one
+ * copy of the operator semantics.
+ */
+export function compileExcelFilter(
+  filter: ExcelFilter,
+  options?: ExcelFilterOptions,
+): CompiledExcelFilter {
+  const locale = options?.locale
+  const fold = (v: unknown) => normalizeForFilter(String(v ?? ''), locale)
+  const needle = fold(filter.value)
+
+  switch (filter.operator) {
+    case 'contains':
+      return (cellValue) => fold(cellValue).includes(needle)
+    case 'notContains':
+      // An empty needle is no constraint (mirrors `contains` returning true),
+      // so nothing is excluded until the user types something.
+      if (needle === '') return () => true
+      return (cellValue) => !fold(cellValue).includes(needle)
+    case 'equals':
+      return (cellValue) => fold(cellValue) === needle
+    case 'notEquals':
+      if (needle === '') return () => true
+      return (cellValue) => fold(cellValue) !== needle
+    case 'startsWith':
+      return (cellValue) => fold(cellValue).startsWith(needle)
+    case 'endsWith':
+      return (cellValue) => fold(cellValue).endsWith(needle)
+    case 'regex': {
+      // Case-insensitive by default (matches the accent/case-folded feel of
+      // the other text operators). An invalid pattern matches nothing rather
+      // than throwing, so a half-typed regex never crashes the row model.
+      // Note this tests the RAW text, not the folded text.
+      const pattern = String(filter.value ?? '')
+      if (!pattern) return () => true
+      let re: RegExp
+      try {
+        re = new RegExp(pattern, 'i')
+      } catch {
+        return () => false
+      }
+      return (cellValue) => re.test(String(cellValue ?? ''))
+    }
+    case 'in':
+    case 'notIn': {
+      const tokens = splitInTokens(filter.value)
+      if (tokens.length === 0) return () => true
+      // A Set of pre-folded tokens turns the old per-row `tokens.some(...)`
+      // (which re-folded every token for every row) into one lookup.
+      const wanted = new Set(tokens.map((t) => normalizeForFilter(t, locale)))
+      if (filter.operator === 'in') return (cellValue) => wanted.has(fold(cellValue))
+      return (cellValue) => !wanted.has(fold(cellValue))
+    }
+    case 'greaterThan': {
+      const b = Number(filter.value)
+      const bFinite = Number.isFinite(b)
+      const bText = String(filter.value ?? '')
+      return (cellValue) => {
+        const a = Number(cellValue)
+        if (bFinite && Number.isFinite(a)) return a > b
+        return String(cellValue ?? '') > bText
+      }
+    }
+    case 'lessThan': {
+      const b = Number(filter.value)
+      const bFinite = Number.isFinite(b)
+      const bText = String(filter.value ?? '')
+      return (cellValue) => {
+        const a = Number(cellValue)
+        if (bFinite && Number.isFinite(a)) return a < b
+        return String(cellValue ?? '') < bText
+      }
+    }
+    case 'between': {
+      // Two paths: numeric (coerce both endpoints with the historical
+      // `?? 0` fallback) and string (used when EITHER endpoint is a
+      // non-numeric string like an ISO date). Numeric wins when its
+      // operands are finite so callers passing numeric ages, prices, etc.
+      // see the historical inclusive-range behaviour.
+      const lo = filter.value   == null ? 0 : Number(filter.value)
+      const hi = filter.valueTo == null ? 0 : Number(filter.valueTo)
+      const rangeFinite = Number.isFinite(lo) && Number.isFinite(hi)
+      // Either endpoint is non-numeric (ISO date string, etc). Compare
+      // lexicographically - YYYY-MM-DD orders chronologically so the
+      // result matches user intent.
+      const sl = String(filter.value ?? '')
+      const sh = String(filter.valueTo ?? '')
+      return (cellValue) => {
+        const a = Number(cellValue ?? 0)
+        if (rangeFinite && Number.isFinite(a)) return a >= lo && a <= hi
+        const s = String(cellValue ?? '')
+        return s >= sl && s <= sh
+      }
+    }
+    case 'isBlank':
+      return (cellValue) => String(cellValue ?? '').trim().length === 0
+    case 'isNotBlank':
+      return (cellValue) => String(cellValue ?? '').trim().length > 0
+  }
+}
+
+export function applyExcelFilter(
+  cellValue: unknown,
+  filter: ExcelFilter,
+  options?: ExcelFilterOptions,
+) {
+  return compileExcelFilter(filter, options)(cellValue)
+}