@pythia-software/query-table-core 0.4.1 → 0.5.0

package/README.md

@@ -44,9 +44,18 @@ Text filters include `matches_regex` / `not_matches_regex`. An order term may
 set `extract: { regex: "..." }` to compare the first capture group (or the whole
 match when there is no capture group); non-matches sort as nulls.
 
+Text fields also support `length_gt`, `length_lt`, and `length_eq` with a
+non-negative integer value. Length counts Unicode code points. `length_eq` with
+`"0"` matches empty strings but excludes NULL; `length_gt` with `"0"` matches
+nonempty strings.
+
 See the [repository README](https://github.com/Pythia-Software/query-table#readme)
 for the complete schema and backend documentation.
 
 ## Computed SELECT definitions
 
 Exports include `ComputedColumnStore`, `httpComputedColumnStore`, `memoryComputedColumnStore`, `compileFormula`, `FORMULA_FUNCTIONS`, and `formulaRuntime`. Shared source definitions are versioned separately from queries; query SELECT entries use `{ field: "@computed/<id>" }`. The React package executes the interpreter in a bounded worker. Do not run user regex on the browser main thread. See the repository’s `docs/computed-columns.md` for semantics and persistence details.
+
+For case-sensitive array keys, set `filter: { arrayCaseSensitive: true }` on the
+`textarray` field. The default remains case-insensitive. This setting applies to
+local filtering and metrics and to PostgreSQL array membership.

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 /** All filter operators across both source projects, unioned. Every value can
  *  be NULL, so `is_null`/`is_not_null` are valid for every field type. */
-type FilterOp = "=" | "!=" | ">" | ">=" | "<" | "<=" | "contains" | "starts_with" | "ends_with" | "matches_regex" | "not_matches_regex" | "includes" | "is_null" | "is_not_null";
+type FilterOp = "=" | "!=" | ">" | ">=" | "<" | "<=" | "contains" | "starts_with" | "ends_with" | "matches_regex" | "not_matches_regex" | "length_gt" | "length_lt" | "length_eq" | "includes" | "is_null" | "is_not_null";
 /** A single filter predicate (a "literal" in CNF terms). `value` is always the
  *  raw string the UI captured; coercion to number/bool/date happens at
  *  apply/compile time based on the field's declared type (never on the value's
@@ -17,12 +17,21 @@ interface WhereClause {
     op: FilterOp;
     value: string;
     negated?: boolean;
+    setFilter?: SetFilterMetadata;
+}
+type SetFilterMode = "any" | "all" | "none" | "empty";
+/** Presentation-only identity; execution always uses the accompanying predicates. */
+interface SetFilterMetadata {
+    id: string;
+    mode: SetFilterMode;
+    values: string[];
 }
 /** A disjunction of predicates — the inner OR of conjunctive normal form. Every
  *  member is a plain literal; nesting is intentionally not allowed yet (CNF
  *  only), but the object shape leaves room to grow. */
 interface OrGroup {
     any: WhereClause[];
+    setFilter?: SetFilterMetadata;
 }
 /** One conjunct of the WHERE clause: either a single predicate or an OR group.
  *  `QueryState.where` is the AND of these terms, so the whole WHERE is
@@ -180,6 +189,10 @@ type FilterValues = {
     source: "freeform";
 };
 interface FilterConfig {
+    /** Opt into the native multi-value editor for textarray fields. */
+    editor?: "set";
+    /** Preserve case when matching textarray values. Default false. */
+    arrayCaseSensitive?: boolean;
     /** Whether this field is filterable. Default: backend ⇒ true, derived ⇒ false. */
     enabled?: boolean;
     /** Evaluate on the backend (predicate pushdown)? Default: backend ⇒ true.
@@ -606,4 +619,17 @@ interface ColumnPreview {
 }
 declare function groupPreview(dependencies: string[], inputs: Record<string, FormulaValue>[], results: FormulaResult[], total: number): ColumnPreview;
 
-export { AGG_OPS_BY_TYPE, AGG_OPS_NEEDING_FIELD, type AggOp, type AggregateConfig, type AggregationBucket, type AggregationClause, type AggregationRequest, type AggregationResult, type AggregationResultEntry, type Align, type ApplyResult, type BackendField, COMPUTED_PREFIX, type CellRenderer, type ColumnPreview, type ComputedCellError, type ComputedColumn, type ComputedColumnDraft, type ComputedColumnStore, type DerivedField, type DistinctValuesQuery, type DistinctValuesResult, EMPTY_QUERY, FORMULA_FUNCTIONS, type FetchRowsResult, type FieldDef, type FieldSchema, type FieldSource, type FieldStats, type FieldType, type FilterConfig, type FilterOp, type FilterValues, type FormulaDiagnostic, FormulaError, type FormulaFunction, type FormulaNode, type FormulaPlan, type FormulaResult, type FormulaType, type FormulaValue, MAX_AGGREGATIONS, MAX_GROUP_BY_FIELDS, MAX_ORDER_BY_TERMS, MAX_QUERY_LIMIT, MAX_QUERY_OFFSET, MAX_QUERY_TOKEN_LENGTH, MAX_SELECT_COLUMNS, MAX_WHERE_CLAUSES, NULLARY_OPS, OPS_BY_TYPE, type OpChoice, type OpPair, type OrGroup, type OrderByClause, type PreviewGroup, type QueryState, type RegexInspection, type RowId, type SavedQuery, type SelectColumn, type SelectConfig, type ServerQuery, type SortConfig, type StorageAdapter, type Transport, type WhereClause, type WhereTerm, aggOpAllowedForType, aggOpNeedsField, aggOpsForField, applyAggregations, applyQuery, coerceValue, compileFormula, computedFieldName, decodeQuery, encodeQuery, filterValues, formulaRuntime, groupPreview, httpComputedColumnStore, indexFields, isComputedCellError, isComputedField, isFilterable, isGroupable, isMeasurable, isNegativePredicate, isOrGroup, isPushdownFilter, isSelectable, isSortable, loadSchema, localStorageAdapter, matchesClause, memoryComputedColumnStore, memoryStorageAdapter, negateClause, normalizeQueryState, opAllowedForType, opPairsForField, opsForField, predicatesOf, queriesEqual, readFieldValue, selectedFields, toAggregationQuery, toServerQuery, validateComputedColumn };
+/** Compile an explicitly identified editor into existing CNF predicates. */
+declare function createSetFilter(field: string, mode: SetFilterMode, values: string[], id: string): WhereTerm[];
+/** Remove presentation metadata at transport boundaries, including inner literals. */
+declare function stripSetFilter(term: WhereTerm): WhereTerm;
+/** Only collapse explicitly marked, intact editor output. Never infer from predicates. */
+declare function readSetFilter(where: WhereTerm[], index: number): {
+    field: string;
+    metadata: SetFilterMetadata;
+    count: number;
+} | null;
+/** Canonicalize consumer-owned value aliases without disconnecting editor metadata. */
+declare function mapSetFilterValues(where: WhereTerm[], canonicalize: (field: string, value: string) => string): WhereTerm[];
+
+export { AGG_OPS_BY_TYPE, AGG_OPS_NEEDING_FIELD, type AggOp, type AggregateConfig, type AggregationBucket, type AggregationClause, type AggregationRequest, type AggregationResult, type AggregationResultEntry, type Align, type ApplyResult, type BackendField, COMPUTED_PREFIX, type CellRenderer, type ColumnPreview, type ComputedCellError, type ComputedColumn, type ComputedColumnDraft, type ComputedColumnStore, type DerivedField, type DistinctValuesQuery, type DistinctValuesResult, EMPTY_QUERY, FORMULA_FUNCTIONS, type FetchRowsResult, type FieldDef, type FieldSchema, type FieldSource, type FieldStats, type FieldType, type FilterConfig, type FilterOp, type FilterValues, type FormulaDiagnostic, FormulaError, type FormulaFunction, type FormulaNode, type FormulaPlan, type FormulaResult, type FormulaType, type FormulaValue, MAX_AGGREGATIONS, MAX_GROUP_BY_FIELDS, MAX_ORDER_BY_TERMS, MAX_QUERY_LIMIT, MAX_QUERY_OFFSET, MAX_QUERY_TOKEN_LENGTH, MAX_SELECT_COLUMNS, MAX_WHERE_CLAUSES, NULLARY_OPS, OPS_BY_TYPE, type OpChoice, type OpPair, type OrGroup, type OrderByClause, type PreviewGroup, type QueryState, type RegexInspection, type RowId, type SavedQuery, type SelectColumn, type SelectConfig, type ServerQuery, type SetFilterMetadata, type SetFilterMode, type SortConfig, type StorageAdapter, type Transport, type WhereClause, type WhereTerm, aggOpAllowedForType, aggOpNeedsField, aggOpsForField, applyAggregations, applyQuery, coerceValue, compileFormula, computedFieldName, createSetFilter, decodeQuery, encodeQuery, filterValues, formulaRuntime, groupPreview, httpComputedColumnStore, indexFields, isComputedCellError, isComputedField, isFilterable, isGroupable, isMeasurable, isNegativePredicate, isOrGroup, isPushdownFilter, isSelectable, isSortable, loadSchema, localStorageAdapter, mapSetFilterValues, matchesClause, memoryComputedColumnStore, memoryStorageAdapter, negateClause, normalizeQueryState, opAllowedForType, opPairsForField, opsForField, predicatesOf, queriesEqual, readFieldValue, readSetFilter, selectedFields, stripSetFilter, toAggregationQuery, toServerQuery, validateComputedColumn };

package/dist/index.js

@@ -179,6 +179,9 @@ var FILTER_OPS = /* @__PURE__ */ new Set([
   "ends_with",
   "matches_regex",
   "not_matches_regex",
+  "length_gt",
+  "length_lt",
+  "length_eq",
   "includes",
   "is_null",
   "is_not_null"
@@ -206,7 +209,16 @@ function normalizeWhereClause(item) {
   return clause;
 }
 function cloneWhereTerm(term) {
-  return isOrGroup(term) ? { any: term.any.map((c) => ({ ...c })) } : { ...term };
+  const copy = isOrGroup(term) ? { ...term, any: term.any.map((c) => ({ ...c })) } : { ...term };
+  if (term.setFilter) copy.setFilter = { ...term.setFilter, values: [...term.setFilter.values] };
+  return copy;
+}
+function normalizeSetFilter(item) {
+  if (!isRecord(item)) return void 0;
+  const id = boundedString(item.id, MAX_FIELD_NAME_LENGTH);
+  if (!id || !["any", "all", "none", "empty"].includes(String(item.mode)) || !Array.isArray(item.values)) return void 0;
+  if (item.values.length > MAX_WHERE_CLAUSES || !item.values.every((value) => typeof value === "string" && value.length <= MAX_FILTER_VALUE_LENGTH)) return void 0;
+  return { id, mode: item.mode, values: [...item.values] };
 }
 function normalizeQueryState(input, fallback = EMPTY_QUERY) {
   const raw = isRecord(input) ? input : {};
@@ -230,6 +242,7 @@ function normalizeQueryState(input, fallback = EMPTY_QUERY) {
     let literalBudget = MAX_WHERE_CLAUSES;
     for (const item of raw.where) {
       if (literalBudget <= 0) break;
+      const start = where.length;
       if (isRecord(item) && Array.isArray(item.any)) {
         const members = [];
         for (const inner of item.any) {
@@ -249,6 +262,10 @@ function normalizeQueryState(input, fallback = EMPTY_QUERY) {
           literalBudget--;
         }
       }
+      if (where.length > start && isRecord(item)) {
+        const metadata = normalizeSetFilter(item.setFilter);
+        if (metadata) where[start].setFilter = metadata;
+      }
     }
   } else {
     where.push(...fallback.where.map(cloneWhereTerm));
@@ -487,6 +504,12 @@ var COMPLEMENT_OP = {
   not_matches_regex: "matches_regex"
 };
 function negateClause(clause, allowedOps) {
+  if (clause.value === "0" && !clause.negated) {
+    const zeroComplement = clause.op === "length_gt" ? "length_eq" : clause.op === "length_eq" ? "length_gt" : null;
+    if (zeroComplement && (!allowedOps || allowedOps.includes(zeroComplement))) {
+      return { field: clause.field, op: zeroComplement, value: "0" };
+    }
+  }
   const complement = COMPLEMENT_OP[clause.op];
   if (complement && (!allowedOps || allowedOps.includes(complement))) {
     const next2 = { field: clause.field, op: complement, value: clause.value };
@@ -499,7 +522,7 @@ function negateClause(clause, allowedOps) {
 function isNegativePredicate(clause) {
   return Boolean(clause.negated) || clause.op === "!=" || clause.op === "not_matches_regex" || clause.op === "is_not_null";
 }
-var POSITIVE_OP_ORDER = ["=", ">", ">=", "contains", "starts_with", "ends_with", "matches_regex", "includes"];
+var POSITIVE_OP_ORDER = ["=", ">", ">=", "contains", "starts_with", "ends_with", "length_gt", "length_lt", "length_eq", "matches_regex", "includes"];
 function opPairsForField(field) {
   const ops = opsForField(field);
   const allow = new Set(ops);
@@ -523,6 +546,9 @@ var OPS_BY_TYPE = {
     "ends_with",
     "matches_regex",
     "not_matches_regex",
+    "length_gt",
+    "length_lt",
+    "length_eq",
     ...NULLITY
   ],
   enum: ["=", "!=", ...NULLITY],
@@ -597,6 +623,51 @@ function isGroupable(field) {
   return field.type === "enum" || field.type === "text" || field.type === "bool";
 }
 
+// src/setFilter.ts
+function createSetFilter(field, mode, values, id) {
+  const keys = [...new Set(values)];
+  if (mode !== "empty" && keys.length === 0) return [];
+  const includes = keys.map((value) => ({ field, op: "includes", value }));
+  let terms;
+  if (mode === "empty") terms = [{ field, op: "is_null", value: "" }];
+  else if (mode === "any") terms = includes.length === 1 ? includes : [{ any: includes }];
+  else if (mode === "all") terms = includes;
+  else terms = includes.map((clause) => ({ any: [{ field, op: "is_null", value: "" }, { ...clause, negated: true }] }));
+  return terms.map((term) => ({ ...term, setFilter: { id, mode, values: mode === "empty" ? [] : [...keys] } }));
+}
+function stripSetFilter(term) {
+  const strip = ({ setFilter: _metadata, ...clause }) => clause;
+  return isOrGroup(term) ? { any: term.any.map(strip) } : strip(term);
+}
+function readSetFilter(where, index) {
+  const term = where[index];
+  const metadata = term?.setFilter;
+  if (!term || !metadata || where[index - 1]?.setFilter?.id === metadata.id) return null;
+  const field = predicatesOf(term)[0]?.field;
+  if (!field) return null;
+  const expected = createSetFilter(field, metadata.mode, metadata.values, metadata.id);
+  if (expected.length === 0) return null;
+  for (let offset = 0; offset < expected.length; offset++) {
+    const actual = where[index + offset];
+    if (!actual || JSON.stringify(actual.setFilter) !== JSON.stringify(expected[offset].setFilter) || JSON.stringify(stripSetFilter(actual)) !== JSON.stringify(stripSetFilter(expected[offset]))) return null;
+  }
+  if (where[index + expected.length]?.setFilter?.id === metadata.id) return null;
+  return { field, metadata, count: expected.length };
+}
+function mapSetFilterValues(where, canonicalize) {
+  const result = [];
+  for (let index = 0; index < where.length; index++) {
+    const set = readSetFilter(where, index);
+    if (!set) {
+      result.push(where[index]);
+      continue;
+    }
+    result.push(...createSetFilter(set.field, set.metadata.mode, set.metadata.values.map((value) => canonicalize(set.field, value)), set.metadata.id));
+    index += set.count - 1;
+  }
+  return result;
+}
+
 // src/encode.ts
 function toBase64Url(json) {
   let b64;
@@ -667,7 +738,7 @@ function toServerQuery(q, schema) {
   q = normalizeQueryState(q);
   const byName = indexFields(schema);
   const resolveField2 = (name) => resolveFieldName(schema, name) ?? name;
-  const where = q.where.filter((term) => termIsPushdown(term, byName, resolveField2));
+  const where = q.where.filter((term) => termIsPushdown(term, byName, resolveField2)).map(stripSetFilter);
   const orderBy = [];
   for (const term of q.orderBy) {
     const field = resolveField2(term.field);
@@ -705,7 +776,7 @@ function toAggregationQuery(q, schema) {
   q = normalizeQueryState(q);
   const byName = indexFields(schema);
   const resolveField2 = (name) => resolveFieldName(schema, name) ?? name;
-  const where = q.where.filter((term) => termIsPushdown(term, byName, resolveField2));
+  const where = q.where.filter((term) => termIsPushdown(term, byName, resolveField2)).map(stripSetFilter);
   const isBackend = (name) => {
     if (name == null) return true;
     const field = resolveField2(name);
@@ -867,6 +938,9 @@ function matchesWith(byName, row, prepared) {
   if (clause.value === "" && clause.op !== "=" && clause.op !== "!=" && clause.op !== "is_null" && clause.op !== "is_not_null") {
     return true;
   }
+  if (clause.op === "length_gt" || clause.op === "length_lt" || clause.op === "length_eq") {
+    if (!/^(0|[1-9]\d*)$/.test(clause.value) || !Number.isSafeInteger(Number(clause.value))) return false;
+  }
   const base = matchesBase(field, v, prepared);
   if (!clause.negated) return base;
   const isNull = Array.isArray(v) ? v.length === 0 : v == null || v === "";
@@ -876,10 +950,11 @@ function matchesBase(field, v, prepared) {
   const { clause, regex } = prepared;
   if (clause.op === "is_null") return Array.isArray(v) ? v.length === 0 : v == null || v === "";
   if (clause.op === "is_not_null") return Array.isArray(v) ? v.length > 0 : v != null && v !== "";
+  const arrayText = (value) => field.filter?.arrayCaseSensitive ? String(value) : String(value).toLowerCase();
   if (clause.op === "includes") {
     const arr = Array.isArray(v) ? v : [];
-    const needle = clause.value.toLowerCase();
-    return arr.some((x) => String(x).toLowerCase() === needle);
+    const needle = arrayText(clause.value);
+    return arr.some((x) => arrayText(x) === needle);
   }
   if (clause.op === "matches_regex" || clause.op === "not_matches_regex") {
     if (v == null || Array.isArray(v)) return false;
@@ -887,9 +962,16 @@ function matchesBase(field, v, prepared) {
     const matches = regex.test(String(v));
     return clause.op === "matches_regex" ? matches : !matches;
   }
+  if (clause.op === "length_gt" || clause.op === "length_lt" || clause.op === "length_eq") {
+    if (typeof v !== "string" || !/^(0|[1-9]\d*)$/.test(clause.value)) return false;
+    const limit = Number(clause.value);
+    if (!Number.isSafeInteger(limit)) return false;
+    const length = Array.from(v).length;
+    return clause.op === "length_gt" ? length > limit : clause.op === "length_lt" ? length < limit : length === limit;
+  }
   if (Array.isArray(v)) {
-    const needle = clause.value.toLowerCase();
-    const hay = v.map((x) => String(x).toLowerCase());
+    const needle = arrayText(clause.value);
+    const hay = v.map(arrayText);
     switch (clause.op) {
       case "contains":
         return hay.some((s) => s.includes(needle));
@@ -2239,6 +2321,7 @@ export {
   coerceValue,
   compileFormula,
   computedFieldName,
+  createSetFilter,
   decodeQuery,
   encodeQuery,
   filterValues,
@@ -2258,6 +2341,7 @@ export {
   isSortable,
   loadSchema,
   localStorageAdapter,
+  mapSetFilterValues,
   matchesClause,
   memoryComputedColumnStore,
   memoryStorageAdapter,
@@ -2269,7 +2353,9 @@ export {
   predicatesOf,
   queriesEqual,
   readFieldValue,
+  readSetFilter,
   selectedFields,
+  stripSetFilter,
   toAggregationQuery,
   toServerQuery,
   validateComputedColumn