@dcupl/common 2.0.0-beta.2 → 2.0.0-beta.21

package/dist/esm/types/model.types.d.ts

@@ -1,10 +1,28 @@
-import { AggregationOptions, DcuplGlobalQueryOptions, DcuplQuery, DcuplQueryGroup, RawItem } from '.';
-import { PivotOptions } from '../pivot';
+import { AggregationOptions, DcuplGlobalQueryOptions, DcuplQuery, DcuplQueryGroup, RawItem } from './index.js';
+import { KeyPropertySpec } from './key-property.types.js';
+import { PivotOptions } from '../pivot.js';
 type AggregationOptionsModel = Omit<AggregationOptions, 'attribute'>;
 export type PropertyType = 'any' | 'Array<date>' | 'Array<float>' | 'Array<int>' | 'Array<string>' | 'boolean' | 'date' | 'float' | 'int' | 'json' | 'string';
 export declare const PropertyTypesAsArray: PropertyType[];
 export declare const NumberPropertyTypes: PropertyType[];
 export declare const DatePropertyTypes: PropertyType[];
+/**
+ * Configures how property auto-generation inspects input data.
+ *
+ * - `true` / `false`: enable/disable auto-generation. When enabled with `true`,
+ *   only the first row of input data is inspected (fast, default behavior).
+ * - Object form lets you scan more rows:
+ *   - `deep: true` walks every row and reconciles types across them.
+ *   - `sampleSize: N` walks at most N rows (implies deep). Useful for large datasets
+ *     where row 1 alone may not represent the schema.
+ *
+ * When more than one row is scanned, empty values (`null`, `undefined`, `''`)
+ * are ignored during inference so a single blank cell does not poison a column's type.
+ */
+export type AutoGeneratePropertiesOption = boolean | {
+    deep?: boolean;
+    sampleSize?: number;
+};
 /**
  *  must be singleValued or  multiValued
  * singleValued contains only one key to an object
@@ -308,8 +326,19 @@ export type ValueMappingConfig = {
 };
 export type ModelAttribute = Property | Reference;
 export type ModelQualityConfig = {
+    /**
+     * Master switch for quality checks on this model. Default: true.
+     * Checks only run when error tracking is also enabled globally
+     * (two-tier enablement).
+     */
     enabled?: boolean;
-    attributes?: AttributeQualityConfig;
+    /**
+     * Model-wide defaults for the attribute quality FLAGS
+     * (required / nullable / forceStrictDataType / validatorHandling),
+     * merged under each attribute's own `quality`.
+     * Concrete `validators` are per-attribute only and cannot be set here.
+     */
+    attributes?: Omit<AttributeQualityConfig, 'validators'>;
 };
 export type ValidatorErrorType = 'loose' | 'strict';
 export type ValidatorConfig = {
@@ -333,6 +362,18 @@ export type AttributeValidatorConfig = {
     unique?: ValidatorConfig;
     startsWith?: ValidatorConfig;
 };
+/**
+ * Quality rules for a single property or reference.
+ *
+ * Defaults: required: true, nullable: false, forceStrictDataType: false,
+ * validatorHandling: 'loose'.
+ *
+ * validatorHandling decides what happens when a validator fails:
+ * 'loose' (default) reports an InvalidValidator error and keeps the value;
+ * 'strict' reports AND drops the value from the loaded data.
+ * forceStrictDataType: true disables type coercion — wrong-typed raw
+ * values are reported as WrongDataType and dropped.
+ */
 export type AttributeQualityConfig = {
     required?: boolean;
     nullable?: boolean;
@@ -355,9 +396,17 @@ export type ModelDefinition<ModelNames extends string = string> = {
     properties?: Property[];
     references?: Reference[];
     data?: RawItem[];
-    keyProperty?: string;
+    /**
+     * Property name(s) on each row that uniquely identify it.
+     * See `KeyPropertySpec` for composite-key behavior.
+     */
+    keyProperty?: KeyPropertySpec;
+    /**
+     * Separator joining composite `keyProperty` parts. Default `'::'`.
+     */
+    keyPropertySeparator?: string;
     autoGenerateKey?: boolean;
-    autoGenerateProperties?: boolean;
+    autoGenerateProperties?: AutoGeneratePropertiesOption;
     supportsAutoCreation?: boolean;
     valueMappings?: ValueMappingConfig[];
     meta?: ModelMetadata;

package/dist/esm/types/quality.types.d.ts

@@ -1,6 +1,64 @@
 export declare namespace QualityAnalyzer {
     type ModelErrorGroup = 'ReferenceDataError' | 'PropertyDataError' | 'DataContainerError' | 'ModelDefinitionError';
-    type ModelErrorType = 'UndefinedAttribute' | 'InvalidValidator' | 'UndefinedValue' | 'NullValue' | 'WrongDataType' | 'NonUniqueKey' | 'MissingModel' | 'MissingProperty' | 'MissingReference' | 'UnknownExpressionVariable' | 'RemoteReferenceKeyNotFound' | 'InvalidModelDefinition';
+    /**
+     * Error types emitted by the quality controller during model registration
+     * and data ingest.
+     *
+     * - `UndefinedAttribute`  — A declared attribute has no corresponding column /
+     *   field in an ingested data container. Fires once per (model, attribute).
+     * - `InvalidValidator`   — A validator on a property could not be parsed or
+     *   evaluated against the data.
+     * - `UndefinedValue`     — A required property has no value on a specific
+     *   data entry (the cell is `undefined`). Fires per-row.
+     * - `NullValue`          — A non-nullable property has an explicit `null` on
+     *   a specific data entry. Fires per-row.
+     * - `WrongDataType`      — A value present on a data entry could not be
+     *   coerced to the declared property type. Fires per-row. Also covers *lossy*
+     *   numeric coercions where coercion succeeds but silently drops information —
+     *   trailing/embedded non-numeric content (e.g. `"1.2 lbs"` → `1.2`) or
+     *   fractional truncation for `int` properties (e.g. `"1.2"` → `1`). Lossy
+     *   records carry `meta.lossy: true` plus `meta.rawValue` and
+     *   `meta.coercedValue`; the coerced value is still kept on the data entry.
+     *   Representation-only differences that preserve the value (leading/trailing
+     *   zeros, surrounding whitespace, `"4.0"` → `4`) are not flagged.
+     *   Aggregation (#197): when one or more cells of a typed column fail to coerce,
+     *   the per-row signals are consolidated into a single `WrongDataType` per
+     *   `(model, attribute)` with `meta.rowCount` = the number of failing rows
+     *   (exact and cap-independent for loader/CSV sources; bounded by
+     *   `maxErrorsPerGroup` for inline data). `getErrorCounts()` reports the stored
+     *   *record* count (1 for an aggregated column), which is intentionally distinct
+     *   from `meta.rowCount`. A column entirely absent from the source still
+     *   produces `UndefinedAttribute`, not `WrongDataType`.
+     * - `NonUniqueKey`       — Two or more data entries share the same key value
+     *   in the same model; later rows are dropped.
+     * - `MissingKey`         — A data row has no value for the model's key
+     *   attribute, so it cannot be identified; the row is dropped on ingest.
+     * - `DuplicatePropertyKey` — A model definition declares the same property
+     *   key more than once; the first definition is kept, the duplicate ignored.
+     * - `MissingModel`       — A reference points to a model key that has not
+     *   been registered.
+     * - `MissingProperty`    — Emitted only during model-definition validation
+     *   when a *derived property* declares `derive.remoteProperty` pointing to a
+     *   property that does not exist on the remote model. Not used in any
+     *   CSV/JSON data-ingest path — those are covered by `UndefinedAttribute`,
+     *   `UndefinedValue`, and `WrongDataType`. See model.helper.ts (derived-
+     *   property validator) for the sole emit site.
+     * - `MissingReference`   — A derived property's `derive.localReference` names
+     *   a reference that does not exist on the local model.
+     * - `UnknownExpressionVariable` — An expression property references a
+     *   variable that is neither a property nor a reference on the local model.
+     * - `RemoteReferenceKeyNotFound` — A foreign-key value does not match any
+     *   key in the referenced remote model. Fires per-row.
+     * - `InvalidModelDefinition` — Catch-all for structural problems detected
+     *   when registering a model.
+     * - `UnknownColumn`          — A CSV (or otherwise source-tracked) data
+     *   container contains a column / attribute that is not declared on the
+     *   target model. Values for the column are silently dropped during ingest;
+     *   this error surfaces that data loss without failing the load. Fires once
+     *   per (model, attribute). Only emitted when the loader registers source
+     *   metadata for the container (so inline `model.data` is unaffected).
+     */
+    type ModelErrorType = 'UndefinedAttribute' | 'InvalidValidator' | 'UndefinedValue' | 'NullValue' | 'WrongDataType' | 'NonUniqueKey' | 'MissingKey' | 'DuplicatePropertyKey' | 'MissingModel' | 'MissingProperty' | 'MissingReference' | 'UnknownExpressionVariable' | 'RemoteReferenceKeyNotFound' | 'InvalidModelDefinition' | 'UnknownColumn';
     type DcuplErrorType = 'model' | 'loader';
     type DcuplErrorBase = {
         key?: string;
@@ -10,6 +68,34 @@ export declare namespace QualityAnalyzer {
         type: DcuplErrorType;
         description?: string;
         meta?: any;
+        /**
+         * Exact number of occurrences in this error's (model, attribute, errorType)
+         * group, stamped when the error set is published (#204). Can exceed the
+         * number of stored records when the group hit `maxErrorsPerGroup`.
+         */
+        groupTotal?: number;
+        /** True when this error's group holds fewer stored records than `groupTotal`. */
+        truncated?: boolean;
+    };
+    /**
+     * Per-group error totals, independent of the `maxErrorsPerGroup` record cap
+     * (#204). Available via `dcupl.quality.errors.summary()` and as the
+     * `$DcuplErrorTrackingSummary` model.
+     */
+    type ErrorGroupSummary = {
+        /** `model::attribute::errorType` */
+        key: string;
+        model?: string;
+        attribute?: string;
+        errorGroup: string;
+        errorType: string;
+        type: DcuplErrorType;
+        /** Exact occurrence count. */
+        totalCount: number;
+        /** Records retained in `$DcuplErrorTrackingErrors` (≤ maxErrorsPerGroup). */
+        storedCount: number;
+        /** `totalCount > storedCount` */
+        truncated: boolean;
     };
     type InternalModelError = DcuplErrorBase & {
         type: 'model';

package/dist/esm/types/query.types.d.ts

@@ -1,5 +1,5 @@
-import { ListItem } from './data.types';
-import { Projection, SortingProjection } from './projection.types';
+import { ListItem } from './data.types.js';
+import { Projection, SortingProjection } from './projection.types.js';
 /**
  * Transform options for query values.
  * - `lowercase`: Convert value to lowercase before comparison
@@ -7,13 +7,14 @@ import { Projection, SortingProjection } from './projection.types';
  * - `removeWhitespace`: Remove all whitespace characters
  */
 export type DcuplQueryTransformOptions = 'lowercase' | 'trim' | 'removeWhitespace';
-export declare const DcuplQueryOperatorsAsArray: readonly ["eq", "find", "gt", "gte", "lt", "lte", "typeof", "isTruthy", "size"];
+export declare const DcuplQueryOperatorsAsArray: readonly ["eq", "in", "find", "gt", "gte", "lt", "lte", "typeof", "isTruthy", "size"];
 /**
  * Query operators for filtering data.
  *
  * Available operators:
  * - `eq`: Equality check - matches exact values
- * - `find`: Partial text or object match - searches within strings or objects
+ * - `in`: Membership check - matches if the value equals any value of a list (`eq`'s equality)
+ * - `find`: Regex / exact / object match - bare string is exact-equality, `/regex/` for substring/pattern, object for property match
  * - `gt`: Greater than - numeric/date comparison (exclusive)
  * - `gte`: Greater than or equal - numeric/date comparison (inclusive)
  * - `lt`: Less than - numeric/date comparison (exclusive)
@@ -27,11 +28,14 @@ export declare const DcuplQueryOperatorsAsArray: readonly ["eq", "find", "gt", "
  * // Equality
  * { operator: 'eq', attribute: 'status', value: 'active' }
  *
+ * // Any of several values
+ * { operator: 'in', attribute: 'status', value: ['active', 'pending'] }
+ *
  * // Numeric comparison
  * { operator: 'gte', attribute: 'price', value: 100 }
  *
- * // Text search
- * { operator: 'find', attribute: 'name', value: 'search term' }
+ * // Substring search (slash-delimited regex; bare string would be exact-match)
+ * { operator: 'find', attribute: 'name', value: '/search term/' }
  * ```
  */
 export type DcuplQueryOperator = (typeof DcuplQueryOperatorsAsArray)[number];
@@ -79,7 +83,13 @@ type DcuplQueryBase = {
          */
         transform?: DcuplQueryTransformOptions[];
         /**
-         * Invert the query result (NOT operation).
+         * Invert the query result (NOT operation). Applies to every built-in operator.
+         * A missing (`undefined`/`null`) value matches neither way, except for `eq`,
+         * `in`, `typeof` and `isTruthy`, which compare against it. A list of values (an
+         * `Array<…>` property or a dotted path through a multi-valued reference) is
+         * checked per value instead: an empty list, or a `null` in it, simply does not
+         * match, so `invert` does match it (`memberships.user.daysInactive gte 60`
+         * inverted includes items with no memberships).
          * @example
          * ```typescript
          * // Find items where status is NOT 'active'
@@ -88,9 +98,14 @@ type DcuplQueryBase = {
          */
         invert?: boolean;
         /**
-         * When querying array values, determines match behavior.
+         * When querying array values, determines match behavior. Array values are
+         * `Array<…>` properties and dotted paths through a multi-valued reference
+         * (`memberships.role` compares the `role` of every membership).
          * - `some`: Match if at least one array element matches (default)
-         * - `every`: Match only if all array elements match
+         * - `every`: Match only if all array elements match — and there is at least
+         *   one: an empty array (or a reference with no items) never matches
+         *
+         * On a reference path, leaves that are `undefined` are skipped.
          *
          * @example
          * ```typescript
@@ -128,13 +143,69 @@ type DcuplQueryEq = DcuplQueryBase & {
     value: any;
 };
 /**
- * Find query - partial text or object match.
- * Searches within strings (substring match) or checks object property existence.
+ * Membership query - matches if the attribute's value equals **any** value of
+ * the list, using the same equality as `eq`: a string (or `{ key }`) matches a
+ * reference by its key and a date by its ISO string. It answers like an `or`
+ * group of one `eq` condition per listed value.
+ *
+ * - On an `Array<…>` property or a dotted path through a multi-valued
+ *   reference, `arrayValueHandling: 'some'` (default) matches if at least one
+ *   value is listed; `'every'` if every value is listed (and there is at least one).
+ * - `options.invert` is NOT IN; like `eq`, a missing value then matches.
+ * - `options.transform` applies to the attribute value and to every listed value.
+ * - `null` in the list matches `null`, like `eq` with `null`.
+ * - An empty list matches nothing (and everything under `invert`).
+ * - `value` must be an array; anything else throws. (`eq` with an array value
+ *   compares the whole array instead.)
  *
  * @example
  * ```typescript
- * // Find items containing 'search' in description
- * { operator: 'find', attribute: 'description', value: 'search' }
+ * // Items whose status is 'active' or 'pending'
+ * { operator: 'in', attribute: 'status', value: ['active', 'pending'] }
+ *
+ * // Through references (2 hops)
+ * { operator: 'in', attribute: 'customer.address.city', value: ['Vienna', 'Graz'] }
+ *
+ * // Reference keys, as strings or { key }
+ * { operator: 'in', attribute: 'category', value: ['cat-1', { key: 'cat-2' }] }
+ *
+ * // Every tag is one of the listed ones
+ * { operator: 'in', attribute: 'tags', value: ['sale', 'new'], options: { arrayValueHandling: 'every' } }
+ *
+ * // NOT IN
+ * { operator: 'in', attribute: 'status', value: ['archived', 'deleted'], options: { invert: true } }
+ * ```
+ */
+type DcuplQueryIn = DcuplQueryBase & {
+    operator: 'in';
+    /**
+     * The values to match; any one of them matching is enough.
+     * Note: For indexed search optimization, use string or { key: string } values.
+     */
+    value: unknown[];
+};
+/**
+ * Find query - regex / exact / object match.
+ *
+ * Matching depends on the shape of `value`:
+ * - **Bare string** (e.g. `'search'`) — exact-equality against the *whole* field
+ *   value, NOT a substring. `find` with `'Ball'` does not match `'Soccer Ball'`.
+ * - **Slash-delimited regex** (e.g. `'/search/'`, `'/^foo/'`) — the text between the
+ *   slashes is a JS regex; use this for substring and pattern matching. Flags go in
+ *   `options.regexFlags` (`'/^foo/'` + `regexFlags: 'i'`), not after the closing
+ *   slash: `'/^foo/i'` is a bare string and matches exactly.
+ * - **Object** (e.g. `{ key: 'value' }`) — property matching against the field.
+ *
+ * @example
+ * ```typescript
+ * // Substring search — slash-delimited regex (bare string would be exact-match)
+ * { operator: 'find', attribute: 'description', value: '/search/' }
+ *
+ * // Case-insensitive substring search
+ * { operator: 'find', attribute: 'description', value: '/search/', options: { regexFlags: 'i' } }
+ *
+ * // Exact-match the whole field value
+ * { operator: 'find', attribute: 'status', value: 'active' }
  *
  * // Find items with specific nested property
  * { operator: 'find', attribute: 'metadata', value: { key: 'value' } }
@@ -143,9 +214,16 @@ type DcuplQueryEq = DcuplQueryBase & {
 type DcuplQueryFind = DcuplQueryBase & {
     operator: 'find';
     /**
-     * Text to search for (substring) or object with properties to match.
+     * Bare string (exact-match), `/regex/` (substring/pattern), or object (property match).
      */
     value: Record<string, unknown> | string;
+    options?: {
+        /**
+         * Flags for a `/regex/` value, e.g. `'i'` for case-insensitive. Ignored for any
+         * other value; invalid flags match nothing.
+         */
+        regexFlags?: string;
+    };
 };
 /**
  * Numeric/date comparison queries.
@@ -251,7 +329,7 @@ type DcuplQuerySize = DcuplQueryBase & {
  * };
  * ```
  */
-export type DcuplQuery = DcuplQueryEq | DcuplQueryNumbers | DcuplQueryTypeOf | DcuplQueryIsTruthy | DcuplQuerySize | DcuplQueryFind;
+export type DcuplQuery = DcuplQueryEq | DcuplQueryIn | DcuplQueryNumbers | DcuplQueryTypeOf | DcuplQueryIsTruthy | DcuplQuerySize | DcuplQueryFind;
 /**
  * Query group - combines multiple conditions with AND/OR logic.
  * Groups can be nested to create complex filter expressions.
@@ -302,13 +380,24 @@ export type DcuplQueryGroup<ChildQuery extends DcuplQuery | DcuplQueryGroup = an
      */
     __type?: 'group';
     /**
-     * Combination logic for conditions in this group.
-     * Defaults to 'and' if not specified.
+     * Combination logic for conditions in this group. Set it explicitly; when omitted,
+     * the effective behaviour depends on where the group ends up:
+     * - Groups created by the query builder (`addGroup` / `setGroup` / `apply(group)` for a
+     *   group not yet present) are stored as `'or'`. When `addGroup` extends an existing keyed
+     *   group, that group keeps its current groupType.
+     * - The root query (`groupKey: 'root'`) is evaluated as `'and'`.
+     * - A nested group that reaches evaluation without a groupType (e.g. passed inside a full
+     *   query via `apply({ groupKey: 'root', ... })`, `init` or `execute`, or stored by
+     *   `setGroup` replacing an existing group) is intersected like `'and'`, with one
+     *   difference: if the running intersection becomes empty, the matches of the next
+     *   condition replace it instead of the result staying empty.
      */
     groupType?: DcuplQueryType;
     /**
      * Optional identifier for this group.
-     * Useful for tracking or removing specific query groups.
+     * Useful for tracking or removing specific query groups. Adding a group with an
+     * existing groupKey extends that group; a group without groupKey is always added
+     * as a new group.
      */
     groupKey?: string;
     /**
@@ -403,7 +492,10 @@ export type DcuplGlobalManyItemOptions<T> = {
 };
 /**
  * Options for executing queries on a list instance.
- * Combines query conditions with data retrieval options.
+ * Combines optional query conditions with data retrieval options.
+ *
+ * `queries` is optional: without it the list's own (live) query is used, so
+ * item options alone (`projection`, `start`, `count`, `sort`) are valid.
  *
  * @see ListQueryOptions - Type alias with clearer naming
  *
@@ -417,9 +509,12 @@ export type DcuplGlobalManyItemOptions<T> = {
  *   count: 10
  * };
  * const results = list.catalog.query.execute(options);
+ *
+ * // The list's own query, projected
+ * const projected = list.catalog.query.execute({ projection: { $: false, name: true } });
  * ```
  */
-export type DcuplListQueryOptions<T> = DcuplItemOptions<T> & DcuplQueryGroup;
+export type DcuplListQueryOptions<T> = DcuplItemOptions<T> & Partial<DcuplQueryGroup>;
 /**
  * Options for retrieving a specific item from a list instance.
  */

package/dist/esm/types/section.types.d.ts

@@ -1,5 +1,5 @@
-import { ListItem } from './data.types';
-import { Aggregation } from './aggregation.types';
+import { ListItem } from './data.types.js';
+import { Aggregation } from './aggregation.types.js';
 export type GroupByResponse = {
     _meta: {
         currentSize: number;

package/dist/esm/types/suggestion.types.d.ts

@@ -1,4 +1,4 @@
-import { DcuplQuery, DcuplQueryGroup, DcuplQueryTransformOptions } from './query.types';
+import { DcuplQuery, DcuplQueryGroup, DcuplQueryTransformOptions } from './query.types.js';
 type SuggestionOptionsBase = {
     attribute: string;
     value: any;
@@ -7,6 +7,11 @@ type SuggestionOptionsBase = {
     excludeUndefineds?: boolean;
     excludeNulls?: boolean;
     transform?: DcuplQueryTransformOptions[];
+    /**
+     * Flags for a `/pattern/` value, e.g. `'i'` for case-insensitive. Ignored for any
+     * other value; invalid flags suggest nothing.
+     */
+    regexFlags?: string;
 };
 type SuggestionOptionsFiltered = SuggestionOptionsBase & {
     relevantData?: 'filtered';