@tstdl/base 0.93.28 → 0.93.29

package/orm/query/parade.d.ts

@@ -0,0 +1,172 @@
+import type { SQL } from 'drizzle-orm';
+import type { LiteralUnion, UnionToIntersection } from 'type-fest';
+import type { Record } from '../../types/index.js';
+import type { BaseEntity } from '../entity.js';
+import type { TargetColumnPath } from '../repository.types.js';
+/** Represents a comparison query using various operators like $eq, $ne, $gt, $in, etc. */
+export type ParadeComparisonQuery = Partial<ComparisonParadeQuery>;
+/** Union of keys representing comparison query operators. */
+export type ParadeComparisonQueryTypes = keyof ParadeComparisonQuery;
+/** Array containing all valid comparison query operator keys. */
+export declare const paradeComparisonQueryTypes: ParadeComparisonQueryTypes[];
+/** Represents specialized query types beyond simple comparisons. */
+export type ParadeSpecialQuery<T extends BaseEntity = BaseEntity> = ParadeSearchQuery<T> | {
+    $parade: ParadeSpecialQueryObject<T>;
+};
+/** Union of keys representing special query operators. */
+export type ParadeSpecialQueryTypes = keyof UnionToIntersection<ParadeSpecialQuery>;
+/** Array containing all valid special query operator keys. */
+export declare const paradeSpecialQueryTypes: ParadeSpecialQueryTypes[];
+/** Represents a ParadeDB query on a specific field. */
+export type ComparisonParadeQuery = {
+    $parade: ParadeComparisonQueryObject;
+};
+/** Represents a term in a proximity query. */
+export type ParadeProximityTerm = string | {
+    regex: string;
+    maxExpansions?: number;
+} | {
+    array: readonly (string | {
+        regex: string;
+        maxExpansions?: number;
+    })[];
+};
+/**
+ * Represents a bound for a ParadeDB range query.
+ */
+type ParadeRangeBound<V> = {
+    included: V;
+} | {
+    excluded: V;
+};
+/** A recursive type representing the rich ParadeDB / Tantivy query DSL for FIELD-LEVEL queries. */
+export type ParadeComparisonQueryObject = {
+    term?: {
+        value: any;
+        boost?: number;
+        const?: number;
+    };
+    fuzzyTerm?: {
+        value: string;
+        distance?: number;
+        transpositionCostOne?: boolean;
+        prefix?: boolean;
+        boost?: number;
+        const?: number;
+    };
+    regex?: {
+        pattern: string;
+        boost?: number;
+        const?: number;
+    };
+    phrase?: {
+        phrases: readonly string[];
+        slop?: number;
+        boost?: number;
+        const?: number;
+    };
+    match?: {
+        value: string;
+        distance?: number;
+        prefix?: boolean;
+        conjunctionMode?: boolean;
+        tokenizer?: ParadeTokenizerObject;
+        transpositionCostOne?: boolean;
+        boost?: number;
+        const?: number;
+    };
+    exists?: null;
+    range?: {
+        lowerBound?: ParadeRangeBound<any> | null;
+        upperBound?: ParadeRangeBound<any> | null;
+        isDatetime?: boolean;
+    };
+    termSet?: {
+        terms: readonly string[];
+        conjunctionMode?: boolean;
+    };
+    phrasePrefix?: {
+        phrases: readonly string[];
+        maxExpansions?: number;
+    };
+    regexPhrase?: {
+        regexes: readonly string[];
+        slop?: number;
+        maxExpansions?: number;
+    };
+    rangeTerm?: {
+        value: any;
+        operator?: 'Intersects' | 'Contains' | 'Within';
+    };
+    proximity?: {
+        ordered?: boolean;
+        distance: number;
+        terms: readonly [ParadeProximityTerm, ParadeProximityTerm, ...ParadeProximityTerm[]];
+    };
+};
+/**
+ * A type representing the rich ParadeDB / Tantivy query DSL for TOP-LEVEL queries.
+ */
+export type ParadeSpecialQueryObject<T extends BaseEntity = BaseEntity> = {
+    all?: null;
+    empty?: null;
+    moreLikeThis?: {
+        keyValue?: any;
+        document?: Record<TargetColumnPath<T>, any>;
+        fields?: readonly TargetColumnPath<T>[];
+        minDocFrequency?: number;
+        maxDocFrequency?: number;
+        minTermFrequency?: number;
+        maxQueryTerms?: number;
+        minWordLength?: number;
+        maxWordLength?: number;
+        stopWords?: readonly string[];
+    };
+};
+/**
+ * Represents a ParadeDB search using its native JSON-based query language,
+ * or a simplified multi-field match query.
+ */
+export type ParadeSearchQuery<T extends BaseEntity = BaseEntity> = {
+    $parade: {
+        /** The fields to search across. */
+        fields: readonly [TargetColumnPath<T>, ...TargetColumnPath<T>[]];
+        /** The search query string. */
+        query: string | SQL<string>;
+        /**
+         * The conjunction mode for token matching within the query string.
+         * `true` for AND (all terms must match), `false` for OR (any term can match).
+         * Defaults to `false` (OR).
+         */
+        conjunction?: boolean;
+        /**
+         * If greater than zero, enables fuzzy matching.
+         * Specifies the maximum Levenshtein distance (edit distance). Max value is 2.
+         * @default 0
+         */
+        distance?: number;
+        /**
+         * When `true` and fuzzy matching is enabled, considers transpositions as a single edit.
+         * @default true
+         */
+        transpositionCostOne?: boolean;
+        /**
+         * When `true` and fuzzy matching is enabled, allows prefix matching.
+         * @default false
+         */
+        prefix?: boolean;
+        /**
+         * The alias of a field configuration to use for the search.
+         * If specified, all fields must have this alias defined.
+         */
+        alias?: string;
+    };
+};
+/**
+ * Represents a ParadeDB tokenizer configuration object for a `match` query.
+ */
+export type ParadeTokenizerObject = {
+    type: LiteralUnion<'default' | 'whitespace' | 'raw' | 'keyword' | 'regex' | 'ngram' | 'source_code' | 'chinese_compatible' | 'chinese_lindera' | 'korean_lindera' | 'japanese_lindera' | 'jieba' | 'icu', string>;
+    [key: string]: any;
+};
+export {};

package/orm/query/parade.js

@@ -0,0 +1,4 @@
+/** Array containing all valid comparison query operator keys. */
+export const paradeComparisonQueryTypes = ['$parade'];
+/** Array containing all valid special query operator keys. */
+export const paradeSpecialQueryTypes = ['$parade'];

package/orm/repository.types.d.ts

@@ -8,7 +8,7 @@ import type { UntaggedDeep } from '../types/tagged.js';
 import type { AnyColumn, SQL, SQLWrapper } from 'drizzle-orm';
 import type { PartialDeep } from 'type-fest';
 import type { BaseEntity, Entity, EntityMetadata } from './entity.js';
-import type { FullTextSearchQuery, Query } from './query.js';
+import type { FullTextSearchQuery, Query } from './query/index.js';
 import type { TsHeadlineOptions } from './sqls.js';
 type WithSql<T> = {
     [P in keyof T]: T[P] extends Record ? WithSql<T[P]> : (T[P] | SQL);
@@ -79,13 +79,23 @@ export type HighlightOptions<T extends BaseEntity> = {
      * The source to generate the highlight from. Can be one or more property paths or a raw SQL expression.
      */
     source: TargetColumnPath<T> | SQL<string>;
+    /**
+     * If true, includes the byte offsets of highlight matches in the result.
+     * @default false
+     */
+    includePositions?: boolean;
 } & (TsHeadlineOptions | ParadeDbHighlightOptions);
 /**
- * Options for highlighting with ParadeDB (`pdb.snippet`).
+ * Options for highlighting with ParadeDB (`pdb.snippet` or `pdb.snippets`).
  */
 export type ParadeDbHighlightOptions = {
     /**
-     * Limits the number of characters in the snippet. Defaults to 150.
+     * If true, retrieves multiple snippets using `pdb.snippets`.
+     * If false (default), retrieves a single best snippet using `pdb.snippet`.
+     */
+    multiple?: boolean;
+    /**
+     * Limits the number of characters in each snippet. Defaults to 150.
      */
     maxNumChars?: number;
     /**
@@ -97,13 +107,19 @@ export type ParadeDbHighlightOptions = {
      */
     endTag?: string;
     /**
-     * Limits the number of highlighted terms returned in the snippet.
+     * The maximum number of snippets to return per document. Only used when `multiple` is true.
+     * @default 5
      */
     limit?: number;
     /**
-     * Ignores the first N highlighted terms.
+     * The number of snippets to skip before returning results. Only used when `multiple` is true.
+     * @default 0
      */
     offset?: number;
+    /**
+     * The order in which to sort the snippets. Can be 'score' (default) or 'position'. Only used when `multiple` is true.
+     */
+    sortBy?: 'score' | 'position';
 };
 /**
  * Options for the `search` method.
@@ -151,7 +167,12 @@ export type SearchOptions<T extends BaseEntity> = SimplifyObject<TypedOmit<LoadM
 export type SearchResult<T extends BaseEntity> = {
     entity: T;
     score?: number;
-    highlight?: string;
+    highlight?: string | string[];
+    /**
+     * Byte offsets of highlight matches. Format is an array of start/end pairs.
+     * e.g., `[{ start: 14, end: 19 }]`
+     */
+    highlightPositions?: [start: number, end: number][];
 };
 /**
  * Options for update operations (currently inherits from LoadOptions, primarily for ordering).

package/orm/schemas/index.d.ts

@@ -1,4 +1,5 @@
 export * from './json.js';
 export * from './numeric-date.js';
 export * from './timestamp.js';
+export * from './tsvector.js';
 export * from './uuid.js';

package/orm/schemas/index.js

@@ -1,4 +1,5 @@
 export * from './json.js';
 export * from './numeric-date.js';
 export * from './timestamp.js';
+export * from './tsvector.js';
 export * from './uuid.js';

package/orm/schemas/tsvector.d.ts

@@ -0,0 +1,12 @@
+import type { JsonPath } from '../../json-path/json-path.js';
+import { Schema, type SchemaDecoratorOptions, type SchemaPropertyDecorator, type SchemaTestOptions, type SchemaTestResult } from '../../schema/index.js';
+export declare class TsVectorSchema extends Schema<string> {
+    readonly name = "TsVector";
+    _test(_value: any, _path: JsonPath, _options: SchemaTestOptions): SchemaTestResult<string>;
+}
+export declare function tsvector(): TsVectorSchema;
+/**
+ * Defines a `tsvector` column.
+ * For automatically generated tsvector columns, use `@GeneratedTsVector()` instead.
+ */
+export declare function TsVectorProperty(options?: SchemaDecoratorOptions): SchemaPropertyDecorator;

package/orm/schemas/tsvector.js

@@ -0,0 +1,20 @@
+import { Property, Schema, SchemaError } from '../../schema/index.js';
+export class TsVectorSchema extends Schema {
+    name = 'TsVector';
+    _test(_value, _path, _options) {
+        return {
+            valid: false,
+            error: SchemaError.expectedButGot('nothing', 'value', _path, { customMessage: `TSVectorSchema does not support validation as tsvector columns are not directly mapped to application data.` })
+        };
+    }
+}
+export function tsvector() {
+    return new TsVectorSchema();
+}
+/**
+ * Defines a `tsvector` column.
+ * For automatically generated tsvector columns, use `@GeneratedTsVector()` instead.
+ */
+export function TsVectorProperty(options) {
+    return Property(tsvector(), options);
+}

package/orm/server/drizzle/schema-converter.js

@@ -1,4 +1,4 @@
-import { SQL } from 'drizzle-orm';
+import { sql, SQL } from 'drizzle-orm';
 import { toCamelCase, toSnakeCase } from 'drizzle-orm/casing';
 import { boolean, check, doublePrecision, foreignKey, index, integer, jsonb, pgSchema, primaryKey, text, unique, uniqueIndex, uuid } from 'drizzle-orm/pg-core';
 import { match, P } from 'ts-pattern';
@@ -13,16 +13,17 @@ import { decodeText, encodeUtf8 } from '../../../utils/encoding.js';
 import { enumValues } from '../../../utils/enum.js';
 import { memoize, memoizeSingle } from '../../../utils/function/memoize.js';
 import { compileDereferencer } from '../../../utils/object/dereference.js';
-import { fromEntries, objectEntries } from '../../../utils/object/object.js';
-import { assertDefined, assertDefinedPass, isArray, isDefined, isNotNullOrUndefined, isNull, isString, isUndefined } from '../../../utils/type-guards.js';
+import { fromEntries, mapObjectKeysToSnakeCase, objectEntries } from '../../../utils/object/object.js';
+import { assertDefined, assertDefinedPass, isArray, isDefined, isNotNull, isNotNullOrUndefined, isNull, isString, isUndefined } from '../../../utils/type-guards.js';
 import { resolveValueOrProvider } from '../../../utils/value-or-provider.js';
-import { bytea, numericDate, timestamp } from '../../data-types/index.js';
+import { bytea, numericDate, timestamp, tsvector } from '../../data-types/index.js';
 import { JsonSchema } from '../../schemas/json.js';
 import { NumericDateSchema } from '../../schemas/numeric-date.js';
 import { TimestampSchema } from '../../schemas/timestamp.js';
+import { TsVectorSchema } from '../../schemas/tsvector.js';
 import { UuidSchema } from '../../schemas/uuid.js';
 import { decryptBytes, encryptBytes } from '../encryption.js';
-import { convertQuery, resolveTargetColumn } from '../query-converter.js';
+import { convertQuery, resolveTargetColumn, resolveTargetColumns } from '../query-converter.js';
 const getDbSchema = memoizeSingle(pgSchema);
 export const getDrizzleTableFromType = memoize(_getDrizzleTableFromType);
 const columnDefinitionsSymbol = Symbol('columnDefinitions');
@@ -59,10 +60,13 @@ export function _getDrizzleTableFromType(type, fallbackSchemaName) {
             if (columnValue instanceof SQL) {
                 return columnValue;
             }
-            assertDefined(columnValue, 'Missing column name for index.');
+            assertDefined(columnValue, 'Missing column for index.');
             const [columnNameOrConfig, columnOrder] = isArray(columnValue) ? columnValue : [columnValue];
             const order = columnOrder ?? data.order ?? 'asc';
-            let column = isString(columnNameOrConfig) ? getColumn(table, columnNameOrConfig) : columnNameOrConfig;
+            let column = getColumn(table, columnNameOrConfig);
+            if (isDefined(data.options?.opclass)) {
+                return sql `${column} ${sql.raw(data.options.opclass)}`;
+            }
             column = column[order]();
             if (data.options?.nulls == 'first') {
                 column = column.nullsFirst();
@@ -89,16 +93,90 @@ export function _getDrizzleTableFromType(type, fallbackSchemaName) {
     function buildPrimaryKey(table) {
         const columns = primaryKeyColumnDefinitions.map((columnDefinition) => table[columnDefinition.name]);
         return primaryKey({
-            name: mergedTableReflectionData.compundPrimaryKeyName ?? getPrimaryKeyName(tableName, columns, { naming: mergedTableReflectionData.compundPrimaryKeyNaming }),
+            name: mergedTableReflectionData.compoundPrimaryKeyName ?? getPrimaryKeyName(tableName, columns, { naming: mergedTableReflectionData.compoundPrimaryKeyNaming }),
             columns,
         });
     }
+    function buildParadeCast(columnSql, paradeOptions) {
+        const { tokenizer, alias } = paradeOptions;
+        const { type, parameters: rawParameters = [], filters = {} } = isString(tokenizer) ? { type: tokenizer, parameters: [], filters: undefined } : (tokenizer ?? {});
+        const parameters = objectEntries({ alias, ...filters })
+            .filter(([_, value]) => isDefined(value))
+            .map(([key, value]) => {
+            return `'${toSnakeCase(key)}=${String(value)}'`;
+        });
+        if (isUndefined(type)) {
+            return sql `${columnSql}`;
+        }
+        if (parameters.length == 0) {
+            return sql `((${columnSql})::pdb.${sql.raw(type)})`;
+        }
+        const allParameters = [...rawParameters.map((param) => String(param)), ...parameters];
+        const parametersSql = sql.join(allParameters.map((parameter) => sql `${sql.raw(parameter)}`), sql `, `);
+        return sql `((${columnSql})::pdb.${sql.raw(type)}(${parametersSql}))`;
+    }
+    function buildParadeIndex(table) {
+        const paradeIndexData = mergedTableReflectionData.paradeIndex;
+        const propertyLevelColumns = columnDefinitions.filter((definition) => isDefined(definition.reflectionData?.paradeField));
+        if (isUndefined(paradeIndexData) && (propertyLevelColumns.length == 0)) {
+            return null;
+        }
+        const { naming, where, concurrently, columns: classLevelColumns, compoundColumns, expressions, ...indexOptions } = paradeIndexData ?? {};
+        const classLevelColumnSqls = (classLevelColumns ?? [])
+            .map((configOrProvider) => resolveValueOrProvider(configOrProvider, table))
+            .map((columnConfig) => {
+            const [columnOrSql, fieldOptions] = isArray(columnConfig) ? columnConfig : [columnConfig, undefined];
+            const resolvedColumnSql = resolveTargetColumn(columnOrSql, table, columnDefinitionsMap);
+            if (isUndefined(fieldOptions)) {
+                return resolvedColumnSql.getSQL();
+            }
+            return buildParadeCast(resolvedColumnSql, fieldOptions);
+        });
+        const compoundColumnSqls = (compoundColumns ?? []).map(([columnsOrProvider, fieldOptions]) => {
+            const columns = resolveValueOrProvider(columnsOrProvider, table);
+            const resolvedColumns = resolveTargetColumns(columns, table, columnDefinitionsMap);
+            const compoundExpression = sql.join(resolvedColumns, sql ` || ' ' || `);
+            return buildParadeCast(compoundExpression, fieldOptions);
+        });
+        const expressionSqls = (expressions ?? []).map(([expressionSqlOrProvider, paradeOptions]) => {
+            const expressionSql = resolveValueOrProvider(expressionSqlOrProvider, table);
+            return buildParadeCast(expressionSql, paradeOptions);
+        });
+        const propertyLevelColumnSqls = propertyLevelColumns
+            .map((columnDef) => {
+            const columnSql = resolveTargetColumn(columnDef, table, columnDefinitionsMap).getSQL();
+            const paradeOptions = columnDef.reflectionData.paradeField;
+            return buildParadeCast(columnSql, paradeOptions);
+        });
+        const indexName = getIndexName(tableName, 'parade', { naming });
+        const indexColumns = [
+            table.id, // this orm always uses 'id' as key field as every entity has it
+            ...classLevelColumnSqls,
+            ...propertyLevelColumnSqls,
+            ...compoundColumnSqls,
+            ...expressionSqls,
+        ];
+        let builder = index(indexName).using('bm25', ...indexColumns);
+        if (isDefined(where)) {
+            const query = convertQuery(where(table), table, columnDefinitionsMap);
+            builder = builder.where(query.inlineParams());
+        }
+        if (concurrently == true) {
+            builder = builder.concurrently();
+        }
+        builder = builder.with({
+            key_field: `'id'`,
+            ...mapObjectKeysToSnakeCase(indexOptions),
+        });
+        return builder;
+    }
     const primaryKeyColumnDefinitions = columnDefinitions.filter((columnDefinition) => columnDefinition.reflectionData?.primaryKey == true);
     const skipPrimaryKey = primaryKeyColumnDefinitions.length > 1;
     const columnEntries = columnDefinitions.map((entry) => [entry.name, entry.buildType({ skipPrimaryKey })]);
     const drizzleSchema = dbSchema.table(tableName, fromEntries(columnEntries), (drizzleTable) => {
         const table = drizzleTable;
         const indexes = tableReflectionDatas.flatMap((tableReflectionData) => tableReflectionData.index).filter(isDefined).map((data) => buildIndex(table, data));
+        const bm25Index = buildParadeIndex(table);
         const checks = tableReflectionDatas.flatMap((tableReflectionData) => tableReflectionData.checks).filter(isDefined).map((data) => check(data.name, data.builder(table)));
         return [
             ...((primaryKeyColumnDefinitions.length > 1)
@@ -130,6 +208,7 @@ export function _getDrizzleTableFromType(type, fallbackSchemaName) {
                 return constraint;
             }),
             ...indexes,
+            ...(isNotNull(bm25Index) ? [bm25Index] : []),
             ...checks,
         ];
     });
@@ -242,6 +321,7 @@ function getPostgresBaseColumn(columnName, dbSchema, schema, reflectionData, con
         .with(P.instanceOf(EnumerationSchema), (s) => getPgEnum(dbSchema, s.enumeration, context)(columnName))
         .with(P.instanceOf(JsonSchema), () => jsonb(columnName))
         .with(P.instanceOf(Uint8ArraySchema), () => bytea(columnName))
+        .with(P.instanceOf(TsVectorSchema), () => tsvector(columnName))
         .otherwise(() => {
         throw new NotSupportedError(`Schema "${schema.constructor.name}" not supported on type "${context.type.name}" for property "${context.property}"`);
     });
@@ -272,23 +352,24 @@ export function getPgEnum(schema, enumeration, context) {
 function getDefaultTableName(type) {
     return toSnakeCase(isString(type.entityName) ? type.entityName : type.name.replace(/\d+$/u, ''));
 }
-function getPrimaryKeyName(tableName, columns, options) {
-    return getIdentifier(tableName, columns, 'pk', options);
+function getPrimaryKeyName(tableName, columnsOrBaseName, options) {
+    return getIdentifier(tableName, columnsOrBaseName, 'pk', options);
 }
-function getIndexName(tableName, columns, options) {
-    return getIdentifier(tableName, columns, 'idx', options);
+function getIndexName(tableName, columnsOrBaseName, options) {
+    return getIdentifier(tableName, columnsOrBaseName, 'idx', options);
 }
-function getUniqueName(tableName, columns, options) {
-    return getIdentifier(tableName, columns, 'unique', options);
+function getUniqueName(tableName, columnsOrBaseName, options) {
+    return getIdentifier(tableName, columnsOrBaseName, 'unique', options);
 }
-function getForeignKeyName(tableName, columns, options) {
-    return getIdentifier(tableName, columns, 'fkey', options);
+function getForeignKeyName(tableName, columnsOrBaseName, options) {
+    return getIdentifier(tableName, columnsOrBaseName, 'fkey', options);
 }
-function getIdentifier(tableName, columns, suffix, options) {
-    const identifier = `${getTablePrefix(tableName, options?.naming)}_${getColumnNames(columns).join('_')}_${suffix}`;
+function getIdentifier(tableName, columnsOrBaseName, suffix, options) {
+    const middle = isString(columnsOrBaseName) ? columnsOrBaseName : getColumnNames(columnsOrBaseName).join('_');
+    const identifier = `${getTablePrefix(tableName, options?.naming)}_${middle}_${suffix}`;
     if (identifier.length > 63) {
         if (options?.naming != 'abbreviated-table') {
-            return getIdentifier(tableName, columns, suffix, { naming: 'abbreviated-table' });
+            return getIdentifier(tableName, columnsOrBaseName, suffix, { naming: 'abbreviated-table' });
         }
         throw new Error(`Identifier "${identifier}" for table "${tableName}" is too long. Maximum length is 63 characters.`);
     }

package/orm/server/query-converter.d.ts

@@ -1,6 +1,6 @@
 import { SQL, type SQLWrapper } from 'drizzle-orm';
 import type { BaseEntity } from '../entity.js';
-import type { Query, TsVectorParser, TsVectorWeight } from '../query.js';
+import type { Query, TsVectorParser, TsVectorWeight } from '../query/index.js';
 import type { TargetColumn } from '../repository.types.js';
 import type { ColumnDefinition, ExtraConfigColumnsFromType, PgTableFromType } from './types.js';
 /**
@@ -16,7 +16,7 @@ export declare function resolveTargetColumn<T extends BaseEntity>(target: Target
  * @param table The Drizzle table object.
  * @param columnDefinitionsMap A map from property names to column definitions.
  */
-export declare function resolveTargetColumns<T extends BaseEntity>(targets: (TargetColumn<T> | ColumnDefinition)[], table: PgTableFromType, columnDefinitionsMap: Map<string, ColumnDefinition>): SQLWrapper[];
+export declare function resolveTargetColumns<T extends BaseEntity>(targets: readonly (TargetColumn<T> | ColumnDefinition)[], table: PgTableFromType | ExtraConfigColumnsFromType, columnDefinitionsMap: Map<string, ColumnDefinition>): SQLWrapper[];
 /**
  * Converts a query object into a Drizzle SQL condition.
  * Recursively handles nested logical operators and maps property names to database columns.